func (s memorySeriesStorage) AppendSample(sample model.Sample) (err error) { var ( metric = sample.Metric fingerprint = model.NewFingerprintFromMetric(metric) series, ok = s.fingerprintToSeries[fingerprint] ) if !ok { series = newStream(metric) s.fingerprintToSeries[fingerprint] = series for k, v := range metric { labelPair := fmt.Sprintf("%s%s%s", k, reservedDelimiter, v) labelPairValues := s.labelPairToFingerprints[labelPair] labelPairValues = append(labelPairValues, fingerprint) s.labelPairToFingerprints[labelPair] = labelPairValues labelNameValues := s.labelNameToFingerprints[k] labelNameValues = append(labelNameValues, fingerprint) s.labelNameToFingerprints[k] = labelNameValues } } series.add(sample.Timestamp, sample.Value) return }
// groupByFingerprint collects all of the provided samples, groups them // together by their respective metric fingerprint, and finally sorts // them chronologically. func groupByFingerprint(samples model.Samples) map[model.Fingerprint]model.Samples { var ( fingerprintToSamples = map[model.Fingerprint]model.Samples{} ) for _, sample := range samples { fingerprint := model.NewFingerprintFromMetric(sample.Metric) samples := fingerprintToSamples[fingerprint] samples = append(samples, sample) fingerprintToSamples[fingerprint] = samples } var ( sortingSemaphore = make(chan bool, sortConcurrency) doneSorting sync.WaitGroup ) for i := 0; i < sortConcurrency; i++ { sortingSemaphore <- true } for _, samples := range fingerprintToSamples { doneSorting.Add(1) go func(samples model.Samples) { <-sortingSemaphore sort.Sort(samples) sortingSemaphore <- true doneSorting.Done() }(samples) } doneSorting.Wait() return fingerprintToSamples }
func GetBoundaryValuesTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) { type value struct { year int month time.Month day int hour int value float32 } type input struct { openYear int openMonth time.Month openDay int openHour int endYear int endMonth time.Month endDay int endHour int staleness time.Duration } type output struct { open model.SampleValue end model.SampleValue } type behavior struct { name string input input output *output } var contexts = []struct { name string values []value behaviors []behavior }{ { name: "no values", values: []value{}, behaviors: []behavior{ { name: "non-existent interval without staleness policy", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(0), }, }, { name: "non-existent interval with staleness policy", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(365*24) * time.Hour, }, }, }, }, { name: "single value", values: []value{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, }, behaviors: []behavior{ { name: "on start but missing end without staleness policy", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(0), }, }, { name: "non-existent interval after within staleness policy", input: input{ openYear: 1984, openMonth: 3, openDay: 31, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(4380) * time.Hour, }, }, { name: "non-existent interval after without staleness policy", input: input{ openYear: 1984, openMonth: 3, openDay: 31, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(0), }, }, { name: "non-existent interval before with staleness policy", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 3, endDay: 29, endHour: 0, staleness: time.Duration(365*24) * time.Hour, }, }, { name: "non-existent interval before without staleness policy", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 3, endDay: 29, endHour: 0, staleness: time.Duration(0), }, }, { name: "on end but not start without staleness policy", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(0), }, }, { name: "on end but not start without staleness policy", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(365*24) * time.Hour, }, }, { name: "before point without staleness policy", input: input{ openYear: 1982, openMonth: 3, openDay: 30, openHour: 0, endYear: 1983, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(0), }, }, { name: "before point with staleness policy", input: input{ openYear: 1982, openMonth: 3, openDay: 30, openHour: 0, endYear: 1983, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(365*24) * time.Hour, }, }, { name: "after point without staleness policy", input: input{ openYear: 1985, openMonth: 3, openDay: 30, openHour: 0, endYear: 1986, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(0), }, }, { name: "after point with staleness policy", input: input{ openYear: 1985, openMonth: 3, openDay: 30, openHour: 0, endYear: 1986, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(365*24) * time.Hour, }, }, { name: "spanning point without staleness policy", input: input{ openYear: 1983, openMonth: 9, openDay: 29, openHour: 12, endYear: 1984, endMonth: 9, endDay: 28, endHour: 12, staleness: time.Duration(0), }, }, { name: "spanning point with staleness policy", input: input{ openYear: 1983, openMonth: 9, openDay: 29, openHour: 12, endYear: 1984, endMonth: 9, endDay: 28, endHour: 12, staleness: time.Duration(365*24) * time.Hour, }, }, }, }, { name: "double values", values: []value{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, }, behaviors: []behavior{ { name: "on points without staleness policy", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(0), }, output: &output{ open: 0, end: 1, }, }, { name: "on points with staleness policy", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ open: 0, end: 1, }, }, { name: "on first before second outside of staleness", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 6, endDay: 29, endHour: 6, staleness: time.Duration(2190) * time.Hour, }, }, { name: "on first before second within staleness", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 6, endDay: 29, endHour: 6, staleness: time.Duration(356*24) * time.Hour, }, }, { name: "on first after second outside of staleness", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 6, endDay: 29, endHour: 6, staleness: time.Duration(1) * time.Hour, }, }, { name: "on first after second within staleness", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 6, endDay: 29, endHour: 6, staleness: time.Duration(356*24) * time.Hour, }, output: &output{ open: 0, end: 1, }, }, }, }, } for i, context := range contexts { // Wrapping in function to enable garbage collection of resources. func() { p, closer := persistenceMaker() defer closer.Close() defer p.Close() m := model.Metric{ model.MetricNameLabel: "age_in_years", } for _, value := range context.values { testAppendSample(p, model.Sample{ Value: model.SampleValue(value.value), Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC), Metric: m, }, t) } for j, behavior := range context.behaviors { input := behavior.input open := time.Date(input.openYear, input.openMonth, input.openDay, input.openHour, 0, 0, 0, time.UTC) end := time.Date(input.endYear, input.endMonth, input.endDay, input.endHour, 0, 0, 0, time.UTC) interval := model.Interval{ OldestInclusive: open, NewestInclusive: end, } po := StalenessPolicy{ DeltaAllowance: input.staleness, } openValue, endValue, err := p.GetBoundaryValues(model.NewFingerprintFromMetric(m), interval, po) if err != nil { t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err) } if behavior.output == nil { if openValue != nil { t.Fatalf("%d.%d(%s). Expected open to be nil but got: %q\n", i, j, behavior.name, openValue) } if endValue != nil { t.Fatalf("%d.%d(%s). Expected end to be nil but got: %q\n", i, j, behavior.name, endValue) } } else { if openValue == nil { t.Fatalf("%d.%d(%s). Expected open to be %s but got nil\n", i, j, behavior.name, behavior.output) } if endValue == nil { t.Fatalf("%d.%d(%s). Expected end to be %s but got nil\n", i, j, behavior.name, behavior.output) } if openValue.Value != behavior.output.open { t.Fatalf("%d.%d(%s). Expected open to be %s but got %s\n", i, j, behavior.name, behavior.output.open, openValue.Value) } if endValue.Value != behavior.output.end { t.Fatalf("%d.%d(%s). Expected end to be %s but got %s\n", i, j, behavior.name, behavior.output.end, endValue.Value) } } } }() } }
func GetValueAtTimeTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) { type value struct { year int month time.Month day int hour int value float32 } type input struct { year int month time.Month day int hour int staleness time.Duration } type output struct { value model.SampleValue } type behavior struct { name string input input output *output } var contexts = []struct { name string values []value behaviors []behavior }{ { name: "no values", values: []value{}, behaviors: []behavior{ { name: "random target", input: input{ year: 1984, month: 3, day: 30, hour: 0, staleness: time.Duration(0), }, }, }, }, { name: "singleton", values: []value{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, }, behaviors: []behavior{ { name: "exact without staleness policy", input: input{ year: 1984, month: 3, day: 30, hour: 0, staleness: time.Duration(0), }, output: &output{ value: 0, }, }, { name: "exact with staleness policy", input: input{ year: 1984, month: 3, day: 30, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 0, }, }, { name: "before without staleness policy", input: input{ year: 1984, month: 3, day: 29, hour: 0, staleness: time.Duration(0), }, }, { name: "before within staleness policy", input: input{ year: 1984, month: 3, day: 29, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, }, { name: "before outside staleness policy", input: input{ year: 1984, month: 3, day: 29, hour: 0, staleness: time.Duration(1) * time.Hour, }, }, { name: "after without staleness policy", input: input{ year: 1984, month: 3, day: 31, hour: 0, staleness: time.Duration(0), }, }, { name: "after within staleness policy", input: input{ year: 1984, month: 3, day: 31, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 0, }, }, { name: "after outside staleness policy", input: input{ year: 1984, month: 4, day: 7, hour: 0, staleness: time.Duration(7*24) * time.Hour, }, }, }, }, { name: "double", values: []value{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, }, behaviors: []behavior{ { name: "exact first without staleness policy", input: input{ year: 1984, month: 3, day: 30, hour: 0, staleness: time.Duration(0), }, output: &output{ value: 0, }, }, { name: "exact first with staleness policy", input: input{ year: 1984, month: 3, day: 30, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 0, }, }, { name: "exact second without staleness policy", input: input{ year: 1985, month: 3, day: 30, hour: 0, staleness: time.Duration(0), }, output: &output{ value: 1, }, }, { name: "exact second with staleness policy", input: input{ year: 1985, month: 3, day: 30, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 1, }, }, { name: "before first without staleness policy", input: input{ year: 1983, month: 9, day: 29, hour: 12, staleness: time.Duration(0), }, }, { name: "before first with staleness policy", input: input{ year: 1983, month: 9, day: 29, hour: 12, staleness: time.Duration(365*24) * time.Hour, }, }, { name: "after second with staleness policy", input: input{ year: 1985, month: 9, day: 28, hour: 12, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 1, }, }, { name: "after second without staleness policy", input: input{ year: 1985, month: 9, day: 28, hour: 12, staleness: time.Duration(0), }, }, { name: "middle without staleness policy", input: input{ year: 1984, month: 9, day: 28, hour: 12, staleness: time.Duration(0), }, }, { name: "middle with insufficient staleness policy", input: input{ year: 1984, month: 9, day: 28, hour: 12, staleness: time.Duration(364*24) * time.Hour, }, }, { name: "middle with sufficient staleness policy", input: input{ year: 1984, month: 9, day: 28, hour: 12, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 0.5, }, }, }, }, { name: "triple", values: []value{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, { year: 1986, month: 3, day: 30, hour: 0, value: 2, }, }, behaviors: []behavior{ { name: "exact first without staleness policy", input: input{ year: 1984, month: 3, day: 30, hour: 0, staleness: time.Duration(0), }, output: &output{ value: 0, }, }, { name: "exact first with staleness policy", input: input{ year: 1984, month: 3, day: 30, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 0, }, }, { name: "exact second without staleness policy", input: input{ year: 1985, month: 3, day: 30, hour: 0, staleness: time.Duration(0), }, output: &output{ value: 1, }, }, { name: "exact second with staleness policy", input: input{ year: 1985, month: 3, day: 30, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 1, }, }, { name: "exact third without staleness policy", input: input{ year: 1986, month: 3, day: 30, hour: 0, staleness: time.Duration(0), }, output: &output{ value: 2, }, }, { name: "exact third with staleness policy", input: input{ year: 1986, month: 3, day: 30, hour: 0, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 2, }, }, { name: "before first without staleness policy", input: input{ year: 1983, month: 9, day: 29, hour: 12, staleness: time.Duration(0), }, }, { name: "before first with staleness policy", input: input{ year: 1983, month: 9, day: 29, hour: 12, staleness: time.Duration(365*24) * time.Hour, }, }, { name: "after third within staleness policy", input: input{ year: 1986, month: 9, day: 28, hour: 12, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 2, }, }, { name: "after third outside staleness policy", input: input{ year: 1986, month: 9, day: 28, hour: 12, staleness: time.Duration(1*24) * time.Hour, }, }, { name: "after third without staleness policy", input: input{ year: 1986, month: 9, day: 28, hour: 12, staleness: time.Duration(0), }, }, { name: "first middle without staleness policy", input: input{ year: 1984, month: 9, day: 28, hour: 12, staleness: time.Duration(0), }, }, { name: "first middle with insufficient staleness policy", input: input{ year: 1984, month: 9, day: 28, hour: 12, staleness: time.Duration(364*24) * time.Hour, }, }, { name: "first middle with sufficient staleness policy", input: input{ year: 1984, month: 9, day: 28, hour: 12, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 0.5, }, }, { name: "second middle without staleness policy", input: input{ year: 1985, month: 9, day: 28, hour: 12, staleness: time.Duration(0), }, }, { name: "second middle with insufficient staleness policy", input: input{ year: 1985, month: 9, day: 28, hour: 12, staleness: time.Duration(364*24) * time.Hour, }, }, { name: "second middle with sufficient staleness policy", input: input{ year: 1985, month: 9, day: 28, hour: 12, staleness: time.Duration(365*24) * time.Hour, }, output: &output{ value: 1.5, }, }, }, }, } for i, context := range contexts { // Wrapping in function to enable garbage collection of resources. func() { p, closer := persistenceMaker() defer closer.Close() defer p.Close() m := model.Metric{ model.MetricNameLabel: "age_in_years", } for _, value := range context.values { testAppendSample(p, model.Sample{ Value: model.SampleValue(value.value), Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC), Metric: m, }, t) } for j, behavior := range context.behaviors { input := behavior.input time := time.Date(input.year, input.month, input.day, input.hour, 0, 0, 0, time.UTC) sp := StalenessPolicy{ DeltaAllowance: input.staleness, } actual, err := p.GetValueAtTime(model.NewFingerprintFromMetric(m), time, sp) if err != nil { t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err) } if behavior.output == nil { if actual != nil { t.Fatalf("%d.%d(%s). Expected nil but got: %q\n", i, j, behavior.name, actual) } } else { if actual == nil { t.Fatalf("%d.%d(%s). Expected %s but got nil\n", i, j, behavior.name, behavior.output) } else { if actual.Value != behavior.output.value { t.Fatalf("%d.%d(%s). Expected %s but got %s\n", i, j, behavior.name, behavior.output, actual) } } } } }() } }
func GetRangeValuesTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) { type value struct { year int month time.Month day int hour int value float32 } type input struct { openYear int openMonth time.Month openDay int openHour int endYear int endMonth time.Month endDay int endHour int } type output struct { year int month time.Month day int hour int value float32 } type behavior struct { name string input input output []output } var contexts = []struct { name string values []value behaviors []behavior }{ { name: "no values", values: []value{}, behaviors: []behavior{ { name: "non-existent interval", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, }, }, }, }, { name: "singleton value", values: []value{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, }, behaviors: []behavior{ { name: "start on first value", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, }, }, { name: "end on first value", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, }, }, { name: "overlap on first value", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, }, }, }, }, { name: "two values", values: []value{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, }, behaviors: []behavior{ { name: "start on first value", input: input{ openYear: 1984, openMonth: 3, openDay: 30, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, }, }, { name: "start on second value", input: input{ openYear: 1985, openMonth: 3, openDay: 30, openHour: 0, endYear: 1986, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, }, }, { name: "end on first value", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1984, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, }, }, { name: "end on second value", input: input{ openYear: 1985, openMonth: 1, openDay: 1, openHour: 0, endYear: 1985, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, }, }, { name: "overlap on values", input: input{ openYear: 1983, openMonth: 3, openDay: 30, openHour: 0, endYear: 1986, endMonth: 3, endDay: 30, endHour: 0, }, output: []output{ { year: 1984, month: 3, day: 30, hour: 0, value: 0, }, { year: 1985, month: 3, day: 30, hour: 0, value: 1, }, }, }, }, }, } for i, context := range contexts { // Wrapping in function to enable garbage collection of resources. func() { p, closer := persistenceMaker() defer closer.Close() defer p.Close() m := model.Metric{ model.MetricNameLabel: "age_in_years", } for _, value := range context.values { testAppendSample(p, model.Sample{ Value: model.SampleValue(value.value), Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC), Metric: m, }, t) } for j, behavior := range context.behaviors { input := behavior.input open := time.Date(input.openYear, input.openMonth, input.openDay, input.openHour, 0, 0, 0, time.UTC) end := time.Date(input.endYear, input.endMonth, input.endDay, input.endHour, 0, 0, 0, time.UTC) in := model.Interval{ OldestInclusive: open, NewestInclusive: end, } values, err := p.GetRangeValues(model.NewFingerprintFromMetric(m), in) if err != nil { t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err) } if values == nil && len(behavior.output) != 0 { t.Fatalf("%d.%d(%s). Expected %s but got: %s\n", i, j, behavior.name, behavior.output, values) } if behavior.output == nil { if values != nil { t.Fatalf("%d.%d(%s). Expected nil values but got: %s\n", i, j, behavior.name, values) } } else { if len(behavior.output) != len(values.Values) { t.Fatalf("%d.%d(%s). Expected length %d but got: %d\n", i, j, behavior.name, len(behavior.output), len(values.Values)) } for k, actual := range values.Values { expected := behavior.output[k] if actual.Value != model.SampleValue(expected.value) { t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.value, actual.Value) } if actual.Timestamp.Year() != expected.year { t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.year, actual.Timestamp.Year()) } if actual.Timestamp.Month() != expected.month { t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.month, actual.Timestamp.Month()) } // XXX: Find problem here. // Mismatches occur in this and have for a long time in the LevelDB // case, however not im-memory. // // if actual.Timestamp.Day() != expected.day { // t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.day, actual.Timestamp.Day()) // } // if actual.Timestamp.Hour() != expected.hour { // t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.hour, actual.Timestamp.Hour()) // } } } } }() } }
func testMakeView(t test.Tester) { type in struct { atTime []getValuesAtTimeOp atInterval []getValuesAtIntervalOp alongRange []getValuesAlongRangeOp } type out struct { atTime [][]model.SamplePair atInterval [][]model.SamplePair alongRange [][]model.SamplePair } var ( instant = time.Date(1984, 3, 30, 0, 0, 0, 0, time.Local) metric = model.Metric{model.MetricNameLabel: "request_count"} fingerprint = model.NewFingerprintFromMetric(metric) scenarios = []struct { data []model.Sample in in out out }{ // No sample, but query asks for one. { in: in{ atTime: []getValuesAtTimeOp{ { time: instant, }, }, }, out: out{ atTime: [][]model.SamplePair{{}}, }, }, // Single sample, query asks for exact sample time. { data: []model.Sample{ { Metric: metric, Value: 0, Timestamp: instant, }, }, in: in{ atTime: []getValuesAtTimeOp{ { time: instant, }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant, Value: 0, }, }, }, }, }, // Single sample, query time before the sample. { data: []model.Sample{ { Metric: metric, Value: 0, Timestamp: instant.Add(time.Second), }, { Metric: metric, Value: 1, Timestamp: instant.Add(time.Second * 2), }, }, in: in{ atTime: []getValuesAtTimeOp{ { time: instant, }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant.Add(time.Second), Value: 0, }, }, }, }, }, // Single sample, query time after the sample. { data: []model.Sample{ { Metric: metric, Value: 0, Timestamp: instant, }, }, in: in{ atTime: []getValuesAtTimeOp{ { time: instant.Add(time.Second), }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant, Value: 0, }, }, }, }, }, // Two samples, query asks for first sample time. { data: []model.Sample{ { Metric: metric, Value: 0, Timestamp: instant, }, { Metric: metric, Value: 1, Timestamp: instant.Add(time.Second), }, }, in: in{ atTime: []getValuesAtTimeOp{ { time: instant, }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant, Value: 0, }, }, }, }, }, // Three samples, query asks for second sample time. { data: []model.Sample{ { Metric: metric, Value: 0, Timestamp: instant, }, { Metric: metric, Value: 1, Timestamp: instant.Add(time.Second), }, { Metric: metric, Value: 2, Timestamp: instant.Add(time.Second * 2), }, }, in: in{ atTime: []getValuesAtTimeOp{ { time: instant.Add(time.Second), }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant.Add(time.Second), Value: 1, }, }, }, }, }, // Three samples, query asks for time between first and second samples. { data: []model.Sample{ { Metric: metric, Value: 0, Timestamp: instant, }, { Metric: metric, Value: 1, Timestamp: instant.Add(time.Second * 2), }, { Metric: metric, Value: 2, Timestamp: instant.Add(time.Second * 4), }, }, in: in{ atTime: []getValuesAtTimeOp{ { time: instant.Add(time.Second), }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant, Value: 0, }, { Timestamp: instant.Add(time.Second * 2), Value: 1, }, }, }, }, }, // Three samples, query asks for time between second and third samples. { data: []model.Sample{ { Metric: metric, Value: 0, Timestamp: instant, }, { Metric: metric, Value: 1, Timestamp: instant.Add(time.Second * 2), }, { Metric: metric, Value: 2, Timestamp: instant.Add(time.Second * 4), }, }, in: in{ atTime: []getValuesAtTimeOp{ { time: instant.Add(time.Second * 3), }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant.Add(time.Second * 2), Value: 1, }, { Timestamp: instant.Add(time.Second * 4), Value: 2, }, }, }, }, }, // Two chunks of samples, query asks for values from first chunk. { data: buildSamples(instant, instant.Add(time.Duration(*leveldbChunkSize*2)*time.Second), time.Second, metric), in: in{ atTime: []getValuesAtTimeOp{ { time: instant.Add(time.Second*time.Duration(*leveldbChunkSize/2) + 1), }, }, }, out: out{ atTime: [][]model.SamplePair{ { { Timestamp: instant.Add(time.Second * time.Duration(*leveldbChunkSize/2)), Value: 100, }, { Timestamp: instant.Add(time.Second * (time.Duration(*leveldbChunkSize/2) + 1)), Value: 101, }, }, }, }, }, } ) for i, scenario := range scenarios { tiered, closer := newTestTieredStorage(t) defer closer.Close() for j, datum := range scenario.data { err := tiered.AppendSample(datum) if err != nil { t.Fatalf("%d.%d. failed to add fixture data: %s", i, j, err) } } tiered.Flush() requestBuilder := NewViewRequestBuilder() for _, atTime := range scenario.in.atTime { requestBuilder.GetMetricAtTime(fingerprint, atTime.time) } for _, atInterval := range scenario.in.atInterval { requestBuilder.GetMetricAtInterval(fingerprint, atInterval.from, atInterval.through, atInterval.interval) } for _, alongRange := range scenario.in.alongRange { requestBuilder.GetMetricRange(fingerprint, alongRange.from, alongRange.through) } v, err := tiered.MakeView(requestBuilder, time.Second*5) if err != nil { t.Fatalf("%d. failed due to %s", i, err) } for j, atTime := range scenario.in.atTime { actual := v.GetValueAtTime(fingerprint, atTime.time) if len(actual) != len(scenario.out.atTime[j]) { t.Fatalf("%d.%d. expected %d output, got %d", i, j, len(scenario.out.atTime[j]), len(actual)) } for k, value := range scenario.out.atTime[j] { if value.Value != actual[k].Value { t.Fatalf("%d.%d.%d expected %v value, got %v", i, j, k, value.Value, actual[k].Value) } if !value.Timestamp.Equal(actual[k].Timestamp) { t.Fatalf("%d.%d.%d expected %s timestamp, got %s", i, j, k, value.Timestamp, actual[k].Timestamp) } } } tiered.Drain() } }
func StochasticTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) { stochastic := func(x int) (success bool) { p, closer := persistenceMaker() defer closer.Close() defer p.Close() seed := rand.NewSource(int64(x)) random := rand.New(seed) numberOfMetrics := random.Intn(stochasticMaximumVariance) + 1 numberOfSharedLabels := random.Intn(stochasticMaximumVariance) numberOfUnsharedLabels := random.Intn(stochasticMaximumVariance) numberOfSamples := random.Intn(stochasticMaximumVariance) + 2 numberOfRangeScans := random.Intn(stochasticMaximumVariance) metricTimestamps := map[int]map[int64]bool{} metricEarliestSample := map[int]int64{} metricNewestSample := map[int]int64{} for metricIndex := 0; metricIndex < numberOfMetrics; metricIndex++ { sample := model.Sample{ Metric: model.Metric{}, } v := model.LabelValue(fmt.Sprintf("metric_index_%d", metricIndex)) sample.Metric[model.MetricNameLabel] = v for sharedLabelIndex := 0; sharedLabelIndex < numberOfSharedLabels; sharedLabelIndex++ { l := model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex)) v := model.LabelValue(fmt.Sprintf("label_%d", sharedLabelIndex)) sample.Metric[l] = v } for unsharedLabelIndex := 0; unsharedLabelIndex < numberOfUnsharedLabels; unsharedLabelIndex++ { l := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, unsharedLabelIndex)) v := model.LabelValue(fmt.Sprintf("private_label_%d", unsharedLabelIndex)) sample.Metric[l] = v } timestamps := map[int64]bool{} metricTimestamps[metricIndex] = timestamps var ( newestSample int64 = math.MinInt64 oldestSample int64 = math.MaxInt64 nextTimestamp func() int64 ) nextTimestamp = func() int64 { var candidate int64 candidate = random.Int63n(math.MaxInt32 - 1) if _, has := timestamps[candidate]; has { // WART candidate = nextTimestamp() } timestamps[candidate] = true if candidate < oldestSample { oldestSample = candidate } if candidate > newestSample { newestSample = candidate } return candidate } for sampleIndex := 0; sampleIndex < numberOfSamples; sampleIndex++ { sample.Timestamp = time.Unix(nextTimestamp(), 0) sample.Value = model.SampleValue(sampleIndex) err := p.AppendSample(sample) if err != nil { t.Error(err) return } } metricEarliestSample[metricIndex] = oldestSample metricNewestSample[metricIndex] = newestSample for sharedLabelIndex := 0; sharedLabelIndex < numberOfSharedLabels; sharedLabelIndex++ { labelPair := model.LabelSet{ model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex)): model.LabelValue(fmt.Sprintf("label_%d", sharedLabelIndex)), } fingerprints, err := p.GetFingerprintsForLabelSet(labelPair) if err != nil { t.Error(err) return } if len(fingerprints) == 0 { t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints)) return } labelName := model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex)) fingerprints, err = p.GetFingerprintsForLabelName(labelName) if err != nil { t.Error(err) return } if len(fingerprints) == 0 { t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints)) return } } } for sharedIndex := 0; sharedIndex < numberOfSharedLabels; sharedIndex++ { labelName := model.LabelName(fmt.Sprintf("shared_label_%d", sharedIndex)) fingerprints, err := p.GetFingerprintsForLabelName(labelName) if err != nil { t.Error(err) return } if len(fingerprints) != numberOfMetrics { t.Errorf("expected fingerprint count of %d, got %d", numberOfMetrics, len(fingerprints)) return } } for metricIndex := 0; metricIndex < numberOfMetrics; metricIndex++ { for unsharedLabelIndex := 0; unsharedLabelIndex < numberOfUnsharedLabels; unsharedLabelIndex++ { labelName := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, unsharedLabelIndex)) labelValue := model.LabelValue(fmt.Sprintf("private_label_%d", unsharedLabelIndex)) labelSet := model.LabelSet{ labelName: labelValue, } fingerprints, err := p.GetFingerprintsForLabelSet(labelSet) if err != nil { t.Error(err) return } if len(fingerprints) != 1 { t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints)) return } fingerprints, err = p.GetFingerprintsForLabelName(labelName) if err != nil { t.Error(err) return } if len(fingerprints) != 1 { t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints)) return } } metric := model.Metric{} metric[model.MetricNameLabel] = model.LabelValue(fmt.Sprintf("metric_index_%d", metricIndex)) for i := 0; i < numberOfSharedLabels; i++ { l := model.LabelName(fmt.Sprintf("shared_label_%d", i)) v := model.LabelValue(fmt.Sprintf("label_%d", i)) metric[l] = v } for i := 0; i < numberOfUnsharedLabels; i++ { l := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, i)) v := model.LabelValue(fmt.Sprintf("private_label_%d", i)) metric[l] = v } for i := 0; i < numberOfRangeScans; i++ { timestamps := metricTimestamps[metricIndex] var first int64 = 0 var second int64 = 0 for { firstCandidate := random.Int63n(int64(len(timestamps))) secondCandidate := random.Int63n(int64(len(timestamps))) smallest := int64(-1) largest := int64(-1) if firstCandidate == secondCandidate { continue } else if firstCandidate > secondCandidate { largest = firstCandidate smallest = secondCandidate } else { largest = secondCandidate smallest = firstCandidate } j := int64(0) for i := range timestamps { if j == smallest { first = i } else if j == largest { second = i break } j++ } break } begin := first end := second if second < first { begin, end = second, first } interval := model.Interval{ OldestInclusive: time.Unix(begin, 0), NewestInclusive: time.Unix(end, 0), } samples, err := p.GetRangeValues(model.NewFingerprintFromMetric(metric), interval) if err != nil { t.Error(err) return } if len(samples.Values) < 2 { t.Errorf("expected sample count less than %d, got %d", 2, len(samples.Values)) return } } } return true } if err := quick.Check(stochastic, nil); err != nil { t.Error(err) } }