Exemplo n.º 1
0
func (s memorySeriesStorage) AppendSample(sample model.Sample) (err error) {
	var (
		metric      = sample.Metric
		fingerprint = model.NewFingerprintFromMetric(metric)
		series, ok  = s.fingerprintToSeries[fingerprint]
	)

	if !ok {
		series = newStream(metric)
		s.fingerprintToSeries[fingerprint] = series

		for k, v := range metric {
			labelPair := fmt.Sprintf("%s%s%s", k, reservedDelimiter, v)

			labelPairValues := s.labelPairToFingerprints[labelPair]
			labelPairValues = append(labelPairValues, fingerprint)
			s.labelPairToFingerprints[labelPair] = labelPairValues

			labelNameValues := s.labelNameToFingerprints[k]
			labelNameValues = append(labelNameValues, fingerprint)
			s.labelNameToFingerprints[k] = labelNameValues
		}
	}

	series.add(sample.Timestamp, sample.Value)

	return
}
Exemplo n.º 2
0
// groupByFingerprint collects all of the provided samples, groups them
// together by their respective metric fingerprint, and finally sorts
// them chronologically.
func groupByFingerprint(samples model.Samples) map[model.Fingerprint]model.Samples {
	var (
		fingerprintToSamples = map[model.Fingerprint]model.Samples{}
	)

	for _, sample := range samples {
		fingerprint := model.NewFingerprintFromMetric(sample.Metric)
		samples := fingerprintToSamples[fingerprint]
		samples = append(samples, sample)
		fingerprintToSamples[fingerprint] = samples
	}

	var (
		sortingSemaphore = make(chan bool, sortConcurrency)
		doneSorting      sync.WaitGroup
	)

	for i := 0; i < sortConcurrency; i++ {
		sortingSemaphore <- true
	}

	for _, samples := range fingerprintToSamples {
		doneSorting.Add(1)

		go func(samples model.Samples) {
			<-sortingSemaphore
			sort.Sort(samples)
			sortingSemaphore <- true
			doneSorting.Done()
		}(samples)
	}

	doneSorting.Wait()

	return fingerprintToSamples
}
func GetBoundaryValuesTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
	type value struct {
		year  int
		month time.Month
		day   int
		hour  int
		value float32
	}

	type input struct {
		openYear  int
		openMonth time.Month
		openDay   int
		openHour  int
		endYear   int
		endMonth  time.Month
		endDay    int
		endHour   int
		staleness time.Duration
	}

	type output struct {
		open model.SampleValue
		end  model.SampleValue
	}

	type behavior struct {
		name   string
		input  input
		output *output
	}

	var contexts = []struct {
		name      string
		values    []value
		behaviors []behavior
	}{
		{
			name:   "no values",
			values: []value{},
			behaviors: []behavior{
				{
					name: "non-existent interval without staleness policy",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "non-existent interval with staleness policy",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
			},
		},
		{
			name: "single value",
			values: []value{
				{
					year:  1984,
					month: 3,
					day:   30,
					hour:  0,
					value: 0,
				},
			},
			behaviors: []behavior{
				{
					name: "on start but missing end without staleness policy",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "non-existent interval after within staleness policy",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   31,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(4380) * time.Hour,
					},
				},
				{
					name: "non-existent interval after without staleness policy",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   31,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "non-existent interval before with staleness policy",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  3,
						endDay:    29,
						endHour:   0,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
				{
					name: "non-existent interval before without staleness policy",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  3,
						endDay:    29,
						endHour:   0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "on end but not start without staleness policy",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "on end but not start without staleness policy",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
				{
					name: "before point without staleness policy",
					input: input{
						openYear:  1982,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1983,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "before point with staleness policy",
					input: input{
						openYear:  1982,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1983,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
				{
					name: "after point without staleness policy",
					input: input{
						openYear:  1985,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1986,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "after point with staleness policy",
					input: input{
						openYear:  1985,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1986,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
				{
					name: "spanning point without staleness policy",
					input: input{
						openYear:  1983,
						openMonth: 9,
						openDay:   29,
						openHour:  12,
						endYear:   1984,
						endMonth:  9,
						endDay:    28,
						endHour:   12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "spanning point with staleness policy",
					input: input{
						openYear:  1983,
						openMonth: 9,
						openDay:   29,
						openHour:  12,
						endYear:   1984,
						endMonth:  9,
						endDay:    28,
						endHour:   12,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
			},
		},
		{
			name: "double values",
			values: []value{
				{
					year:  1984,
					month: 3,
					day:   30,
					hour:  0,
					value: 0,
				},
				{
					year:  1985,
					month: 3,
					day:   30,
					hour:  0,
					value: 1,
				},
			},
			behaviors: []behavior{
				{
					name: "on points without staleness policy",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(0),
					},
					output: &output{
						open: 0,
						end:  1,
					},
				},
				{
					name: "on points with staleness policy",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						open: 0,
						end:  1,
					},
				},
				{
					name: "on first before second outside of staleness",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  6,
						endDay:    29,
						endHour:   6,
						staleness: time.Duration(2190) * time.Hour,
					},
				},
				{
					name: "on first before second within staleness",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  6,
						endDay:    29,
						endHour:   6,
						staleness: time.Duration(356*24) * time.Hour,
					},
				},
				{
					name: "on first after second outside of staleness",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  6,
						endDay:    29,
						endHour:   6,
						staleness: time.Duration(1) * time.Hour,
					},
				},
				{
					name: "on first after second within staleness",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  6,
						endDay:    29,
						endHour:   6,
						staleness: time.Duration(356*24) * time.Hour,
					},
					output: &output{
						open: 0,
						end:  1,
					},
				},
			},
		},
	}

	for i, context := range contexts {
		// Wrapping in function to enable garbage collection of resources.
		func() {
			p, closer := persistenceMaker()

			defer closer.Close()
			defer p.Close()

			m := model.Metric{
				model.MetricNameLabel: "age_in_years",
			}

			for _, value := range context.values {
				testAppendSample(p, model.Sample{
					Value:     model.SampleValue(value.value),
					Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC),
					Metric:    m,
				}, t)
			}

			for j, behavior := range context.behaviors {
				input := behavior.input
				open := time.Date(input.openYear, input.openMonth, input.openDay, input.openHour, 0, 0, 0, time.UTC)
				end := time.Date(input.endYear, input.endMonth, input.endDay, input.endHour, 0, 0, 0, time.UTC)
				interval := model.Interval{
					OldestInclusive: open,
					NewestInclusive: end,
				}
				po := StalenessPolicy{
					DeltaAllowance: input.staleness,
				}

				openValue, endValue, err := p.GetBoundaryValues(model.NewFingerprintFromMetric(m), interval, po)
				if err != nil {
					t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err)
				}

				if behavior.output == nil {
					if openValue != nil {
						t.Fatalf("%d.%d(%s). Expected open to be nil but got: %q\n", i, j, behavior.name, openValue)
					}
					if endValue != nil {
						t.Fatalf("%d.%d(%s). Expected end to be nil but got: %q\n", i, j, behavior.name, endValue)
					}
				} else {
					if openValue == nil {
						t.Fatalf("%d.%d(%s). Expected open to be %s but got nil\n", i, j, behavior.name, behavior.output)
					}
					if endValue == nil {
						t.Fatalf("%d.%d(%s). Expected end to be %s but got nil\n", i, j, behavior.name, behavior.output)
					}
					if openValue.Value != behavior.output.open {
						t.Fatalf("%d.%d(%s). Expected open to be %s but got %s\n", i, j, behavior.name, behavior.output.open, openValue.Value)
					}

					if endValue.Value != behavior.output.end {
						t.Fatalf("%d.%d(%s). Expected end to be %s but got %s\n", i, j, behavior.name, behavior.output.end, endValue.Value)
					}
				}
			}
		}()
	}
}
func GetValueAtTimeTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
	type value struct {
		year  int
		month time.Month
		day   int
		hour  int
		value float32
	}

	type input struct {
		year      int
		month     time.Month
		day       int
		hour      int
		staleness time.Duration
	}

	type output struct {
		value model.SampleValue
	}

	type behavior struct {
		name   string
		input  input
		output *output
	}

	var contexts = []struct {
		name      string
		values    []value
		behaviors []behavior
	}{
		{
			name:   "no values",
			values: []value{},
			behaviors: []behavior{
				{
					name: "random target",
					input: input{
						year:      1984,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(0),
					},
				},
			},
		},
		{
			name: "singleton",
			values: []value{
				{
					year:  1984,
					month: 3,
					day:   30,
					hour:  0,
					value: 0,
				},
			},
			behaviors: []behavior{
				{
					name: "exact without staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(0),
					},
					output: &output{
						value: 0,
					},
				},
				{
					name: "exact with staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 0,
					},
				},
				{
					name: "before without staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       29,
						hour:      0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "before within staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       29,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
				{
					name: "before outside staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       29,
						hour:      0,
						staleness: time.Duration(1) * time.Hour,
					},
				},
				{
					name: "after without staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       31,
						hour:      0,
						staleness: time.Duration(0),
					},
				},
				{
					name: "after within staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       31,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 0,
					},
				},
				{
					name: "after outside staleness policy",
					input: input{
						year:      1984,
						month:     4,
						day:       7,
						hour:      0,
						staleness: time.Duration(7*24) * time.Hour,
					},
				},
			},
		},
		{
			name: "double",
			values: []value{
				{
					year:  1984,
					month: 3,
					day:   30,
					hour:  0,
					value: 0,
				},
				{
					year:  1985,
					month: 3,
					day:   30,
					hour:  0,
					value: 1,
				},
			},
			behaviors: []behavior{
				{
					name: "exact first without staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(0),
					},
					output: &output{
						value: 0,
					},
				},
				{
					name: "exact first with staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 0,
					},
				},
				{
					name: "exact second without staleness policy",
					input: input{
						year:      1985,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(0),
					},
					output: &output{
						value: 1,
					},
				},
				{
					name: "exact second with staleness policy",
					input: input{
						year:      1985,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 1,
					},
				},
				{
					name: "before first without staleness policy",
					input: input{
						year:      1983,
						month:     9,
						day:       29,
						hour:      12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "before first with staleness policy",
					input: input{
						year:      1983,
						month:     9,
						day:       29,
						hour:      12,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
				{
					name: "after second with staleness policy",
					input: input{
						year:      1985,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 1,
					},
				},
				{
					name: "after second without staleness policy",
					input: input{
						year:      1985,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "middle without staleness policy",
					input: input{
						year:      1984,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "middle with insufficient staleness policy",
					input: input{
						year:      1984,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(364*24) * time.Hour,
					},
				},
				{
					name: "middle with sufficient staleness policy",
					input: input{
						year:      1984,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 0.5,
					},
				},
			},
		},
		{
			name: "triple",
			values: []value{
				{
					year:  1984,
					month: 3,
					day:   30,
					hour:  0,
					value: 0,
				},
				{
					year:  1985,
					month: 3,
					day:   30,
					hour:  0,
					value: 1,
				},
				{
					year:  1986,
					month: 3,
					day:   30,
					hour:  0,
					value: 2,
				},
			},
			behaviors: []behavior{
				{
					name: "exact first without staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(0),
					},
					output: &output{
						value: 0,
					},
				},
				{
					name: "exact first with staleness policy",
					input: input{
						year:      1984,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 0,
					},
				},
				{
					name: "exact second without staleness policy",
					input: input{
						year:      1985,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(0),
					},
					output: &output{
						value: 1,
					},
				},
				{
					name: "exact second with staleness policy",
					input: input{
						year:      1985,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 1,
					},
				},
				{
					name: "exact third without staleness policy",
					input: input{
						year:      1986,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(0),
					},
					output: &output{
						value: 2,
					},
				},
				{
					name: "exact third with staleness policy",
					input: input{
						year:      1986,
						month:     3,
						day:       30,
						hour:      0,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 2,
					},
				},
				{
					name: "before first without staleness policy",
					input: input{
						year:      1983,
						month:     9,
						day:       29,
						hour:      12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "before first with staleness policy",
					input: input{
						year:      1983,
						month:     9,
						day:       29,
						hour:      12,
						staleness: time.Duration(365*24) * time.Hour,
					},
				},
				{
					name: "after third within staleness policy",
					input: input{
						year:      1986,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 2,
					},
				},
				{
					name: "after third outside staleness policy",
					input: input{
						year:      1986,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(1*24) * time.Hour,
					},
				},
				{
					name: "after third without staleness policy",
					input: input{
						year:      1986,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "first middle without staleness policy",
					input: input{
						year:      1984,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "first middle with insufficient staleness policy",
					input: input{
						year:      1984,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(364*24) * time.Hour,
					},
				},
				{
					name: "first middle with sufficient staleness policy",
					input: input{
						year:      1984,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 0.5,
					},
				},
				{
					name: "second middle without staleness policy",
					input: input{
						year:      1985,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(0),
					},
				},
				{
					name: "second middle with insufficient staleness policy",
					input: input{
						year:      1985,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(364*24) * time.Hour,
					},
				},
				{
					name: "second middle with sufficient staleness policy",
					input: input{
						year:      1985,
						month:     9,
						day:       28,
						hour:      12,
						staleness: time.Duration(365*24) * time.Hour,
					},
					output: &output{
						value: 1.5,
					},
				},
			},
		},
	}

	for i, context := range contexts {
		// Wrapping in function to enable garbage collection of resources.
		func() {
			p, closer := persistenceMaker()

			defer closer.Close()
			defer p.Close()

			m := model.Metric{
				model.MetricNameLabel: "age_in_years",
			}

			for _, value := range context.values {
				testAppendSample(p, model.Sample{
					Value:     model.SampleValue(value.value),
					Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC),
					Metric:    m,
				}, t)
			}

			for j, behavior := range context.behaviors {
				input := behavior.input
				time := time.Date(input.year, input.month, input.day, input.hour, 0, 0, 0, time.UTC)
				sp := StalenessPolicy{
					DeltaAllowance: input.staleness,
				}

				actual, err := p.GetValueAtTime(model.NewFingerprintFromMetric(m), time, sp)
				if err != nil {
					t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err)
				}

				if behavior.output == nil {
					if actual != nil {
						t.Fatalf("%d.%d(%s). Expected nil but got: %q\n", i, j, behavior.name, actual)
					}
				} else {
					if actual == nil {
						t.Fatalf("%d.%d(%s). Expected %s but got nil\n", i, j, behavior.name, behavior.output)
					} else {
						if actual.Value != behavior.output.value {
							t.Fatalf("%d.%d(%s). Expected %s but got %s\n", i, j, behavior.name, behavior.output, actual)

						}
					}
				}
			}
		}()
	}
}
func GetRangeValuesTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
	type value struct {
		year  int
		month time.Month
		day   int
		hour  int
		value float32
	}

	type input struct {
		openYear  int
		openMonth time.Month
		openDay   int
		openHour  int
		endYear   int
		endMonth  time.Month
		endDay    int
		endHour   int
	}

	type output struct {
		year  int
		month time.Month
		day   int
		hour  int
		value float32
	}

	type behavior struct {
		name   string
		input  input
		output []output
	}

	var contexts = []struct {
		name      string
		values    []value
		behaviors []behavior
	}{
		{
			name:   "no values",
			values: []value{},
			behaviors: []behavior{
				{
					name: "non-existent interval",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
				},
			},
		},
		{
			name: "singleton value",
			values: []value{
				{
					year:  1984,
					month: 3,
					day:   30,
					hour:  0,
					value: 0,
				},
			},
			behaviors: []behavior{
				{
					name: "start on first value",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1984,
							month: 3,
							day:   30,
							hour:  0,
							value: 0,
						},
					},
				},
				{
					name: "end on first value",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1984,
							month: 3,
							day:   30,
							hour:  0,
							value: 0,
						},
					},
				},
				{
					name: "overlap on first value",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1984,
							month: 3,
							day:   30,
							hour:  0,
							value: 0,
						},
					},
				},
			},
		},
		{
			name: "two values",
			values: []value{
				{
					year:  1984,
					month: 3,
					day:   30,
					hour:  0,
					value: 0,
				},
				{
					year:  1985,
					month: 3,
					day:   30,
					hour:  0,
					value: 1,
				},
			},
			behaviors: []behavior{
				{
					name: "start on first value",
					input: input{
						openYear:  1984,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1984,
							month: 3,
							day:   30,
							hour:  0,
							value: 0,
						},
						{
							year:  1985,
							month: 3,
							day:   30,
							hour:  0,
							value: 1,
						},
					},
				},
				{
					name: "start on second value",
					input: input{
						openYear:  1985,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1986,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1985,
							month: 3,
							day:   30,
							hour:  0,
							value: 1,
						},
					},
				},
				{
					name: "end on first value",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1984,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1984,
							month: 3,
							day:   30,
							hour:  0,
							value: 0,
						},
					},
				},
				{
					name: "end on second value",
					input: input{
						openYear:  1985,
						openMonth: 1,
						openDay:   1,
						openHour:  0,
						endYear:   1985,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1985,
							month: 3,
							day:   30,
							hour:  0,
							value: 1,
						},
					},
				},
				{
					name: "overlap on values",
					input: input{
						openYear:  1983,
						openMonth: 3,
						openDay:   30,
						openHour:  0,
						endYear:   1986,
						endMonth:  3,
						endDay:    30,
						endHour:   0,
					},
					output: []output{
						{
							year:  1984,
							month: 3,
							day:   30,
							hour:  0,
							value: 0,
						},
						{
							year:  1985,
							month: 3,
							day:   30,
							hour:  0,
							value: 1,
						},
					},
				},
			},
		},
	}

	for i, context := range contexts {
		// Wrapping in function to enable garbage collection of resources.
		func() {
			p, closer := persistenceMaker()

			defer closer.Close()
			defer p.Close()

			m := model.Metric{
				model.MetricNameLabel: "age_in_years",
			}

			for _, value := range context.values {
				testAppendSample(p, model.Sample{
					Value:     model.SampleValue(value.value),
					Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC),
					Metric:    m,
				}, t)
			}

			for j, behavior := range context.behaviors {
				input := behavior.input
				open := time.Date(input.openYear, input.openMonth, input.openDay, input.openHour, 0, 0, 0, time.UTC)
				end := time.Date(input.endYear, input.endMonth, input.endDay, input.endHour, 0, 0, 0, time.UTC)
				in := model.Interval{
					OldestInclusive: open,
					NewestInclusive: end,
				}

				values, err := p.GetRangeValues(model.NewFingerprintFromMetric(m), in)
				if err != nil {
					t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err)
				}

				if values == nil && len(behavior.output) != 0 {
					t.Fatalf("%d.%d(%s). Expected %s but got: %s\n", i, j, behavior.name, behavior.output, values)
				}

				if behavior.output == nil {
					if values != nil {
						t.Fatalf("%d.%d(%s). Expected nil values but got: %s\n", i, j, behavior.name, values)
					}
				} else {
					if len(behavior.output) != len(values.Values) {
						t.Fatalf("%d.%d(%s). Expected length %d but got: %d\n", i, j, behavior.name, len(behavior.output), len(values.Values))
					}

					for k, actual := range values.Values {
						expected := behavior.output[k]

						if actual.Value != model.SampleValue(expected.value) {
							t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.value, actual.Value)
						}

						if actual.Timestamp.Year() != expected.year {
							t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.year, actual.Timestamp.Year())
						}
						if actual.Timestamp.Month() != expected.month {
							t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.month, actual.Timestamp.Month())
						}
						// XXX: Find problem here.
						// Mismatches occur in this and have for a long time in the LevelDB
						// case, however not im-memory.
						//
						// if actual.Timestamp.Day() != expected.day {
						// 	t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.day, actual.Timestamp.Day())
						// }
						// if actual.Timestamp.Hour() != expected.hour {
						// 	t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.hour, actual.Timestamp.Hour())
						// }
					}
				}
			}
		}()
	}
}
Exemplo n.º 6
0
func testMakeView(t test.Tester) {
	type in struct {
		atTime     []getValuesAtTimeOp
		atInterval []getValuesAtIntervalOp
		alongRange []getValuesAlongRangeOp
	}

	type out struct {
		atTime     [][]model.SamplePair
		atInterval [][]model.SamplePair
		alongRange [][]model.SamplePair
	}
	var (
		instant     = time.Date(1984, 3, 30, 0, 0, 0, 0, time.Local)
		metric      = model.Metric{model.MetricNameLabel: "request_count"}
		fingerprint = model.NewFingerprintFromMetric(metric)
		scenarios   = []struct {
			data []model.Sample
			in   in
			out  out
		}{
			// No sample, but query asks for one.
			{
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant,
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{{}},
				},
			},
			// Single sample, query asks for exact sample time.
			{
				data: []model.Sample{
					{
						Metric:    metric,
						Value:     0,
						Timestamp: instant,
					},
				},
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant,
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant,
								Value:     0,
							},
						},
					},
				},
			},
			// Single sample, query time before the sample.
			{
				data: []model.Sample{
					{
						Metric:    metric,
						Value:     0,
						Timestamp: instant.Add(time.Second),
					},
					{
						Metric:    metric,
						Value:     1,
						Timestamp: instant.Add(time.Second * 2),
					},
				},
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant,
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant.Add(time.Second),
								Value:     0,
							},
						},
					},
				},
			},
			// Single sample, query time after the sample.
			{
				data: []model.Sample{
					{
						Metric:    metric,
						Value:     0,
						Timestamp: instant,
					},
				},
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant.Add(time.Second),
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant,
								Value:     0,
							},
						},
					},
				},
			},
			// Two samples, query asks for first sample time.
			{
				data: []model.Sample{
					{
						Metric:    metric,
						Value:     0,
						Timestamp: instant,
					},
					{
						Metric:    metric,
						Value:     1,
						Timestamp: instant.Add(time.Second),
					},
				},
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant,
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant,
								Value:     0,
							},
						},
					},
				},
			},
			// Three samples, query asks for second sample time.
			{
				data: []model.Sample{
					{
						Metric:    metric,
						Value:     0,
						Timestamp: instant,
					},
					{
						Metric:    metric,
						Value:     1,
						Timestamp: instant.Add(time.Second),
					},
					{
						Metric:    metric,
						Value:     2,
						Timestamp: instant.Add(time.Second * 2),
					},
				},
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant.Add(time.Second),
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant.Add(time.Second),
								Value:     1,
							},
						},
					},
				},
			},
			// Three samples, query asks for time between first and second samples.
			{
				data: []model.Sample{
					{
						Metric:    metric,
						Value:     0,
						Timestamp: instant,
					},
					{
						Metric:    metric,
						Value:     1,
						Timestamp: instant.Add(time.Second * 2),
					},
					{
						Metric:    metric,
						Value:     2,
						Timestamp: instant.Add(time.Second * 4),
					},
				},
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant.Add(time.Second),
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant,
								Value:     0,
							},
							{
								Timestamp: instant.Add(time.Second * 2),
								Value:     1,
							},
						},
					},
				},
			},
			// Three samples, query asks for time between second and third samples.
			{
				data: []model.Sample{
					{
						Metric:    metric,
						Value:     0,
						Timestamp: instant,
					},
					{
						Metric:    metric,
						Value:     1,
						Timestamp: instant.Add(time.Second * 2),
					},
					{
						Metric:    metric,
						Value:     2,
						Timestamp: instant.Add(time.Second * 4),
					},
				},
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant.Add(time.Second * 3),
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant.Add(time.Second * 2),
								Value:     1,
							},
							{
								Timestamp: instant.Add(time.Second * 4),
								Value:     2,
							},
						},
					},
				},
			},
			// Two chunks of samples, query asks for values from first chunk.
			{
				data: buildSamples(instant, instant.Add(time.Duration(*leveldbChunkSize*2)*time.Second), time.Second, metric),
				in: in{
					atTime: []getValuesAtTimeOp{
						{
							time: instant.Add(time.Second*time.Duration(*leveldbChunkSize/2) + 1),
						},
					},
				},
				out: out{
					atTime: [][]model.SamplePair{
						{
							{
								Timestamp: instant.Add(time.Second * time.Duration(*leveldbChunkSize/2)),
								Value:     100,
							},
							{
								Timestamp: instant.Add(time.Second * (time.Duration(*leveldbChunkSize/2) + 1)),
								Value:     101,
							},
						},
					},
				},
			},
		}
	)

	for i, scenario := range scenarios {
		tiered, closer := newTestTieredStorage(t)
		defer closer.Close()

		for j, datum := range scenario.data {
			err := tiered.AppendSample(datum)
			if err != nil {
				t.Fatalf("%d.%d. failed to add fixture data: %s", i, j, err)
			}
		}

		tiered.Flush()

		requestBuilder := NewViewRequestBuilder()

		for _, atTime := range scenario.in.atTime {
			requestBuilder.GetMetricAtTime(fingerprint, atTime.time)
		}

		for _, atInterval := range scenario.in.atInterval {
			requestBuilder.GetMetricAtInterval(fingerprint, atInterval.from, atInterval.through, atInterval.interval)
		}

		for _, alongRange := range scenario.in.alongRange {
			requestBuilder.GetMetricRange(fingerprint, alongRange.from, alongRange.through)
		}

		v, err := tiered.MakeView(requestBuilder, time.Second*5)

		if err != nil {
			t.Fatalf("%d. failed due to %s", i, err)
		}

		for j, atTime := range scenario.in.atTime {
			actual := v.GetValueAtTime(fingerprint, atTime.time)

			if len(actual) != len(scenario.out.atTime[j]) {
				t.Fatalf("%d.%d. expected %d output, got %d", i, j, len(scenario.out.atTime[j]), len(actual))
			}

			for k, value := range scenario.out.atTime[j] {
				if value.Value != actual[k].Value {
					t.Fatalf("%d.%d.%d expected %v value, got %v", i, j, k, value.Value, actual[k].Value)
				}
				if !value.Timestamp.Equal(actual[k].Timestamp) {
					t.Fatalf("%d.%d.%d expected %s timestamp, got %s", i, j, k, value.Timestamp, actual[k].Timestamp)
				}
			}
		}

		tiered.Drain()
	}
}
Exemplo n.º 7
0
func StochasticTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
	stochastic := func(x int) (success bool) {
		p, closer := persistenceMaker()
		defer closer.Close()
		defer p.Close()

		seed := rand.NewSource(int64(x))
		random := rand.New(seed)

		numberOfMetrics := random.Intn(stochasticMaximumVariance) + 1
		numberOfSharedLabels := random.Intn(stochasticMaximumVariance)
		numberOfUnsharedLabels := random.Intn(stochasticMaximumVariance)
		numberOfSamples := random.Intn(stochasticMaximumVariance) + 2
		numberOfRangeScans := random.Intn(stochasticMaximumVariance)

		metricTimestamps := map[int]map[int64]bool{}
		metricEarliestSample := map[int]int64{}
		metricNewestSample := map[int]int64{}

		for metricIndex := 0; metricIndex < numberOfMetrics; metricIndex++ {
			sample := model.Sample{
				Metric: model.Metric{},
			}

			v := model.LabelValue(fmt.Sprintf("metric_index_%d", metricIndex))
			sample.Metric[model.MetricNameLabel] = v

			for sharedLabelIndex := 0; sharedLabelIndex < numberOfSharedLabels; sharedLabelIndex++ {
				l := model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex))
				v := model.LabelValue(fmt.Sprintf("label_%d", sharedLabelIndex))

				sample.Metric[l] = v
			}

			for unsharedLabelIndex := 0; unsharedLabelIndex < numberOfUnsharedLabels; unsharedLabelIndex++ {
				l := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, unsharedLabelIndex))
				v := model.LabelValue(fmt.Sprintf("private_label_%d", unsharedLabelIndex))

				sample.Metric[l] = v
			}

			timestamps := map[int64]bool{}
			metricTimestamps[metricIndex] = timestamps
			var (
				newestSample  int64 = math.MinInt64
				oldestSample  int64 = math.MaxInt64
				nextTimestamp func() int64
			)

			nextTimestamp = func() int64 {
				var candidate int64
				candidate = random.Int63n(math.MaxInt32 - 1)

				if _, has := timestamps[candidate]; has {
					// WART
					candidate = nextTimestamp()
				}

				timestamps[candidate] = true

				if candidate < oldestSample {
					oldestSample = candidate
				}

				if candidate > newestSample {
					newestSample = candidate
				}

				return candidate
			}

			for sampleIndex := 0; sampleIndex < numberOfSamples; sampleIndex++ {
				sample.Timestamp = time.Unix(nextTimestamp(), 0)
				sample.Value = model.SampleValue(sampleIndex)

				err := p.AppendSample(sample)

				if err != nil {
					t.Error(err)
					return
				}
			}

			metricEarliestSample[metricIndex] = oldestSample
			metricNewestSample[metricIndex] = newestSample

			for sharedLabelIndex := 0; sharedLabelIndex < numberOfSharedLabels; sharedLabelIndex++ {
				labelPair := model.LabelSet{
					model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex)): model.LabelValue(fmt.Sprintf("label_%d", sharedLabelIndex)),
				}

				fingerprints, err := p.GetFingerprintsForLabelSet(labelPair)
				if err != nil {
					t.Error(err)
					return
				}
				if len(fingerprints) == 0 {
					t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints))
					return
				}

				labelName := model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex))
				fingerprints, err = p.GetFingerprintsForLabelName(labelName)
				if err != nil {
					t.Error(err)
					return
				}
				if len(fingerprints) == 0 {
					t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints))
					return
				}
			}
		}

		for sharedIndex := 0; sharedIndex < numberOfSharedLabels; sharedIndex++ {
			labelName := model.LabelName(fmt.Sprintf("shared_label_%d", sharedIndex))
			fingerprints, err := p.GetFingerprintsForLabelName(labelName)
			if err != nil {
				t.Error(err)
				return
			}

			if len(fingerprints) != numberOfMetrics {
				t.Errorf("expected fingerprint count of %d, got %d", numberOfMetrics, len(fingerprints))
				return
			}
		}

		for metricIndex := 0; metricIndex < numberOfMetrics; metricIndex++ {
			for unsharedLabelIndex := 0; unsharedLabelIndex < numberOfUnsharedLabels; unsharedLabelIndex++ {
				labelName := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, unsharedLabelIndex))
				labelValue := model.LabelValue(fmt.Sprintf("private_label_%d", unsharedLabelIndex))
				labelSet := model.LabelSet{
					labelName: labelValue,
				}

				fingerprints, err := p.GetFingerprintsForLabelSet(labelSet)
				if err != nil {
					t.Error(err)
					return
				}
				if len(fingerprints) != 1 {
					t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
					return
				}

				fingerprints, err = p.GetFingerprintsForLabelName(labelName)
				if err != nil {
					t.Error(err)
					return
				}
				if len(fingerprints) != 1 {
					t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
					return
				}
			}

			metric := model.Metric{}
			metric[model.MetricNameLabel] = model.LabelValue(fmt.Sprintf("metric_index_%d", metricIndex))

			for i := 0; i < numberOfSharedLabels; i++ {
				l := model.LabelName(fmt.Sprintf("shared_label_%d", i))
				v := model.LabelValue(fmt.Sprintf("label_%d", i))

				metric[l] = v
			}

			for i := 0; i < numberOfUnsharedLabels; i++ {
				l := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, i))
				v := model.LabelValue(fmt.Sprintf("private_label_%d", i))

				metric[l] = v
			}

			for i := 0; i < numberOfRangeScans; i++ {
				timestamps := metricTimestamps[metricIndex]

				var first int64 = 0
				var second int64 = 0

				for {
					firstCandidate := random.Int63n(int64(len(timestamps)))
					secondCandidate := random.Int63n(int64(len(timestamps)))

					smallest := int64(-1)
					largest := int64(-1)

					if firstCandidate == secondCandidate {
						continue
					} else if firstCandidate > secondCandidate {
						largest = firstCandidate
						smallest = secondCandidate
					} else {
						largest = secondCandidate
						smallest = firstCandidate
					}

					j := int64(0)
					for i := range timestamps {
						if j == smallest {
							first = i
						} else if j == largest {
							second = i
							break
						}
						j++
					}

					break
				}

				begin := first
				end := second

				if second < first {
					begin, end = second, first
				}

				interval := model.Interval{
					OldestInclusive: time.Unix(begin, 0),
					NewestInclusive: time.Unix(end, 0),
				}

				samples, err := p.GetRangeValues(model.NewFingerprintFromMetric(metric), interval)
				if err != nil {
					t.Error(err)
					return
				}

				if len(samples.Values) < 2 {
					t.Errorf("expected sample count less than %d, got %d", 2, len(samples.Values))
					return
				}
			}
		}

		return true
	}

	if err := quick.Check(stochastic, nil); err != nil {
		t.Error(err)
	}
}