func (s memorySeriesStorage) AppendSample(sample model.Sample) (err error) {
var (
metric = sample.Metric
fingerprint = model.NewFingerprintFromMetric(metric)
series, ok = s.fingerprintToSeries[fingerprint]
)
if !ok {
series = newStream(metric)
s.fingerprintToSeries[fingerprint] = series
for k, v := range metric {
labelPair := fmt.Sprintf("%s%s%s", k, reservedDelimiter, v)
labelPairValues := s.labelPairToFingerprints[labelPair]
labelPairValues = append(labelPairValues, fingerprint)
s.labelPairToFingerprints[labelPair] = labelPairValues
labelNameValues := s.labelNameToFingerprints[k]
labelNameValues = append(labelNameValues, fingerprint)
s.labelNameToFingerprints[k] = labelNameValues
}
}
series.add(sample.Timestamp, sample.Value)
return
}
// groupByFingerprint collects all of the provided samples, groups them
// together by their respective metric fingerprint, and finally sorts
// them chronologically.
func groupByFingerprint(samples model.Samples) map[model.Fingerprint]model.Samples {
var (
fingerprintToSamples = map[model.Fingerprint]model.Samples{}
)
for _, sample := range samples {
fingerprint := model.NewFingerprintFromMetric(sample.Metric)
samples := fingerprintToSamples[fingerprint]
samples = append(samples, sample)
fingerprintToSamples[fingerprint] = samples
}
var (
sortingSemaphore = make(chan bool, sortConcurrency)
doneSorting sync.WaitGroup
)
for i := 0; i < sortConcurrency; i++ {
sortingSemaphore <- true
}
for _, samples := range fingerprintToSamples {
doneSorting.Add(1)
go func(samples model.Samples) {
<-sortingSemaphore
sort.Sort(samples)
sortingSemaphore <- true
doneSorting.Done()
}(samples)
}
doneSorting.Wait()
return fingerprintToSamples
}
func GetBoundaryValuesTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
type value struct {
year int
month time.Month
day int
hour int
value float32
}
type input struct {
openYear int
openMonth time.Month
openDay int
openHour int
endYear int
endMonth time.Month
endDay int
endHour int
staleness time.Duration
}
type output struct {
open model.SampleValue
end model.SampleValue
}
type behavior struct {
name string
input input
output *output
}
var contexts = []struct {
name string
values []value
behaviors []behavior
}{
{
name: "no values",
values: []value{},
behaviors: []behavior{
{
name: "non-existent interval without staleness policy",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(0),
},
},
{
name: "non-existent interval with staleness policy",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
},
},
},
{
name: "single value",
values: []value{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
},
behaviors: []behavior{
{
name: "on start but missing end without staleness policy",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(0),
},
},
{
name: "non-existent interval after within staleness policy",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 31,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(4380) * time.Hour,
},
},
{
name: "non-existent interval after without staleness policy",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 31,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(0),
},
},
{
name: "non-existent interval before with staleness policy",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 3,
endDay: 29,
endHour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
},
{
name: "non-existent interval before without staleness policy",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 3,
endDay: 29,
endHour: 0,
staleness: time.Duration(0),
},
},
{
name: "on end but not start without staleness policy",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(0),
},
},
{
name: "on end but not start without staleness policy",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
},
{
name: "before point without staleness policy",
input: input{
openYear: 1982,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1983,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(0),
},
},
{
name: "before point with staleness policy",
input: input{
openYear: 1982,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1983,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
},
{
name: "after point without staleness policy",
input: input{
openYear: 1985,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1986,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(0),
},
},
{
name: "after point with staleness policy",
input: input{
openYear: 1985,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1986,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
},
{
name: "spanning point without staleness policy",
input: input{
openYear: 1983,
openMonth: 9,
openDay: 29,
openHour: 12,
endYear: 1984,
endMonth: 9,
endDay: 28,
endHour: 12,
staleness: time.Duration(0),
},
},
{
name: "spanning point with staleness policy",
input: input{
openYear: 1983,
openMonth: 9,
openDay: 29,
openHour: 12,
endYear: 1984,
endMonth: 9,
endDay: 28,
endHour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
},
},
},
{
name: "double values",
values: []value{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
},
behaviors: []behavior{
{
name: "on points without staleness policy",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(0),
},
output: &output{
open: 0,
end: 1,
},
},
{
name: "on points with staleness policy",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
open: 0,
end: 1,
},
},
{
name: "on first before second outside of staleness",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 6,
endDay: 29,
endHour: 6,
staleness: time.Duration(2190) * time.Hour,
},
},
{
name: "on first before second within staleness",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 6,
endDay: 29,
endHour: 6,
staleness: time.Duration(356*24) * time.Hour,
},
},
{
name: "on first after second outside of staleness",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 6,
endDay: 29,
endHour: 6,
staleness: time.Duration(1) * time.Hour,
},
},
{
name: "on first after second within staleness",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 6,
endDay: 29,
endHour: 6,
staleness: time.Duration(356*24) * time.Hour,
},
output: &output{
open: 0,
end: 1,
},
},
},
},
}
for i, context := range contexts {
// Wrapping in function to enable garbage collection of resources.
func() {
p, closer := persistenceMaker()
defer closer.Close()
defer p.Close()
m := model.Metric{
model.MetricNameLabel: "age_in_years",
}
for _, value := range context.values {
testAppendSample(p, model.Sample{
Value: model.SampleValue(value.value),
Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC),
Metric: m,
}, t)
}
for j, behavior := range context.behaviors {
input := behavior.input
open := time.Date(input.openYear, input.openMonth, input.openDay, input.openHour, 0, 0, 0, time.UTC)
end := time.Date(input.endYear, input.endMonth, input.endDay, input.endHour, 0, 0, 0, time.UTC)
interval := model.Interval{
OldestInclusive: open,
NewestInclusive: end,
}
po := StalenessPolicy{
DeltaAllowance: input.staleness,
}
openValue, endValue, err := p.GetBoundaryValues(model.NewFingerprintFromMetric(m), interval, po)
if err != nil {
t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err)
}
if behavior.output == nil {
if openValue != nil {
t.Fatalf("%d.%d(%s). Expected open to be nil but got: %q\n", i, j, behavior.name, openValue)
}
if endValue != nil {
t.Fatalf("%d.%d(%s). Expected end to be nil but got: %q\n", i, j, behavior.name, endValue)
}
} else {
if openValue == nil {
t.Fatalf("%d.%d(%s). Expected open to be %s but got nil\n", i, j, behavior.name, behavior.output)
}
if endValue == nil {
t.Fatalf("%d.%d(%s). Expected end to be %s but got nil\n", i, j, behavior.name, behavior.output)
}
if openValue.Value != behavior.output.open {
t.Fatalf("%d.%d(%s). Expected open to be %s but got %s\n", i, j, behavior.name, behavior.output.open, openValue.Value)
}
if endValue.Value != behavior.output.end {
t.Fatalf("%d.%d(%s). Expected end to be %s but got %s\n", i, j, behavior.name, behavior.output.end, endValue.Value)
}
}
}
}()
}
}
func GetValueAtTimeTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
type value struct {
year int
month time.Month
day int
hour int
value float32
}
type input struct {
year int
month time.Month
day int
hour int
staleness time.Duration
}
type output struct {
value model.SampleValue
}
type behavior struct {
name string
input input
output *output
}
var contexts = []struct {
name string
values []value
behaviors []behavior
}{
{
name: "no values",
values: []value{},
behaviors: []behavior{
{
name: "random target",
input: input{
year: 1984,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(0),
},
},
},
},
{
name: "singleton",
values: []value{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
},
behaviors: []behavior{
{
name: "exact without staleness policy",
input: input{
year: 1984,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(0),
},
output: &output{
value: 0,
},
},
{
name: "exact with staleness policy",
input: input{
year: 1984,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 0,
},
},
{
name: "before without staleness policy",
input: input{
year: 1984,
month: 3,
day: 29,
hour: 0,
staleness: time.Duration(0),
},
},
{
name: "before within staleness policy",
input: input{
year: 1984,
month: 3,
day: 29,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
},
{
name: "before outside staleness policy",
input: input{
year: 1984,
month: 3,
day: 29,
hour: 0,
staleness: time.Duration(1) * time.Hour,
},
},
{
name: "after without staleness policy",
input: input{
year: 1984,
month: 3,
day: 31,
hour: 0,
staleness: time.Duration(0),
},
},
{
name: "after within staleness policy",
input: input{
year: 1984,
month: 3,
day: 31,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 0,
},
},
{
name: "after outside staleness policy",
input: input{
year: 1984,
month: 4,
day: 7,
hour: 0,
staleness: time.Duration(7*24) * time.Hour,
},
},
},
},
{
name: "double",
values: []value{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
},
behaviors: []behavior{
{
name: "exact first without staleness policy",
input: input{
year: 1984,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(0),
},
output: &output{
value: 0,
},
},
{
name: "exact first with staleness policy",
input: input{
year: 1984,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 0,
},
},
{
name: "exact second without staleness policy",
input: input{
year: 1985,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(0),
},
output: &output{
value: 1,
},
},
{
name: "exact second with staleness policy",
input: input{
year: 1985,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 1,
},
},
{
name: "before first without staleness policy",
input: input{
year: 1983,
month: 9,
day: 29,
hour: 12,
staleness: time.Duration(0),
},
},
{
name: "before first with staleness policy",
input: input{
year: 1983,
month: 9,
day: 29,
hour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
},
{
name: "after second with staleness policy",
input: input{
year: 1985,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 1,
},
},
{
name: "after second without staleness policy",
input: input{
year: 1985,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(0),
},
},
{
name: "middle without staleness policy",
input: input{
year: 1984,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(0),
},
},
{
name: "middle with insufficient staleness policy",
input: input{
year: 1984,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(364*24) * time.Hour,
},
},
{
name: "middle with sufficient staleness policy",
input: input{
year: 1984,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 0.5,
},
},
},
},
{
name: "triple",
values: []value{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
{
year: 1986,
month: 3,
day: 30,
hour: 0,
value: 2,
},
},
behaviors: []behavior{
{
name: "exact first without staleness policy",
input: input{
year: 1984,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(0),
},
output: &output{
value: 0,
},
},
{
name: "exact first with staleness policy",
input: input{
year: 1984,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 0,
},
},
{
name: "exact second without staleness policy",
input: input{
year: 1985,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(0),
},
output: &output{
value: 1,
},
},
{
name: "exact second with staleness policy",
input: input{
year: 1985,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 1,
},
},
{
name: "exact third without staleness policy",
input: input{
year: 1986,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(0),
},
output: &output{
value: 2,
},
},
{
name: "exact third with staleness policy",
input: input{
year: 1986,
month: 3,
day: 30,
hour: 0,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 2,
},
},
{
name: "before first without staleness policy",
input: input{
year: 1983,
month: 9,
day: 29,
hour: 12,
staleness: time.Duration(0),
},
},
{
name: "before first with staleness policy",
input: input{
year: 1983,
month: 9,
day: 29,
hour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
},
{
name: "after third within staleness policy",
input: input{
year: 1986,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 2,
},
},
{
name: "after third outside staleness policy",
input: input{
year: 1986,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(1*24) * time.Hour,
},
},
{
name: "after third without staleness policy",
input: input{
year: 1986,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(0),
},
},
{
name: "first middle without staleness policy",
input: input{
year: 1984,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(0),
},
},
{
name: "first middle with insufficient staleness policy",
input: input{
year: 1984,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(364*24) * time.Hour,
},
},
{
name: "first middle with sufficient staleness policy",
input: input{
year: 1984,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 0.5,
},
},
{
name: "second middle without staleness policy",
input: input{
year: 1985,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(0),
},
},
{
name: "second middle with insufficient staleness policy",
input: input{
year: 1985,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(364*24) * time.Hour,
},
},
{
name: "second middle with sufficient staleness policy",
input: input{
year: 1985,
month: 9,
day: 28,
hour: 12,
staleness: time.Duration(365*24) * time.Hour,
},
output: &output{
value: 1.5,
},
},
},
},
}
for i, context := range contexts {
// Wrapping in function to enable garbage collection of resources.
func() {
p, closer := persistenceMaker()
defer closer.Close()
defer p.Close()
m := model.Metric{
model.MetricNameLabel: "age_in_years",
}
for _, value := range context.values {
testAppendSample(p, model.Sample{
Value: model.SampleValue(value.value),
Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC),
Metric: m,
}, t)
}
for j, behavior := range context.behaviors {
input := behavior.input
time := time.Date(input.year, input.month, input.day, input.hour, 0, 0, 0, time.UTC)
sp := StalenessPolicy{
DeltaAllowance: input.staleness,
}
actual, err := p.GetValueAtTime(model.NewFingerprintFromMetric(m), time, sp)
if err != nil {
t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err)
}
if behavior.output == nil {
if actual != nil {
t.Fatalf("%d.%d(%s). Expected nil but got: %q\n", i, j, behavior.name, actual)
}
} else {
if actual == nil {
t.Fatalf("%d.%d(%s). Expected %s but got nil\n", i, j, behavior.name, behavior.output)
} else {
if actual.Value != behavior.output.value {
t.Fatalf("%d.%d(%s). Expected %s but got %s\n", i, j, behavior.name, behavior.output, actual)
}
}
}
}
}()
}
}
func GetRangeValuesTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
type value struct {
year int
month time.Month
day int
hour int
value float32
}
type input struct {
openYear int
openMonth time.Month
openDay int
openHour int
endYear int
endMonth time.Month
endDay int
endHour int
}
type output struct {
year int
month time.Month
day int
hour int
value float32
}
type behavior struct {
name string
input input
output []output
}
var contexts = []struct {
name string
values []value
behaviors []behavior
}{
{
name: "no values",
values: []value{},
behaviors: []behavior{
{
name: "non-existent interval",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
},
},
},
},
{
name: "singleton value",
values: []value{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
},
behaviors: []behavior{
{
name: "start on first value",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
},
},
{
name: "end on first value",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
},
},
{
name: "overlap on first value",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
},
},
},
},
{
name: "two values",
values: []value{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
},
behaviors: []behavior{
{
name: "start on first value",
input: input{
openYear: 1984,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
},
},
{
name: "start on second value",
input: input{
openYear: 1985,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1986,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
},
},
{
name: "end on first value",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1984,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
},
},
{
name: "end on second value",
input: input{
openYear: 1985,
openMonth: 1,
openDay: 1,
openHour: 0,
endYear: 1985,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
},
},
{
name: "overlap on values",
input: input{
openYear: 1983,
openMonth: 3,
openDay: 30,
openHour: 0,
endYear: 1986,
endMonth: 3,
endDay: 30,
endHour: 0,
},
output: []output{
{
year: 1984,
month: 3,
day: 30,
hour: 0,
value: 0,
},
{
year: 1985,
month: 3,
day: 30,
hour: 0,
value: 1,
},
},
},
},
},
}
for i, context := range contexts {
// Wrapping in function to enable garbage collection of resources.
func() {
p, closer := persistenceMaker()
defer closer.Close()
defer p.Close()
m := model.Metric{
model.MetricNameLabel: "age_in_years",
}
for _, value := range context.values {
testAppendSample(p, model.Sample{
Value: model.SampleValue(value.value),
Timestamp: time.Date(value.year, value.month, value.day, value.hour, 0, 0, 0, time.UTC),
Metric: m,
}, t)
}
for j, behavior := range context.behaviors {
input := behavior.input
open := time.Date(input.openYear, input.openMonth, input.openDay, input.openHour, 0, 0, 0, time.UTC)
end := time.Date(input.endYear, input.endMonth, input.endDay, input.endHour, 0, 0, 0, time.UTC)
in := model.Interval{
OldestInclusive: open,
NewestInclusive: end,
}
values, err := p.GetRangeValues(model.NewFingerprintFromMetric(m), in)
if err != nil {
t.Fatalf("%d.%d(%s). Could not query for value: %q\n", i, j, behavior.name, err)
}
if values == nil && len(behavior.output) != 0 {
t.Fatalf("%d.%d(%s). Expected %s but got: %s\n", i, j, behavior.name, behavior.output, values)
}
if behavior.output == nil {
if values != nil {
t.Fatalf("%d.%d(%s). Expected nil values but got: %s\n", i, j, behavior.name, values)
}
} else {
if len(behavior.output) != len(values.Values) {
t.Fatalf("%d.%d(%s). Expected length %d but got: %d\n", i, j, behavior.name, len(behavior.output), len(values.Values))
}
for k, actual := range values.Values {
expected := behavior.output[k]
if actual.Value != model.SampleValue(expected.value) {
t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.value, actual.Value)
}
if actual.Timestamp.Year() != expected.year {
t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.year, actual.Timestamp.Year())
}
if actual.Timestamp.Month() != expected.month {
t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.month, actual.Timestamp.Month())
}
// XXX: Find problem here.
// Mismatches occur in this and have for a long time in the LevelDB
// case, however not im-memory.
//
// if actual.Timestamp.Day() != expected.day {
// t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.day, actual.Timestamp.Day())
// }
// if actual.Timestamp.Hour() != expected.hour {
// t.Fatalf("%d.%d.%d(%s). Expected %d but got: %d\n", i, j, k, behavior.name, expected.hour, actual.Timestamp.Hour())
// }
}
}
}
}()
}
}
func testMakeView(t test.Tester) {
type in struct {
atTime []getValuesAtTimeOp
atInterval []getValuesAtIntervalOp
alongRange []getValuesAlongRangeOp
}
type out struct {
atTime [][]model.SamplePair
atInterval [][]model.SamplePair
alongRange [][]model.SamplePair
}
var (
instant = time.Date(1984, 3, 30, 0, 0, 0, 0, time.Local)
metric = model.Metric{model.MetricNameLabel: "request_count"}
fingerprint = model.NewFingerprintFromMetric(metric)
scenarios = []struct {
data []model.Sample
in in
out out
}{
// No sample, but query asks for one.
{
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant,
},
},
},
out: out{
atTime: [][]model.SamplePair{{}},
},
},
// Single sample, query asks for exact sample time.
{
data: []model.Sample{
{
Metric: metric,
Value: 0,
Timestamp: instant,
},
},
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant,
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant,
Value: 0,
},
},
},
},
},
// Single sample, query time before the sample.
{
data: []model.Sample{
{
Metric: metric,
Value: 0,
Timestamp: instant.Add(time.Second),
},
{
Metric: metric,
Value: 1,
Timestamp: instant.Add(time.Second * 2),
},
},
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant,
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant.Add(time.Second),
Value: 0,
},
},
},
},
},
// Single sample, query time after the sample.
{
data: []model.Sample{
{
Metric: metric,
Value: 0,
Timestamp: instant,
},
},
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant.Add(time.Second),
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant,
Value: 0,
},
},
},
},
},
// Two samples, query asks for first sample time.
{
data: []model.Sample{
{
Metric: metric,
Value: 0,
Timestamp: instant,
},
{
Metric: metric,
Value: 1,
Timestamp: instant.Add(time.Second),
},
},
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant,
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant,
Value: 0,
},
},
},
},
},
// Three samples, query asks for second sample time.
{
data: []model.Sample{
{
Metric: metric,
Value: 0,
Timestamp: instant,
},
{
Metric: metric,
Value: 1,
Timestamp: instant.Add(time.Second),
},
{
Metric: metric,
Value: 2,
Timestamp: instant.Add(time.Second * 2),
},
},
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant.Add(time.Second),
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant.Add(time.Second),
Value: 1,
},
},
},
},
},
// Three samples, query asks for time between first and second samples.
{
data: []model.Sample{
{
Metric: metric,
Value: 0,
Timestamp: instant,
},
{
Metric: metric,
Value: 1,
Timestamp: instant.Add(time.Second * 2),
},
{
Metric: metric,
Value: 2,
Timestamp: instant.Add(time.Second * 4),
},
},
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant.Add(time.Second),
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant,
Value: 0,
},
{
Timestamp: instant.Add(time.Second * 2),
Value: 1,
},
},
},
},
},
// Three samples, query asks for time between second and third samples.
{
data: []model.Sample{
{
Metric: metric,
Value: 0,
Timestamp: instant,
},
{
Metric: metric,
Value: 1,
Timestamp: instant.Add(time.Second * 2),
},
{
Metric: metric,
Value: 2,
Timestamp: instant.Add(time.Second * 4),
},
},
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant.Add(time.Second * 3),
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant.Add(time.Second * 2),
Value: 1,
},
{
Timestamp: instant.Add(time.Second * 4),
Value: 2,
},
},
},
},
},
// Two chunks of samples, query asks for values from first chunk.
{
data: buildSamples(instant, instant.Add(time.Duration(*leveldbChunkSize*2)*time.Second), time.Second, metric),
in: in{
atTime: []getValuesAtTimeOp{
{
time: instant.Add(time.Second*time.Duration(*leveldbChunkSize/2) + 1),
},
},
},
out: out{
atTime: [][]model.SamplePair{
{
{
Timestamp: instant.Add(time.Second * time.Duration(*leveldbChunkSize/2)),
Value: 100,
},
{
Timestamp: instant.Add(time.Second * (time.Duration(*leveldbChunkSize/2) + 1)),
Value: 101,
},
},
},
},
},
}
)
for i, scenario := range scenarios {
tiered, closer := newTestTieredStorage(t)
defer closer.Close()
for j, datum := range scenario.data {
err := tiered.AppendSample(datum)
if err != nil {
t.Fatalf("%d.%d. failed to add fixture data: %s", i, j, err)
}
}
tiered.Flush()
requestBuilder := NewViewRequestBuilder()
for _, atTime := range scenario.in.atTime {
requestBuilder.GetMetricAtTime(fingerprint, atTime.time)
}
for _, atInterval := range scenario.in.atInterval {
requestBuilder.GetMetricAtInterval(fingerprint, atInterval.from, atInterval.through, atInterval.interval)
}
for _, alongRange := range scenario.in.alongRange {
requestBuilder.GetMetricRange(fingerprint, alongRange.from, alongRange.through)
}
v, err := tiered.MakeView(requestBuilder, time.Second*5)
if err != nil {
t.Fatalf("%d. failed due to %s", i, err)
}
for j, atTime := range scenario.in.atTime {
actual := v.GetValueAtTime(fingerprint, atTime.time)
if len(actual) != len(scenario.out.atTime[j]) {
t.Fatalf("%d.%d. expected %d output, got %d", i, j, len(scenario.out.atTime[j]), len(actual))
}
for k, value := range scenario.out.atTime[j] {
if value.Value != actual[k].Value {
t.Fatalf("%d.%d.%d expected %v value, got %v", i, j, k, value.Value, actual[k].Value)
}
if !value.Timestamp.Equal(actual[k].Timestamp) {
t.Fatalf("%d.%d.%d expected %s timestamp, got %s", i, j, k, value.Timestamp, actual[k].Timestamp)
}
}
}
tiered.Drain()
}
}
func StochasticTests(persistenceMaker func() (MetricPersistence, test.Closer), t test.Tester) {
stochastic := func(x int) (success bool) {
p, closer := persistenceMaker()
defer closer.Close()
defer p.Close()
seed := rand.NewSource(int64(x))
random := rand.New(seed)
numberOfMetrics := random.Intn(stochasticMaximumVariance) + 1
numberOfSharedLabels := random.Intn(stochasticMaximumVariance)
numberOfUnsharedLabels := random.Intn(stochasticMaximumVariance)
numberOfSamples := random.Intn(stochasticMaximumVariance) + 2
numberOfRangeScans := random.Intn(stochasticMaximumVariance)
metricTimestamps := map[int]map[int64]bool{}
metricEarliestSample := map[int]int64{}
metricNewestSample := map[int]int64{}
for metricIndex := 0; metricIndex < numberOfMetrics; metricIndex++ {
sample := model.Sample{
Metric: model.Metric{},
}
v := model.LabelValue(fmt.Sprintf("metric_index_%d", metricIndex))
sample.Metric[model.MetricNameLabel] = v
for sharedLabelIndex := 0; sharedLabelIndex < numberOfSharedLabels; sharedLabelIndex++ {
l := model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex))
v := model.LabelValue(fmt.Sprintf("label_%d", sharedLabelIndex))
sample.Metric[l] = v
}
for unsharedLabelIndex := 0; unsharedLabelIndex < numberOfUnsharedLabels; unsharedLabelIndex++ {
l := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, unsharedLabelIndex))
v := model.LabelValue(fmt.Sprintf("private_label_%d", unsharedLabelIndex))
sample.Metric[l] = v
}
timestamps := map[int64]bool{}
metricTimestamps[metricIndex] = timestamps
var (
newestSample int64 = math.MinInt64
oldestSample int64 = math.MaxInt64
nextTimestamp func() int64
)
nextTimestamp = func() int64 {
var candidate int64
candidate = random.Int63n(math.MaxInt32 - 1)
if _, has := timestamps[candidate]; has {
// WART
candidate = nextTimestamp()
}
timestamps[candidate] = true
if candidate < oldestSample {
oldestSample = candidate
}
if candidate > newestSample {
newestSample = candidate
}
return candidate
}
for sampleIndex := 0; sampleIndex < numberOfSamples; sampleIndex++ {
sample.Timestamp = time.Unix(nextTimestamp(), 0)
sample.Value = model.SampleValue(sampleIndex)
err := p.AppendSample(sample)
if err != nil {
t.Error(err)
return
}
}
metricEarliestSample[metricIndex] = oldestSample
metricNewestSample[metricIndex] = newestSample
for sharedLabelIndex := 0; sharedLabelIndex < numberOfSharedLabels; sharedLabelIndex++ {
labelPair := model.LabelSet{
model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex)): model.LabelValue(fmt.Sprintf("label_%d", sharedLabelIndex)),
}
fingerprints, err := p.GetFingerprintsForLabelSet(labelPair)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) == 0 {
t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints))
return
}
labelName := model.LabelName(fmt.Sprintf("shared_label_%d", sharedLabelIndex))
fingerprints, err = p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) == 0 {
t.Errorf("expected fingerprint count of %d, got %d", 0, len(fingerprints))
return
}
}
}
for sharedIndex := 0; sharedIndex < numberOfSharedLabels; sharedIndex++ {
labelName := model.LabelName(fmt.Sprintf("shared_label_%d", sharedIndex))
fingerprints, err := p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) != numberOfMetrics {
t.Errorf("expected fingerprint count of %d, got %d", numberOfMetrics, len(fingerprints))
return
}
}
for metricIndex := 0; metricIndex < numberOfMetrics; metricIndex++ {
for unsharedLabelIndex := 0; unsharedLabelIndex < numberOfUnsharedLabels; unsharedLabelIndex++ {
labelName := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, unsharedLabelIndex))
labelValue := model.LabelValue(fmt.Sprintf("private_label_%d", unsharedLabelIndex))
labelSet := model.LabelSet{
labelName: labelValue,
}
fingerprints, err := p.GetFingerprintsForLabelSet(labelSet)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) != 1 {
t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
return
}
fingerprints, err = p.GetFingerprintsForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(fingerprints) != 1 {
t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
return
}
}
metric := model.Metric{}
metric[model.MetricNameLabel] = model.LabelValue(fmt.Sprintf("metric_index_%d", metricIndex))
for i := 0; i < numberOfSharedLabels; i++ {
l := model.LabelName(fmt.Sprintf("shared_label_%d", i))
v := model.LabelValue(fmt.Sprintf("label_%d", i))
metric[l] = v
}
for i := 0; i < numberOfUnsharedLabels; i++ {
l := model.LabelName(fmt.Sprintf("metric_index_%d_private_label_%d", metricIndex, i))
v := model.LabelValue(fmt.Sprintf("private_label_%d", i))
metric[l] = v
}
for i := 0; i < numberOfRangeScans; i++ {
timestamps := metricTimestamps[metricIndex]
var first int64 = 0
var second int64 = 0
for {
firstCandidate := random.Int63n(int64(len(timestamps)))
secondCandidate := random.Int63n(int64(len(timestamps)))
smallest := int64(-1)
largest := int64(-1)
if firstCandidate == secondCandidate {
continue
} else if firstCandidate > secondCandidate {
largest = firstCandidate
smallest = secondCandidate
} else {
largest = secondCandidate
smallest = firstCandidate
}
j := int64(0)
for i := range timestamps {
if j == smallest {
first = i
} else if j == largest {
second = i
break
}
j++
}
break
}
begin := first
end := second
if second < first {
begin, end = second, first
}
interval := model.Interval{
OldestInclusive: time.Unix(begin, 0),
NewestInclusive: time.Unix(end, 0),
}
samples, err := p.GetRangeValues(model.NewFingerprintFromMetric(metric), interval)
if err != nil {
t.Error(err)
return
}
if len(samples.Values) < 2 {
t.Errorf("expected sample count less than %d, got %d", 2, len(samples.Values))
return
}
}
}
return true
}
if err := quick.Check(stochastic, nil); err != nil {
t.Error(err)
}
}
