func TestFederation(t *testing.T) {
suite, err := promql.NewTest(t, `
load 1m
test_metric1{foo="bar"} 0+100x100
test_metric1{foo="boo"} 1+0x100
test_metric2{foo="boo"} 1+0x100
test_metric_without_labels 1+10x100
`)
if err != nil {
t.Fatal(err)
}
defer suite.Close()
if err := suite.Run(); err != nil {
t.Fatal(err)
}
h := &Handler{
storage: suite.Storage(),
queryEngine: suite.QueryEngine(),
now: func() model.Time { return 101 * 60 * 1000 }, // 101min after epoch.
}
for name, scenario := range scenarios {
req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(
"GET http://example.org/federate?" + scenario.params + " HTTP/1.0\r\n\r\n",
)))
if err != nil {
t.Fatal(err)
}
// HTTP/1.0 was used above to avoid needing a Host field. Change it to 1.1 here.
req.Proto = "HTTP/1.1"
req.ProtoMinor = 1
req.Close = false
// 192.0.2.0/24 is "TEST-NET" in RFC 5737 for use solely in
// documentation and example source code and should not be
// used publicly.
req.RemoteAddr = "192.0.2.1:1234"
// TODO(beorn7): Once we are completely on Go1.7, replace the lines above by the following:
// req := httptest.NewRequest("GET", "http://example.org/federate?"+scenario.params, nil)
res := httptest.NewRecorder()
h.federation(res, req)
if got, want := res.Code, scenario.code; got != want {
t.Errorf("Scenario %q: got code %d, want %d", name, got, want)
}
if got, want := normalizeBody(res.Body), scenario.body; got != want {
t.Errorf("Scenario %q: got body %q, want %q", name, got, want)
}
}
}
func TestEndpoints(t *testing.T) {
suite, err := promql.NewTest(t, `
load 1m
test_metric1{foo="bar"} 0+100x100
test_metric1{foo="boo"} 1+0x100
test_metric2{foo="boo"} 1+0x100
`)
if err != nil {
t.Fatal(err)
}
defer suite.Close()
if err := suite.Run(); err != nil {
t.Fatal(err)
}
api := &API{
Storage: suite.Storage(),
QueryEngine: suite.QueryEngine(),
}
start := clientmodel.Timestamp(0)
var tests = []struct {
endpoint apiFunc
params map[string]string
query url.Values
response interface{}
errType errorType
}{
{
endpoint: api.query,
query: url.Values{
"query": []string{"2"},
"time": []string{"123.3"},
},
response: &queryData{
ResultType: promql.ExprScalar,
Result: &promql.Scalar{
Value: 2,
Timestamp: start.Add(123*time.Second + 300*time.Millisecond),
},
},
},
{
endpoint: api.query,
query: url.Values{
"query": []string{"0.333"},
"time": []string{"1970-01-01T00:02:03Z"},
},
response: &queryData{
ResultType: promql.ExprScalar,
Result: &promql.Scalar{
Value: 0.333,
Timestamp: start.Add(123 * time.Second),
},
},
},
{
endpoint: api.query,
query: url.Values{
"query": []string{"0.333"},
"time": []string{"1970-01-01T01:02:03+01:00"},
},
response: &queryData{
ResultType: promql.ExprScalar,
Result: &promql.Scalar{
Value: 0.333,
Timestamp: start.Add(123 * time.Second),
},
},
},
{
endpoint: api.queryRange,
query: url.Values{
"query": []string{"time()"},
"start": []string{"0"},
"end": []string{"2"},
"step": []string{"1"},
},
response: &queryData{
ResultType: promql.ExprMatrix,
Result: promql.Matrix{
&promql.SampleStream{
Values: metric.Values{
{Value: 0, Timestamp: start},
{Value: 1, Timestamp: start.Add(1 * time.Second)},
{Value: 2, Timestamp: start.Add(2 * time.Second)},
},
},
},
},
},
// Missing query params in range queries.
{
endpoint: api.queryRange,
query: url.Values{
"query": []string{"time()"},
"end": []string{"2"},
"step": []string{"1"},
},
errType: errorBadData,
},
{
endpoint: api.queryRange,
query: url.Values{
"query": []string{"time()"},
"start": []string{"0"},
"step": []string{"1"},
},
errType: errorBadData,
},
{
endpoint: api.queryRange,
query: url.Values{
"query": []string{"time()"},
"start": []string{"0"},
"end": []string{"2"},
},
errType: errorBadData,
},
// Missing evaluation time.
{
endpoint: api.query,
query: url.Values{
"query": []string{"0.333"},
},
errType: errorBadData,
},
// Bad query expression.
{
endpoint: api.query,
query: url.Values{
"query": []string{"invalid][query"},
"time": []string{"1970-01-01T01:02:03+01:00"},
},
errType: errorBadData,
},
{
endpoint: api.queryRange,
query: url.Values{
"query": []string{"invalid][query"},
"start": []string{"0"},
"end": []string{"100"},
"step": []string{"1"},
},
errType: errorBadData,
},
{
endpoint: api.labelValues,
params: map[string]string{
"name": "__name__",
},
response: clientmodel.LabelValues{
"test_metric1",
"test_metric2",
},
},
{
endpoint: api.labelValues,
params: map[string]string{
"name": "foo",
},
response: clientmodel.LabelValues{
"bar",
"boo",
},
},
// Bad name parameter.
{
endpoint: api.labelValues,
params: map[string]string{
"name": "not!!!allowed",
},
errType: errorBadData,
},
{
endpoint: api.series,
query: url.Values{
"match[]": []string{`test_metric2`},
},
response: []clientmodel.Metric{
{
"__name__": "test_metric2",
"foo": "boo",
},
},
},
{
endpoint: api.series,
query: url.Values{
"match[]": []string{`test_metric1{foo=~"o$"}`},
},
response: []clientmodel.Metric{
{
"__name__": "test_metric1",
"foo": "boo",
},
},
},
{
endpoint: api.series,
query: url.Values{
"match[]": []string{`test_metric1{foo=~"o$"}`, `test_metric1{foo=~"o$"}`},
},
response: []clientmodel.Metric{
{
"__name__": "test_metric1",
"foo": "boo",
},
},
},
{
endpoint: api.series,
query: url.Values{
"match[]": []string{`test_metric1{foo=~"o$"}`, `none`},
},
response: []clientmodel.Metric{
{
"__name__": "test_metric1",
"foo": "boo",
},
},
},
// Missing match[] query params in series requests.
{
endpoint: api.series,
errType: errorBadData,
},
{
endpoint: api.dropSeries,
errType: errorBadData,
},
// The following tests delete time series from the test storage. They
// must remain at the end and are fixed in their order.
{
endpoint: api.dropSeries,
query: url.Values{
"match[]": []string{`test_metric1{foo=~"o$"}`},
},
response: struct {
NumDeleted int `json:"numDeleted"`
}{1},
},
{
endpoint: api.series,
query: url.Values{
"match[]": []string{`test_metric1`},
},
response: []clientmodel.Metric{
{
"__name__": "test_metric1",
"foo": "bar",
},
},
}, {
endpoint: api.dropSeries,
query: url.Values{
"match[]": []string{`{__name__=~".*"}`},
},
response: struct {
NumDeleted int `json:"numDeleted"`
}{2},
},
}
for _, test := range tests {
// Build a context with the correct request params.
ctx := context.Background()
for p, v := range test.params {
ctx = route.WithParam(ctx, p, v)
}
api.context = func(r *http.Request) context.Context {
return ctx
}
req, err := http.NewRequest("ANY", fmt.Sprintf("http://example.com?%s", test.query.Encode()), nil)
if err != nil {
t.Fatal(err)
}
resp, apiErr := test.endpoint(req)
if apiErr != nil {
if test.errType == errorNone {
t.Fatalf("Unexpected error: %s", apiErr)
}
if test.errType != apiErr.typ {
t.Fatalf("Expected error of type %q but got type %q", test.errType, apiErr.typ)
}
continue
}
if apiErr == nil && test.errType != errorNone {
t.Fatalf("Expected error of type %q but got none", test.errType)
}
if !reflect.DeepEqual(resp, test.response) {
t.Fatalf("Response does not match, expected:\n%#v\ngot:\n%#v", test.response, resp)
}
// Ensure that removed metrics are unindexed before the next request.
suite.Storage().WaitForIndexing()
}
}
func TestAlertingRule(t *testing.T) {
suite, err := promql.NewTest(t, `
load 5m
http_requests{job="app-server", instance="0", group="canary", severity="overwrite-me"} 75 85 95 105 105 95 85
http_requests{job="app-server", instance="1", group="canary", severity="overwrite-me"} 80 90 100 110 120 130 140
`)
if err != nil {
t.Fatal(err)
}
defer suite.Close()
if err := suite.Run(); err != nil {
t.Fatal(err)
}
expr, err := promql.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
if err != nil {
t.Fatalf("Unable to parse alert expression: %s", err)
}
rule := NewAlertingRule(
"HTTPRequestRateLow",
expr,
time.Minute,
model.LabelSet{"severity": "{{\"c\"}}ritical"},
model.LabelSet{},
)
var tests = []struct {
time time.Duration
result []string
}{
{
time: 0,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
},
}, {
time: 5 * time.Minute,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
},
}, {
time: 10 * time.Minute,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
},
},
{
time: 15 * time.Minute,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
},
},
{
time: 20 * time.Minute,
result: []string{},
},
{
time: 25 * time.Minute,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
},
},
{
time: 30 * time.Minute,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
},
},
}
for i, test := range tests {
evalTime := model.Time(0).Add(test.time)
res, err := rule.eval(evalTime, suite.QueryEngine(), "")
if err != nil {
t.Fatalf("Error during alerting rule evaluation: %s", err)
}
actual := strings.Split(res.String(), "\n")
expected := annotateWithTime(test.result, evalTime)
if actual[0] == "" {
actual = []string{}
}
if len(actual) != len(expected) {
t.Errorf("%d. Number of samples in expected and actual output don't match (%d vs. %d)", i, len(expected), len(actual))
}
for j, expectedSample := range expected {
found := false
for _, actualSample := range actual {
if actualSample == expectedSample {
found = true
}
}
if !found {
t.Errorf("%d.%d. Couldn't find expected sample in output: '%v'", i, j, expectedSample)
}
}
if t.Failed() {
t.Errorf("%d. Expected and actual outputs don't match:", i)
t.Fatalf("Expected:\n%v\n----\nActual:\n%v", strings.Join(expected, "\n"), strings.Join(actual, "\n"))
}
for _, aa := range rule.ActiveAlerts() {
if _, ok := aa.Labels[model.MetricNameLabel]; ok {
t.Fatalf("%s label set on active alert: %s", model.MetricNameLabel, aa.Labels)
}
}
}
}
func TestAlertingRule(t *testing.T) {
suite, err := promql.NewTest(t, `
load 5m
http_requests{job="api-server", instance="0", group="production"} 0+10x10
http_requests{job="api-server", instance="1", group="production"} 0+20x10
http_requests{job="api-server", instance="0", group="canary"} 0+30x10
http_requests{job="api-server", instance="1", group="canary"} 0+40x10
http_requests{job="app-server", instance="0", group="production"} 0+50x10
http_requests{job="app-server", instance="1", group="production"} 0+60x10
http_requests{job="app-server", instance="0", group="canary"} 0+70x10
http_requests{job="app-server", instance="1", group="canary"} 0+80x10
`)
if err != nil {
t.Fatal(err)
}
defer suite.Close()
if err := suite.Run(); err != nil {
t.Fatal(err)
}
expr, err := promql.ParseExpr(`http_requests{group="canary", job="app-server"} < 100`)
if err != nil {
t.Fatalf("Unable to parse alert expression: %s", err)
}
rule := NewAlertingRule(
"HTTPRequestRateLow",
expr,
time.Minute,
model.LabelSet{"severity": "critical"},
"summary", "description", "runbook",
)
var tests = []struct {
time time.Duration
result []string
}{
{
time: 0,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
},
}, {
time: 5 * time.Minute,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 1 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="pending", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 1 @[%v]`,
},
}, {
time: 10 * time.Minute,
result: []string{
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="1", job="app-server", severity="critical"} => 0 @[%v]`,
`ALERTS{alertname="HTTPRequestRateLow", alertstate="firing", group="canary", instance="0", job="app-server", severity="critical"} => 0 @[%v]`,
},
},
{
time: 15 * time.Minute,
result: nil,
},
{
time: 20 * time.Minute,
result: nil,
},
}
for i, test := range tests {
evalTime := model.Time(0).Add(test.time)
res, err := rule.eval(evalTime, suite.QueryEngine())
if err != nil {
t.Fatalf("Error during alerting rule evaluation: %s", err)
}
actual := strings.Split(res.String(), "\n")
expected := annotateWithTime(test.result, evalTime)
if actual[0] == "" {
actual = []string{}
}
if len(actual) != len(expected) {
t.Errorf("%d. Number of samples in expected and actual output don't match (%d vs. %d)", i, len(expected), len(actual))
}
for j, expectedSample := range expected {
found := false
for _, actualSample := range actual {
if actualSample == expectedSample {
found = true
}
}
if !found {
t.Errorf("%d.%d. Couldn't find expected sample in output: '%v'", i, j, expectedSample)
}
}
if t.Failed() {
t.Errorf("%d. Expected and actual outputs don't match:", i)
t.Fatalf("Expected:\n%v\n----\nActual:\n%v", strings.Join(expected, "\n"), strings.Join(actual, "\n"))
}
}
}