func (p *prometheus) delete(olderThan time.Duration, batchSize int) error {
select {
case s, ok := <-p.curationSema:
if !ok {
glog.Warning("Prometheus is shutting down; no more curation runs are allowed.")
return nil
}
defer func() {
p.curationSema <- s
}()
default:
glog.Warningf("Deferred deletion for %s due to existing operation.", olderThan)
return nil
}
processor := tiered.NewDeletionProcessor(&tiered.DeletionProcessorOptions{
MaximumMutationPoolBatch: batchSize,
})
defer processor.Close()
curator := tiered.NewCurator(&tiered.CuratorOptions{
Stop: p.stopBackgroundOperations,
ViewQueue: p.storage.ViewQueue,
})
defer curator.Close()
return curator.Run(olderThan, clientmodel.Now(), processor, p.storage.DiskStorage.CurationRemarks, p.storage.DiskStorage.MetricSamples, p.storage.DiskStorage.MetricHighWatermarks, p.curationState)
}
func (s *memorySeriesStorage) loop() {
checkpointTimer := time.NewTimer(s.checkpointInterval)
dirtySeriesCount := 0
defer func() {
checkpointTimer.Stop()
log.Info("Maintenance loop stopped.")
close(s.loopStopped)
}()
memoryFingerprints := s.cycleThroughMemoryFingerprints()
archivedFingerprints := s.cycleThroughArchivedFingerprints()
loop:
for {
select {
case <-s.loopStopping:
break loop
case <-checkpointTimer.C:
s.persistence.checkpointSeriesMapAndHeads(s.fpToSeries, s.fpLocker)
dirtySeriesCount = 0
checkpointTimer.Reset(s.checkpointInterval)
case fp := <-memoryFingerprints:
if s.maintainMemorySeries(fp, clientmodel.Now().Add(-s.dropAfter)) {
dirtySeriesCount++
// Check if we have enough "dirty" series so that we need an early checkpoint.
// However, if we are already behind persisting chunks, creating a checkpoint
// would be counterproductive, as it would slow down chunk persisting even more,
// while in a situation like that, where we are clearly lacking speed of disk
// maintenance, the best we can do for crash recovery is to persist chunks as
// quickly as possible. So only checkpoint if the storage is not in "graceful
// degratadion mode".
if dirtySeriesCount >= s.checkpointDirtySeriesLimit && !s.isDegraded() {
checkpointTimer.Reset(0)
}
}
case fp := <-archivedFingerprints:
s.maintainArchivedSeries(fp, clientmodel.Now().Add(-s.dropAfter))
}
}
// Wait until both channels are closed.
for range memoryFingerprints {
}
for range archivedFingerprints {
}
}
func (h *Handler) executeTemplate(w http.ResponseWriter, name string, data interface{}) {
text, err := h.getTemplate(name)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
tmpl := template.NewTemplateExpander(text, name, data, clientmodel.Now(), h.queryEngine, h.options.PathPrefix)
tmpl.Funcs(template_text.FuncMap{
"since": time.Since,
"getConsoles": h.getConsoles,
"pathPrefix": func() string { return h.options.PathPrefix },
"stripLabels": func(lset clientmodel.LabelSet, labels ...clientmodel.LabelName) clientmodel.LabelSet {
for _, ln := range labels {
delete(lset, ln)
}
return lset
},
"globalURL": func(u *url.URL) *url.URL {
for _, lhr := range localhostRepresentations {
if strings.HasPrefix(u.Host, lhr+":") {
u.Host = strings.Replace(u.Host, lhr+":", h.options.Hostname+":", 1)
}
}
return u
},
"healthToClass": func(th retrieval.TargetHealth) string {
switch th {
case retrieval.HealthUnknown:
return "warning"
case retrieval.HealthGood:
return "success"
default:
return "danger"
}
},
"alertStateToClass": func(as rules.AlertState) string {
switch as {
case rules.StateInactive:
return "success"
case rules.StatePending:
return "warning"
case rules.StateFiring:
return "danger"
default:
panic("unknown alert state")
}
},
})
result, err := tmpl.ExpandHTML(nil)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
io.WriteString(w, result)
}
func TestDeltaWithEmptyElementDoesNotCrash(t *testing.T) {
now := clientmodel.Now()
vector := deltaImpl(now, nil, []Node{emptyRangeNode{}, &ScalarLiteral{value: 0}}).(Vector)
if len(vector) != 0 {
t.Fatalf("Expected empty result vector, got: %v", vector)
}
vector = deltaImpl(now, nil, []Node{emptyRangeNode{}, &ScalarLiteral{value: 1}}).(Vector)
if len(vector) != 0 {
t.Fatalf("Expected empty result vector, got: %v", vector)
}
}
func (m *ruleManager) runIteration(results chan<- clientmodel.Samples) {
now := clientmodel.Now()
wg := sync.WaitGroup{}
m.Lock()
rulesSnapshot := make([]rules.Rule, len(m.rules))
copy(rulesSnapshot, m.rules)
m.Unlock()
for _, rule := range rulesSnapshot {
wg.Add(1)
// BUG(julius): Look at fixing thundering herd.
go func(rule rules.Rule) {
defer wg.Done()
start := time.Now()
vector, err := rule.Eval(now, m.storage)
duration := time.Since(start)
samples := make(clientmodel.Samples, len(vector))
for i, s := range vector {
samples[i] = &clientmodel.Sample{
Metric: s.Metric.Metric,
Value: s.Value,
Timestamp: s.Timestamp,
}
}
if err != nil {
evalFailures.Inc()
glog.Warningf("Error while evaluating rule %q: %s", rule, err)
} else {
m.results <- samples
}
switch r := rule.(type) {
case *rules.AlertingRule:
m.queueAlertNotifications(r, now)
evalDuration.WithLabelValues(alertingRuleType).Observe(
float64(duration / time.Millisecond),
)
case *rules.RecordingRule:
evalDuration.WithLabelValues(recordingRuleType).Observe(
float64(duration / time.Millisecond),
)
default:
panic(fmt.Sprintf("Unknown rule type: %T", rule))
}
}(rule)
}
wg.Wait()
}
func (t *target) scrape(ingester extraction.Ingester) (err error) {
timestamp := clientmodel.Now()
defer func(start time.Time) {
t.Lock() // Writing t.state and t.lastError requires the lock.
if err == nil {
t.state = Alive
} else {
t.state = Unreachable
}
t.lastError = err
t.Unlock()
t.recordScrapeHealth(ingester, timestamp, err == nil, time.Since(start))
}(time.Now())
req, err := http.NewRequest("GET", t.URL(), nil)
if err != nil {
panic(err)
}
req.Header.Add("Accept", acceptHeader)
resp, err := t.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("server returned HTTP status %s", resp.Status)
}
processor, err := extraction.ProcessorForRequestHeader(resp.Header)
if err != nil {
return err
}
baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.URL())}
for baseLabel, baseValue := range t.baseLabels {
baseLabels[baseLabel] = baseValue
}
i := &MergeLabelsIngester{
Labels: baseLabels,
CollisionPrefix: clientmodel.ExporterLabelPrefix,
Ingester: ingester,
}
processOptions := &extraction.ProcessOptions{
Timestamp: timestamp,
}
return processor.ProcessSingle(resp.Body, i, processOptions)
}
func (m *Manager) runIteration() {
now := clientmodel.Now()
wg := sync.WaitGroup{}
m.Lock()
rulesSnapshot := make([]Rule, len(m.rules))
copy(rulesSnapshot, m.rules)
m.Unlock()
for _, rule := range rulesSnapshot {
wg.Add(1)
// BUG(julius): Look at fixing thundering herd.
go func(rule Rule) {
defer wg.Done()
start := time.Now()
vector, err := rule.eval(now, m.queryEngine)
duration := time.Since(start)
if err != nil {
evalFailures.Inc()
log.Warnf("Error while evaluating rule %q: %s", rule, err)
return
}
switch r := rule.(type) {
case *AlertingRule:
m.queueAlertNotifications(r, now)
evalDuration.WithLabelValues(ruleTypeAlerting).Observe(
float64(duration / time.Millisecond),
)
case *RecordingRule:
evalDuration.WithLabelValues(ruleTypeRecording).Observe(
float64(duration / time.Millisecond),
)
default:
panic(fmt.Errorf("Unknown rule type: %T", rule))
}
for _, s := range vector {
m.sampleAppender.Append(&clientmodel.Sample{
Metric: s.Metric.Metric,
Value: s.Value,
Timestamp: s.Timestamp,
})
}
}(rule)
}
wg.Wait()
}
func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) {
ctx := route.Context(r)
name := route.Param(ctx, "filepath")
file, err := http.Dir(h.options.ConsoleTemplatesPath).Open(name)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
text, err := ioutil.ReadAll(file)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Provide URL parameters as a map for easy use. Advanced users may have need for
// parameters beyond the first, so provide RawParams.
rawParams, err := url.ParseQuery(r.URL.RawQuery)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
params := map[string]string{}
for k, v := range rawParams {
params[k] = v[0]
}
data := struct {
RawParams url.Values
Params map[string]string
Path string
}{
RawParams: rawParams,
Params: params,
Path: name,
}
tmpl := template.NewTemplateExpander(string(text), "__console_"+name, data, clientmodel.Now(), h.queryEngine, h.options.PathPrefix)
filenames, err := filepath.Glob(h.options.ConsoleLibrariesPath + "/*.lib")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
result, err := tmpl.ExpandHTML(filenames)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
io.WriteString(w, result)
}
func (h *Handler) executeTemplate(w http.ResponseWriter, name string, data interface{}) {
text, err := h.getTemplate(name)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
tmpl := template.NewTemplateExpander(text, name, data, clientmodel.Now(), h.queryEngine, h.options.ExternalURL.Path)
tmpl.Funcs(tmplFuncs(h.consolesPath(), h.options))
result, err := tmpl.ExpandHTML(nil)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
io.WriteString(w, result)
}
func TestTargetRecordScrapeHealth(t *testing.T) {
testTarget := target{
url: "http://example.url",
baseLabels: clientmodel.LabelSet{clientmodel.JobLabel: "testjob"},
httpClient: utility.NewDeadlineClient(0),
}
now := clientmodel.Now()
ingester := &collectResultIngester{}
testTarget.recordScrapeHealth(ingester, now, true, 2*time.Second)
result := ingester.result
if len(result) != 2 {
t.Fatalf("Expected two samples, got %d", len(result))
}
actual := result[0]
expected := &clientmodel.Sample{
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: scrapeHealthMetricName,
InstanceLabel: "http://example.url",
clientmodel.JobLabel: "testjob",
},
Timestamp: now,
Value: 1,
}
if !actual.Equal(expected) {
t.Fatalf("Expected and actual samples not equal. Expected: %v, actual: %v", expected, actual)
}
actual = result[1]
expected = &clientmodel.Sample{
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: scrapeDurationMetricName,
InstanceLabel: "http://example.url",
clientmodel.JobLabel: "testjob",
},
Timestamp: now,
Value: 2.0,
}
if !actual.Equal(expected) {
t.Fatalf("Expected and actual samples not equal. Expected: %v, actual: %v", expected, actual)
}
}
// NewIterator implements Storage.
func (s *memorySeriesStorage) NewIterator(fp clientmodel.Fingerprint) SeriesIterator {
s.fpLocker.Lock(fp)
defer s.fpLocker.Unlock(fp)
series, ok := s.fpToSeries.get(fp)
if !ok {
// Oops, no series for fp found. That happens if, after
// preloading is done, the whole series is identified as old
// enough for purging and hence purged for good. As there is no
// data left to iterate over, return an iterator that will never
// return any values.
return nopSeriesIterator{}
}
return &boundedIterator{
it: series.newIterator(),
start: clientmodel.Now().Add(-s.dropAfter),
}
}
func (t *target) Scrape(earliest time.Time, ingester extraction.Ingester) error {
now := clientmodel.Now()
futureState := t.state
err := t.scrape(now, ingester)
if err != nil {
t.recordScrapeHealth(ingester, now, false)
futureState = UNREACHABLE
} else {
t.recordScrapeHealth(ingester, now, true)
futureState = ALIVE
}
t.scheduler.Reschedule(earliest, futureState)
t.state = futureState
t.lastError = err
return err
}
func TestTargetRecordScrapeHealth(t *testing.T) {
testTarget := newTestTarget("example.url:80", 0, clientmodel.LabelSet{clientmodel.JobLabel: "testjob"})
now := clientmodel.Now()
appender := &collectResultAppender{}
testTarget.status.setLastError(nil)
recordScrapeHealth(appender, now, testTarget.BaseLabels(), testTarget.status.Health(), 2*time.Second)
result := appender.result
if len(result) != 2 {
t.Fatalf("Expected two samples, got %d", len(result))
}
actual := result[0]
expected := &clientmodel.Sample{
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: scrapeHealthMetricName,
clientmodel.InstanceLabel: "example.url:80",
clientmodel.JobLabel: "testjob",
},
Timestamp: now,
Value: 1,
}
if !actual.Equal(expected) {
t.Fatalf("Expected and actual samples not equal. Expected: %v, actual: %v", expected, actual)
}
actual = result[1]
expected = &clientmodel.Sample{
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: scrapeDurationMetricName,
clientmodel.InstanceLabel: "example.url:80",
clientmodel.JobLabel: "testjob",
},
Timestamp: now,
Value: 2.0,
}
if !actual.Equal(expected) {
t.Fatalf("Expected and actual samples not equal. Expected: %v, actual: %v", expected, actual)
}
}
func (t *TieredStorage) flushMemory(ttl time.Duration) {
flushOlderThan := clientmodel.Now().Add(-1 * ttl)
glog.Info("Flushing samples to disk...")
t.memoryArena.Flush(flushOlderThan, t.appendToDiskQueue)
queueLength := len(t.appendToDiskQueue)
if queueLength > 0 {
samples := clientmodel.Samples{}
for i := 0; i < queueLength; i++ {
chunk := <-t.appendToDiskQueue
samples = append(samples, chunk...)
}
glog.Infof("Writing %d samples...", len(samples))
t.DiskStorage.AppendSamples(samples)
}
glog.Info("Done flushing.")
}
func (m *ruleManager) runIteration(results chan<- *extraction.Result) {
now := clientmodel.Now()
wg := sync.WaitGroup{}
m.Lock()
rules := make([]Rule, len(m.rules))
copy(rules, m.rules)
m.Unlock()
for _, rule := range rules {
wg.Add(1)
// BUG(julius): Look at fixing thundering herd.
go func(rule Rule) {
defer wg.Done()
start := time.Now()
vector, err := rule.Eval(now, m.storage)
duration := time.Since(start)
samples := make(clientmodel.Samples, len(vector))
copy(samples, vector)
m.results <- &extraction.Result{
Samples: samples,
Err: err,
}
switch r := rule.(type) {
case *AlertingRule:
m.queueAlertNotifications(r)
recordOutcome(alertingRuleType, duration)
case *RecordingRule:
recordOutcome(recordingRuleType, duration)
default:
panic(fmt.Sprintf("Unknown rule type: %T", rule))
}
}(rule)
}
wg.Wait()
}
func TestGetMetricForFingerprintCachesCopyOfMetric(t *testing.T) {
ts, closer := NewTestTieredStorage(t)
defer closer.Close()
m := clientmodel.Metric{
clientmodel.MetricNameLabel: "testmetric",
}
samples := clientmodel.Samples{
&clientmodel.Sample{
Metric: m,
Value: 0,
Timestamp: clientmodel.Now(),
},
}
if err := ts.AppendSamples(samples); err != nil {
t.Fatal(err)
}
ts.Flush()
fp := &clientmodel.Fingerprint{}
fp.LoadFromMetric(m)
m, err := ts.GetMetricForFingerprint(fp)
if err != nil {
t.Fatal(err)
}
m[clientmodel.MetricNameLabel] = "changedmetric"
m, err = ts.GetMetricForFingerprint(fp)
if err != nil {
t.Fatal(err)
}
if m[clientmodel.MetricNameLabel] != "testmetric" {
t.Fatal("Metric name label value has changed: ", m[clientmodel.MetricNameLabel])
}
}
// cycleThroughArchivedFingerprints returns a channel that emits fingerprints
// for archived series in a throttled fashion. It continues to cycle through all
// archived fingerprints until s.loopStopping is closed.
func (s *memorySeriesStorage) cycleThroughArchivedFingerprints() chan clientmodel.Fingerprint {
archivedFingerprints := make(chan clientmodel.Fingerprint)
go func() {
defer close(archivedFingerprints)
for {
archivedFPs, err := s.persistence.fingerprintsModifiedBefore(
clientmodel.Now().Add(-s.dropAfter),
)
if err != nil {
log.Error("Failed to lookup archived fingerprint ranges: ", err)
s.waitForNextFP(0, 1)
continue
}
// Initial wait, also important if there are no FPs yet.
if !s.waitForNextFP(len(archivedFPs), 1) {
return
}
begin := time.Now()
for _, fp := range archivedFPs {
select {
case archivedFingerprints <- fp:
case <-s.loopStopping:
return
}
// Never speed up maintenance of archived FPs.
s.waitForNextFP(len(archivedFPs), 1)
}
if len(archivedFPs) > 0 {
log.Infof(
"Completed maintenance sweep through %d archived fingerprints in %v.",
len(archivedFPs), time.Since(begin),
)
}
}
}()
return archivedFingerprints
}
func TestTargetRecordScrapeHealth(t *testing.T) {
testTarget := target{
scheduler: literalScheduler{},
address: "http://example.url",
baseLabels: clientmodel.LabelSet{clientmodel.JobLabel: "testjob"},
httpClient: utility.NewDeadlineClient(0),
}
now := clientmodel.Now()
ingester := &collectResultIngester{}
testTarget.recordScrapeHealth(ingester, now, true)
result := ingester.result
if len(result.Samples) != 1 {
t.Fatalf("Expected one sample, got %d", len(result.Samples))
}
actual := result.Samples[0]
expected := &clientmodel.Sample{
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: ScrapeHealthMetricName,
InstanceLabel: "http://example.url",
clientmodel.JobLabel: "testjob",
},
Timestamp: now,
Value: 1,
}
if result.Err != nil {
t.Fatalf("Got unexpected error: %v", result.Err)
}
if !actual.Equal(expected) {
t.Fatalf("Expected and actual samples not equal. Expected: %v, actual: %v", expected, actual)
}
}
func GetFingerprintsForLabelSetUsesAndForLabelMatchingTests(p metric.Persistence, t test.Tester) {
metrics := []clientmodel.LabelSet{
{clientmodel.MetricNameLabel: "request_metrics_latency_equal_tallying_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "requests_metrics_latency_equal_accumulating_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "requests_metrics_latency_logarithmic_accumulating_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "requests_metrics_latency_logarithmic_tallying_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "targets_healthy_scrape_latency_ms", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
}
for _, metric := range metrics {
m := clientmodel.Metric{}
for k, v := range metric {
m[clientmodel.LabelName(k)] = clientmodel.LabelValue(v)
}
testAppendSamples(p, &clientmodel.Sample{
Value: clientmodel.SampleValue(0.0),
Timestamp: clientmodel.Now(),
Metric: m,
}, t)
}
labelSet := clientmodel.LabelSet{
clientmodel.MetricNameLabel: "targets_healthy_scrape_latency_ms",
"percentile": "0.010000",
}
fingerprints, err := p.GetFingerprintsForLabelMatchers(labelMatchersFromLabelSet(labelSet))
if err != nil {
t.Errorf("could not get labels: %s", err)
}
if len(fingerprints) != 1 {
t.Errorf("did not get a single metric as is expected, got %s", fingerprints)
}
}
package extraction
import (
"bytes"
"errors"
"io/ioutil"
"os"
"path"
"runtime"
"sort"
"testing"
"github.com/prometheus/client_golang/model"
)
var test002Time = model.Now()
type testProcessor002ProcessScenario struct {
in string
expected, actual []model.Samples
err error
}
func (s *testProcessor002ProcessScenario) Ingest(samples model.Samples) error {
s.actual = append(s.actual, samples)
return nil
}
func (s *testProcessor002ProcessScenario) test(t testing.TB, set int) {
reader, err := os.Open(path.Join("fixtures", s.in))
if err != nil {
func TestTruncateBefore(t *testing.T) {
type in struct {
values metric.Values
time clientmodel.Timestamp
}
instant := clientmodel.Now()
var scenarios = []struct {
in in
out metric.Values
}{
{
in: in{
time: instant,
values: metric.Values{
{
Value: 0,
Timestamp: instant,
},
{
Value: 1,
Timestamp: instant.Add(time.Second),
},
{
Value: 2,
Timestamp: instant.Add(2 * time.Second),
},
{
Value: 3,
Timestamp: instant.Add(3 * time.Second),
},
{
Value: 4,
Timestamp: instant.Add(4 * time.Second),
},
},
},
out: metric.Values{
{
Value: 0,
Timestamp: instant,
},
{
Value: 1,
Timestamp: instant.Add(time.Second),
},
{
Value: 2,
Timestamp: instant.Add(2 * time.Second),
},
{
Value: 3,
Timestamp: instant.Add(3 * time.Second),
},
{
Value: 4,
Timestamp: instant.Add(4 * time.Second),
},
},
},
{
in: in{
time: instant.Add(2 * time.Second),
values: metric.Values{
{
Value: 0,
Timestamp: instant,
},
{
Value: 1,
Timestamp: instant.Add(time.Second),
},
{
Value: 2,
Timestamp: instant.Add(2 * time.Second),
},
{
Value: 3,
Timestamp: instant.Add(3 * time.Second),
},
{
Value: 4,
Timestamp: instant.Add(4 * time.Second),
},
},
},
out: metric.Values{
{
Value: 1,
Timestamp: instant.Add(time.Second),
},
{
Value: 2,
Timestamp: instant.Add(2 * time.Second),
},
{
Value: 3,
Timestamp: instant.Add(3 * time.Second),
},
{
Value: 4,
Timestamp: instant.Add(4 * time.Second),
},
},
},
{
in: in{
time: instant.Add(5 * time.Second),
values: metric.Values{
{
Value: 0,
Timestamp: instant,
},
{
Value: 1,
Timestamp: instant.Add(time.Second),
},
{
Value: 2,
Timestamp: instant.Add(2 * time.Second),
},
{
Value: 3,
Timestamp: instant.Add(3 * time.Second),
},
{
Value: 4,
Timestamp: instant.Add(4 * time.Second),
},
},
},
out: metric.Values{
// Preserve the last value in case it needs to be used for the next set.
{
Value: 4,
Timestamp: instant.Add(4 * time.Second),
},
},
},
}
for i, scenario := range scenarios {
actual := chunk(scenario.in.values).TruncateBefore(scenario.in.time)
if len(actual) != len(scenario.out) {
t.Fatalf("%d. expected length of %d, got %d", i, len(scenario.out), len(actual))
}
for j, actualValue := range actual {
if !actualValue.Equal(&scenario.out[j]) {
t.Fatalf("%d.%d. expected %s, got %s", i, j, scenario.out[j], actualValue)
}
}
}
}
// See the License for the specific language governing permissions and
// limitations under the License.
package extraction
import (
"errors"
"os"
"path"
"sort"
"testing"
"github.com/prometheus/client_golang/model"
)
var test001Time = model.Now()
type testProcessor001ProcessScenario struct {
in string
expected, actual []model.Samples
err error
}
func (s *testProcessor001ProcessScenario) Ingest(samples model.Samples) error {
s.actual = append(s.actual, samples)
return nil
}
func (s *testProcessor001ProcessScenario) test(t testing.TB, set int) {
reader, err := os.Open(path.Join("fixtures", s.in))
if err != nil {
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extraction
import (
"sort"
"strings"
"testing"
"github.com/prometheus/client_golang/model"
)
var testTime = model.Now()
type metricFamilyProcessorScenario struct {
in string
expected, actual []model.Samples
}
func (s *metricFamilyProcessorScenario) Ingest(samples model.Samples) error {
s.actual = append(s.actual, samples)
return nil
}
func (s *metricFamilyProcessorScenario) test(t *testing.T, set int) {
i := strings.NewReader(s.in)
o := &ProcessOptions{
func createRandomSamples(metricName string, minLen int) clientmodel.Samples {
type valueCreator func() clientmodel.SampleValue
type deltaApplier func(clientmodel.SampleValue) clientmodel.SampleValue
var (
maxMetrics = 5
maxStreakLength = 500
maxTimeDelta = 10000
maxTimeDeltaFactor = 10
timestamp = clientmodel.Now() - clientmodel.Timestamp(maxTimeDelta*maxTimeDeltaFactor*minLen/4) // So that some timestamps are in the future.
generators = []struct {
createValue valueCreator
applyDelta []deltaApplier
}{
{ // "Boolean".
createValue: func() clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(2))
},
applyDelta: []deltaApplier{
func(_ clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(2))
},
},
},
{ // Integer with int deltas of various byte length.
createValue: func() clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Int63() - 1<<62)
},
applyDelta: []deltaApplier{
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(1<<8) - 1<<7 + int(v))
},
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(1<<16) - 1<<15 + int(v))
},
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Int63n(1<<32) - 1<<31 + int64(v))
},
},
},
{ // Float with float32 and float64 deltas.
createValue: func() clientmodel.SampleValue {
return clientmodel.SampleValue(rand.NormFloat64())
},
applyDelta: []deltaApplier{
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return v + clientmodel.SampleValue(float32(rand.NormFloat64()))
},
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return v + clientmodel.SampleValue(rand.NormFloat64())
},
},
},
}
)
// Prefill result with two samples with colliding metrics (to test fingerprint mapping).
result := clientmodel.Samples{
&clientmodel.Sample{
Metric: clientmodel.Metric{
"instance": "ip-10-33-84-73.l05.ams5.s-cloud.net:24483",
"status": "503",
},
Value: 42,
Timestamp: timestamp,
},
&clientmodel.Sample{
Metric: clientmodel.Metric{
"instance": "ip-10-33-84-73.l05.ams5.s-cloud.net:24480",
"status": "500",
},
Value: 2010,
Timestamp: timestamp + 1,
},
}
metrics := []clientmodel.Metric{}
for n := rand.Intn(maxMetrics); n >= 0; n-- {
metrics = append(metrics, clientmodel.Metric{
clientmodel.MetricNameLabel: clientmodel.LabelValue(metricName),
clientmodel.LabelName(fmt.Sprintf("labelname_%d", n+1)): clientmodel.LabelValue(fmt.Sprintf("labelvalue_%d", rand.Int())),
})
}
for len(result) < minLen {
// Pick a metric for this cycle.
metric := metrics[rand.Intn(len(metrics))]
timeDelta := rand.Intn(maxTimeDelta) + 1
generator := generators[rand.Intn(len(generators))]
createValue := generator.createValue
applyDelta := generator.applyDelta[rand.Intn(len(generator.applyDelta))]
incTimestamp := func() { timestamp += clientmodel.Timestamp(timeDelta * (rand.Intn(maxTimeDeltaFactor) + 1)) }
switch rand.Intn(4) {
case 0: // A single sample.
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: createValue(),
Timestamp: timestamp,
})
incTimestamp()
case 1: // A streak of random sample values.
for n := rand.Intn(maxStreakLength); n >= 0; n-- {
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: createValue(),
Timestamp: timestamp,
})
incTimestamp()
}
case 2: // A streak of sample values with incremental changes.
value := createValue()
for n := rand.Intn(maxStreakLength); n >= 0; n-- {
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: value,
Timestamp: timestamp,
})
incTimestamp()
value = applyDelta(value)
}
case 3: // A streak of constant sample values.
value := createValue()
for n := rand.Intn(maxStreakLength); n >= 0; n-- {
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: value,
Timestamp: timestamp,
})
incTimestamp()
}
}
}
return result
}
func (serv MetricsService) QueryRange(w http.ResponseWriter, r *http.Request) {
setAccessControlHeaders(w)
w.Header().Set("Content-Type", "application/json")
params := http_utils.GetQueryParams(r)
expr := params.Get("expr")
end, _ := strconv.ParseInt(params.Get("end"), 0, 64)
duration, _ := strconv.ParseInt(params.Get("range"), 0, 64)
step, _ := strconv.ParseInt(params.Get("step"), 0, 64)
exprNode, err := rules.LoadExprFromString(expr)
if err != nil {
fmt.Fprint(w, ast.ErrorToJSON(err))
return
}
if exprNode.Type() != ast.VECTOR {
fmt.Fprint(w, ast.ErrorToJSON(errors.New("Expression does not evaluate to vector type")))
return
}
if end == 0 {
end = clientmodel.Now().Unix()
}
if step < 1 {
step = 1
}
if end-duration < 0 {
duration = end
}
// Align the start to step "tick" boundary.
end -= end % step
queryStats := stats.NewTimerGroup()
evalTimer := queryStats.GetTimer(stats.TotalEvalTime).Start()
matrix, err := ast.EvalVectorRange(
exprNode.(ast.VectorNode),
clientmodel.TimestampFromUnix(end-duration),
clientmodel.TimestampFromUnix(end),
time.Duration(step)*time.Second,
serv.Storage,
queryStats)
if err != nil {
fmt.Fprint(w, ast.ErrorToJSON(err))
return
}
evalTimer.Stop()
sortTimer := queryStats.GetTimer(stats.ResultSortTime).Start()
sort.Sort(matrix)
sortTimer.Stop()
jsonTimer := queryStats.GetTimer(stats.JsonEncodeTime).Start()
result := ast.TypedValueToJSON(matrix, "matrix")
jsonTimer.Stop()
glog.Infof("Range query: %s\nQuery stats:\n%s\n", expr, queryStats)
fmt.Fprint(w, result)
}
// QueryRange handles the /api/query_range endpoint.
func (serv MetricsService) QueryRange(w http.ResponseWriter, r *http.Request) {
setAccessControlHeaders(w)
w.Header().Set("Content-Type", "application/json")
params := httputils.GetQueryParams(r)
expr := params.Get("expr")
// Input times and durations are in seconds and get converted to nanoseconds.
endFloat, _ := strconv.ParseFloat(params.Get("end"), 64)
durationFloat, _ := strconv.ParseFloat(params.Get("range"), 64)
stepFloat, _ := strconv.ParseFloat(params.Get("step"), 64)
nanosPerSecond := int64(time.Second / time.Nanosecond)
end := int64(endFloat) * nanosPerSecond
duration := int64(durationFloat) * nanosPerSecond
step := int64(stepFloat) * nanosPerSecond
exprNode, err := rules.LoadExprFromString(expr)
if err != nil {
fmt.Fprint(w, ast.ErrorToJSON(err))
return
}
if exprNode.Type() != ast.VectorType {
fmt.Fprint(w, ast.ErrorToJSON(errors.New("expression does not evaluate to vector type")))
return
}
if end == 0 {
end = clientmodel.Now().UnixNano()
}
if step <= 0 {
step = nanosPerSecond
}
if end-duration < 0 {
duration = end
}
// For safety, limit the number of returned points per timeseries.
// This is sufficient for 60s resolution for a week or 1h resolution for a year.
if duration/step > 11000 {
fmt.Fprint(w, ast.ErrorToJSON(errors.New("exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)")))
return
}
// Align the start to step "tick" boundary.
end -= end % step
queryStats := stats.NewTimerGroup()
evalTimer := queryStats.GetTimer(stats.TotalEvalTime).Start()
matrix, err := ast.EvalVectorRange(
exprNode.(ast.VectorNode),
clientmodel.TimestampFromUnixNano(end-duration),
clientmodel.TimestampFromUnixNano(end),
time.Duration(step),
serv.Storage,
queryStats)
if err != nil {
fmt.Fprint(w, ast.ErrorToJSON(err))
return
}
evalTimer.Stop()
sortTimer := queryStats.GetTimer(stats.ResultSortTime).Start()
sort.Sort(matrix)
sortTimer.Stop()
jsonTimer := queryStats.GetTimer(stats.JSONEncodeTime).Start()
result := ast.TypedValueToJSON(matrix, "matrix")
jsonTimer.Stop()
glog.V(1).Infof("Range query: %s\nQuery stats:\n%s\n", expr, queryStats)
fmt.Fprint(w, result)
}
func testEvictAndLoadChunkDescs(t *testing.T, encoding chunkEncoding) {
samples := make(clientmodel.Samples, 10000)
for i := range samples {
samples[i] = &clientmodel.Sample{
Timestamp: clientmodel.Timestamp(2 * i),
Value: clientmodel.SampleValue(float64(i * i)),
}
}
// Give last sample a timestamp of now so that the head chunk will not
// be closed (which would then archive the time series later as
// everything will get evicted).
samples[len(samples)-1] = &clientmodel.Sample{
Timestamp: clientmodel.Now(),
Value: clientmodel.SampleValue(3.14),
}
s, closer := NewTestStorage(t, encoding)
defer closer.Close()
// Adjust memory chunks to lower value to see evictions.
s.maxMemoryChunks = 1
for _, sample := range samples {
s.Append(sample)
}
s.WaitForIndexing()
fp := clientmodel.Metric{}.FastFingerprint()
series, ok := s.fpToSeries.get(fp)
if !ok {
t.Fatal("could not find series")
}
oldLen := len(series.chunkDescs)
// Maintain series without any dropped chunks.
s.maintainMemorySeries(fp, 0)
// Give the evict goroutine an opportunity to run.
time.Sleep(10 * time.Millisecond)
// Maintain series again to trigger chunkDesc eviction
s.maintainMemorySeries(fp, 0)
if oldLen <= len(series.chunkDescs) {
t.Errorf("Expected number of chunkDescs to decrease, old number %d, current number %d.", oldLen, len(series.chunkDescs))
}
// Load everything back.
p := s.NewPreloader()
p.PreloadRange(fp, 0, 100000, time.Hour)
if oldLen != len(series.chunkDescs) {
t.Errorf("Expected number of chunkDescs to have reached old value again, old number %d, current number %d.", oldLen, len(series.chunkDescs))
}
p.Close()
// Now maintain series with drops to make sure nothing crazy happens.
s.maintainMemorySeries(fp, 100000)
if len(series.chunkDescs) != 1 {
t.Errorf("Expected exactly one chunkDesc left, got %d.", len(series.chunkDescs))
}
}
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package extraction
import (
"sort"
"strings"
"testing"
"github.com/prometheus/client_golang/model"
)
var (
ts = model.Now()
in = `
# Only a quite simple scenario with two metric families.
# More complicated tests of the parser itself can be found in the text package.
# TYPE mf2 counter
mf2 3
mf1{label="value1"} -3.14 123456
mf1{label="value2"} 42
mf2 4
`
out = map[model.LabelValue]*Result{
"mf1": {
Samples: model.Samples{
&model.Sample{
Metric: model.Metric{model.MetricNameLabel: "mf1", "label": "value1"},
Value: -3.14,
func TestRetentionCutoff(t *testing.T) {
now := clientmodel.Now()
insertStart := now.Add(-2 * time.Hour)
s, closer := NewTestStorage(t, 1)
defer closer.Close()
// Stop maintenance loop to prevent actual purging.
s.loopStopping <- struct{}{}
s.dropAfter = 1 * time.Hour
for i := 0; i < 120; i++ {
smpl := &clientmodel.Sample{
Metric: clientmodel.Metric{"job": "test"},
Timestamp: insertStart.Add(time.Duration(i) * time.Minute), // 1 minute intervals.
Value: 1,
}
s.Append(smpl)
}
s.WaitForIndexing()
var fp clientmodel.Fingerprint
for f := range s.fingerprintsForLabelPairs(metric.LabelPair{Name: "job", Value: "test"}) {
fp = f
break
}
pl := s.NewPreloader()
defer pl.Close()
// Preload everything.
err := pl.PreloadRange(fp, insertStart, now, 5*time.Minute)
if err != nil {
t.Fatalf("Error preloading outdated chunks: %s", err)
}
it := s.NewIterator(fp)
vals := it.ValueAtTime(now.Add(-61 * time.Minute))
if len(vals) != 0 {
t.Errorf("unexpected result for timestamp before retention period")
}
vals = it.RangeValues(metric.Interval{OldestInclusive: insertStart, NewestInclusive: now})
// We get 59 values here because the clientmodel.Now() is slightly later
// than our now.
if len(vals) != 59 {
t.Errorf("expected 59 values but got %d", len(vals))
}
if expt := now.Add(-1 * time.Hour).Add(time.Minute); vals[0].Timestamp != expt {
t.Errorf("unexpected timestamp for first sample: %v, expected %v", vals[0].Timestamp.Time(), expt.Time())
}
vals = it.BoundaryValues(metric.Interval{OldestInclusive: insertStart, NewestInclusive: now})
if len(vals) != 2 {
t.Errorf("expected 2 values but got %d", len(vals))
}
if expt := now.Add(-1 * time.Hour).Add(time.Minute); vals[0].Timestamp != expt {
t.Errorf("unexpected timestamp for first sample: %v, expected %v", vals[0].Timestamp.Time(), expt.Time())
}
}
func TestDropMetrics(t *testing.T) {
now := clientmodel.Now()
insertStart := now.Add(-2 * time.Hour)
s, closer := NewTestStorage(t, 1)
defer closer.Close()
m1 := clientmodel.Metric{clientmodel.MetricNameLabel: "test", "n1": "v1"}
m2 := clientmodel.Metric{clientmodel.MetricNameLabel: "test", "n1": "v2"}
N := 120000
for j, m := range []clientmodel.Metric{m1, m2} {
for i := 0; i < N; i++ {
smpl := &clientmodel.Sample{
Metric: m,
Timestamp: insertStart.Add(time.Duration(i) * time.Millisecond), // 1 minute intervals.
Value: clientmodel.SampleValue(j),
}
s.Append(smpl)
}
}
s.WaitForIndexing()
fps := s.fingerprintsForLabelPairs(metric.LabelPair{Name: clientmodel.MetricNameLabel, Value: "test"})
if len(fps) != 2 {
t.Fatalf("unexpected number of fingerprints: %d", len(fps))
}
var fpList clientmodel.Fingerprints
for fp := range fps {
it := s.NewIterator(fp)
if vals := it.RangeValues(metric.Interval{OldestInclusive: insertStart, NewestInclusive: now}); len(vals) != N {
t.Fatalf("unexpected number of samples: %d", len(vals))
}
fpList = append(fpList, fp)
}
s.DropMetricsForFingerprints(fpList[0])
s.WaitForIndexing()
fps2 := s.fingerprintsForLabelPairs(metric.LabelPair{
Name: clientmodel.MetricNameLabel, Value: "test",
})
if len(fps2) != 1 {
t.Fatalf("unexpected number of fingerprints: %d", len(fps2))
}
it := s.NewIterator(fpList[0])
if vals := it.RangeValues(metric.Interval{OldestInclusive: insertStart, NewestInclusive: now}); len(vals) != 0 {
t.Fatalf("unexpected number of samples: %d", len(vals))
}
it = s.NewIterator(fpList[1])
if vals := it.RangeValues(metric.Interval{OldestInclusive: insertStart, NewestInclusive: now}); len(vals) != N {
t.Fatalf("unexpected number of samples: %d", len(vals))
}
s.DropMetricsForFingerprints(fpList...)
s.WaitForIndexing()
fps3 := s.fingerprintsForLabelPairs(metric.LabelPair{
Name: clientmodel.MetricNameLabel, Value: "test",
})
if len(fps3) != 0 {
t.Fatalf("unexpected number of fingerprints: %d", len(fps3))
}
it = s.NewIterator(fpList[0])
if vals := it.RangeValues(metric.Interval{OldestInclusive: insertStart, NewestInclusive: now}); len(vals) != 0 {
t.Fatalf("unexpected number of samples: %d", len(vals))
}
it = s.NewIterator(fpList[1])
if vals := it.RangeValues(metric.Interval{OldestInclusive: insertStart, NewestInclusive: now}); len(vals) != 0 {
t.Fatalf("unexpected number of samples: %d", len(vals))
}
}