func ReadEmptyTests(p metric.Persistence, t test.Tester) {
hasLabelPair := func(x int) (success bool) {
fingerprints, err := p.GetFingerprintsForLabelMatchers(metric.LabelMatchers{{
Type: metric.Equal,
Name: clientmodel.LabelName(string(x)),
Value: clientmodel.LabelValue(string(x)),
}})
if err != nil {
t.Error(err)
return
}
success = len(fingerprints) == 0
if !success {
t.Errorf("unexpected fingerprint length %d, got %d", 0, len(fingerprints))
}
return
}
err := quick.Check(hasLabelPair, nil)
if err != nil {
t.Error(err)
return
}
hasLabelName := func(x int) (success bool) {
labelName := clientmodel.LabelName(string(x))
values, err := p.GetLabelValuesForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
success = len(values) == 0
if !success {
t.Errorf("unexpected values length %d, got %d", 0, len(values))
}
return
}
err = quick.Check(hasLabelName, nil)
if err != nil {
t.Error(err)
return
}
}
func AppendSampleAsPureSparseAppendTests(p metric.Persistence, t test.Tester) {
appendSample := func(x int) (success bool) {
v := clientmodel.SampleValue(x)
ts := clientmodel.TimestampFromUnix(int64(x))
labelName := clientmodel.LabelName(x)
labelValue := clientmodel.LabelValue(x)
l := clientmodel.Metric{labelName: labelValue}
sample := &clientmodel.Sample{
Value: v,
Timestamp: ts,
Metric: l,
}
err := p.AppendSamples(clientmodel.Samples{sample})
success = err == nil
if !success {
t.Error(err)
}
return
}
if err := quick.Check(appendSample, nil); err != nil {
t.Error(err)
}
}
func (s *testNotificationScenario) test(i int, t *testing.T) {
h := NewNotificationHandler(&NotificationHandlerOptions{
AlertmanagerURL: "alertmanager_url",
QueueCapacity: 0,
Deadline: 10 * time.Second,
})
defer h.Stop()
receivedPost := make(chan bool, 1)
poster := testHTTPPoster{receivedPost: receivedPost}
h.httpClient = &poster
go h.Run()
h.SubmitReqs(NotificationReqs{
{
Summary: s.summary,
Description: s.description,
Labels: clientmodel.LabelSet{
clientmodel.LabelName("instance"): clientmodel.LabelValue("testinstance"),
},
Value: clientmodel.SampleValue(1.0 / 3.0),
ActiveSince: time.Time{},
RuleString: "Test rule string",
GeneratorURL: "prometheus_url",
},
})
<-receivedPost
if poster.message != s.message {
t.Fatalf("%d. Expected '%s', received '%s'", i, s.message, poster.message)
}
}
func (s *testNotificationScenario) test(i int, t *testing.T) {
notifications := make(chan NotificationReqs)
defer close(notifications)
h := NewNotificationHandler("alertmanager_url", notifications)
receivedPost := make(chan bool, 1)
poster := testHttpPoster{receivedPost: receivedPost}
h.httpClient = &poster
go h.Run()
notifications <- NotificationReqs{
{
Summary: s.summary,
Description: s.description,
Labels: clientmodel.LabelSet{
clientmodel.LabelName("instance"): clientmodel.LabelValue("testinstance"),
},
Value: clientmodel.SampleValue(1.0 / 3.0),
ActiveSince: time.Time{},
RuleString: "Test rule string",
GeneratorUrl: "prometheus_url",
},
}
<-receivedPost
if poster.message != s.message {
t.Fatalf("%d. Expected '%s', received '%s'", i, s.message, poster.message)
}
}
func parseServersetMember(data []byte, path string) (*clientmodel.LabelSet, error) {
member := serversetMember{}
err := json.Unmarshal(data, &member)
if err != nil {
return nil, fmt.Errorf("error unmarshaling serverset member %q: %s", path, err)
}
labels := clientmodel.LabelSet{}
labels[serversetPathLabel] = clientmodel.LabelValue(path)
labels[clientmodel.AddressLabel] = clientmodel.LabelValue(
fmt.Sprintf("%s:%d", member.ServiceEndpoint.Host, member.ServiceEndpoint.Port))
labels[serversetEndpointLabelPrefix+"_host"] = clientmodel.LabelValue(member.ServiceEndpoint.Host)
labels[serversetEndpointLabelPrefix+"_port"] = clientmodel.LabelValue(fmt.Sprintf("%d", member.ServiceEndpoint.Port))
for name, endpoint := range member.AdditionalEndpoints {
cleanName := clientmodel.LabelName(invalidLabelCharRE.ReplaceAllString(name, "_"))
labels[serversetEndpointLabelPrefix+"_host_"+cleanName] = clientmodel.LabelValue(
endpoint.Host)
labels[serversetEndpointLabelPrefix+"_port_"+cleanName] = clientmodel.LabelValue(
fmt.Sprintf("%d", endpoint.Port))
}
labels[serversetStatusLabel] = clientmodel.LabelValue(member.Status)
return &labels, nil
}
func extractCounter(out Ingester, o *ProcessOptions, f *dto.MetricFamily) error {
samples := make(model.Samples, 0, len(f.Metric))
for _, m := range f.Metric {
if m.Counter == nil {
continue
}
sample := new(model.Sample)
samples = append(samples, sample)
if m.TimestampMs != nil {
sample.Timestamp = model.TimestampFromUnix(*m.TimestampMs / 1000)
} else {
sample.Timestamp = o.Timestamp
}
sample.Metric = model.Metric{}
metric := sample.Metric
for _, p := range m.Label {
metric[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
}
metric[model.MetricNameLabel] = model.LabelValue(f.GetName())
sample.Value = model.SampleValue(m.Counter.GetValue())
}
return out.Ingest(&Result{Samples: samples})
}
func extractUntyped(out Ingester, o *ProcessOptions, f *dto.MetricFamily) error {
samples := make(model.Samples, 0, len(f.Metric))
for _, m := range f.Metric {
if m.Untyped == nil {
continue
}
sample := &model.Sample{
Metric: model.Metric{},
Value: model.SampleValue(m.Untyped.GetValue()),
}
samples = append(samples, sample)
if m.TimestampMs != nil {
sample.Timestamp = model.TimestampFromUnixNano(*m.TimestampMs * 1000000)
} else {
sample.Timestamp = o.Timestamp
}
metric := sample.Metric
for _, p := range m.Label {
metric[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
}
metric[model.MetricNameLabel] = model.LabelValue(f.GetName())
}
return out.Ingest(samples)
}
func dumpMetric(d *dto.Metric, m clientmodel.Metric) {
d.Reset()
metricLength := len(m)
labelNames := make([]string, 0, metricLength)
for labelName := range m {
labelNames = append(labelNames, string(labelName))
}
sort.Strings(labelNames)
pairs := make([]*dto.LabelPair, 0, metricLength)
for _, labelName := range labelNames {
l := clientmodel.LabelName(labelName)
labelValue := m[l]
labelPair := &dto.LabelPair{
Name: proto.String(string(labelName)),
Value: proto.String(string(labelValue)),
}
pairs = append(pairs, labelPair)
}
d.LabelPair = pairs
}
func (m *targetManager) AddTargetsFromConfig(config config.Config) {
for _, job := range config.Jobs() {
if job.SdName != nil {
m.Lock()
m.targetPoolForJob(job)
m.Unlock()
continue
}
for _, targetGroup := range job.TargetGroup {
baseLabels := clientmodel.LabelSet{
clientmodel.JobLabel: clientmodel.LabelValue(job.GetName()),
}
if targetGroup.Labels != nil {
for _, label := range targetGroup.Labels.Label {
baseLabels[clientmodel.LabelName(label.GetName())] = clientmodel.LabelValue(label.GetValue())
}
}
for _, endpoint := range targetGroup.Target {
target := NewTarget(endpoint, job.ScrapeTimeout(), baseLabels)
m.AddTarget(job, target)
}
}
}
}
// Helper function to convert map[string]string into LabelSet.
//
// NOTE: This should be deleted when support for go 1.0.3 is removed; 1.1 is
// smart enough to unmarshal JSON objects into LabelSet directly.
func labelSet(labels map[string]string) model.LabelSet {
labelset := make(model.LabelSet, len(labels))
for k, v := range labels {
labelset[model.LabelName(k)] = model.LabelValue(v)
}
return labelset
}
// Return the global labels as a LabelSet.
func (c Config) GlobalLabels() clientmodel.LabelSet {
labels := clientmodel.LabelSet{}
if c.Global.Labels != nil {
for _, label := range c.Global.Labels.Label {
labels[clientmodel.LabelName(label.GetName())] = clientmodel.LabelValue(label.GetValue())
}
}
return labels
}
// === label_replace(vector ExprVector, dst_label, replacement, src_labelname, regex ExprString) Vector ===
func funcLabelReplace(ev *evaluator, args Expressions) Value {
var (
vector = ev.evalVector(args[0])
dst = clientmodel.LabelName(ev.evalString(args[1]).Value)
repl = ev.evalString(args[2]).Value
src = clientmodel.LabelName(ev.evalString(args[3]).Value)
regexStr = ev.evalString(args[4]).Value
)
regex, err := regexp.Compile(regexStr)
if err != nil {
ev.errorf("invalid regular expression in label_replace(): %s", regexStr)
}
if !clientmodel.LabelNameRE.MatchString(string(dst)) {
ev.errorf("invalid destination label name in label_replace(): %s", dst)
}
outSet := make(map[clientmodel.Fingerprint]struct{}, len(vector))
for _, el := range vector {
srcVal := string(el.Metric.Metric[src])
indexes := regex.FindStringSubmatchIndex(srcVal)
// If there is no match, no replacement should take place.
if indexes == nil {
continue
}
res := regex.ExpandString([]byte{}, repl, srcVal, indexes)
if len(res) == 0 {
el.Metric.Delete(dst)
} else {
el.Metric.Set(dst, clientmodel.LabelValue(res))
}
fp := el.Metric.Metric.Fingerprint()
if _, exists := outSet[fp]; exists {
ev.errorf("duplicated label set in output of label_replace(): %s", el.Metric.Metric)
} else {
outSet[fp] = struct{}{}
}
}
return vector
}
func (api *API) labelValues(r *http.Request) (interface{}, *apiError) {
name := route.Param(api.context(r), "name")
if !clientmodel.LabelNameRE.MatchString(name) {
return nil, &apiError{errorBadData, fmt.Errorf("invalid label name: %q", name)}
}
vals := api.Storage.LabelValuesForLabelName(clientmodel.LabelName(name))
sort.Sort(vals)
return vals, nil
}
func AppendSampleAsSparseAppendWithReadsTests(p metric.Persistence, t test.Tester) {
appendSample := func(x int) (success bool) {
v := clientmodel.SampleValue(x)
ts := clientmodel.TimestampFromUnix(int64(x))
labelName := clientmodel.LabelName(x)
labelValue := clientmodel.LabelValue(x)
l := clientmodel.Metric{labelName: labelValue}
sample := &clientmodel.Sample{
Value: v,
Timestamp: ts,
Metric: l,
}
err := p.AppendSamples(clientmodel.Samples{sample})
if err != nil {
t.Error(err)
return
}
values, err := p.GetLabelValuesForLabelName(labelName)
if err != nil {
t.Error(err)
return
}
if len(values) != 1 {
t.Errorf("expected label values count of %d, got %d", 1, len(values))
return
}
fingerprints, err := p.GetFingerprintsForLabelMatchers(metric.LabelMatchers{{
Type: metric.Equal,
Name: labelName,
Value: labelValue,
}})
if err != nil {
t.Error(err)
return
}
if len(fingerprints) != 1 {
t.Errorf("expected fingerprint count of %d, got %d", 1, len(fingerprints))
return
}
return true
}
if err := quick.Check(appendSample, nil); err != nil {
t.Error(err)
}
}
func (kd *KubernetesDiscovery) updateServiceTargetGroup(service *Service, endpoints *Endpoints) *config.TargetGroup {
tg := &config.TargetGroup{
Source: serviceSource(service),
Labels: clientmodel.LabelSet{
serviceNamespaceLabel: clientmodel.LabelValue(service.ObjectMeta.Namespace),
serviceNameLabel: clientmodel.LabelValue(service.ObjectMeta.Name),
},
}
for k, v := range service.ObjectMeta.Labels {
labelName := strutil.SanitizeLabelName(serviceLabelPrefix + k)
tg.Labels[clientmodel.LabelName(labelName)] = clientmodel.LabelValue(v)
}
for k, v := range service.ObjectMeta.Annotations {
labelName := strutil.SanitizeLabelName(serviceAnnotationPrefix + k)
tg.Labels[clientmodel.LabelName(labelName)] = clientmodel.LabelValue(v)
}
// Now let's loop through the endpoints & add them to the target group with appropriate labels.
for _, eps := range endpoints.Subsets {
epPort := eps.Ports[0].Port
for _, addr := range eps.Addresses {
ipAddr := addr.IP
if len(ipAddr) == net.IPv6len {
ipAddr = "[" + ipAddr + "]"
}
address := fmt.Sprintf("%s:%d", ipAddr, epPort)
t := clientmodel.LabelSet{clientmodel.AddressLabel: clientmodel.LabelValue(address)}
tg.Targets = append(tg.Targets, t)
}
}
return tg
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (lp *LabelPair) UnmarshalBinary(buf []byte) error {
r := bytes.NewReader(buf)
n, err := decodeString(r)
if err != nil {
return err
}
v, err := decodeString(r)
if err != nil {
return err
}
lp.Name = clientmodel.LabelName(n)
lp.Value = clientmodel.LabelValue(v)
return nil
}
func relabel(labels clientmodel.LabelSet, cfg *config.RelabelConfig) (clientmodel.LabelSet, error) {
values := make([]string, 0, len(cfg.SourceLabels))
for _, ln := range cfg.SourceLabels {
values = append(values, string(labels[ln]))
}
val := strings.Join(values, cfg.Separator)
switch cfg.Action {
case config.RelabelDrop:
if cfg.Regex.MatchString(val) {
return nil, nil
}
case config.RelabelKeep:
if !cfg.Regex.MatchString(val) {
return nil, nil
}
case config.RelabelReplace:
indexes := cfg.Regex.FindStringSubmatchIndex(val)
// If there is no match no replacement must take place.
if indexes == nil {
break
}
res := cfg.Regex.ExpandString([]byte{}, cfg.Replacement, val, indexes)
if len(res) == 0 {
delete(labels, cfg.TargetLabel)
} else {
labels[cfg.TargetLabel] = clientmodel.LabelValue(res)
}
case config.RelabelHashMod:
mod := sum64(md5.Sum([]byte(val))) % cfg.Modulus
labels[cfg.TargetLabel] = clientmodel.LabelValue(fmt.Sprintf("%d", mod))
case config.RelabelLabelMap:
out := make(clientmodel.LabelSet, len(labels))
// Take a copy to avoid infinite loops.
for ln, lv := range labels {
out[ln] = lv
}
for ln, lv := range labels {
if cfg.Regex.MatchString(string(ln)) {
res := cfg.Regex.ReplaceAllString(string(ln), cfg.Replacement)
out[clientmodel.LabelName(res)] = lv
}
}
labels = out
default:
panic(fmt.Errorf("retrieval.relabel: unknown relabel action type %q", cfg.Action))
}
return labels, nil
}
// Lookup implements FingerprintMetricIndex.
func (i *LevelDBFingerprintMetricIndex) Lookup(f *clientmodel.Fingerprint) (m clientmodel.Metric, ok bool, err error) {
k := &dto.Fingerprint{}
dumpFingerprint(k, f)
v := &dto.Metric{}
if ok, err := i.LevelDBPersistence.Get(k, v); !ok {
return nil, false, nil
} else if err != nil {
return nil, false, err
}
m = clientmodel.Metric{}
for _, pair := range v.LabelPair {
m[clientmodel.LabelName(pair.GetName())] = clientmodel.LabelValue(pair.GetValue())
}
return m, true, nil
}
// labels parses a list of labelnames.
//
// '(' <label_name>, ... ')'
//
func (p *parser) labels() clientmodel.LabelNames {
const ctx = "grouping opts"
p.expect(itemLeftParen, ctx)
labels := clientmodel.LabelNames{}
for {
id := p.expect(itemIdentifier, ctx)
labels = append(labels, clientmodel.LabelName(id.val))
if p.peek().typ != itemComma {
break
}
p.next()
}
p.expect(itemRightParen, ctx)
return labels
}
// UnmarshalFromReader unmarshals a Metric from a reader that implements
// both, io.Reader and io.ByteReader. It can be used with the zero value of
// Metric.
func (m *Metric) UnmarshalFromReader(r byteReader) error {
numLabelPairs, err := binary.ReadVarint(r)
if err != nil {
return err
}
*m = make(Metric, numLabelPairs)
for ; numLabelPairs > 0; numLabelPairs-- {
ln, err := decodeString(r)
if err != nil {
return err
}
lv, err := decodeString(r)
if err != nil {
return err
}
(*m)[clientmodel.LabelName(ln)] = clientmodel.LabelValue(lv)
}
return nil
}
func createTargetGroup(app *App) *config.TargetGroup {
var (
targets = targetsForApp(app)
appName = clientmodel.LabelValue(app.ID)
image = clientmodel.LabelValue(app.Container.Docker.Image)
)
tg := &config.TargetGroup{
Targets: targets,
Labels: clientmodel.LabelSet{
appLabel: appName,
imageLabel: image,
},
Source: app.ID,
}
for ln, lv := range app.Labels {
ln = appLabelPrefix + ln
tg.Labels[clientmodel.LabelName(ln)] = clientmodel.LabelValue(lv)
}
return tg
}
// SetTargets handles the /api/targets endpoint.
func (serv MetricsService) SetTargets(w http.ResponseWriter, r *http.Request) {
params := httputils.GetQueryParams(r)
jobName := params.Get("job")
decoder := json.NewDecoder(r.Body)
var targetGroups []TargetGroup
err := decoder.Decode(&targetGroups)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
job := serv.Config.GetJobByName(jobName)
if job == nil {
http.Error(w, "job not found", http.StatusNotFound)
return
}
newTargets := []retrieval.Target{}
for _, targetGroup := range targetGroups {
// Do mandatory map type conversion due to Go shortcomings.
baseLabels := clientmodel.LabelSet{
clientmodel.JobLabel: clientmodel.LabelValue(job.GetName()),
}
for label, value := range targetGroup.BaseLabels {
baseLabels[clientmodel.LabelName(label)] = clientmodel.LabelValue(value)
}
for _, endpoint := range targetGroup.Endpoints {
newTarget := retrieval.NewTarget(endpoint, job.ScrapeTimeout(), baseLabels)
newTargets = append(newTargets, newTarget)
}
}
// BUG(julius): Validate that this ScrapeInterval is in fact the proper one
// for the job.
serv.TargetManager.ReplaceTargets(*job, newTargets)
}
func benchmarkAppendSamples(b *testing.B, labels int) {
b.StopTimer()
s := NewMemorySeriesStorage(MemorySeriesOptions{})
metric := clientmodel.Metric{}
for i := 0; i < labels; i++ {
metric[clientmodel.LabelName(fmt.Sprintf("label_%d", i))] = clientmodel.LabelValue(fmt.Sprintf("value_%d", i))
}
samples := make(clientmodel.Samples, 0, b.N)
for i := 0; i < b.N; i++ {
samples = append(samples, &clientmodel.Sample{
Metric: metric,
Value: clientmodel.SampleValue(i),
Timestamp: clientmodel.TimestampFromTime(time.Date(i, 0, 0, 0, 0, 0, 0, time.UTC)),
})
}
b.StartTimer()
for i := 0; i < b.N; i++ {
s.AppendSample(samples[i])
}
}
func (kd *KubernetesDiscovery) updateNodesTargetGroup() *config.TargetGroup {
kd.nodesMu.Lock()
defer kd.nodesMu.Unlock()
tg := &config.TargetGroup{Source: nodesTargetGroupName}
// Now let's loop through the nodes & add them to the target group with appropriate labels.
for nodeName, node := range kd.nodes {
address := fmt.Sprintf("%s:%d", node.Status.Addresses[0].Address, kd.Conf.KubeletPort)
t := clientmodel.LabelSet{
clientmodel.AddressLabel: clientmodel.LabelValue(address),
nodeLabel: clientmodel.LabelValue(nodeName),
}
for k, v := range node.ObjectMeta.Labels {
labelName := strutil.SanitizeLabelName(nodeLabelPrefix + k)
t[clientmodel.LabelName(labelName)] = clientmodel.LabelValue(v)
}
tg.Targets = append(tg.Targets, t)
}
return tg
}
func GetFingerprintsForLabelSetUsesAndForLabelMatchingTests(p metric.Persistence, t test.Tester) {
metrics := []clientmodel.LabelSet{
{clientmodel.MetricNameLabel: "request_metrics_latency_equal_tallying_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "requests_metrics_latency_equal_accumulating_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "requests_metrics_latency_logarithmic_accumulating_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "requests_metrics_latency_logarithmic_tallying_microseconds", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
{clientmodel.MetricNameLabel: "targets_healthy_scrape_latency_ms", "instance": "http://localhost:9090/metrics.json", "percentile": "0.010000"},
}
for _, metric := range metrics {
m := clientmodel.Metric{}
for k, v := range metric {
m[clientmodel.LabelName(k)] = clientmodel.LabelValue(v)
}
testAppendSamples(p, &clientmodel.Sample{
Value: clientmodel.SampleValue(0.0),
Timestamp: clientmodel.Now(),
Metric: m,
}, t)
}
labelSet := clientmodel.LabelSet{
clientmodel.MetricNameLabel: "targets_healthy_scrape_latency_ms",
"percentile": "0.010000",
}
fingerprints, err := p.GetFingerprintsForLabelMatchers(labelMatchersFromLabelSet(labelSet))
if err != nil {
t.Errorf("could not get labels: %s", err)
}
if len(fingerprints) != 1 {
t.Errorf("did not get a single metric as is expected, got %s", fingerprints)
}
}
func TestRelabel(t *testing.T) {
tests := []struct {
input clientmodel.LabelSet
relabel []*config.RelabelConfig
output clientmodel.LabelSet
}{
{
input: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"a"},
Regex: &config.Regexp{*regexp.MustCompile("f(.*)")},
TargetLabel: clientmodel.LabelName("d"),
Separator: ";",
Replacement: "ch${1}-ch${1}",
Action: config.RelabelReplace,
},
},
output: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
"d": "choo-choo",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"a", "b"},
Regex: &config.Regexp{*regexp.MustCompile("^f(.*);(.*)r$")},
TargetLabel: clientmodel.LabelName("a"),
Separator: ";",
Replacement: "b${1}${2}m", // boobam
Action: config.RelabelReplace,
},
{
SourceLabels: clientmodel.LabelNames{"c", "a"},
Regex: &config.Regexp{*regexp.MustCompile("(b).*b(.*)ba(.*)")},
TargetLabel: clientmodel.LabelName("d"),
Separator: ";",
Replacement: "$1$2$2$3",
Action: config.RelabelReplace,
},
},
output: clientmodel.LabelSet{
"a": "boobam",
"b": "bar",
"c": "baz",
"d": "boooom",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"a"},
Regex: &config.Regexp{*regexp.MustCompile("o$")},
Action: config.RelabelDrop,
}, {
SourceLabels: clientmodel.LabelNames{"a"},
Regex: &config.Regexp{*regexp.MustCompile("f(.*)")},
TargetLabel: clientmodel.LabelName("d"),
Separator: ";",
Replacement: "ch$1-ch$1",
Action: config.RelabelReplace,
},
},
output: nil,
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"a"},
Regex: &config.Regexp{*regexp.MustCompile("no-match")},
Action: config.RelabelDrop,
},
},
output: clientmodel.LabelSet{
"a": "foo",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"a"},
Regex: &config.Regexp{*regexp.MustCompile("no-match")},
Action: config.RelabelKeep,
},
},
output: nil,
},
{
input: clientmodel.LabelSet{
"a": "foo",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"a"},
Regex: &config.Regexp{*regexp.MustCompile("^f")},
Action: config.RelabelKeep,
},
},
output: clientmodel.LabelSet{
"a": "foo",
},
},
{
// No replacement must be applied if there is no match.
input: clientmodel.LabelSet{
"a": "boo",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"a"},
Regex: &config.Regexp{*regexp.MustCompile("^f")},
TargetLabel: clientmodel.LabelName("b"),
Replacement: "bar",
Action: config.RelabelReplace,
},
},
output: clientmodel.LabelSet{
"a": "boo",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
},
relabel: []*config.RelabelConfig{
{
SourceLabels: clientmodel.LabelNames{"c"},
TargetLabel: clientmodel.LabelName("d"),
Separator: ";",
Action: config.RelabelHashMod,
Modulus: 1000,
},
},
output: clientmodel.LabelSet{
"a": "foo",
"b": "bar",
"c": "baz",
"d": "976",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
"b1": "bar",
"b2": "baz",
},
relabel: []*config.RelabelConfig{
{
Regex: &config.Regexp{*regexp.MustCompile("^(b.*)")},
Replacement: "bar_${1}",
Action: config.RelabelLabelMap,
},
},
output: clientmodel.LabelSet{
"a": "foo",
"b1": "bar",
"b2": "baz",
"bar_b1": "bar",
"bar_b2": "baz",
},
},
{
input: clientmodel.LabelSet{
"a": "foo",
"__meta_my_bar": "aaa",
"__meta_my_baz": "bbb",
"__meta_other": "ccc",
},
relabel: []*config.RelabelConfig{
{
Regex: &config.Regexp{*regexp.MustCompile("^__meta_(my.*)")},
Replacement: "${1}",
Action: config.RelabelLabelMap,
},
},
output: clientmodel.LabelSet{
"a": "foo",
"__meta_my_bar": "aaa",
"__meta_my_baz": "bbb",
"__meta_other": "ccc",
"my_bar": "aaa",
"my_baz": "bbb",
},
},
}
for i, test := range tests {
res, err := Relabel(test.input, test.relabel...)
if err != nil {
t.Errorf("Test %d: error relabeling: %s", i+1, err)
}
if !reflect.DeepEqual(res, test.output) {
t.Errorf("Test %d: relabel output mismatch: expected %#v, got %#v", i+1, test.output, res)
}
}
}
func createRandomSamples(metricName string, minLen int) clientmodel.Samples {
type valueCreator func() clientmodel.SampleValue
type deltaApplier func(clientmodel.SampleValue) clientmodel.SampleValue
var (
maxMetrics = 5
maxStreakLength = 500
maxTimeDelta = 10000
maxTimeDeltaFactor = 10
timestamp = clientmodel.Now() - clientmodel.Timestamp(maxTimeDelta*maxTimeDeltaFactor*minLen/4) // So that some timestamps are in the future.
generators = []struct {
createValue valueCreator
applyDelta []deltaApplier
}{
{ // "Boolean".
createValue: func() clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(2))
},
applyDelta: []deltaApplier{
func(_ clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(2))
},
},
},
{ // Integer with int deltas of various byte length.
createValue: func() clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Int63() - 1<<62)
},
applyDelta: []deltaApplier{
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(1<<8) - 1<<7 + int(v))
},
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Intn(1<<16) - 1<<15 + int(v))
},
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return clientmodel.SampleValue(rand.Int63n(1<<32) - 1<<31 + int64(v))
},
},
},
{ // Float with float32 and float64 deltas.
createValue: func() clientmodel.SampleValue {
return clientmodel.SampleValue(rand.NormFloat64())
},
applyDelta: []deltaApplier{
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return v + clientmodel.SampleValue(float32(rand.NormFloat64()))
},
func(v clientmodel.SampleValue) clientmodel.SampleValue {
return v + clientmodel.SampleValue(rand.NormFloat64())
},
},
},
}
)
// Prefill result with two samples with colliding metrics (to test fingerprint mapping).
result := clientmodel.Samples{
&clientmodel.Sample{
Metric: clientmodel.Metric{
"instance": "ip-10-33-84-73.l05.ams5.s-cloud.net:24483",
"status": "503",
},
Value: 42,
Timestamp: timestamp,
},
&clientmodel.Sample{
Metric: clientmodel.Metric{
"instance": "ip-10-33-84-73.l05.ams5.s-cloud.net:24480",
"status": "500",
},
Value: 2010,
Timestamp: timestamp + 1,
},
}
metrics := []clientmodel.Metric{}
for n := rand.Intn(maxMetrics); n >= 0; n-- {
metrics = append(metrics, clientmodel.Metric{
clientmodel.MetricNameLabel: clientmodel.LabelValue(metricName),
clientmodel.LabelName(fmt.Sprintf("labelname_%d", n+1)): clientmodel.LabelValue(fmt.Sprintf("labelvalue_%d", rand.Int())),
})
}
for len(result) < minLen {
// Pick a metric for this cycle.
metric := metrics[rand.Intn(len(metrics))]
timeDelta := rand.Intn(maxTimeDelta) + 1
generator := generators[rand.Intn(len(generators))]
createValue := generator.createValue
applyDelta := generator.applyDelta[rand.Intn(len(generator.applyDelta))]
incTimestamp := func() { timestamp += clientmodel.Timestamp(timeDelta * (rand.Intn(maxTimeDeltaFactor) + 1)) }
switch rand.Intn(4) {
case 0: // A single sample.
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: createValue(),
Timestamp: timestamp,
})
incTimestamp()
case 1: // A streak of random sample values.
for n := rand.Intn(maxStreakLength); n >= 0; n-- {
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: createValue(),
Timestamp: timestamp,
})
incTimestamp()
}
case 2: // A streak of sample values with incremental changes.
value := createValue()
for n := rand.Intn(maxStreakLength); n >= 0; n-- {
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: value,
Timestamp: timestamp,
})
incTimestamp()
value = applyDelta(value)
}
case 3: // A streak of constant sample values.
value := createValue()
for n := rand.Intn(maxStreakLength); n >= 0; n-- {
result = append(result, &clientmodel.Sample{
Metric: metric,
Value: value,
Timestamp: timestamp,
})
incTimestamp()
}
}
}
return result
}
// labelMatchers parses a set of label matchers.
//
// '{' [ <labelname> <match_op> <match_string>, ... ] '}'
//
func (p *parser) labelMatchers(operators ...itemType) metric.LabelMatchers {
const ctx = "label matching"
matchers := metric.LabelMatchers{}
p.expect(itemLeftBrace, ctx)
// Check if no matchers are provided.
if p.peek().typ == itemRightBrace {
p.next()
return matchers
}
for {
label := p.expect(itemIdentifier, ctx)
op := p.next().typ
if !op.isOperator() {
p.errorf("expected label matching operator but got %s", op)
}
var validOp = false
for _, allowedOp := range operators {
if op == allowedOp {
validOp = true
}
}
if !validOp {
p.errorf("operator must be one of %q, is %q", operators, op)
}
val := trimOne(p.expect(itemString, ctx).val)
// Map the item to the respective match type.
var matchType metric.MatchType
switch op {
case itemEQL:
matchType = metric.Equal
case itemNEQ:
matchType = metric.NotEqual
case itemEQLRegex:
matchType = metric.RegexMatch
case itemNEQRegex:
matchType = metric.RegexNoMatch
default:
p.errorf("item %q is not a metric match type", op)
}
m, err := metric.NewLabelMatcher(
matchType,
clientmodel.LabelName(label.val),
clientmodel.LabelValue(val),
)
if err != nil {
p.error(err)
}
matchers = append(matchers, m)
// Terminate list if last matcher.
if p.peek().typ != itemComma {
break
}
p.next()
}
p.expect(itemRightBrace, ctx)
return matchers
}
func (p *processor002) ProcessSingle(in io.Reader, out Ingester, o *ProcessOptions) error {
// Processor for telemetry schema version 0.0.2.
// container for telemetry data
var entities []struct {
BaseLabels map[string]string `json:"baseLabels"`
Docstring string `json:"docstring"`
Metric struct {
Type string `json:"type"`
Values json.RawMessage `json:"value"`
} `json:"metric"`
}
if err := json.NewDecoder(in).Decode(&entities); err != nil {
return err
}
pendingSamples := model.Samples{}
for _, entity := range entities {
switch entity.Metric.Type {
case "counter", "gauge":
var values []counter002
if err := json.Unmarshal(entity.Metric.Values, &values); err != nil {
return fmt.Errorf("could not extract %s value: %s", entity.Metric.Type, err)
}
for _, counter := range values {
labels := labelSet(entity.BaseLabels).Merge(labelSet(counter.Labels))
pendingSamples = append(pendingSamples, &model.Sample{
Metric: model.Metric(labels),
Timestamp: o.Timestamp,
Value: counter.Value,
})
}
case "histogram":
var values []histogram002
if err := json.Unmarshal(entity.Metric.Values, &values); err != nil {
return fmt.Errorf("could not extract %s value: %s", entity.Metric.Type, err)
}
for _, histogram := range values {
for percentile, value := range histogram.Values {
labels := labelSet(entity.BaseLabels).Merge(labelSet(histogram.Labels))
labels[model.LabelName("percentile")] = model.LabelValue(percentile)
pendingSamples = append(pendingSamples, &model.Sample{
Metric: model.Metric(labels),
Timestamp: o.Timestamp,
Value: value,
})
}
}
default:
return fmt.Errorf("unknown metric type %q", entity.Metric.Type)
}
}
if len(pendingSamples) > 0 {
return out.Ingest(pendingSamples)
}
return nil
}
func (p *processor001) ProcessSingle(in io.Reader, out Ingester, o *ProcessOptions) error {
// TODO(matt): Replace with plain-jane JSON unmarshalling.
buffer, err := ioutil.ReadAll(in)
if err != nil {
return err
}
entities := entity001{}
if err = json.Unmarshal(buffer, &entities); err != nil {
return err
}
// TODO(matt): This outer loop is a great basis for parallelization.
pendingSamples := model.Samples{}
for _, entity := range entities {
for _, value := range entity.Metric.Value {
labels := labelSet(entity.BaseLabels).Merge(labelSet(value.Labels))
switch entity.Metric.MetricType {
case gauge001, counter001:
sampleValue, ok := value.Value.(float64)
if !ok {
return fmt.Errorf("could not convert value from %s %s to float64", entity, value)
}
pendingSamples = append(pendingSamples, &model.Sample{
Metric: model.Metric(labels),
Timestamp: o.Timestamp,
Value: model.SampleValue(sampleValue),
})
break
case histogram001:
sampleValue, ok := value.Value.(map[string]interface{})
if !ok {
return fmt.Errorf("could not convert value from %q to a map[string]interface{}", value.Value)
}
for percentile, percentileValue := range sampleValue {
individualValue, ok := percentileValue.(float64)
if !ok {
return fmt.Errorf("could not convert value from %q to a float64", percentileValue)
}
childMetric := make(map[model.LabelName]model.LabelValue, len(labels)+1)
for k, v := range labels {
childMetric[k] = v
}
childMetric[model.LabelName(percentile001)] = model.LabelValue(percentile)
pendingSamples = append(pendingSamples, &model.Sample{
Metric: model.Metric(childMetric),
Timestamp: o.Timestamp,
Value: model.SampleValue(individualValue),
})
}
break
}
}
}
if len(pendingSamples) > 0 {
return out.Ingest(pendingSamples)
}
return nil
}