func (m *targetManager) AddTargetsFromConfig(config config.Config) {
for _, job := range config.Jobs() {
if job.SdName != nil {
m.Lock()
m.targetPoolForJob(job)
m.Unlock()
continue
}
for _, targetGroup := range job.TargetGroup {
baseLabels := clientmodel.LabelSet{
clientmodel.JobLabel: clientmodel.LabelValue(job.GetName()),
}
if targetGroup.Labels != nil {
for _, label := range targetGroup.Labels.Label {
baseLabels[clientmodel.LabelName(label.GetName())] = clientmodel.LabelValue(label.GetValue())
}
}
for _, endpoint := range targetGroup.Target {
target := NewTarget(endpoint, job.ScrapeTimeout(), baseLabels)
m.AddTarget(job, target)
}
}
}
}
func relabel(labels clientmodel.LabelSet, cfg *config.RelabelConfig) (clientmodel.LabelSet, error) {
values := make([]string, 0, len(cfg.SourceLabels))
for _, ln := range cfg.SourceLabels {
values = append(values, string(labels[ln]))
}
val := strings.Join(values, cfg.Separator)
switch cfg.Action {
case config.RelabelDrop:
if cfg.Regex.MatchString(val) {
return nil, nil
}
case config.RelabelKeep:
if !cfg.Regex.MatchString(val) {
return nil, nil
}
case config.RelabelReplace:
// If there is no match no replacement must take place.
if !cfg.Regex.MatchString(val) {
break
}
res := cfg.Regex.ReplaceAllString(val, cfg.Replacement)
if res == "" {
delete(labels, cfg.TargetLabel)
} else {
labels[cfg.TargetLabel] = clientmodel.LabelValue(res)
}
case config.RelabelHashMod:
mod := sum64(md5.Sum([]byte(val))) % cfg.Modulus
labels[cfg.TargetLabel] = clientmodel.LabelValue(fmt.Sprintf("%d", mod))
default:
panic(fmt.Errorf("retrieval.relabel: unknown relabel action type %q", cfg.Action))
}
return labels, nil
}
func (t *target) recordScrapeHealth(ingester extraction.Ingester, timestamp clientmodel.Timestamp, healthy bool, scrapeDuration time.Duration) {
healthMetric := clientmodel.Metric{}
durationMetric := clientmodel.Metric{}
for label, value := range t.baseLabels {
healthMetric[label] = value
durationMetric[label] = value
}
healthMetric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(scrapeHealthMetricName)
durationMetric[clientmodel.MetricNameLabel] = clientmodel.LabelValue(scrapeDurationMetricName)
healthMetric[InstanceLabel] = clientmodel.LabelValue(t.URL())
durationMetric[InstanceLabel] = clientmodel.LabelValue(t.URL())
healthValue := clientmodel.SampleValue(0)
if healthy {
healthValue = clientmodel.SampleValue(1)
}
healthSample := &clientmodel.Sample{
Metric: healthMetric,
Timestamp: timestamp,
Value: healthValue,
}
durationSample := &clientmodel.Sample{
Metric: durationMetric,
Timestamp: timestamp,
Value: clientmodel.SampleValue(float64(scrapeDuration) / float64(time.Second)),
}
ingester.Ingest(clientmodel.Samples{healthSample, durationSample})
}
func extractCounter(out Ingester, o *ProcessOptions, f *dto.MetricFamily) error {
samples := make(model.Samples, 0, len(f.Metric))
for _, m := range f.Metric {
if m.Counter == nil {
continue
}
sample := new(model.Sample)
samples = append(samples, sample)
if m.TimestampMs != nil {
sample.Timestamp = model.TimestampFromUnix(*m.TimestampMs / 1000)
} else {
sample.Timestamp = o.Timestamp
}
sample.Metric = model.Metric{}
metric := sample.Metric
for _, p := range m.Label {
metric[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
}
metric[model.MetricNameLabel] = model.LabelValue(f.GetName())
sample.Value = model.SampleValue(m.Counter.GetValue())
}
return out.Ingest(&Result{Samples: samples})
}
// watchServices retrieves updates from Consul's services endpoint and sends
// potential updates to the update channel.
func (cd *ConsulDiscovery) watchServices(update chan<- *consulService) {
var lastIndex uint64
for {
catalog := cd.client.Catalog()
srvs, meta, err := catalog.Services(&consul.QueryOptions{
WaitIndex: lastIndex,
WaitTime: consulWatchTimeout,
})
if err != nil {
log.Errorf("Error refreshing service list: %s", err)
<-time.After(consulRetryInterval)
continue
}
// If the index equals the previous one, the watch timed out with no update.
if meta.LastIndex == lastIndex {
continue
}
lastIndex = meta.LastIndex
cd.mu.Lock()
select {
case <-cd.srvsDone:
cd.mu.Unlock()
return
default:
// Continue.
}
// Check for new services.
for name := range srvs {
if _, ok := cd.scrapedServices[name]; !ok {
continue
}
srv, ok := cd.services[name]
if !ok {
srv = &consulService{
name: name,
tgroup: &config.TargetGroup{},
done: make(chan struct{}, 1),
}
srv.tgroup.Source = consulSourcePrefix + ":" + name
cd.services[name] = srv
}
srv.tgroup.Labels = clientmodel.LabelSet{
ConsulServiceLabel: clientmodel.LabelValue(name),
ConsulDCLabel: clientmodel.LabelValue(cd.clientConf.Datacenter),
}
update <- srv
}
// Check for removed services.
for name, srv := range cd.services {
if _, ok := srvs[name]; !ok {
srv.removed = true
update <- srv
srv.done <- struct{}{}
delete(cd.services, name)
}
}
cd.mu.Unlock()
}
}
func extractUntyped(out Ingester, o *ProcessOptions, f *dto.MetricFamily) error {
samples := make(model.Samples, 0, len(f.Metric))
for _, m := range f.Metric {
if m.Untyped == nil {
continue
}
sample := &model.Sample{
Metric: model.Metric{},
Value: model.SampleValue(m.Untyped.GetValue()),
}
samples = append(samples, sample)
if m.TimestampMs != nil {
sample.Timestamp = model.TimestampFromUnixNano(*m.TimestampMs * 1000000)
} else {
sample.Timestamp = o.Timestamp
}
metric := sample.Metric
for _, p := range m.Label {
metric[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
}
metric[model.MetricNameLabel] = model.LabelValue(f.GetName())
}
return out.Ingest(samples)
}
func (dd *DNSDiscovery) refresh(name string, ch chan<- *config.TargetGroup) error {
response, err := lookupSRV(name)
dnsSDLookupsCount.Inc()
if err != nil {
dnsSDLookupFailuresCount.Inc()
return err
}
tg := &config.TargetGroup{}
for _, record := range response.Answer {
addr, ok := record.(*dns.SRV)
if !ok {
log.Warnf("%q is not a valid SRV record", record)
continue
}
// Remove the final dot from rooted DNS names to make them look more usual.
addr.Target = strings.TrimRight(addr.Target, ".")
target := clientmodel.LabelValue(fmt.Sprintf("%s:%d", addr.Target, addr.Port))
tg.Targets = append(tg.Targets, clientmodel.LabelSet{
clientmodel.AddressLabel: target,
DNSNameLabel: clientmodel.LabelValue(name),
})
}
tg.Source = dnsSourcePrefix + ":" + name
ch <- tg
return nil
}
// fullLabels returns the base labels plus internal labels defining the target.
func (t *Target) fullLabels() clientmodel.LabelSet {
t.RLock()
defer t.RUnlock()
lset := make(clientmodel.LabelSet, len(t.baseLabels)+2)
for ln, lv := range t.baseLabels {
lset[ln] = lv
}
lset[clientmodel.MetricsPathLabel] = clientmodel.LabelValue(t.url.Path)
lset[clientmodel.AddressLabel] = clientmodel.LabelValue(t.url.Host)
return lset
}
func targetsForApp(app *App) []clientmodel.LabelSet {
targets := make([]clientmodel.LabelSet, 0, len(app.Tasks))
for _, t := range app.Tasks {
target := targetForTask(&t)
targets = append(targets, clientmodel.LabelSet{
clientmodel.AddressLabel: clientmodel.LabelValue(target),
taskLabel: clientmodel.LabelValue(t.ID),
})
}
return targets
}
func relabel(labels clientmodel.LabelSet, cfg *config.RelabelConfig) (clientmodel.LabelSet, error) {
values := make([]string, 0, len(cfg.SourceLabels))
for _, ln := range cfg.SourceLabels {
values = append(values, string(labels[ln]))
}
val := strings.Join(values, cfg.Separator)
switch cfg.Action {
case config.RelabelDrop:
if cfg.Regex.MatchString(val) {
return nil, nil
}
case config.RelabelKeep:
if !cfg.Regex.MatchString(val) {
return nil, nil
}
case config.RelabelReplace:
indexes := cfg.Regex.FindStringSubmatchIndex(val)
// If there is no match no replacement must take place.
if indexes == nil {
break
}
res := cfg.Regex.ExpandString([]byte{}, cfg.Replacement, val, indexes)
if len(res) == 0 {
delete(labels, cfg.TargetLabel)
} else {
labels[cfg.TargetLabel] = clientmodel.LabelValue(res)
}
case config.RelabelHashMod:
mod := sum64(md5.Sum([]byte(val))) % cfg.Modulus
labels[cfg.TargetLabel] = clientmodel.LabelValue(fmt.Sprintf("%d", mod))
case config.RelabelLabelMap:
out := make(clientmodel.LabelSet, len(labels))
// Take a copy to avoid infinite loops.
for ln, lv := range labels {
out[ln] = lv
}
for ln, lv := range labels {
if cfg.Regex.MatchString(string(ln)) {
res := cfg.Regex.ReplaceAllString(string(ln), cfg.Replacement)
out[clientmodel.LabelName(res)] = lv
}
}
labels = out
default:
panic(fmt.Errorf("retrieval.relabel: unknown relabel action type %q", cfg.Action))
}
return labels, nil
}
func BenchmarkGetFingerprintsForNotEqualMatcher1000(b *testing.B) {
numSeries := 1000
samples := make(clientmodel.Samples, 0, numSeries)
for i := 0; i < numSeries; i++ {
samples = append(samples, &clientmodel.Sample{
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: "testmetric",
"instance": clientmodel.LabelValue(fmt.Sprint("instance_", i)),
},
Value: 1,
Timestamp: clientmodel.TimestampFromTime(time.Date(2000, 0, 0, 0, 0, 0, 0, time.UTC)),
})
}
s := NewMemorySeriesStorage(MemorySeriesOptions{})
if err := s.AppendSamples(samples); err != nil {
b.Fatal(err)
}
m, err := metric.NewLabelMatcher(metric.NotEqual, "instance", "foo")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.GetFingerprintsForLabelMatchers(metric.LabelMatchers{m})
}
}
func (m *ruleManager) queueAlertNotifications(rule *AlertingRule) {
activeAlerts := rule.ActiveAlerts()
if len(activeAlerts) == 0 {
return
}
notifications := make(notification.NotificationReqs, 0, len(activeAlerts))
for _, aa := range activeAlerts {
if aa.State != FIRING {
// BUG: In the future, make AlertManager support pending alerts?
continue
}
notifications = append(notifications, ¬ification.NotificationReq{
Summary: rule.Summary,
Description: rule.Description,
Labels: aa.Labels.Merge(clientmodel.LabelSet{
AlertNameLabel: clientmodel.LabelValue(rule.Name()),
}),
Value: aa.Value,
ActiveSince: aa.ActiveSince.Time(),
RuleString: rule.String(),
GeneratorUrl: m.prometheusUrl + ConsoleLinkForExpression(rule.vector.String()),
})
}
m.notifications <- notifications
}
func createTargetGroup(app *App) *config.TargetGroup {
var (
targets = targetsForApp(app)
source = targetGroupName(app)
appName = clientmodel.LabelValue(sanitizeName(app.ID))
image = clientmodel.LabelValue(imageName(app))
)
return &config.TargetGroup{
Targets: targets,
Labels: clientmodel.LabelSet{
AppLabel: appName,
ImageLabel: image,
},
Source: source,
}
}
func AppendSampleAsPureSparseAppendTests(p metric.Persistence, t test.Tester) {
appendSample := func(x int) (success bool) {
v := clientmodel.SampleValue(x)
ts := clientmodel.TimestampFromUnix(int64(x))
labelName := clientmodel.LabelName(x)
labelValue := clientmodel.LabelValue(x)
l := clientmodel.Metric{labelName: labelValue}
sample := &clientmodel.Sample{
Value: v,
Timestamp: ts,
Metric: l,
}
err := p.AppendSamples(clientmodel.Samples{sample})
success = err == nil
if !success {
t.Error(err)
}
return
}
if err := quick.Check(appendSample, nil); err != nil {
t.Error(err)
}
}
// readFile reads a JSON or YAML list of targets groups from the file, depending on its
// file extension. It returns full configuration target groups.
func readFile(filename string) ([]*config.TargetGroup, error) {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
var targetGroups []*config.TargetGroup
switch ext := filepath.Ext(filename); strings.ToLower(ext) {
case ".json":
if err := json.Unmarshal(content, &targetGroups); err != nil {
return nil, err
}
case ".yml", ".yaml":
if err := yaml.Unmarshal(content, &targetGroups); err != nil {
return nil, err
}
default:
panic(fmt.Errorf("retrieval.FileDiscovery.readFile: unhandled file extension %q", ext))
}
for i, tg := range targetGroups {
tg.Source = fileSource(filename, i)
if tg.Labels == nil {
tg.Labels = clientmodel.LabelSet{}
}
tg.Labels[FileSDFilepathLabel] = clientmodel.LabelValue(filename)
}
return targetGroups, nil
}
func (s *testNotificationScenario) test(i int, t *testing.T) {
notifications := make(chan NotificationReqs)
defer close(notifications)
h := NewNotificationHandler("alertmanager_url", notifications)
receivedPost := make(chan bool, 1)
poster := testHttpPoster{receivedPost: receivedPost}
h.httpClient = &poster
go h.Run()
notifications <- NotificationReqs{
{
Summary: s.summary,
Description: s.description,
Labels: clientmodel.LabelSet{
clientmodel.LabelName("instance"): clientmodel.LabelValue("testinstance"),
},
Value: clientmodel.SampleValue(1.0 / 3.0),
ActiveSince: time.Time{},
RuleString: "Test rule string",
GeneratorUrl: "prometheus_url",
},
}
<-receivedPost
if poster.message != s.message {
t.Fatalf("%d. Expected '%s', received '%s'", i, s.message, poster.message)
}
}
// Update overwrites settings in the target that are derived from the job config
// it belongs to.
func (t *Target) Update(cfg *config.ScrapeConfig, baseLabels, metaLabels clientmodel.LabelSet) {
t.Lock()
defer t.Unlock()
t.url.Scheme = cfg.Scheme
t.url.Path = string(baseLabels[clientmodel.MetricsPathLabel])
if cfg.BasicAuth != nil {
t.url.User = url.UserPassword(cfg.BasicAuth.Username, cfg.BasicAuth.Password)
}
t.url.RawQuery = cfg.Params.Encode()
t.scrapeInterval = time.Duration(cfg.ScrapeInterval)
t.deadline = time.Duration(cfg.ScrapeTimeout)
t.httpClient = httputil.NewDeadlineClient(time.Duration(cfg.ScrapeTimeout))
t.honorLabels = cfg.HonorLabels
t.metaLabels = metaLabels
t.baseLabels = clientmodel.LabelSet{}
// All remaining internal labels will not be part of the label set.
for name, val := range baseLabels {
if !strings.HasPrefix(string(name), clientmodel.ReservedLabelPrefix) {
t.baseLabels[name] = val
}
}
if _, ok := t.baseLabels[clientmodel.InstanceLabel]; !ok {
t.baseLabels[clientmodel.InstanceLabel] = clientmodel.LabelValue(t.InstanceIdentifier())
}
t.metricRelabelConfigs = cfg.MetricRelabelConfigs
}
// sample returns a Sample suitable for recording the alert.
func (a Alert) sample(timestamp clientmodel.Timestamp, value clientmodel.SampleValue) *clientmodel.Sample {
recordedMetric := clientmodel.Metric{}
for label, value := range a.Labels {
recordedMetric[label] = value
}
recordedMetric[clientmodel.MetricNameLabel] = AlertMetricName
recordedMetric[AlertNameLabel] = clientmodel.LabelValue(a.Name)
recordedMetric[AlertStateLabel] = clientmodel.LabelValue(a.State.String())
return &clientmodel.Sample{
Metric: recordedMetric,
Value: value,
Timestamp: timestamp,
}
}
func newTestTarget(targetURL string, deadline time.Duration, baseLabels clientmodel.LabelSet) *Target {
cfg := &config.ScrapeConfig{
ScrapeTimeout: config.Duration(deadline),
}
c, _ := newHTTPClient(cfg)
t := &Target{
url: &url.URL{
Scheme: "http",
Host: strings.TrimLeft(targetURL, "http://"),
Path: "/metrics",
},
deadline: deadline,
status: &TargetStatus{},
scrapeInterval: 1 * time.Millisecond,
httpClient: c,
scraperStopping: make(chan struct{}),
scraperStopped: make(chan struct{}),
}
t.baseLabels = clientmodel.LabelSet{
clientmodel.InstanceLabel: clientmodel.LabelValue(t.InstanceIdentifier()),
}
for baseLabel, baseValue := range baseLabels {
t.baseLabels[baseLabel] = baseValue
}
return t
}
func (s *testNotificationScenario) test(i int, t *testing.T) {
h := NewNotificationHandler(&NotificationHandlerOptions{
AlertmanagerURL: "alertmanager_url",
QueueCapacity: 0,
Deadline: 10 * time.Second,
})
defer h.Stop()
receivedPost := make(chan bool, 1)
poster := testHTTPPoster{receivedPost: receivedPost}
h.httpClient = &poster
go h.Run()
h.SubmitReqs(NotificationReqs{
{
Summary: s.summary,
Description: s.description,
Labels: clientmodel.LabelSet{
clientmodel.LabelName("instance"): clientmodel.LabelValue("testinstance"),
},
Value: clientmodel.SampleValue(1.0 / 3.0),
ActiveSince: time.Time{},
RuleString: "Test rule string",
GeneratorURL: "prometheus_url",
},
})
<-receivedPost
if poster.message != s.message {
t.Fatalf("%d. Expected '%s', received '%s'", i, s.message, poster.message)
}
}
func parseServersetMember(data []byte, path string) (*clientmodel.LabelSet, error) {
member := serversetMember{}
err := json.Unmarshal(data, &member)
if err != nil {
return nil, fmt.Errorf("error unmarshaling serverset member %q: %s", path, err)
}
labels := clientmodel.LabelSet{}
labels[serversetPathLabel] = clientmodel.LabelValue(path)
labels[clientmodel.AddressLabel] = clientmodel.LabelValue(
fmt.Sprintf("%s:%d", member.ServiceEndpoint.Host, member.ServiceEndpoint.Port))
labels[serversetEndpointLabelPrefix+"_host"] = clientmodel.LabelValue(member.ServiceEndpoint.Host)
labels[serversetEndpointLabelPrefix+"_port"] = clientmodel.LabelValue(fmt.Sprintf("%d", member.ServiceEndpoint.Port))
for name, endpoint := range member.AdditionalEndpoints {
cleanName := clientmodel.LabelName(invalidLabelCharRE.ReplaceAllString(name, "_"))
labels[serversetEndpointLabelPrefix+"_host_"+cleanName] = clientmodel.LabelValue(
endpoint.Host)
labels[serversetEndpointLabelPrefix+"_port_"+cleanName] = clientmodel.LabelValue(
fmt.Sprintf("%d", endpoint.Port))
}
labels[serversetStatusLabel] = clientmodel.LabelValue(member.Status)
return &labels, nil
}
// Return the global labels as a LabelSet.
func (c Config) GlobalLabels() clientmodel.LabelSet {
labels := clientmodel.LabelSet{}
if c.Global.Labels != nil {
for _, label := range c.Global.Labels.Label {
labels[clientmodel.LabelName(label.GetName())] = clientmodel.LabelValue(label.GetValue())
}
}
return labels
}
// Helper function to convert map[string]string into LabelSet.
//
// NOTE: This should be deleted when support for go 1.0.3 is removed; 1.1 is
// smart enough to unmarshal JSON objects into LabelSet directly.
func labelSet(labels map[string]string) model.LabelSet {
labelset := make(model.LabelSet, len(labels))
for k, v := range labels {
labelset[model.LabelName(k)] = model.LabelValue(v)
}
return labelset
}
// sample returns a Sample suitable for recording the alert.
func (a Alert) sample(timestamp clientmodel.Timestamp, value clientmodel.SampleValue) *promql.Sample {
recordedMetric := clientmodel.Metric{}
for label, value := range a.Labels {
recordedMetric[label] = value
}
recordedMetric[clientmodel.MetricNameLabel] = alertMetricName
recordedMetric[alertNameLabel] = clientmodel.LabelValue(a.Name)
recordedMetric[alertStateLabel] = clientmodel.LabelValue(a.State.String())
return &promql.Sample{
Metric: clientmodel.COWMetric{
Metric: recordedMetric,
Copied: true,
},
Value: value,
Timestamp: timestamp,
}
}
func TestURLParams(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
w.Write([]byte{})
r.ParseForm()
if r.Form["foo"][0] != "bar" {
t.Fatalf("URL parameter 'foo' had unexpected first value '%v'", r.Form["foo"][0])
}
if r.Form["foo"][1] != "baz" {
t.Fatalf("URL parameter 'foo' had unexpected second value '%v'", r.Form["foo"][1])
}
},
),
)
defer server.Close()
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
target := NewTarget(
&config.ScrapeConfig{
JobName: "test_job1",
ScrapeInterval: config.Duration(1 * time.Minute),
ScrapeTimeout: config.Duration(1 * time.Second),
Scheme: serverURL.Scheme,
Params: url.Values{
"foo": []string{"bar", "baz"},
},
},
clientmodel.LabelSet{
clientmodel.SchemeLabel: clientmodel.LabelValue(serverURL.Scheme),
clientmodel.AddressLabel: clientmodel.LabelValue(serverURL.Host),
"__param_foo": "bar",
},
nil)
app := &collectResultAppender{}
if err = target.scrape(app); err != nil {
t.Fatal(err)
}
}
func BenchmarkAppend(b *testing.B) {
samples := make(clientmodel.Samples, b.N)
for i := range samples {
samples[i] = &clientmodel.Sample{
Metric: clientmodel.Metric{
clientmodel.MetricNameLabel: clientmodel.LabelValue(fmt.Sprintf("test_metric_%d", i%10)),
"label1": clientmodel.LabelValue(fmt.Sprintf("test_metric_%d", i%10)),
"label2": clientmodel.LabelValue(fmt.Sprintf("test_metric_%d", i%10)),
},
Timestamp: clientmodel.Timestamp(i),
Value: clientmodel.SampleValue(i),
}
}
b.ResetTimer()
s, closer := NewTestStorage(b)
defer closer.Close()
s.AppendSamples(samples)
}
func (t *target) scrape(timestamp clientmodel.Timestamp, ingester extraction.Ingester) (err error) {
defer func(start time.Time) {
ms := float64(time.Since(start)) / float64(time.Millisecond)
labels := map[string]string{address: t.Address(), outcome: success}
if err != nil {
labels[outcome] = failure
}
targetOperationLatencies.Add(labels, ms)
targetOperations.Increment(labels)
}(time.Now())
req, err := http.NewRequest("GET", t.Address(), nil)
if err != nil {
panic(err)
}
req.Header.Add("Accept", acceptHeader)
resp, err := t.httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
processor, err := extraction.ProcessorForRequestHeader(resp.Header)
if err != nil {
return err
}
// XXX: This is a wart; we need to handle this more gracefully down the
// road, especially once we have service discovery support.
baseLabels := clientmodel.LabelSet{InstanceLabel: clientmodel.LabelValue(t.Address())}
for baseLabel, baseValue := range t.baseLabels {
baseLabels[baseLabel] = baseValue
}
// N.B. - It is explicitly required to extract the entire payload before
// attempting to deserialize, as the underlying reader will interpret
// pending data as a truncated message.
buf := new(bytes.Buffer)
if _, err := buf.ReadFrom(resp.Body); err != nil {
return err
}
i := &MergeLabelsIngester{
Labels: baseLabels,
CollisionPrefix: clientmodel.ExporterLabelPrefix,
Ingester: ingester,
}
processOptions := &extraction.ProcessOptions{
Timestamp: timestamp,
}
return processor.ProcessSingle(buf, i, processOptions)
}
// watchService retrieves updates about srv from Consul's service endpoint.
// On a potential update the resulting target group is sent to ch.
func (cd *ConsulDiscovery) watchService(srv *consulService, ch chan<- *config.TargetGroup) {
catalog := cd.client.Catalog()
for {
nodes, meta, err := catalog.Service(srv.name, "", &consul.QueryOptions{
WaitIndex: srv.lastIndex,
WaitTime: consulWatchTimeout,
})
if err != nil {
log.Errorf("Error refreshing service %s: %s", srv.name, err)
<-time.After(consulRetryInterval)
continue
}
// If the index equals the previous one, the watch timed out with no update.
if meta.LastIndex == srv.lastIndex {
continue
}
srv.lastIndex = meta.LastIndex
srv.tgroup.Targets = make([]clientmodel.LabelSet, 0, len(nodes))
for _, node := range nodes {
addr := fmt.Sprintf("%s:%d", node.Address, node.ServicePort)
// We surround the separated list with the separator as well. This way regular expressions
// in relabeling rules don't have to consider tag positions.
tags := cd.tagSeparator + strings.Join(node.ServiceTags, cd.tagSeparator) + cd.tagSeparator
srv.tgroup.Targets = append(srv.tgroup.Targets, clientmodel.LabelSet{
clientmodel.AddressLabel: clientmodel.LabelValue(addr),
ConsulNodeLabel: clientmodel.LabelValue(node.Node),
ConsulTagsLabel: clientmodel.LabelValue(tags),
})
}
cd.mu.Lock()
select {
case <-srv.done:
return
default:
// Continue.
}
ch <- srv.tgroup
cd.mu.Unlock()
}
}
func (m *Manager) queueAlertNotifications(rule *AlertingRule, timestamp clientmodel.Timestamp) {
activeAlerts := rule.ActiveAlerts()
if len(activeAlerts) == 0 {
return
}
notifications := make(notification.NotificationReqs, 0, len(activeAlerts))
for _, aa := range activeAlerts {
if aa.State != StateFiring {
// BUG: In the future, make AlertManager support pending alerts?
continue
}
// Provide the alert information to the template.
l := map[string]string{}
for k, v := range aa.Labels {
l[string(k)] = string(v)
}
tmplData := struct {
Labels map[string]string
Value clientmodel.SampleValue
}{
Labels: l,
Value: aa.Value,
}
// Inject some convenience variables that are easier to remember for users
// who are not used to Go's templating system.
defs := "{{$labels := .Labels}}{{$value := .Value}}"
expand := func(text string) string {
tmpl := template.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path)
result, err := tmpl.Expand()
if err != nil {
result = err.Error()
log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err)
}
return result
}
notifications = append(notifications, ¬ification.NotificationReq{
Summary: expand(rule.summary),
Description: expand(rule.description),
Runbook: rule.runbook,
Labels: aa.Labels.Merge(clientmodel.LabelSet{
alertNameLabel: clientmodel.LabelValue(rule.Name()),
}),
Value: aa.Value,
ActiveSince: aa.ActiveSince.Time(),
RuleString: rule.String(),
GeneratorURL: m.externalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()),
})
}
m.notificationHandler.SubmitReqs(notifications)
}
func createTargetGroup(app *App) *config.TargetGroup {
var (
targets = targetsForApp(app)
appName = clientmodel.LabelValue(app.ID)
image = clientmodel.LabelValue(app.Container.Docker.Image)
)
tg := &config.TargetGroup{
Targets: targets,
Labels: clientmodel.LabelSet{
appLabel: appName,
imageLabel: image,
},
Source: app.ID,
}
for ln, lv := range app.Labels {
ln = appLabelPrefix + ln
tg.Labels[clientmodel.LabelName(ln)] = clientmodel.LabelValue(lv)
}
return tg
}