func (kd *Discovery) updateAPIServersTargetGroup() *config.TargetGroup {
tg := &config.TargetGroup{
Source: apiServersTargetGroupName,
Labels: model.LabelSet{
roleLabel: model.LabelValue("apiserver"),
},
}
for _, apiServer := range kd.apiServers {
apiServerAddress := apiServer.Host
_, _, err := net.SplitHostPort(apiServerAddress)
// If error then no port is specified - use default for scheme.
if err != nil {
switch apiServer.Scheme {
case "http":
apiServerAddress = net.JoinHostPort(apiServerAddress, "80")
case "https":
apiServerAddress = net.JoinHostPort(apiServerAddress, "443")
}
}
t := model.LabelSet{
model.AddressLabel: model.LabelValue(apiServerAddress),
model.SchemeLabel: model.LabelValue(apiServer.Scheme),
}
tg.Targets = append(tg.Targets, t)
}
return tg
}
func extractUntyped(o *DecodeOptions, f *dto.MetricFamily) model.Vector {
samples := make(model.Vector, 0, len(f.Metric))
for _, m := range f.Metric {
if m.Untyped == nil {
continue
}
lset := make(model.LabelSet, len(m.Label)+1)
for _, p := range m.Label {
lset[model.LabelName(p.GetName())] = model.LabelValue(p.GetValue())
}
lset[model.MetricNameLabel] = model.LabelValue(f.GetName())
smpl := &model.Sample{
Metric: model.Metric(lset),
Value: model.SampleValue(m.Untyped.GetValue()),
}
if m.TimestampMs != nil {
smpl.Timestamp = model.TimeFromUnixNano(*m.TimestampMs * 1000000)
} else {
smpl.Timestamp = o.Timestamp
}
samples = append(samples, smpl)
}
return samples
}
func TestPrometheus(t *testing.T) {
Convey("Prometheus", t, func() {
Convey("converting metric name", func() {
metric := map[p.LabelName]p.LabelValue{
p.LabelName("app"): p.LabelValue("backend"),
p.LabelName("device"): p.LabelValue("mobile"),
}
query := &PrometheusQuery{
LegendFormat: "legend {{app}} {{ device }} {{broken}}",
}
So(formatLegend(metric, query), ShouldEqual, "legend backend mobile {{broken}}")
})
Convey("build full serie name", func() {
metric := map[p.LabelName]p.LabelValue{
p.LabelName(p.MetricNameLabel): p.LabelValue("http_request_total"),
p.LabelName("app"): p.LabelValue("backend"),
p.LabelName("device"): p.LabelValue("mobile"),
}
query := &PrometheusQuery{
LegendFormat: "",
}
So(formatLegend(metric, query), ShouldEqual, `http_request_total{app="backend", device="mobile"}`)
})
})
}
func (kd *Discovery) updateNodesTargetGroup() *config.TargetGroup {
kd.nodesMu.Lock()
defer kd.nodesMu.Unlock()
tg := &config.TargetGroup{
Source: nodesTargetGroupName,
Labels: model.LabelSet{
roleLabel: model.LabelValue("node"),
},
}
// Now let's loop through the nodes & add them to the target group with appropriate labels.
for nodeName, node := range kd.nodes {
address := fmt.Sprintf("%s:%d", node.Status.Addresses[0].Address, kd.Conf.KubeletPort)
t := model.LabelSet{
model.AddressLabel: model.LabelValue(address),
model.InstanceLabel: model.LabelValue(nodeName),
}
for k, v := range node.ObjectMeta.Labels {
labelName := strutil.SanitizeLabelName(nodeLabelPrefix + k)
t[model.LabelName(labelName)] = model.LabelValue(v)
}
tg.Targets = append(tg.Targets, t)
}
return tg
}
// alertmanagersFromGroup extracts a list of alertmanagers from a target group and an associcated
// AlertmanagerConfig.
func alertmanagerFromGroup(tg *config.TargetGroup, cfg *config.AlertmanagerConfig) ([]alertmanager, error) {
var res []alertmanager
for _, lset := range tg.Targets {
// Set configured scheme as the initial scheme label for overwrite.
lset[model.SchemeLabel] = model.LabelValue(cfg.Scheme)
lset[pathLabel] = model.LabelValue(postPath(cfg.PathPrefix))
// Combine target labels with target group labels.
for ln, lv := range tg.Labels {
if _, ok := lset[ln]; !ok {
lset[ln] = lv
}
}
lset := relabel.Process(lset, cfg.RelabelConfigs...)
if lset == nil {
continue
}
// addPort checks whether we should add a default port to the address.
// If the address is not valid, we don't append a port either.
addPort := func(s string) bool {
// If we can split, a port exists and we don't have to add one.
if _, _, err := net.SplitHostPort(s); err == nil {
return false
}
// If adding a port makes it valid, the previous error
// was not due to an invalid address and we can append a port.
_, _, err := net.SplitHostPort(s + ":1234")
return err == nil
}
// If it's an address with no trailing port, infer it based on the used scheme.
if addr := string(lset[model.AddressLabel]); addPort(addr) {
// Addresses reaching this point are already wrapped in [] if necessary.
switch lset[model.SchemeLabel] {
case "http", "":
addr = addr + ":80"
case "https":
addr = addr + ":443"
default:
return nil, fmt.Errorf("invalid scheme: %q", cfg.Scheme)
}
lset[model.AddressLabel] = model.LabelValue(addr)
}
if err := config.CheckTargetAddress(lset[model.AddressLabel]); err != nil {
return nil, err
}
// Meta labels are deleted after relabelling. Other internal labels propagate to
// the target which decides whether they will be part of their label set.
for ln := range lset {
if strings.HasPrefix(string(ln), model.MetaLabelPrefix) {
delete(lset, ln)
}
}
res = append(res, alertmanagerLabels(lset))
}
return res, nil
}
func TestTargetScrapeScrapeNotFound(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}),
)
defer server.Close()
serverURL, err := url.Parse(server.URL)
if err != nil {
panic(err)
}
ts := &targetScraper{
Target: &Target{
labels: model.LabelSet{
model.SchemeLabel: model.LabelValue(serverURL.Scheme),
model.AddressLabel: model.LabelValue(serverURL.Host),
},
},
client: http.DefaultClient,
}
if _, err := ts.scrape(context.Background(), time.Now()); !strings.Contains(err.Error(), "404") {
t.Fatalf("Expected \"404 NotFound\" error but got: %s", err)
}
}
func (ed *EC2Discovery) refresh() (*config.TargetGroup, error) {
ec2s := ec2.New(ed.aws)
tg := &config.TargetGroup{
Source: *ed.aws.Region,
}
if err := ec2s.DescribeInstancesPages(nil, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool {
for _, r := range p.Reservations {
for _, inst := range r.Instances {
if inst.PrivateIpAddress == nil {
continue
}
labels := model.LabelSet{
ec2LabelInstanceID: model.LabelValue(*inst.InstanceId),
}
if inst.PublicIpAddress != nil {
labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress)
}
labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress)
addr := fmt.Sprintf("%s:%d", *inst.PrivateIpAddress, ed.port)
labels[model.AddressLabel] = model.LabelValue(addr)
for _, t := range inst.Tags {
name := strutil.SanitizeLabelName(*t.Key)
labels[ec2LabelTag+model.LabelName(name)] = model.LabelValue(*t.Value)
}
tg.Targets = append(tg.Targets, labels)
}
}
return true
}); err != nil {
return nil, fmt.Errorf("could not describe instances: %s", err)
}
return tg, nil
}
func (kd *Discovery) updateNodesTargetGroup() *config.TargetGroup {
kd.nodesMu.RLock()
defer kd.nodesMu.RUnlock()
tg := &config.TargetGroup{
Source: nodesTargetGroupName,
Labels: model.LabelSet{
roleLabel: model.LabelValue("node"),
},
}
// Now let's loop through the nodes & add them to the target group with appropriate labels.
for nodeName, node := range kd.nodes {
nodeAddress, err := nodeHostIP(node)
if err != nil {
log.Debugf("Skipping node %s: %s", node.Name, err)
continue
}
address := fmt.Sprintf("%s:%d", nodeAddress.String(), kd.Conf.KubeletPort)
t := model.LabelSet{
model.AddressLabel: model.LabelValue(address),
model.InstanceLabel: model.LabelValue(nodeName),
}
for k, v := range node.ObjectMeta.Labels {
labelName := strutil.SanitizeLabelName(nodeLabelPrefix + k)
t[model.LabelName(labelName)] = model.LabelValue(v)
}
tg.Targets = append(tg.Targets, t)
}
return tg
}
// Decode implements the Decoder interface.
func (d *protoDecoder) Decode(v *dto.MetricFamily) error {
_, err := pbutil.ReadDelimited(d.r, v)
if err != nil {
return err
}
if !model.IsValidMetricName(model.LabelValue(v.GetName())) {
return fmt.Errorf("invalid metric name %q", v.GetName())
}
for _, m := range v.GetMetric() {
if m == nil {
continue
}
for _, l := range m.GetLabel() {
if l == nil {
continue
}
if !model.LabelValue(l.GetValue()).IsValid() {
return fmt.Errorf("invalid label value %q", l.GetValue())
}
if !model.LabelName(l.GetName()).IsValid() {
return fmt.Errorf("invalid label name %q", l.GetName())
}
}
}
return nil
}
// watchServices retrieves updates from Consul's services endpoint and sends
// potential updates to the update channel.
func (cd *ConsulDiscovery) watchServices(update chan<- *consulService, done <-chan struct{}) {
var lastIndex uint64
for {
catalog := cd.client.Catalog()
srvs, meta, err := catalog.Services(&consul.QueryOptions{
WaitIndex: lastIndex,
WaitTime: consulWatchTimeout,
})
if err != nil {
log.Errorf("Error refreshing service list: %s", err)
time.Sleep(consulRetryInterval)
continue
}
// If the index equals the previous one, the watch timed out with no update.
if meta.LastIndex == lastIndex {
continue
}
lastIndex = meta.LastIndex
cd.mu.Lock()
select {
case <-done:
cd.mu.Unlock()
return
default:
// Continue.
}
// Check for new services.
for name := range srvs {
if _, ok := cd.scrapedServices[name]; len(cd.scrapedServices) > 0 && !ok {
continue
}
srv, ok := cd.services[name]
if !ok {
srv = &consulService{
name: name,
tgroup: &config.TargetGroup{},
done: make(chan struct{}),
}
srv.tgroup.Source = name
cd.services[name] = srv
}
srv.tgroup.Labels = model.LabelSet{
consulServiceLabel: model.LabelValue(name),
consulDCLabel: model.LabelValue(cd.clientDatacenter),
}
update <- srv
}
// Check for removed services.
for name, srv := range cd.services {
if _, ok := srvs[name]; !ok {
srv.removed = true
update <- srv
delete(cd.services, name)
}
}
cd.mu.Unlock()
}
}
func TestHandlerSend(t *testing.T) {
var (
expected model.Alerts
status int
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != alertPushEndpoint {
t.Fatalf("Bad endpoint %q used, expected %q", r.URL.Path, alertPushEndpoint)
}
defer r.Body.Close()
var alerts model.Alerts
if err := json.NewDecoder(r.Body).Decode(&alerts); err != nil {
t.Fatalf("Unexpected error on input decoding: %s", err)
}
if !alertsEqual(alerts, expected) {
t.Errorf("%#v %#v", *alerts[0], *expected[0])
t.Fatalf("Unexpected alerts received %v exp %v", alerts, expected)
}
w.WriteHeader(status)
}))
defer server.Close()
h := New(&HandlerOptions{
AlertmanagerURL: server.URL,
Timeout: time.Minute,
ExternalLabels: model.LabelSet{"a": "b"},
})
for i := range make([]struct{}, maxBatchSize) {
h.queue = append(h.queue, &model.Alert{
Labels: model.LabelSet{
"alertname": model.LabelValue(fmt.Sprintf("%d", i)),
},
})
expected = append(expected, &model.Alert{
Labels: model.LabelSet{
"alertname": model.LabelValue(fmt.Sprintf("%d", i)),
"a": "b",
},
})
}
status = http.StatusOK
if err := h.send(h.queue...); err != nil {
t.Fatalf("Unexpected error: %s", err)
}
status = 500
if err := h.send(h.queue...); err == nil {
t.Fatalf("Expected error but got none")
}
}
func (gd *GCEDiscovery) refresh() (tg *config.TargetGroup, err error) {
t0 := time.Now()
defer func() {
gceSDScrapeDuration.Observe(time.Since(t0).Seconds())
gceSDScrapesCount.Inc()
if err != nil {
gceSDScrapeFailuresCount.Inc()
}
}()
tg = &config.TargetGroup{
Source: fmt.Sprintf("GCE_%s_%s", gd.project, gd.zone),
}
ilc := gd.isvc.List(gd.project, gd.zone)
if len(gd.filter) > 0 {
ilc = ilc.Filter(gd.filter)
}
err = ilc.Pages(nil, func(l *compute.InstanceList) error {
for _, inst := range l.Items {
if len(inst.NetworkInterfaces) == 0 {
continue
}
labels := model.LabelSet{
gceLabelProject: model.LabelValue(gd.project),
gceLabelZone: model.LabelValue(inst.Zone),
gceLabelInstanceName: model.LabelValue(inst.Name),
gceLabelInstanceStatus: model.LabelValue(inst.Status),
}
priIface := inst.NetworkInterfaces[0]
labels[gceLabelNetwork] = model.LabelValue(priIface.Network)
labels[gceLabelSubnetwork] = model.LabelValue(priIface.Subnetwork)
labels[gceLabelPrivateIP] = model.LabelValue(priIface.NetworkIP)
addr := fmt.Sprintf("%s:%d", priIface.NetworkIP, gd.port)
labels[model.AddressLabel] = model.LabelValue(addr)
if inst.Tags != nil && len(inst.Tags.Items) > 0 {
// We surround the separated list with the separator as well. This way regular expressions
// in relabeling rules don't have to consider tag positions.
tags := gd.tagSeparator + strings.Join(inst.Tags.Items, gd.tagSeparator) + gd.tagSeparator
labels[gceLabelTags] = model.LabelValue(tags)
}
if len(priIface.AccessConfigs) > 0 {
ac := priIface.AccessConfigs[0]
if ac.Type == "ONE_TO_ONE_NAT" {
labels[gceLabelPublicIP] = model.LabelValue(ac.NatIP)
}
}
tg.Targets = append(tg.Targets, labels)
}
return nil
})
if err != nil {
return tg, fmt.Errorf("error retrieving scrape targets from gce: %s", err)
}
return tg, nil
}
func (ed *EC2Discovery) refresh() (*config.TargetGroup, error) {
ec2s := ec2.New(ed.aws)
tg := &config.TargetGroup{
Source: *ed.aws.Region,
}
if err := ec2s.DescribeInstancesPages(nil, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool {
for _, r := range p.Reservations {
for _, inst := range r.Instances {
if inst.PrivateIpAddress == nil {
continue
}
labels := model.LabelSet{
ec2LabelInstanceID: model.LabelValue(*inst.InstanceId),
}
labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress)
addr := net.JoinHostPort(*inst.PrivateIpAddress, fmt.Sprintf("%d", ed.port))
labels[model.AddressLabel] = model.LabelValue(addr)
if inst.PublicIpAddress != nil {
labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress)
labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName)
}
labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone)
labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name)
if inst.VpcId != nil {
labels[ec2LabelVPCID] = model.LabelValue(*inst.VpcId)
subnetsMap := make(map[string]struct{})
for _, eni := range inst.NetworkInterfaces {
subnetsMap[*eni.SubnetId] = struct{}{}
}
subnets := []string{}
for k := range subnetsMap {
subnets = append(subnets, k)
}
labels[ec2LabelSubnetID] = model.LabelValue(
subnetSeparator +
strings.Join(subnets, subnetSeparator) +
subnetSeparator)
}
for _, t := range inst.Tags {
name := strutil.SanitizeLabelName(*t.Key)
labels[ec2LabelTag+model.LabelName(name)] = model.LabelValue(*t.Value)
}
tg.Targets = append(tg.Targets, labels)
}
}
return true
}); err != nil {
return nil, fmt.Errorf("could not describe instances: %s", err)
}
return tg, nil
}
func relabel(labels model.LabelSet, cfg *config.RelabelConfig) model.LabelSet {
values := make([]string, 0, len(cfg.SourceLabels))
for _, ln := range cfg.SourceLabels {
values = append(values, string(labels[ln]))
}
val := strings.Join(values, cfg.Separator)
switch cfg.Action {
case config.RelabelDrop:
if cfg.Regex.MatchString(val) {
return nil
}
case config.RelabelKeep:
if !cfg.Regex.MatchString(val) {
return nil
}
case config.RelabelReplace:
indexes := cfg.Regex.FindStringSubmatchIndex(val)
// If there is no match no replacement must take place.
if indexes == nil {
break
}
target := model.LabelName(cfg.Regex.ExpandString([]byte{}, cfg.TargetLabel, val, indexes))
if !target.IsValid() {
delete(labels, model.LabelName(cfg.TargetLabel))
break
}
res := cfg.Regex.ExpandString([]byte{}, cfg.Replacement, val, indexes)
if len(res) == 0 {
delete(labels, model.LabelName(cfg.TargetLabel))
break
}
labels[target] = model.LabelValue(res)
case config.RelabelHashMod:
mod := sum64(md5.Sum([]byte(val))) % cfg.Modulus
labels[model.LabelName(cfg.TargetLabel)] = model.LabelValue(fmt.Sprintf("%d", mod))
case config.RelabelLabelMap:
out := make(model.LabelSet, len(labels))
// Take a copy to avoid infinite loops.
for ln, lv := range labels {
out[ln] = lv
}
for ln, lv := range labels {
if cfg.Regex.MatchString(string(ln)) {
res := cfg.Regex.ReplaceAllString(string(ln), cfg.Replacement)
out[model.LabelName(res)] = lv
}
}
labels = out
default:
panic(fmt.Errorf("retrieval.relabel: unknown relabel action type %q", cfg.Action))
}
return labels
}
// Update overwrites settings in the target that are derived from the job config
// it belongs to.
func (t *Target) Update(cfg *config.ScrapeConfig, baseLabels, metaLabels model.LabelSet) {
t.Lock()
defer t.Unlock()
httpClient, err := newHTTPClient(cfg)
if err != nil {
log.Errorf("cannot create HTTP client: %v", err)
return
}
t.httpClient = httpClient
t.url.Scheme = string(baseLabels[model.SchemeLabel])
t.url.Path = string(baseLabels[model.MetricsPathLabel])
t.internalLabels = model.LabelSet{}
t.internalLabels[model.SchemeLabel] = baseLabels[model.SchemeLabel]
t.internalLabels[model.MetricsPathLabel] = baseLabels[model.MetricsPathLabel]
t.internalLabels[model.AddressLabel] = model.LabelValue(t.url.Host)
params := url.Values{}
for k, v := range cfg.Params {
params[k] = make([]string, len(v))
copy(params[k], v)
}
for k, v := range baseLabels {
if strings.HasPrefix(string(k), model.ParamLabelPrefix) {
if len(params[string(k[len(model.ParamLabelPrefix):])]) > 0 {
params[string(k[len(model.ParamLabelPrefix):])][0] = string(v)
} else {
params[string(k[len(model.ParamLabelPrefix):])] = []string{string(v)}
}
t.internalLabels[model.ParamLabelPrefix+k[len(model.ParamLabelPrefix):]] = v
}
}
t.url.RawQuery = params.Encode()
t.scrapeInterval = time.Duration(cfg.ScrapeInterval)
t.deadline = time.Duration(cfg.ScrapeTimeout)
t.honorLabels = cfg.HonorLabels
t.metaLabels = metaLabels
t.baseLabels = model.LabelSet{}
// All remaining internal labels will not be part of the label set.
for name, val := range baseLabels {
if !strings.HasPrefix(string(name), model.ReservedLabelPrefix) {
t.baseLabels[name] = val
}
}
if _, ok := t.baseLabels[model.InstanceLabel]; !ok {
t.baseLabels[model.InstanceLabel] = model.LabelValue(t.InstanceIdentifier())
}
t.metricRelabelConfigs = cfg.MetricRelabelConfigs
}
func targetsForApp(app *App) []model.LabelSet {
targets := make([]model.LabelSet, 0, len(app.Tasks))
for _, t := range app.Tasks {
target := targetForTask(&t)
targets = append(targets, model.LabelSet{
model.AddressLabel: model.LabelValue(target),
taskLabel: model.LabelValue(t.ID),
})
}
return targets
}
// fullLabels returns the base labels plus internal labels defining the target.
func (t *Target) fullLabels() model.LabelSet {
t.RLock()
defer t.RUnlock()
lset := make(model.LabelSet, len(t.baseLabels)+2)
for ln, lv := range t.baseLabels {
lset[ln] = lv
}
lset[model.MetricsPathLabel] = model.LabelValue(t.url.Path)
lset[model.AddressLabel] = model.LabelValue(t.url.Host)
lset[model.SchemeLabel] = model.LabelValue(t.url.Scheme)
return lset
}
func TestTargetScrapeScrapeCancel(t *testing.T) {
block := make(chan struct{})
server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
<-block
}),
)
defer server.Close()
serverURL, err := url.Parse(server.URL)
if err != nil {
panic(err)
}
ts := &targetScraper{
Target: &Target{
labels: model.LabelSet{
model.SchemeLabel: model.LabelValue(serverURL.Scheme),
model.AddressLabel: model.LabelValue(serverURL.Host),
},
},
client: http.DefaultClient,
}
ctx, cancel := context.WithCancel(context.Background())
errc := make(chan error)
go func() {
time.Sleep(1 * time.Second)
cancel()
}()
go func() {
if _, err := ts.scrape(ctx, time.Now()); err != context.Canceled {
errc <- fmt.Errorf("Expected context cancelation error but got: %s", err)
}
close(errc)
}()
select {
case <-time.After(5 * time.Second):
t.Fatalf("Scrape function did not return unexpectedly")
case err := <-errc:
if err != nil {
t.Fatalf(err.Error())
}
}
// If this is closed in a defer above the function the test server
// does not terminate and the test doens't complete.
close(block)
}
func TestTargetScraperScrapeOK(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
w.Write([]byte("metric_a 1\nmetric_b 2\n"))
}),
)
defer server.Close()
serverURL, err := url.Parse(server.URL)
if err != nil {
panic(err)
}
ts := &targetScraper{
Target: &Target{
labels: model.LabelSet{
model.SchemeLabel: model.LabelValue(serverURL.Scheme),
model.AddressLabel: model.LabelValue(serverURL.Host),
},
},
client: http.DefaultClient,
}
now := time.Now()
samples, err := ts.scrape(context.Background(), now)
if err != nil {
t.Fatalf("Unexpected scrape error: %s", err)
}
expectedSamples := model.Samples{
{
Metric: model.Metric{"__name__": "metric_a"},
Timestamp: model.TimeFromUnixNano(now.UnixNano()),
Value: 1,
},
{
Metric: model.Metric{"__name__": "metric_b"},
Timestamp: model.TimeFromUnixNano(now.UnixNano()),
Value: 2,
},
}
sort.Sort(expectedSamples)
sort.Sort(samples)
if !reflect.DeepEqual(samples, expectedSamples) {
t.Errorf("Scraped samples did not match served metrics")
t.Errorf("Expected: %v", expectedSamples)
t.Fatalf("Got: %v", samples)
}
}
func parseServersetMember(data []byte, path string) (model.LabelSet, error) {
member := serversetMember{}
if err := json.Unmarshal(data, &member); err != nil {
return nil, fmt.Errorf("error unmarshaling serverset member %q: %s", path, err)
}
labels := model.LabelSet{}
labels[serversetPathLabel] = model.LabelValue(path)
labels[model.AddressLabel] = model.LabelValue(
net.JoinHostPort(member.ServiceEndpoint.Host, fmt.Sprintf("%d", member.ServiceEndpoint.Port)))
labels[serversetEndpointLabelPrefix+"_host"] = model.LabelValue(member.ServiceEndpoint.Host)
labels[serversetEndpointLabelPrefix+"_port"] = model.LabelValue(fmt.Sprintf("%d", member.ServiceEndpoint.Port))
for name, endpoint := range member.AdditionalEndpoints {
cleanName := model.LabelName(strutil.SanitizeLabelName(name))
labels[serversetEndpointLabelPrefix+"_host_"+cleanName] = model.LabelValue(
endpoint.Host)
labels[serversetEndpointLabelPrefix+"_port_"+cleanName] = model.LabelValue(
fmt.Sprintf("%d", endpoint.Port))
}
labels[serversetStatusLabel] = model.LabelValue(member.Status)
labels[serversetShardLabel] = model.LabelValue(strconv.Itoa(member.Shard))
return labels, nil
}
// sample returns a Sample suitable for recording the alert.
func (a Alert) sample(timestamp model.Time, value model.SampleValue) *model.Sample {
recordedMetric := make(model.Metric, len(a.Labels)+3)
for label, value := range a.Labels {
recordedMetric[label] = value
}
recordedMetric[model.MetricNameLabel] = alertMetricName
recordedMetric[alertNameLabel] = model.LabelValue(a.Name)
recordedMetric[alertStateLabel] = model.LabelValue(a.State.String())
return &model.Sample{
Metric: recordedMetric,
Value: value,
Timestamp: timestamp,
}
}
func TestSampleDelivery(t *testing.T) {
// Let's create an even number of send batches so we don't run into the
// batch timeout case.
cfg := defaultConfig
n := cfg.QueueCapacity * 2
cfg.Shards = 1
samples := make(model.Samples, 0, n)
for i := 0; i < n; i++ {
name := model.LabelValue(fmt.Sprintf("test_metric_%d", i))
samples = append(samples, &model.Sample{
Metric: model.Metric{
model.MetricNameLabel: name,
},
Value: model.SampleValue(i),
})
}
c := NewTestStorageClient()
c.expectSamples(samples[:len(samples)/2])
m := NewStorageQueueManager(c, &cfg)
// These should be received by the client.
for _, s := range samples[:len(samples)/2] {
m.Append(s)
}
// These will be dropped because the queue is full.
for _, s := range samples[len(samples)/2:] {
m.Append(s)
}
go m.Run()
defer m.Stop()
c.waitForExpectedSamples(t)
}
func TestSampleDeliveryOrder(t *testing.T) {
cfg := defaultConfig
ts := 10
n := cfg.MaxSamplesPerSend * ts
// Ensure we don't drop samples in this test.
cfg.QueueCapacity = n
samples := make(model.Samples, 0, n)
for i := 0; i < n; i++ {
name := model.LabelValue(fmt.Sprintf("test_metric_%d", i%ts))
samples = append(samples, &model.Sample{
Metric: model.Metric{
model.MetricNameLabel: name,
},
Value: model.SampleValue(i),
Timestamp: model.Time(i),
})
}
c := NewTestStorageClient()
c.expectSamples(samples)
m := NewStorageQueueManager(c, &cfg)
// These should be received by the client.
for _, s := range samples {
m.Append(s)
}
go m.Run()
defer m.Stop()
c.waitForExpectedSamples(t)
}
// readFile reads a JSON or YAML list of targets groups from the file, depending on its
// file extension. It returns full configuration target groups.
func readFile(filename string) ([]*config.TargetGroup, error) {
content, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
var targetGroups []*config.TargetGroup
switch ext := filepath.Ext(filename); strings.ToLower(ext) {
case ".json":
if err := json.Unmarshal(content, &targetGroups); err != nil {
return nil, err
}
case ".yml", ".yaml":
if err := yaml.Unmarshal(content, &targetGroups); err != nil {
return nil, err
}
default:
panic(fmt.Errorf("retrieval.FileDiscovery.readFile: unhandled file extension %q", ext))
}
for i, tg := range targetGroups {
tg.Source = fileSource(filename, i)
if tg.Labels == nil {
tg.Labels = model.LabelSet{}
}
tg.Labels[fileSDFilepathLabel] = model.LabelValue(filename)
}
return targetGroups, nil
}
func TestRoutedNotifier(t *testing.T) {
router := Router{
"1": &recordNotifier{},
"2": &recordNotifier{},
"3": &recordNotifier{},
}
for _, route := range []string{"3", "2", "1"} {
var (
ctx = WithReceiver(context.Background(), route)
alert = &types.Alert{
Alert: model.Alert{
Labels: model.LabelSet{"route": model.LabelValue(route)},
},
}
)
err := router.Notify(ctx, alert)
if err != nil {
t.Fatal(err)
}
rn := router[route].(*recordNotifier)
if len(rn.alerts) != 1 && alert != rn.alerts[0] {
t.Fatalf("Expeceted alert %v, got %v", alert, rn.alerts)
}
}
}
func main() {
http.HandleFunc("/receive", func(w http.ResponseWriter, r *http.Request) {
reqBuf, err := ioutil.ReadAll(snappy.NewReader(r.Body))
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
var req remote.WriteRequest
if err := proto.Unmarshal(reqBuf, &req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
for _, ts := range req.Timeseries {
m := make(model.Metric, len(ts.Labels))
for _, l := range ts.Labels {
m[model.LabelName(l.Name)] = model.LabelValue(l.Value)
}
fmt.Println(m)
for _, s := range ts.Samples {
fmt.Printf(" %f %d\n", s.Value, s.TimestampMs)
}
}
})
http.ListenAndServe(":1234", nil)
}
func newTestTarget(targetURL string, deadline time.Duration, baseLabels model.LabelSet) *Target {
cfg := &config.ScrapeConfig{
ScrapeTimeout: config.Duration(deadline),
}
c, _ := newHTTPClient(cfg)
t := &Target{
url: &url.URL{
Scheme: "http",
Host: strings.TrimLeft(targetURL, "http://"),
Path: "/metrics",
},
deadline: deadline,
status: &TargetStatus{},
scrapeInterval: 1 * time.Millisecond,
httpClient: c,
scraperStopping: make(chan struct{}),
scraperStopped: make(chan struct{}),
}
t.baseLabels = model.LabelSet{
model.InstanceLabel: model.LabelValue(t.InstanceIdentifier()),
}
for baseLabel, baseValue := range baseLabels {
t.baseLabels[baseLabel] = baseValue
}
return t
}
func (s *testNotificationScenario) test(i int, t *testing.T) {
h := NewNotificationHandler(&NotificationHandlerOptions{
AlertmanagerURL: "alertmanager_url",
QueueCapacity: 0,
Deadline: 10 * time.Second,
})
defer h.Stop()
receivedPost := make(chan bool, 1)
poster := testHTTPPoster{receivedPost: receivedPost}
h.httpClient = &poster
go h.Run()
h.SubmitReqs(NotificationReqs{
{
Summary: s.summary,
Description: s.description,
Runbook: s.runbook,
Labels: model.LabelSet{
model.LabelName("instance"): model.LabelValue("testinstance"),
},
Value: model.SampleValue(1.0 / 3.0),
ActiveSince: time.Time{},
RuleString: "Test rule string",
GeneratorURL: "prometheus_url",
},
})
<-receivedPost
if poster.message != s.message {
t.Fatalf("%d. Expected '%s', received '%s'", i, s.message, poster.message)
}
}
func TestURLParams(t *testing.T) {
server := httptest.NewServer(
http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", `text/plain; version=0.0.4`)
w.Write([]byte{})
r.ParseForm()
if r.Form["foo"][0] != "bar" {
t.Fatalf("URL parameter 'foo' had unexpected first value '%v'", r.Form["foo"][0])
}
if r.Form["foo"][1] != "baz" {
t.Fatalf("URL parameter 'foo' had unexpected second value '%v'", r.Form["foo"][1])
}
},
),
)
defer server.Close()
serverURL, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
target, err := NewTarget(
&config.ScrapeConfig{
JobName: "test_job1",
ScrapeInterval: model.Duration(1 * time.Minute),
ScrapeTimeout: model.Duration(1 * time.Second),
Scheme: serverURL.Scheme,
Params: url.Values{
"foo": []string{"bar", "baz"},
},
},
model.LabelSet{
model.SchemeLabel: model.LabelValue(serverURL.Scheme),
model.AddressLabel: model.LabelValue(serverURL.Host),
"__param_foo": "bar",
},
nil,
)
if err != nil {
t.Fatal(err)
}
app := &collectResultAppender{}
if err = target.scrape(app); err != nil {
t.Fatal(err)
}
}
func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints) *config.TargetGroup {
tg := &config.TargetGroup{
Source: serviceSource(service),
Labels: model.LabelSet{
serviceNamespaceLabel: model.LabelValue(service.ObjectMeta.Namespace),
serviceNameLabel: model.LabelValue(service.ObjectMeta.Name),
},
}
for k, v := range service.ObjectMeta.Labels {
labelName := strutil.SanitizeLabelName(serviceLabelPrefix + k)
tg.Labels[model.LabelName(labelName)] = model.LabelValue(v)
}
for k, v := range service.ObjectMeta.Annotations {
labelName := strutil.SanitizeLabelName(serviceAnnotationPrefix + k)
tg.Labels[model.LabelName(labelName)] = model.LabelValue(v)
}
serviceAddress := service.ObjectMeta.Name + "." + service.ObjectMeta.Namespace + ".svc"
// Append the first TCP service port if one exists.
for _, port := range service.Spec.Ports {
if port.Protocol == ProtocolTCP {
serviceAddress += fmt.Sprintf(":%d", port.Port)
break
}
}
t := model.LabelSet{
model.AddressLabel: model.LabelValue(serviceAddress),
roleLabel: model.LabelValue("service"),
}
tg.Targets = append(tg.Targets, t)
// Now let's loop through the endpoints & add them to the target group with appropriate labels.
for _, ss := range eps.Subsets {
epPort := ss.Ports[0].Port
for _, addr := range ss.Addresses {
ipAddr := addr.IP
if len(ipAddr) == net.IPv6len {
ipAddr = "[" + ipAddr + "]"
}
address := fmt.Sprintf("%s:%d", ipAddr, epPort)
t := model.LabelSet{
model.AddressLabel: model.LabelValue(address),
roleLabel: model.LabelValue("endpoint"),
}
tg.Targets = append(tg.Targets, t)
}
}
return tg
}