func (h *Handler) executeTemplate(w http.ResponseWriter, name string, data interface{}) { text, err := h.getTemplate(name) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } tmpl := template.NewTemplateExpander(text, name, data, clientmodel.Now(), h.queryEngine, h.options.PathPrefix) tmpl.Funcs(template_text.FuncMap{ "since": time.Since, "getConsoles": h.getConsoles, "pathPrefix": func() string { return h.options.PathPrefix }, "stripLabels": func(lset clientmodel.LabelSet, labels ...clientmodel.LabelName) clientmodel.LabelSet { for _, ln := range labels { delete(lset, ln) } return lset }, "globalURL": func(u *url.URL) *url.URL { for _, lhr := range localhostRepresentations { if strings.HasPrefix(u.Host, lhr+":") { u.Host = strings.Replace(u.Host, lhr+":", h.options.Hostname+":", 1) } } return u }, "healthToClass": func(th retrieval.TargetHealth) string { switch th { case retrieval.HealthUnknown: return "warning" case retrieval.HealthGood: return "success" default: return "danger" } }, "alertStateToClass": func(as rules.AlertState) string { switch as { case rules.StateInactive: return "success" case rules.StatePending: return "warning" case rules.StateFiring: return "danger" default: panic("unknown alert state") } }, }) result, err := tmpl.ExpandHTML(nil) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } io.WriteString(w, result) }
func (m *Manager) queueAlertNotifications(rule *AlertingRule, timestamp clientmodel.Timestamp) { activeAlerts := rule.ActiveAlerts() if len(activeAlerts) == 0 { return } notifications := make(notification.NotificationReqs, 0, len(activeAlerts)) for _, aa := range activeAlerts { if aa.State != StateFiring { // BUG: In the future, make AlertManager support pending alerts? continue } // Provide the alert information to the template. l := map[string]string{} for k, v := range aa.Labels { l[string(k)] = string(v) } tmplData := struct { Labels map[string]string Value clientmodel.SampleValue }{ Labels: l, Value: aa.Value, } // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}" expand := func(text string) string { tmpl := template.NewTemplateExpander(defs+text, "__alert_"+rule.Name(), tmplData, timestamp, m.queryEngine, m.externalURL.Path) result, err := tmpl.Expand() if err != nil { result = err.Error() log.Warnf("Error expanding alert template %v with data '%v': %v", rule.Name(), tmplData, err) } return result } notifications = append(notifications, ¬ification.NotificationReq{ Summary: expand(rule.summary), Description: expand(rule.description), Runbook: rule.runbook, Labels: aa.Labels.Merge(clientmodel.LabelSet{ alertNameLabel: clientmodel.LabelValue(rule.Name()), }), Value: aa.Value, ActiveSince: aa.ActiveSince.Time(), RuleString: rule.String(), GeneratorURL: m.externalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()), }) } m.notificationHandler.SubmitReqs(notifications) }
func (h *Handler) consoles(w http.ResponseWriter, r *http.Request) { ctx := route.Context(r) name := route.Param(ctx, "filepath") file, err := http.Dir(h.options.ConsoleTemplatesPath).Open(name) if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return } text, err := ioutil.ReadAll(file) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Provide URL parameters as a map for easy use. Advanced users may have need for // parameters beyond the first, so provide RawParams. rawParams, err := url.ParseQuery(r.URL.RawQuery) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } params := map[string]string{} for k, v := range rawParams { params[k] = v[0] } data := struct { RawParams url.Values Params map[string]string Path string }{ RawParams: rawParams, Params: params, Path: strings.TrimLeft(name, "/"), } tmpl := template.NewTemplateExpander(string(text), "__console_"+name, data, model.Now(), h.queryEngine, h.options.ExternalURL.Path) filenames, err := filepath.Glob(h.options.ConsoleLibrariesPath + "/*.lib") if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } result, err := tmpl.ExpandHTML(filenames) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } io.WriteString(w, result) }
func (h *Handler) executeTemplate(w http.ResponseWriter, name string, data interface{}) { text, err := h.getTemplate(name) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) } tmpl := template.NewTemplateExpander(text, name, data, model.Now(), h.queryEngine, h.options.ExternalURL.Path) tmpl.Funcs(tmplFuncs(h.consolesPath(), h.options)) result, err := tmpl.ExpandHTML(nil) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } io.WriteString(w, result) }
// eval evaluates the rule expression and then creates pending alerts and fires // or removes previously pending alerts accordingly. func (r *AlertingRule) eval(ts model.Time, engine *promql.Engine, externalURLPath string) (model.Vector, error) { query, err := engine.NewInstantQuery(r.vector.String(), ts) if err != nil { return nil, err } res, err := query.Exec().Vector() if err != nil { return nil, err } r.mtx.Lock() defer r.mtx.Unlock() // Create pending alerts for any new vector elements in the alert expression // or update the expression value for existing elements. resultFPs := map[model.Fingerprint]struct{}{} for _, smpl := range res { // Provide the alert information to the template. l := make(map[string]string, len(smpl.Metric)) for k, v := range smpl.Metric { l[string(k)] = string(v) } tmplData := struct { Labels map[string]string Value float64 }{ Labels: l, Value: float64(smpl.Value), } // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}" expand := func(text model.LabelValue) model.LabelValue { tmpl := template.NewTemplateExpander( defs+string(text), "__alert_"+r.Name(), tmplData, ts, engine, externalURLPath, ) result, err := tmpl.Expand() if err != nil { result = fmt.Sprintf("<error expanding template: %s>", err) log.Warnf("Error expanding alert template %v with data '%v': %s", r.Name(), tmplData, err) } return model.LabelValue(result) } labels := make(model.LabelSet, len(smpl.Metric)+len(r.labels)+1) for ln, lv := range smpl.Metric { labels[ln] = lv } for ln, lv := range r.labels { labels[ln] = expand(lv) } labels[model.AlertNameLabel] = model.LabelValue(r.Name()) annotations := make(model.LabelSet, len(r.annotations)) for an, av := range r.annotations { annotations[an] = expand(av) } fp := smpl.Metric.Fingerprint() resultFPs[fp] = struct{}{} if alert, ok := r.active[fp]; ok && alert.State != StateInactive { alert.Value = smpl.Value continue } delete(smpl.Metric, model.MetricNameLabel) r.active[fp] = &Alert{ Labels: labels, Annotations: annotations, ActiveAt: ts, State: StatePending, Value: smpl.Value, } } var vec model.Vector // Check if any pending alerts should be removed or fire now. Write out alert timeseries. for fp, a := range r.active { if _, ok := resultFPs[fp]; !ok { if a.State != StateInactive { vec = append(vec, r.sample(a, ts, false)) } // If the alert was previously firing, keep it around for a given // retention time so it is reported as resolved to the AlertManager. if a.State == StatePending || (a.ResolvedAt != 0 && ts.Sub(a.ResolvedAt) > resolvedRetention) { delete(r.active, fp) } if a.State != StateInactive { a.State = StateInactive a.ResolvedAt = ts } continue } if a.State == StatePending && ts.Sub(a.ActiveAt) >= r.holdDuration { vec = append(vec, r.sample(a, ts, false)) a.State = StateFiring } vec = append(vec, r.sample(a, ts, true)) } return vec, nil }
// sendAlerts sends alert notifications for the given rule. func (g *Group) sendAlerts(rule *AlertingRule, timestamp model.Time) error { var alerts model.Alerts for _, alert := range rule.currentAlerts() { // Only send actually firing alerts. if alert.State == StatePending { continue } // Provide the alert information to the template. l := make(map[string]string, len(alert.Labels)) for k, v := range alert.Labels { l[string(k)] = string(v) } tmplData := struct { Labels map[string]string Value float64 }{ Labels: l, Value: float64(alert.Value), } // Inject some convenience variables that are easier to remember for users // who are not used to Go's templating system. defs := "{{$labels := .Labels}}{{$value := .Value}}" expand := func(text model.LabelValue) model.LabelValue { tmpl := template.NewTemplateExpander( defs+string(text), "__alert_"+rule.Name(), tmplData, timestamp, g.opts.QueryEngine, g.opts.ExternalURL.Path, ) result, err := tmpl.Expand() if err != nil { result = fmt.Sprintf("<error expanding template: %s>", err) log.Warnf("Error expanding alert template %v with data '%v': %s", rule.Name(), tmplData, err) } return model.LabelValue(result) } labels := make(model.LabelSet, len(alert.Labels)+1) for ln, lv := range alert.Labels { labels[ln] = expand(lv) } labels[model.AlertNameLabel] = model.LabelValue(rule.Name()) annotations := make(model.LabelSet, len(rule.annotations)) for an, av := range rule.annotations { annotations[an] = expand(av) } a := &model.Alert{ StartsAt: alert.ActiveAt.Add(rule.holdDuration).Time(), Labels: labels, Annotations: annotations, GeneratorURL: g.opts.ExternalURL.String() + strutil.GraphLinkForExpression(rule.vector.String()), } if alert.ResolvedAt != 0 { a.EndsAt = alert.ResolvedAt.Time() } alerts = append(alerts, a) } if len(alerts) > 0 { g.opts.NotificationHandler.Send(alerts...) } return nil }