func (api *API) listAlerts(w http.ResponseWriter, r *http.Request) { alerts := api.alerts.GetPending() defer alerts.Close() var ( err error res []*types.Alert ) // TODO(fabxc): enforce a sensible timeout. for a := range alerts.Next() { if err = alerts.Err(); err != nil { break } res = append(res, a) } if err != nil { respondError(w, apiError{ typ: errorInternal, err: err, }, nil) return } respond(w, types.Alerts(res...)) }
// Notify implements the Notifier interface. func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) error { as := types.Alerts(alerts...) // If there are no annotations, instantiate so // {} is sent rather than null. for _, a := range as { if a.Annotations == nil { a.Annotations = model.LabelSet{} } } msg := &WebhookMessage{ Version: "2", Status: as.Status(), Alerts: as, } var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(msg); err != nil { return err } resp, err := ctxhttp.Post(ctx, http.DefaultClient, w.URL, contentTypeJSON, &buf) if err != nil { return err } resp.Body.Close() if resp.StatusCode/100 != 2 { return fmt.Errorf("unexpected status code %v", resp.StatusCode) } return nil }
// Notify implements the Notifier interface. func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error { key, ok := GroupKey(ctx) if !ok { return fmt.Errorf("group key missing") } data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...) log.With("incident", key).Debugln("notifying OpsGenie") var err error tmpl := tmplText(n.tmpl, data, &err) details := make(map[string]string, len(n.conf.Details)) for k, v := range n.conf.Details { details[k] = tmpl(v) } var ( msg interface{} apiURL string apiMsg = opsGenieMessage{ APIKey: string(n.conf.APIKey), Alias: key, } alerts = types.Alerts(as...) ) switch alerts.Status() { case model.AlertResolved: apiURL = n.conf.APIHost + "v1/json/alert/close" msg = &opsGenieCloseMessage{&apiMsg} default: apiURL = n.conf.APIHost + "v1/json/alert" msg = &opsGenieCreateMessage{ opsGenieMessage: &apiMsg, Message: tmpl(n.conf.Description), Details: details, Source: tmpl(n.conf.Source), } } if err != nil { return fmt.Errorf("templating error: %s", err) } var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(msg); err != nil { return err } resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf) if err != nil { return err } resp.Body.Close() if resp.StatusCode/100 != 2 { return fmt.Errorf("unexpected status code %v", resp.StatusCode) } return nil }
// Data assembles data for template expansion. func (t *Template) Data(recv string, groupLabels model.LabelSet, alerts ...*types.Alert) *Data { data := &Data{ Receiver: strings.SplitN(recv, "/", 2)[0], Status: string(types.Alerts(alerts...).Status()), Alerts: make(Alerts, 0, len(alerts)), GroupLabels: KV{}, CommonLabels: KV{}, CommonAnnotations: KV{}, ExternalURL: t.ExternalURL.String(), } for _, a := range alerts { alert := Alert{ Status: string(a.Status()), Labels: make(KV, len(a.Labels)), Annotations: make(KV, len(a.Annotations)), WasSilenced: a.WasSilenced, WasInhibited: a.WasInhibited, } for k, v := range a.Labels { alert.Labels[string(k)] = string(v) } for k, v := range a.Annotations { alert.Annotations[string(k)] = string(v) } data.Alerts = append(data.Alerts, alert) } for k, v := range groupLabels { data.GroupLabels[string(k)] = string(v) } if len(alerts) >= 1 { var ( commonLabels = alerts[0].Labels.Clone() commonAnnotations = alerts[0].Annotations.Clone() ) for _, a := range alerts[1:] { for ln, lv := range commonLabels { if a.Labels[ln] != lv { delete(commonLabels, ln) } } for an, av := range commonAnnotations { if a.Annotations[an] != av { delete(commonAnnotations, an) } } } for k, v := range commonLabels { data.CommonLabels[string(k)] = string(v) } for k, v := range commonAnnotations { data.CommonAnnotations[string(k)] = string(v) } } return data }
// Notify implements the Notifier interface. // // http://developer.pagerduty.com/documentation/integration/events/trigger func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error { key, ok := GroupKey(ctx) if !ok { return fmt.Errorf("group key missing") } var err error var ( alerts = types.Alerts(as...) data = n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...) tmpl = tmplText(n.tmpl, data, &err) eventType = pagerDutyEventTrigger ) if alerts.Status() == model.AlertResolved { eventType = pagerDutyEventResolve } log.With("incident", key).With("eventType", eventType).Debugln("notifying PagerDuty") details := make(map[string]string, len(n.conf.Details)) for k, v := range n.conf.Details { details[k] = tmpl(v) } msg := &pagerDutyMessage{ ServiceKey: tmpl(string(n.conf.ServiceKey)), EventType: eventType, IncidentKey: key, Description: tmpl(n.conf.Description), Details: details, } if eventType == pagerDutyEventTrigger { msg.Client = tmpl(n.conf.Client) msg.ClientURL = tmpl(n.conf.ClientURL) } if err != nil { return err } var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(msg); err != nil { return err } resp, err := ctxhttp.Post(ctx, http.DefaultClient, n.conf.URL, contentTypeJSON, &buf) if err != nil { return err } resp.Body.Close() if resp.StatusCode/100 != 2 { return fmt.Errorf("unexpected status code %v", resp.StatusCode) } return nil }
// Groups populates an AlertOverview from the dispatcher's internal state. func (d *Dispatcher) Groups() AlertOverview { var overview AlertOverview d.mtx.RLock() defer d.mtx.RUnlock() seen := map[model.Fingerprint]*AlertGroup{} for route, ags := range d.aggrGroups { for _, ag := range ags { alertGroup, ok := seen[ag.fingerprint()] if !ok { alertGroup = &AlertGroup{Labels: ag.labels} alertGroup.GroupKey = ag.GroupKey() seen[ag.fingerprint()] = alertGroup overview = append(overview, alertGroup) } now := time.Now() var apiAlerts []*APIAlert for _, a := range types.Alerts(ag.alertSlice()...) { if !a.EndsAt.IsZero() && a.EndsAt.Before(now) { continue } aa := &APIAlert{ Alert: a, Inhibited: d.marker.Inhibited(a.Fingerprint()), } if sid, ok := d.marker.Silenced(a.Fingerprint()); ok { aa.Silenced = sid } apiAlerts = append(apiAlerts, aa) } if len(apiAlerts) == 0 { continue } alertGroup.Blocks = append(alertGroup.Blocks, &AlertBlock{ RouteOpts: &route.RouteOpts, Alerts: apiAlerts, }) } } sort.Sort(overview) return overview }
// Data assembles data for template expansion. func (t *Template) Data(recv string, groupLabels model.LabelSet, alerts ...*types.Alert) *Data { data := &Data{ Receiver: strings.SplitN(recv, "/", 2)[0], Status: string(types.Alerts(alerts...).Status()), Alerts: make(Alerts, 0, len(alerts)), GroupLabels: KV{}, CommonLabels: KV{}, CommonAnnotations: KV{}, ExternalURL: t.ExternalURL.String(), } // The call to types.Alert is necessary to correctly resolve the internal // representation to the user representation. for _, a := range types.Alerts(alerts...) { alert := Alert{ Status: string(a.Status()), Labels: make(KV, len(a.Labels)), Annotations: make(KV, len(a.Annotations)), StartsAt: a.StartsAt, EndsAt: a.EndsAt, GeneratorURL: a.GeneratorURL, } for k, v := range a.Labels { alert.Labels[string(k)] = string(v) } for k, v := range a.Annotations { alert.Annotations[string(k)] = string(v) } data.Alerts = append(data.Alerts, alert) } for k, v := range groupLabels { data.GroupLabels[string(k)] = string(v) } if len(alerts) >= 1 { var ( commonLabels = alerts[0].Labels.Clone() commonAnnotations = alerts[0].Annotations.Clone() ) for _, a := range alerts[1:] { for ln, lv := range commonLabels { if a.Labels[ln] != lv { delete(commonLabels, ln) } } for an, av := range commonAnnotations { if a.Annotations[an] != av { delete(commonAnnotations, an) } } } for k, v := range commonLabels { data.CommonLabels[string(k)] = string(v) } for k, v := range commonAnnotations { data.CommonAnnotations[string(k)] = string(v) } } return data }
// Notify implements the Notifier interface. func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error { key, ok := GroupKey(ctx) if !ok { return fmt.Errorf("group key missing") } data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...) log.With("incident", key).Debugln("notifying OpsGenie") var err error tmpl := tmplText(n.tmpl, data, &err) details := make(map[string]string, len(n.conf.Details)) for k, v := range n.conf.Details { details[k] = tmpl(v) } var ( msg interface{} apiURL string apiMsg = opsGenieMessage{ APIKey: string(n.conf.APIKey), Alias: key, } alerts = types.Alerts(as...) ) switch alerts.Status() { case model.AlertResolved: apiURL = n.conf.APIHost + "v1/json/alert/close" msg = &opsGenieCloseMessage{&apiMsg} default: apiURL = n.conf.APIHost + "v1/json/alert" msg = &opsGenieCreateMessage{ opsGenieMessage: &apiMsg, Message: tmpl(n.conf.Description), Details: details, Source: tmpl(n.conf.Source), Teams: tmpl(n.conf.Teams), Tags: tmpl(n.conf.Tags), } } if err != nil { return fmt.Errorf("templating error: %s", err) } var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(msg); err != nil { return err } resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode == 400 && alerts.Status() == model.AlertResolved { body, _ := ioutil.ReadAll(resp.Body) var responseMessage opsGenieErrorResponse if err := json.Unmarshal(body, &responseMessage); err != nil { return fmt.Errorf("could not parse error response %q", body) } const alreadyClosedError = 5 if responseMessage.Code == alreadyClosedError { return nil } return fmt.Errorf("error when closing alert: code %d, error %q", responseMessage.Code, responseMessage.Error) } else if resp.StatusCode/100 != 2 { body, _ := ioutil.ReadAll(resp.Body) log.With("incident", key).Debugf("unexpected OpsGenie response from %s (POSTed %s), %s: %s", apiURL, msg, resp.Status, body) return fmt.Errorf("unexpected status code %v", resp.StatusCode) } return nil }
// Notify implements the Notifier interface. func (n *VictorOps) Notify(ctx context.Context, as ...*types.Alert) (bool, error) { victorOpsAllowedEvents := map[string]bool{ "INFO": true, "WARNING": true, "CRITICAL": true, } key, ok := GroupKey(ctx) if !ok { return false, fmt.Errorf("group key missing") } var err error var ( alerts = types.Alerts(as...) data = n.tmpl.Data(receiverName(ctx), groupLabels(ctx), as...) tmpl = tmplText(n.tmpl, data, &err) apiURL = fmt.Sprintf("%s%s/%s", n.conf.APIURL, n.conf.APIKey, n.conf.RoutingKey) messageType = n.conf.MessageType ) if alerts.Status() == model.AlertFiring && !victorOpsAllowedEvents[messageType] { messageType = victorOpsEventTrigger } if alerts.Status() == model.AlertResolved { messageType = victorOpsEventResolve } msg := &victorOpsMessage{ MessageType: messageType, EntityID: key, StateMessage: tmpl(n.conf.StateMessage), From: tmpl(n.conf.From), } if err != nil { return false, fmt.Errorf("templating error: %s", err) } var buf bytes.Buffer if err := json.NewEncoder(&buf).Encode(msg); err != nil { return false, err } resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf) if err != nil { return true, err } defer resp.Body.Close() // Missing documentation therefore assuming only 5xx response codes are // recoverable. if resp.StatusCode/100 == 5 { return true, fmt.Errorf("unexpected status code %v", resp.StatusCode) } if resp.StatusCode/100 != 2 { body, _ := ioutil.ReadAll(resp.Body) var responseMessage victorOpsErrorResponse if err := json.Unmarshal(body, &responseMessage); err != nil { return false, fmt.Errorf("could not parse error response %q", body) } log.With("incident", key).Debugf("unexpected VictorOps response from %s (POSTed %s), %s: %s", apiURL, msg, resp.Status, body) return false, fmt.Errorf("error when posting alert: result %q, message %q", responseMessage.Result, responseMessage.Message) } return false, nil }