func (api *API) listAlerts(w http.ResponseWriter, r *http.Request) {
alerts := api.alerts.GetPending()
defer alerts.Close()
var (
err error
res []*types.Alert
)
// TODO(fabxc): enforce a sensible timeout.
for a := range alerts.Next() {
if err = alerts.Err(); err != nil {
break
}
res = append(res, a)
}
if err != nil {
respondError(w, apiError{
typ: errorInternal,
err: err,
}, nil)
return
}
respond(w, types.Alerts(res...))
}
// Notify implements the Notifier interface.
func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) error {
as := types.Alerts(alerts...)
// If there are no annotations, instantiate so
// {} is sent rather than null.
for _, a := range as {
if a.Annotations == nil {
a.Annotations = model.LabelSet{}
}
}
msg := &WebhookMessage{
Version: "2",
Status: as.Status(),
Alerts: as,
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, w.URL, contentTypeJSON, &buf)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
if !ok {
return fmt.Errorf("group key missing")
}
data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
log.With("incident", key).Debugln("notifying OpsGenie")
var err error
tmpl := tmplText(n.tmpl, data, &err)
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
details[k] = tmpl(v)
}
var (
msg interface{}
apiURL string
apiMsg = opsGenieMessage{
APIKey: string(n.conf.APIKey),
Alias: key,
}
alerts = types.Alerts(as...)
)
switch alerts.Status() {
case model.AlertResolved:
apiURL = n.conf.APIHost + "v1/json/alert/close"
msg = &opsGenieCloseMessage{&apiMsg}
default:
apiURL = n.conf.APIHost + "v1/json/alert"
msg = &opsGenieCreateMessage{
opsGenieMessage: &apiMsg,
Message: tmpl(n.conf.Description),
Details: details,
Source: tmpl(n.conf.Source),
}
}
if err != nil {
return fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Data assembles data for template expansion.
func (t *Template) Data(recv string, groupLabels model.LabelSet, alerts ...*types.Alert) *Data {
data := &Data{
Receiver: strings.SplitN(recv, "/", 2)[0],
Status: string(types.Alerts(alerts...).Status()),
Alerts: make(Alerts, 0, len(alerts)),
GroupLabels: KV{},
CommonLabels: KV{},
CommonAnnotations: KV{},
ExternalURL: t.ExternalURL.String(),
}
for _, a := range alerts {
alert := Alert{
Status: string(a.Status()),
Labels: make(KV, len(a.Labels)),
Annotations: make(KV, len(a.Annotations)),
WasSilenced: a.WasSilenced,
WasInhibited: a.WasInhibited,
}
for k, v := range a.Labels {
alert.Labels[string(k)] = string(v)
}
for k, v := range a.Annotations {
alert.Annotations[string(k)] = string(v)
}
data.Alerts = append(data.Alerts, alert)
}
for k, v := range groupLabels {
data.GroupLabels[string(k)] = string(v)
}
if len(alerts) >= 1 {
var (
commonLabels = alerts[0].Labels.Clone()
commonAnnotations = alerts[0].Annotations.Clone()
)
for _, a := range alerts[1:] {
for ln, lv := range commonLabels {
if a.Labels[ln] != lv {
delete(commonLabels, ln)
}
}
for an, av := range commonAnnotations {
if a.Annotations[an] != av {
delete(commonAnnotations, an)
}
}
}
for k, v := range commonLabels {
data.CommonLabels[string(k)] = string(v)
}
for k, v := range commonAnnotations {
data.CommonAnnotations[string(k)] = string(v)
}
}
return data
}
// Notify implements the Notifier interface.
//
// http://developer.pagerduty.com/documentation/integration/events/trigger
func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
if !ok {
return fmt.Errorf("group key missing")
}
var err error
var (
alerts = types.Alerts(as...)
data = n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
tmpl = tmplText(n.tmpl, data, &err)
eventType = pagerDutyEventTrigger
)
if alerts.Status() == model.AlertResolved {
eventType = pagerDutyEventResolve
}
log.With("incident", key).With("eventType", eventType).Debugln("notifying PagerDuty")
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
details[k] = tmpl(v)
}
msg := &pagerDutyMessage{
ServiceKey: tmpl(string(n.conf.ServiceKey)),
EventType: eventType,
IncidentKey: key,
Description: tmpl(n.conf.Description),
Details: details,
}
if eventType == pagerDutyEventTrigger {
msg.Client = tmpl(n.conf.Client)
msg.ClientURL = tmpl(n.conf.ClientURL)
}
if err != nil {
return err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, n.conf.URL, contentTypeJSON, &buf)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Groups populates an AlertOverview from the dispatcher's internal state.
func (d *Dispatcher) Groups() AlertOverview {
var overview AlertOverview
d.mtx.RLock()
defer d.mtx.RUnlock()
seen := map[model.Fingerprint]*AlertGroup{}
for route, ags := range d.aggrGroups {
for _, ag := range ags {
alertGroup, ok := seen[ag.fingerprint()]
if !ok {
alertGroup = &AlertGroup{Labels: ag.labels}
alertGroup.GroupKey = ag.GroupKey()
seen[ag.fingerprint()] = alertGroup
overview = append(overview, alertGroup)
}
now := time.Now()
var apiAlerts []*APIAlert
for _, a := range types.Alerts(ag.alertSlice()...) {
if !a.EndsAt.IsZero() && a.EndsAt.Before(now) {
continue
}
aa := &APIAlert{
Alert: a,
Inhibited: d.marker.Inhibited(a.Fingerprint()),
}
if sid, ok := d.marker.Silenced(a.Fingerprint()); ok {
aa.Silenced = sid
}
apiAlerts = append(apiAlerts, aa)
}
if len(apiAlerts) == 0 {
continue
}
alertGroup.Blocks = append(alertGroup.Blocks, &AlertBlock{
RouteOpts: &route.RouteOpts,
Alerts: apiAlerts,
})
}
}
sort.Sort(overview)
return overview
}
// Data assembles data for template expansion.
func (t *Template) Data(recv string, groupLabels model.LabelSet, alerts ...*types.Alert) *Data {
data := &Data{
Receiver: strings.SplitN(recv, "/", 2)[0],
Status: string(types.Alerts(alerts...).Status()),
Alerts: make(Alerts, 0, len(alerts)),
GroupLabels: KV{},
CommonLabels: KV{},
CommonAnnotations: KV{},
ExternalURL: t.ExternalURL.String(),
}
// The call to types.Alert is necessary to correctly resolve the internal
// representation to the user representation.
for _, a := range types.Alerts(alerts...) {
alert := Alert{
Status: string(a.Status()),
Labels: make(KV, len(a.Labels)),
Annotations: make(KV, len(a.Annotations)),
StartsAt: a.StartsAt,
EndsAt: a.EndsAt,
GeneratorURL: a.GeneratorURL,
}
for k, v := range a.Labels {
alert.Labels[string(k)] = string(v)
}
for k, v := range a.Annotations {
alert.Annotations[string(k)] = string(v)
}
data.Alerts = append(data.Alerts, alert)
}
for k, v := range groupLabels {
data.GroupLabels[string(k)] = string(v)
}
if len(alerts) >= 1 {
var (
commonLabels = alerts[0].Labels.Clone()
commonAnnotations = alerts[0].Annotations.Clone()
)
for _, a := range alerts[1:] {
for ln, lv := range commonLabels {
if a.Labels[ln] != lv {
delete(commonLabels, ln)
}
}
for an, av := range commonAnnotations {
if a.Annotations[an] != av {
delete(commonAnnotations, an)
}
}
}
for k, v := range commonLabels {
data.CommonLabels[string(k)] = string(v)
}
for k, v := range commonAnnotations {
data.CommonAnnotations[string(k)] = string(v)
}
}
return data
}
// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
if !ok {
return fmt.Errorf("group key missing")
}
data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
log.With("incident", key).Debugln("notifying OpsGenie")
var err error
tmpl := tmplText(n.tmpl, data, &err)
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
details[k] = tmpl(v)
}
var (
msg interface{}
apiURL string
apiMsg = opsGenieMessage{
APIKey: string(n.conf.APIKey),
Alias: key,
}
alerts = types.Alerts(as...)
)
switch alerts.Status() {
case model.AlertResolved:
apiURL = n.conf.APIHost + "v1/json/alert/close"
msg = &opsGenieCloseMessage{&apiMsg}
default:
apiURL = n.conf.APIHost + "v1/json/alert"
msg = &opsGenieCreateMessage{
opsGenieMessage: &apiMsg,
Message: tmpl(n.conf.Description),
Details: details,
Source: tmpl(n.conf.Source),
Teams: tmpl(n.conf.Teams),
Tags: tmpl(n.conf.Tags),
}
}
if err != nil {
return fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == 400 && alerts.Status() == model.AlertResolved {
body, _ := ioutil.ReadAll(resp.Body)
var responseMessage opsGenieErrorResponse
if err := json.Unmarshal(body, &responseMessage); err != nil {
return fmt.Errorf("could not parse error response %q", body)
}
const alreadyClosedError = 5
if responseMessage.Code == alreadyClosedError {
return nil
}
return fmt.Errorf("error when closing alert: code %d, error %q",
responseMessage.Code, responseMessage.Error)
} else if resp.StatusCode/100 != 2 {
body, _ := ioutil.ReadAll(resp.Body)
log.With("incident", key).Debugf("unexpected OpsGenie response from %s (POSTed %s), %s: %s",
apiURL, msg, resp.Status, body)
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Notify implements the Notifier interface.
func (n *VictorOps) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
victorOpsAllowedEvents := map[string]bool{
"INFO": true,
"WARNING": true,
"CRITICAL": true,
}
key, ok := GroupKey(ctx)
if !ok {
return false, fmt.Errorf("group key missing")
}
var err error
var (
alerts = types.Alerts(as...)
data = n.tmpl.Data(receiverName(ctx), groupLabels(ctx), as...)
tmpl = tmplText(n.tmpl, data, &err)
apiURL = fmt.Sprintf("%s%s/%s", n.conf.APIURL, n.conf.APIKey, n.conf.RoutingKey)
messageType = n.conf.MessageType
)
if alerts.Status() == model.AlertFiring && !victorOpsAllowedEvents[messageType] {
messageType = victorOpsEventTrigger
}
if alerts.Status() == model.AlertResolved {
messageType = victorOpsEventResolve
}
msg := &victorOpsMessage{
MessageType: messageType,
EntityID: key,
StateMessage: tmpl(n.conf.StateMessage),
From: tmpl(n.conf.From),
}
if err != nil {
return false, fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
if err != nil {
return true, err
}
defer resp.Body.Close()
// Missing documentation therefore assuming only 5xx response codes are
// recoverable.
if resp.StatusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
if resp.StatusCode/100 != 2 {
body, _ := ioutil.ReadAll(resp.Body)
var responseMessage victorOpsErrorResponse
if err := json.Unmarshal(body, &responseMessage); err != nil {
return false, fmt.Errorf("could not parse error response %q", body)
}
log.With("incident", key).Debugf("unexpected VictorOps response from %s (POSTed %s), %s: %s", apiURL, msg, resp.Status, body)
return false, fmt.Errorf("error when posting alert: result %q, message %q",
responseMessage.Result, responseMessage.Message)
}
return false, nil
}