Exemplo n.º 1
0
func (api *API) listAlerts(w http.ResponseWriter, r *http.Request) {
	alerts := api.alerts.GetPending()
	defer alerts.Close()

	var (
		err error
		res []*types.Alert
	)
	// TODO(fabxc): enforce a sensible timeout.
	for a := range alerts.Next() {
		if err = alerts.Err(); err != nil {
			break
		}
		res = append(res, a)
	}

	if err != nil {
		respondError(w, apiError{
			typ: errorInternal,
			err: err,
		}, nil)
		return
	}
	respond(w, types.Alerts(res...))
}
Exemplo n.º 2
0
// Notify implements the Notifier interface.
func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) error {
	as := types.Alerts(alerts...)

	// If there are no annotations, instantiate so
	// {} is sent rather than null.
	for _, a := range as {
		if a.Annotations == nil {
			a.Annotations = model.LabelSet{}
		}
	}

	msg := &WebhookMessage{
		Version: "2",
		Status:  as.Status(),
		Alerts:  as,
	}

	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
		return err
	}

	resp, err := ctxhttp.Post(ctx, http.DefaultClient, w.URL, contentTypeJSON, &buf)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode/100 != 2 {
		return fmt.Errorf("unexpected status code %v", resp.StatusCode)
	}

	return nil
}
Exemplo n.º 3
0
// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
	key, ok := GroupKey(ctx)
	if !ok {
		return fmt.Errorf("group key missing")
	}
	data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)

	log.With("incident", key).Debugln("notifying OpsGenie")

	var err error
	tmpl := tmplText(n.tmpl, data, &err)

	details := make(map[string]string, len(n.conf.Details))
	for k, v := range n.conf.Details {
		details[k] = tmpl(v)
	}

	var (
		msg    interface{}
		apiURL string

		apiMsg = opsGenieMessage{
			APIKey: string(n.conf.APIKey),
			Alias:  key,
		}
		alerts = types.Alerts(as...)
	)
	switch alerts.Status() {
	case model.AlertResolved:
		apiURL = n.conf.APIHost + "v1/json/alert/close"
		msg = &opsGenieCloseMessage{&apiMsg}
	default:
		apiURL = n.conf.APIHost + "v1/json/alert"
		msg = &opsGenieCreateMessage{
			opsGenieMessage: &apiMsg,
			Message:         tmpl(n.conf.Description),
			Details:         details,
			Source:          tmpl(n.conf.Source),
		}
	}
	if err != nil {
		return fmt.Errorf("templating error: %s", err)
	}

	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
		return err
	}

	resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode/100 != 2 {
		return fmt.Errorf("unexpected status code %v", resp.StatusCode)
	}
	return nil
}
Exemplo n.º 4
0
// Data assembles data for template expansion.
func (t *Template) Data(recv string, groupLabels model.LabelSet, alerts ...*types.Alert) *Data {
	data := &Data{
		Receiver:          strings.SplitN(recv, "/", 2)[0],
		Status:            string(types.Alerts(alerts...).Status()),
		Alerts:            make(Alerts, 0, len(alerts)),
		GroupLabels:       KV{},
		CommonLabels:      KV{},
		CommonAnnotations: KV{},
		ExternalURL:       t.ExternalURL.String(),
	}

	for _, a := range alerts {
		alert := Alert{
			Status:       string(a.Status()),
			Labels:       make(KV, len(a.Labels)),
			Annotations:  make(KV, len(a.Annotations)),
			WasSilenced:  a.WasSilenced,
			WasInhibited: a.WasInhibited,
		}
		for k, v := range a.Labels {
			alert.Labels[string(k)] = string(v)
		}
		for k, v := range a.Annotations {
			alert.Annotations[string(k)] = string(v)
		}
		data.Alerts = append(data.Alerts, alert)
	}

	for k, v := range groupLabels {
		data.GroupLabels[string(k)] = string(v)
	}

	if len(alerts) >= 1 {
		var (
			commonLabels      = alerts[0].Labels.Clone()
			commonAnnotations = alerts[0].Annotations.Clone()
		)
		for _, a := range alerts[1:] {
			for ln, lv := range commonLabels {
				if a.Labels[ln] != lv {
					delete(commonLabels, ln)
				}
			}
			for an, av := range commonAnnotations {
				if a.Annotations[an] != av {
					delete(commonAnnotations, an)
				}
			}
		}
		for k, v := range commonLabels {
			data.CommonLabels[string(k)] = string(v)
		}
		for k, v := range commonAnnotations {
			data.CommonAnnotations[string(k)] = string(v)
		}
	}

	return data
}
Exemplo n.º 5
0
// Notify implements the Notifier interface.
//
// http://developer.pagerduty.com/documentation/integration/events/trigger
func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error {
	key, ok := GroupKey(ctx)
	if !ok {
		return fmt.Errorf("group key missing")
	}

	var err error
	var (
		alerts    = types.Alerts(as...)
		data      = n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
		tmpl      = tmplText(n.tmpl, data, &err)
		eventType = pagerDutyEventTrigger
	)
	if alerts.Status() == model.AlertResolved {
		eventType = pagerDutyEventResolve
	}

	log.With("incident", key).With("eventType", eventType).Debugln("notifying PagerDuty")

	details := make(map[string]string, len(n.conf.Details))
	for k, v := range n.conf.Details {
		details[k] = tmpl(v)
	}

	msg := &pagerDutyMessage{
		ServiceKey:  tmpl(string(n.conf.ServiceKey)),
		EventType:   eventType,
		IncidentKey: key,
		Description: tmpl(n.conf.Description),
		Details:     details,
	}
	if eventType == pagerDutyEventTrigger {
		msg.Client = tmpl(n.conf.Client)
		msg.ClientURL = tmpl(n.conf.ClientURL)
	}
	if err != nil {
		return err
	}

	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
		return err
	}

	resp, err := ctxhttp.Post(ctx, http.DefaultClient, n.conf.URL, contentTypeJSON, &buf)
	if err != nil {
		return err
	}
	resp.Body.Close()

	if resp.StatusCode/100 != 2 {
		return fmt.Errorf("unexpected status code %v", resp.StatusCode)
	}
	return nil
}
Exemplo n.º 6
0
// Groups populates an AlertOverview from the dispatcher's internal state.
func (d *Dispatcher) Groups() AlertOverview {
	var overview AlertOverview

	d.mtx.RLock()
	defer d.mtx.RUnlock()

	seen := map[model.Fingerprint]*AlertGroup{}

	for route, ags := range d.aggrGroups {
		for _, ag := range ags {
			alertGroup, ok := seen[ag.fingerprint()]
			if !ok {
				alertGroup = &AlertGroup{Labels: ag.labels}
				alertGroup.GroupKey = ag.GroupKey()

				seen[ag.fingerprint()] = alertGroup
				overview = append(overview, alertGroup)
			}

			now := time.Now()

			var apiAlerts []*APIAlert
			for _, a := range types.Alerts(ag.alertSlice()...) {
				if !a.EndsAt.IsZero() && a.EndsAt.Before(now) {
					continue
				}
				aa := &APIAlert{
					Alert:     a,
					Inhibited: d.marker.Inhibited(a.Fingerprint()),
				}
				if sid, ok := d.marker.Silenced(a.Fingerprint()); ok {
					aa.Silenced = sid
				}
				apiAlerts = append(apiAlerts, aa)
			}
			if len(apiAlerts) == 0 {
				continue
			}

			alertGroup.Blocks = append(alertGroup.Blocks, &AlertBlock{
				RouteOpts: &route.RouteOpts,
				Alerts:    apiAlerts,
			})
		}
	}

	sort.Sort(overview)

	return overview
}
Exemplo n.º 7
0
// Data assembles data for template expansion.
func (t *Template) Data(recv string, groupLabels model.LabelSet, alerts ...*types.Alert) *Data {
	data := &Data{
		Receiver:          strings.SplitN(recv, "/", 2)[0],
		Status:            string(types.Alerts(alerts...).Status()),
		Alerts:            make(Alerts, 0, len(alerts)),
		GroupLabels:       KV{},
		CommonLabels:      KV{},
		CommonAnnotations: KV{},
		ExternalURL:       t.ExternalURL.String(),
	}

	// The call to types.Alert is necessary to correctly resolve the internal
	// representation to the user representation.
	for _, a := range types.Alerts(alerts...) {
		alert := Alert{
			Status:       string(a.Status()),
			Labels:       make(KV, len(a.Labels)),
			Annotations:  make(KV, len(a.Annotations)),
			StartsAt:     a.StartsAt,
			EndsAt:       a.EndsAt,
			GeneratorURL: a.GeneratorURL,
		}
		for k, v := range a.Labels {
			alert.Labels[string(k)] = string(v)
		}
		for k, v := range a.Annotations {
			alert.Annotations[string(k)] = string(v)
		}
		data.Alerts = append(data.Alerts, alert)
	}

	for k, v := range groupLabels {
		data.GroupLabels[string(k)] = string(v)
	}

	if len(alerts) >= 1 {
		var (
			commonLabels      = alerts[0].Labels.Clone()
			commonAnnotations = alerts[0].Annotations.Clone()
		)
		for _, a := range alerts[1:] {
			for ln, lv := range commonLabels {
				if a.Labels[ln] != lv {
					delete(commonLabels, ln)
				}
			}
			for an, av := range commonAnnotations {
				if a.Annotations[an] != av {
					delete(commonAnnotations, an)
				}
			}
		}
		for k, v := range commonLabels {
			data.CommonLabels[string(k)] = string(v)
		}
		for k, v := range commonAnnotations {
			data.CommonAnnotations[string(k)] = string(v)
		}
	}

	return data
}
Exemplo n.º 8
0
// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
	key, ok := GroupKey(ctx)
	if !ok {
		return fmt.Errorf("group key missing")
	}
	data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)

	log.With("incident", key).Debugln("notifying OpsGenie")

	var err error
	tmpl := tmplText(n.tmpl, data, &err)

	details := make(map[string]string, len(n.conf.Details))
	for k, v := range n.conf.Details {
		details[k] = tmpl(v)
	}

	var (
		msg    interface{}
		apiURL string

		apiMsg = opsGenieMessage{
			APIKey: string(n.conf.APIKey),
			Alias:  key,
		}
		alerts = types.Alerts(as...)
	)
	switch alerts.Status() {
	case model.AlertResolved:
		apiURL = n.conf.APIHost + "v1/json/alert/close"
		msg = &opsGenieCloseMessage{&apiMsg}
	default:
		apiURL = n.conf.APIHost + "v1/json/alert"
		msg = &opsGenieCreateMessage{
			opsGenieMessage: &apiMsg,
			Message:         tmpl(n.conf.Description),
			Details:         details,
			Source:          tmpl(n.conf.Source),
			Teams:           tmpl(n.conf.Teams),
			Tags:            tmpl(n.conf.Tags),
		}
	}
	if err != nil {
		return fmt.Errorf("templating error: %s", err)
	}

	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
		return err
	}

	resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode == 400 && alerts.Status() == model.AlertResolved {
		body, _ := ioutil.ReadAll(resp.Body)

		var responseMessage opsGenieErrorResponse
		if err := json.Unmarshal(body, &responseMessage); err != nil {
			return fmt.Errorf("could not parse error response %q", body)
		}
		const alreadyClosedError = 5
		if responseMessage.Code == alreadyClosedError {
			return nil
		}
		return fmt.Errorf("error when closing alert: code %d, error %q",
			responseMessage.Code, responseMessage.Error)
	} else if resp.StatusCode/100 != 2 {
		body, _ := ioutil.ReadAll(resp.Body)
		log.With("incident", key).Debugf("unexpected OpsGenie response from %s (POSTed %s), %s: %s",
			apiURL, msg, resp.Status, body)
		return fmt.Errorf("unexpected status code %v", resp.StatusCode)
	}
	return nil
}
Exemplo n.º 9
0
// Notify implements the Notifier interface.
func (n *VictorOps) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
	victorOpsAllowedEvents := map[string]bool{
		"INFO":     true,
		"WARNING":  true,
		"CRITICAL": true,
	}

	key, ok := GroupKey(ctx)
	if !ok {
		return false, fmt.Errorf("group key missing")
	}

	var err error
	var (
		alerts      = types.Alerts(as...)
		data        = n.tmpl.Data(receiverName(ctx), groupLabels(ctx), as...)
		tmpl        = tmplText(n.tmpl, data, &err)
		apiURL      = fmt.Sprintf("%s%s/%s", n.conf.APIURL, n.conf.APIKey, n.conf.RoutingKey)
		messageType = n.conf.MessageType
	)

	if alerts.Status() == model.AlertFiring && !victorOpsAllowedEvents[messageType] {
		messageType = victorOpsEventTrigger
	}

	if alerts.Status() == model.AlertResolved {
		messageType = victorOpsEventResolve
	}

	msg := &victorOpsMessage{
		MessageType:  messageType,
		EntityID:     key,
		StateMessage: tmpl(n.conf.StateMessage),
		From:         tmpl(n.conf.From),
	}

	if err != nil {
		return false, fmt.Errorf("templating error: %s", err)
	}

	var buf bytes.Buffer
	if err := json.NewEncoder(&buf).Encode(msg); err != nil {
		return false, err
	}

	resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
	if err != nil {
		return true, err
	}

	defer resp.Body.Close()

	// Missing documentation therefore assuming only 5xx response codes are
	// recoverable.
	if resp.StatusCode/100 == 5 {
		return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
	}

	if resp.StatusCode/100 != 2 {
		body, _ := ioutil.ReadAll(resp.Body)

		var responseMessage victorOpsErrorResponse
		if err := json.Unmarshal(body, &responseMessage); err != nil {
			return false, fmt.Errorf("could not parse error response %q", body)
		}

		log.With("incident", key).Debugf("unexpected VictorOps response from %s (POSTed %s), %s: %s", apiURL, msg, resp.Status, body)

		return false, fmt.Errorf("error when posting alert: result %q, message %q",
			responseMessage.Result, responseMessage.Message)
	}

	return false, nil
}