// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
if !ok {
return fmt.Errorf("group key missing")
}
data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
log.With("incident", key).Debugln("notifying OpsGenie")
var err error
tmpl := tmplText(n.tmpl, data, &err)
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
details[k] = tmpl(v)
}
var (
msg interface{}
apiURL string
apiMsg = opsGenieMessage{
APIKey: string(n.conf.APIKey),
Alias: key,
}
alerts = types.Alerts(as...)
)
switch alerts.Status() {
case model.AlertResolved:
apiURL = n.conf.APIHost + "v1/json/alert/close"
msg = &opsGenieCloseMessage{&apiMsg}
default:
apiURL = n.conf.APIHost + "v1/json/alert"
msg = &opsGenieCreateMessage{
opsGenieMessage: &apiMsg,
Message: tmpl(n.conf.Description),
Details: details,
Source: tmpl(n.conf.Source),
}
}
if err != nil {
return fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Notify implements the Notifier interface.
func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) error {
data := w.tmpl.Data(receiver(ctx), groupLabels(ctx), alerts...)
groupKey, ok := GroupKey(ctx)
if !ok {
log.Errorf("group key missing")
}
msg := &WebhookMessage{
Version: "3",
Data: data,
GroupKey: uint64(groupKey),
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, w.URL, contentTypeJSON, &buf)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v from %s", resp.StatusCode, w.URL)
}
return nil
}
// Notify implements the Notifier interface.
func (n *Pushover) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
if !ok {
return fmt.Errorf("group key missing")
}
data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
log.With("incident", key).Debugln("notifying Pushover")
var err error
tmpl := tmplText(n.tmpl, data, &err)
parameters := url.Values{}
parameters.Add("token", tmpl(string(n.conf.Token)))
parameters.Add("user", tmpl(string(n.conf.UserKey)))
title := tmpl(n.conf.Title)
message := tmpl(n.conf.Message)
parameters.Add("title", title)
if len(title) > 512 {
title = title[:512]
log.With("incident", key).Debugf("Truncated title to %q due to Pushover message limit", title)
}
if len(title)+len(message) > 512 {
message = message[:512-len(title)]
log.With("incident", key).Debugf("Truncated message to %q due to Pushover message limit", message)
}
message = strings.TrimSpace(message)
if message == "" {
// Pushover rejects empty messages.
message = "(no details)"
}
parameters.Add("message", message)
parameters.Add("url", tmpl(n.conf.URL))
parameters.Add("priority", tmpl(n.conf.Priority))
parameters.Add("retry", fmt.Sprintf("%d", int64(time.Duration(n.conf.Retry).Seconds())))
parameters.Add("expire", fmt.Sprintf("%d", int64(time.Duration(n.conf.Expire).Seconds())))
apiURL := "https://api.pushover.net/1/messages.json"
u, err := url.Parse(apiURL)
if err != nil {
return err
}
u.RawQuery = parameters.Encode()
log.With("incident", key).Debugf("Pushover URL = %q", u.String())
resp, err := ctxhttp.Post(ctx, http.DefaultClient, u.String(), "text/plain", nil)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf("unexpected status code %v (body: %s)", resp.StatusCode, string(body))
}
return nil
}
// Notify implements the Notifier interface.
func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) error {
as := types.Alerts(alerts...)
// If there are no annotations, instantiate so
// {} is sent rather than null.
for _, a := range as {
if a.Annotations == nil {
a.Annotations = model.LabelSet{}
}
}
msg := &WebhookMessage{
Version: "2",
Status: as.Status(),
Alerts: as,
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, w.URL, contentTypeJSON, &buf)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Notify implements the Notifier interface.
func (w *Webhook) Notify(ctx context.Context, alerts ...*types.Alert) (bool, error) {
data := w.tmpl.Data(receiverName(ctx), groupLabels(ctx), alerts...)
groupKey, ok := GroupKey(ctx)
if !ok {
log.Errorf("group key missing")
}
msg := &WebhookMessage{
Version: "3",
Data: data,
GroupKey: uint64(groupKey),
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, w.URL, contentTypeJSON, &buf)
if err != nil {
return true, err
}
resp.Body.Close()
return w.retry(resp.StatusCode)
}
func (n *Handler) send(alerts ...*model.Alert) error {
// Attach external labels before sending alerts.
for _, a := range alerts {
for ln, lv := range n.opts.ExternalLabels {
if _, ok := a.Labels[ln]; !ok {
a.Labels[ln] = lv
}
}
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(alerts); err != nil {
return err
}
ctx, _ := context.WithTimeout(context.Background(), n.opts.Timeout)
resp, err := ctxhttp.Post(ctx, http.DefaultClient, n.postURL(), contentTypeJSON, &buf)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("bad response status %v", resp.Status)
}
return nil
}
func makeExternalSignRequest(ctx context.Context, client *http.Client, url string, csrJSON []byte) (cert []byte, err error) {
resp, err := ctxhttp.Post(ctx, client, url, "application/json", bytes.NewReader(csrJSON))
if err != nil {
return nil, recoverableErr{err: errors.Wrap(err, "unable to perform certificate signing request")}
}
doneReading := make(chan struct{})
bodyClosed := make(chan struct{})
go func() {
select {
case <-ctx.Done():
case <-doneReading:
}
resp.Body.Close()
close(bodyClosed)
}()
body, err := ioutil.ReadAll(resp.Body)
close(doneReading)
<-bodyClosed
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
if err != nil {
return nil, recoverableErr{err: errors.Wrap(err, "unable to read CSR response body")}
}
if resp.StatusCode != http.StatusOK {
return nil, recoverableErr{err: errors.Errorf("unexpected status code in CSR response: %d - %s", resp.StatusCode, string(body))}
}
var apiResponse api.Response
if err := json.Unmarshal(body, &apiResponse); err != nil {
logrus.Debugf("unable to JSON-parse CFSSL API response body: %s", string(body))
return nil, recoverableErr{err: errors.Wrap(err, "unable to parse JSON response")}
}
if !apiResponse.Success || apiResponse.Result == nil {
if len(apiResponse.Errors) > 0 {
return nil, errors.Errorf("response errors: %v", apiResponse.Errors)
}
return nil, errors.New("certificate signing request failed")
}
result, ok := apiResponse.Result.(map[string]interface{})
if !ok {
return nil, errors.Errorf("invalid result type: %T", apiResponse.Result)
}
certPEM, ok := result["certificate"].(string)
if !ok {
return nil, errors.Errorf("invalid result certificate field type: %T", result["certificate"])
}
return []byte(certPEM), nil
}
// sendAll sends the alerts to all configured Alertmanagers at concurrently.
// It returns the number of sends that have failed.
func (n *Notifier) sendAll(alerts ...*model.Alert) int {
begin := time.Now()
// Attach external labels before sending alerts.
for _, a := range alerts {
for ln, lv := range n.opts.ExternalLabels {
if _, ok := a.Labels[ln]; !ok {
a.Labels[ln] = lv
}
}
}
b, err := json.Marshal(alerts)
if err != nil {
log.Errorf("Encoding alerts failed: %s", err)
return len(n.opts.AlertmanagerURLs)
}
ctx, _ := context.WithTimeout(context.Background(), n.opts.Timeout)
send := func(u string) error {
resp, err := ctxhttp.Post(ctx, http.DefaultClient, postURL(u), contentTypeJSON, bytes.NewReader(b))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("bad response status %v", resp.Status)
}
return err
}
var (
wg sync.WaitGroup
numErrors uint64
)
for _, u := range n.opts.AlertmanagerURLs {
wg.Add(1)
go func(u string) {
if err := send(u); err != nil {
log.With("alertmanager", u).With("count", fmt.Sprintf("%d", len(alerts))).Errorf("Error sending alerts: %s", err)
n.errors.WithLabelValues(u).Inc()
atomic.AddUint64(&numErrors, 1)
}
n.latency.WithLabelValues(u).Observe(float64(time.Since(begin)) / float64(time.Second))
n.sent.WithLabelValues(u).Add(float64(len(alerts)))
wg.Done()
}(u)
}
wg.Wait()
return int(numErrors)
}
// Notify implements the Notifier interface.
//
// http://developer.pagerduty.com/documentation/integration/events/trigger
func (n *PagerDuty) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
if !ok {
return fmt.Errorf("group key missing")
}
var err error
var (
alerts = types.Alerts(as...)
data = n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
tmpl = tmplText(n.tmpl, data, &err)
eventType = pagerDutyEventTrigger
)
if alerts.Status() == model.AlertResolved {
eventType = pagerDutyEventResolve
}
log.With("incident", key).With("eventType", eventType).Debugln("notifying PagerDuty")
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
details[k] = tmpl(v)
}
msg := &pagerDutyMessage{
ServiceKey: tmpl(string(n.conf.ServiceKey)),
EventType: eventType,
IncidentKey: key,
Description: tmpl(n.conf.Description),
Details: details,
}
if eventType == pagerDutyEventTrigger {
msg.Client = tmpl(n.conf.Client)
msg.ClientURL = tmpl(n.conf.ClientURL)
}
if err != nil {
return err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, n.conf.URL, contentTypeJSON, &buf)
if err != nil {
return err
}
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// postJWS signs the body with the given key and POSTs it to the provided url.
// The body argument must be JSON-serializable.
func postJWS(ctx context.Context, client *http.Client, key crypto.Signer, url string, body interface{}) (*http.Response, error) {
nonce, err := fetchNonce(ctx, client, url)
if err != nil {
return nil, err
}
b, err := jwsEncodeJSON(body, key, nonce)
if err != nil {
return nil, err
}
return ctxhttp.Post(ctx, client, url, "application/jose+json", bytes.NewReader(b))
}
func (n *Notifier) sendOne(ctx context.Context, c *http.Client, url string, b []byte) error {
resp, err := ctxhttp.Post(ctx, c, url, contentTypeJSON, bytes.NewReader(b))
if err != nil {
return err
}
defer resp.Body.Close()
// Any HTTP status 2xx is OK.
if resp.StatusCode/100 != 2 {
return fmt.Errorf("bad response status %v", resp.Status)
}
return err
}
func CreateReq(pflag *arg.Info) (output chan *Response) {
var (
wg sync.WaitGroup
root = ctx.Background()
v = make(chan *Response, 1)
)
go func() {
defer close(v)
for _, endpoint := range Endpoint {
wg.Add(1)
go func(ep string) {
defer wg.Done()
var body *bytes.Reader
if buf, err := json.Marshal(pflag); err != nil {
v <- &Response{Host: ep, Err: err}
return
} else {
body = bytes.NewReader(buf)
}
wk, abort := ctx.WithTimeout(root, DefaultTimeout)
defer abort()
resp, err := ctxhttp.Post(wk, nil, ep+"/proxy", "application/json", body)
if err != nil {
v <- &Response{Host: ep, Err: err}
return
}
defer resp.Body.Close()
inn := new(bytes.Buffer)
io.Copy(inn, resp.Body)
if ans := inn.String(); ans != "done" {
v <- &Response{Host: ep, Err: errors.New(ans)}
return
}
v <- &Response{Host: ep}
}(endpoint)
}
wg.Wait()
}()
return v
}
func (c *ArtifactStoreClient) postAPI(path string, contentType string, body io.Reader) (io.ReadCloser, *ArtifactsError) {
url := c.server + path
ctx, cancel := context.WithTimeout(c.ctx, c.timeout)
defer cancel()
if resp, err := ctxhttp.Post(ctx, nil, url, contentType, body); err != nil {
// If there was an error connecting to the server, it is likely to be transient and should be
// retried.
return nil, NewRetriableError(err.Error())
} else {
if resp.StatusCode != http.StatusOK {
return nil, determineResponseError(resp, url, "POST")
}
return resp.Body, nil
}
}
// Notify implements the Notifier interface.
func (n *Hipchat) Notify(ctx context.Context, as ...*types.Alert) error {
var err error
var msg string
var (
data = n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
tmplText = tmplText(n.tmpl, data, &err)
tmplHTML = tmplHTML(n.tmpl, data, &err)
url = fmt.Sprintf("%sv2/room/%s/notification?auth_token=%s", n.conf.APIURL, n.conf.RoomID, n.conf.AuthToken)
)
if n.conf.MessageFormat == "html" {
msg = tmplHTML(n.conf.Message)
} else {
msg = tmplText(n.conf.Message)
}
req := &hipchatReq{
From: tmplText(n.conf.From),
Notify: n.conf.Notify,
Message: msg,
MessageFormat: n.conf.MessageFormat,
Color: tmplText(n.conf.Color),
}
if err != nil {
return err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(req); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, url, contentTypeJSON, &buf)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Notify implements the Notifier interface.
func (n *Slack) Notify(ctx context.Context, as ...*types.Alert) error {
var err error
var (
data = n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
tmplText = tmplText(n.tmpl, data, &err)
tmplHTML = tmplHTML(n.tmpl, data, &err)
)
attachment := &slackAttachment{
Title: tmplText(n.conf.Title),
TitleLink: tmplText(n.conf.TitleLink),
Pretext: tmplText(n.conf.Pretext),
Text: tmplHTML(n.conf.Text),
Fallback: tmplText(n.conf.Fallback),
Color: tmplText(n.conf.Color),
MrkdwnIn: []string{"fallback", "pretext"},
}
req := &slackReq{
Channel: tmplText(n.conf.Channel),
Username: tmplText(n.conf.Username),
Attachments: []slackAttachment{*attachment},
}
if err != nil {
return err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(req); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, string(n.conf.APIURL), contentTypeJSON, &buf)
if err != nil {
return err
}
// TODO(fabxc): is 2xx status code really indicator for success for Slack API?
resp.Body.Close()
if resp.StatusCode/100 != 2 {
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
// Notify implements the Notifier interface.
func (n *Slack) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
var err error
var (
data = n.tmpl.Data(receiverName(ctx), groupLabels(ctx), as...)
tmplText = tmplText(n.tmpl, data, &err)
)
attachment := &slackAttachment{
Title: tmplText(n.conf.Title),
TitleLink: tmplText(n.conf.TitleLink),
Pretext: tmplText(n.conf.Pretext),
Text: tmplText(n.conf.Text),
Fallback: tmplText(n.conf.Fallback),
Color: tmplText(n.conf.Color),
MrkdwnIn: []string{"fallback", "pretext", "text"},
}
req := &slackReq{
Channel: tmplText(n.conf.Channel),
Username: tmplText(n.conf.Username),
IconEmoji: tmplText(n.conf.IconEmoji),
IconURL: tmplText(n.conf.IconURL),
Attachments: []slackAttachment{*attachment},
}
if err != nil {
return false, err
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(req); err != nil {
return false, err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, string(n.conf.APIURL), contentTypeJSON, &buf)
if err != nil {
return true, err
}
resp.Body.Close()
return n.retry(resp.StatusCode)
}
// Notify implements the Notifier interface.
func (n *VictorOps) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
victorOpsAllowedEvents := map[string]bool{
"INFO": true,
"WARNING": true,
"CRITICAL": true,
}
key, ok := GroupKey(ctx)
if !ok {
return false, fmt.Errorf("group key missing")
}
var err error
var (
alerts = types.Alerts(as...)
data = n.tmpl.Data(receiverName(ctx), groupLabels(ctx), as...)
tmpl = tmplText(n.tmpl, data, &err)
apiURL = fmt.Sprintf("%s%s/%s", n.conf.APIURL, n.conf.APIKey, n.conf.RoutingKey)
messageType = n.conf.MessageType
)
if alerts.Status() == model.AlertFiring && !victorOpsAllowedEvents[messageType] {
messageType = victorOpsEventTrigger
}
if alerts.Status() == model.AlertResolved {
messageType = victorOpsEventResolve
}
msg := &victorOpsMessage{
MessageType: messageType,
EntityID: key,
StateMessage: tmpl(n.conf.StateMessage),
From: tmpl(n.conf.From),
}
if err != nil {
return false, fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return false, err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
if err != nil {
return true, err
}
defer resp.Body.Close()
// Missing documentation therefore assuming only 5xx response codes are
// recoverable.
if resp.StatusCode/100 == 5 {
return true, fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
if resp.StatusCode/100 != 2 {
body, _ := ioutil.ReadAll(resp.Body)
var responseMessage victorOpsErrorResponse
if err := json.Unmarshal(body, &responseMessage); err != nil {
return false, fmt.Errorf("could not parse error response %q", body)
}
log.With("incident", key).Debugf("unexpected VictorOps response from %s (POSTed %s), %s: %s", apiURL, msg, resp.Status, body)
return false, fmt.Errorf("error when posting alert: result %q, message %q",
responseMessage.Result, responseMessage.Message)
}
return false, nil
}
// Notify implements the Notifier interface.
func (n *OpsGenie) Notify(ctx context.Context, as ...*types.Alert) error {
key, ok := GroupKey(ctx)
if !ok {
return fmt.Errorf("group key missing")
}
data := n.tmpl.Data(receiver(ctx), groupLabels(ctx), as...)
log.With("incident", key).Debugln("notifying OpsGenie")
var err error
tmpl := tmplText(n.tmpl, data, &err)
details := make(map[string]string, len(n.conf.Details))
for k, v := range n.conf.Details {
details[k] = tmpl(v)
}
var (
msg interface{}
apiURL string
apiMsg = opsGenieMessage{
APIKey: string(n.conf.APIKey),
Alias: key,
}
alerts = types.Alerts(as...)
)
switch alerts.Status() {
case model.AlertResolved:
apiURL = n.conf.APIHost + "v1/json/alert/close"
msg = &opsGenieCloseMessage{&apiMsg}
default:
apiURL = n.conf.APIHost + "v1/json/alert"
msg = &opsGenieCreateMessage{
opsGenieMessage: &apiMsg,
Message: tmpl(n.conf.Description),
Details: details,
Source: tmpl(n.conf.Source),
Teams: tmpl(n.conf.Teams),
Tags: tmpl(n.conf.Tags),
}
}
if err != nil {
return fmt.Errorf("templating error: %s", err)
}
var buf bytes.Buffer
if err := json.NewEncoder(&buf).Encode(msg); err != nil {
return err
}
resp, err := ctxhttp.Post(ctx, http.DefaultClient, apiURL, contentTypeJSON, &buf)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode == 400 && alerts.Status() == model.AlertResolved {
body, _ := ioutil.ReadAll(resp.Body)
var responseMessage opsGenieErrorResponse
if err := json.Unmarshal(body, &responseMessage); err != nil {
return fmt.Errorf("could not parse error response %q", body)
}
const alreadyClosedError = 5
if responseMessage.Code == alreadyClosedError {
return nil
}
return fmt.Errorf("error when closing alert: code %d, error %q",
responseMessage.Code, responseMessage.Error)
} else if resp.StatusCode/100 != 2 {
body, _ := ioutil.ReadAll(resp.Body)
log.With("incident", key).Debugf("unexpected OpsGenie response from %s (POSTed %s), %s: %s",
apiURL, msg, resp.Status, body)
return fmt.Errorf("unexpected status code %v", resp.StatusCode)
}
return nil
}
