forked from marcw/libpoller
/
alert.go
111 lines (95 loc) · 2.78 KB
/
alert.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
package poller
import (
"fmt"
"github.com/marcw/ezmail"
"github.com/marcw/pagerduty"
"net"
"net/smtp"
"os"
"strings"
"time"
)
// SMTP
type smtpAlerter struct {
addr string
auth smtp.Auth
message ezmail.Message
}
func NewSmtpAlerter() (Alerter, error) {
envHost := os.Getenv("SMTP_HOST")
envPort := os.Getenv("SMTP_PORT")
envAuth := os.Getenv("SMTP_AUTH")
envUsername := os.Getenv("SMTP_USERNAME")
envPassword := os.Getenv("SMTP_PASSWORD")
envIdentity := os.Getenv("SMTP_PLAIN_IDENTITY")
envTo := os.Getenv("SMTP_RECIPIENT")
envFrom := os.Getenv("SMTP_FROM")
if envHost == "" {
return nil, fmt.Errorf("Please define SMTP_HOST env var")
}
if envPort == "" {
return nil, fmt.Errorf("Please define SMTP_PORT env var")
}
if envAuth != "" && envAuth != "MD5" && envAuth != "PLAIN" {
return nil, fmt.Errorf("Please either leave SMTP_AUTH env empty or set it to MD5 or PLAIN")
}
var addr string
if envPort != "" {
addr = net.JoinHostPort(envHost, envPort)
} else {
addr = envHost
}
var auth smtp.Auth
if envAuth == "MD5" {
auth = smtp.CRAMMD5Auth(envUsername, envPassword)
} else if envAuth == "PLAIN" {
auth = smtp.PlainAuth(envIdentity, envUsername, envPassword, envHost)
}
message := ezmail.NewMessage()
message.SetFrom("", envFrom)
for _, v := range strings.Split(envTo, ";") {
message.AddTo("", v)
}
smtp := &smtpAlerter{
addr: addr,
auth: auth,
message: *message}
return smtp, nil
}
func (m *smtpAlerter) Alert(event *Event) {
msg := m.message
msg.Subject = fmt.Sprintf("[ALERT] %s is down", event.Check.Key)
msg.Body = fmt.Sprintf("Poller alert: %s is down since %s", event.Check.AlertDescription(), event.Check.DownSince.Format(time.RFC822))
if err := smtp.SendMail(m.addr, m.auth, msg.From.String(), msg.Recipients(), msg.Bytes()); err != nil {
println(err)
}
}
// PagerDuty
type pagerDutyAlerter struct {
serviceKey string
}
func NewPagerDutyAlerter() (Alerter, error) {
envServiceKey := os.Getenv("PAGERDUTY_SERVICE_KEY")
if envServiceKey == "" {
return nil, fmt.Errorf("Please define the PAGERDUTY_SERVICE_KEY environment variable.")
}
return &pagerDutyAlerter{envServiceKey}, nil
}
func (pda *pagerDutyAlerter) Alert(event *Event) {
description := fmt.Sprintf("%s is DOWN since %s.", event.Check.AlertDescription(), event.Check.DownSince.Format(time.RFC3339))
e := pagerduty.NewTriggerEvent(pda.serviceKey, description)
e.Details["checked_at"] = event.Time.Format(time.RFC3339)
e.Details["duration"] = event.Duration.String()
e.Details["status_code"] = event.StatusCode
e.Details["was_up_for"] = event.Check.WasUpFor.String()
e.IncidentKey = event.Check.Key
for {
_, statusCode, _ := pagerduty.Submit(e)
if statusCode < 500 {
break
} else {
// Wait a bit before trying again
time.Sleep(3 * time.Second)
}
}
}