func (ep *EventProcessor) handleEvents(events []consul.Event) {
for _, event := range events {
log.Println("----------------------------------------")
log.Printf("Processing event %s:\n", event.ID)
log.Println("----------------------------------------")
eventHandlers := consulClient.EventHandlers(event.Name)
for _, eventHandler := range eventHandlers {
data, err := json.Marshal(&event)
if err != nil {
log.Println("Unable to read event: ", event)
// then what?
}
input := bytes.NewReader(data)
output := new(bytes.Buffer)
cmd := exec.Command(eventHandler)
cmd.Stdin = input
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Run(); err != nil {
log.Println("error running handler: ", err)
} else {
log.Printf(">>> \n%s -> %s:\n %s\n", event.ID, eventHandler, output)
}
}
log.Printf("Event Processed.\n\n")
}
}
func (c *CheckProcessor) handleChecks(checks []consul.Check) {
consulClient.LoadConfig()
retryCount := 0
for !hasLeader() {
if retryCount >= 6 {
return
}
log.Println("There is current no consul-alerts leader... waiting for one.")
time.Sleep(5 * time.Second)
retryCount++
}
if !c.leaderElection.leader {
log.Println("Currently not the leader. Ignoring checks.")
return
}
log.Println("Running health check.")
changeThreshold := consulClient.CheckChangeThreshold()
for elapsed := 0; elapsed < changeThreshold; elapsed += 10 {
consulClient.UpdateCheckData()
time.Sleep(10 * time.Second)
}
consulClient.UpdateCheckData()
log.Println("Processing health checks for notification.")
alerts := consulClient.NewAlerts()
if len(alerts) > 0 {
c.notify(alerts)
}
}
func (influxdb *InfluxdbNotifier) Notify(messages Messages) bool {
config := &client.ClientConfig{
Host: influxdb.Host,
Username: influxdb.Username,
Password: influxdb.Password,
Database: influxdb.Database,
}
influxdbClient, err := client.New(config)
if err != nil {
log.Println("unable to access influxdb. can't send notification. ", err)
return false
}
seriesList := influxdb.toSeries(messages)
err = influxdbClient.WriteSeries(seriesList)
if err != nil {
log.Println("unable to send notifications: ", err)
return false
}
log.Println("influxdb notification sent.")
return true
}
func (slack *SlackNotifier) postToSlack() bool {
data, err := json.Marshal(slack)
if err != nil {
log.Println("Unable to marshal slack payload:", err)
return false
}
log.Debugf("struct = %+v, json = %s", slack, string(data))
b := bytes.NewBuffer(data)
if res, err := http.Post(slack.Url, "application/json", b); err != nil {
log.Println("Unable to send data to slack:", err)
return false
} else {
defer res.Body.Close()
statusCode := res.StatusCode
if statusCode != 200 {
body, _ := ioutil.ReadAll(res.Body)
log.Println("Unable to notify slack:", string(body))
return false
} else {
log.Println("Slack notification sent.")
return true
}
}
}
func runWatcher(address, datacenter, watchType string) {
consulAlert := os.Args[0]
cmd := exec.Command(
"consul", "watch",
"-http-addr", address,
"-datacenter", datacenter,
"-type", watchType,
consulAlert, "watch", watchType)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
var exitCode int
switch err.(type) {
case *exec.ExitError:
exitError, _ := err.(*exec.ExitError)
status, _ := exitError.Sys().(syscall.WaitStatus)
exitCode = status.ExitStatus()
log.Println("Shutting down watcher --> Exit Code: ", exitCode)
case *exec.Error:
exitCode = 1
log.Println("Shutting down watcher --> Something went wrong running consul watch: ", err.Error())
default:
exitCode = 127
log.Println("Shutting down watcher --> Unknown error: ", err.Error())
}
os.Exit(exitCode)
} else {
log.Printf("Execution complete.")
}
}
func toWatchObject(reader io.Reader, v interface{}) {
data, err := ioutil.ReadAll(reader)
if err != nil {
log.Println("stdin read error: ", err)
// todo: what to do when can't read?
}
err = json.Unmarshal(data, v)
if err != nil {
log.Println("json unmarshall error: ", err)
// todo: what if we can't serialise?
}
}
func (emailNotifier *EmailNotifier) Notify(alerts Messages) bool {
overAllStatus, pass, warn, fail := alerts.Summary()
nodeMap := mapByNodes(alerts)
e := EmailData{
ClusterName: emailNotifier.ClusterName,
SystemStatus: overAllStatus,
FailCount: fail,
WarnCount: warn,
PassCount: pass,
Nodes: nodeMap,
}
var tmpl *template.Template
var err error
if emailNotifier.Template == "" {
tmpl, err = template.New("base").Parse(defaultTemplate)
} else {
tmpl, err = template.ParseFiles(emailNotifier.Template)
}
if err != nil {
log.Println("Template error, unable to send email notification: ", err)
return false
}
var body bytes.Buffer
if err := tmpl.Execute(&body, e); err != nil {
log.Println("Template error, unable to send email notification: ", err)
return false
}
msg := ""
msg += fmt.Sprintf("From: \"%s\" <%s>\n", emailNotifier.SenderAlias, emailNotifier.SenderEmail)
msg += fmt.Sprintf("Subject: %s is %s\n", emailNotifier.ClusterName, overAllStatus)
msg += "MIME-version: 1.0;\nContent-Type: text/html; charset=\"UTF-8\";\n\n"
msg += body.String()
addr := fmt.Sprintf("%s:%d", emailNotifier.Url, emailNotifier.Port)
auth := smtp.PlainAuth("", emailNotifier.Username, emailNotifier.Password, emailNotifier.Url)
if err := smtp.SendMail(addr, auth, emailNotifier.SenderEmail, emailNotifier.Receivers, []byte(msg)); err != nil {
log.Println("Unable to send notification:", err)
return false
}
log.Println("Email notification sent.")
return true
}
func healthHandler(w http.ResponseWriter, r *http.Request) {
node := r.URL.Query().Get("node")
service := r.URL.Query().Get("service")
check := r.URL.Query().Get("check")
log.Println(node, service, check)
status, output := consulClient.CheckStatus(node, service, check)
var code int
switch status {
case "passing":
code = 200
case "warning", "critical":
code = 503
default:
status = "unknown"
code = 404
}
log.Printf("health status check result for node=%s,service=%s,check=%s: %d", node, service, check, code)
var result string
if output == "" {
result = ""
} else {
result = fmt.Sprintf("output: %s\n", output)
}
body := fmt.Sprintf("status: %s\n%s", status, result)
w.WriteHeader(code)
w.Write([]byte(body))
}
func (ep *EventProcessor) eventHandler(w http.ResponseWriter, r *http.Request) {
consulClient.LoadConfig()
if ep.firstRun {
log.Println("Now watching for events.")
ep.firstRun = false
// set status to OK
return
}
if !consulClient.EventsEnabled() {
log.Println("Event handling disabled. Event ignored.")
// set to OK?
return
}
var events []consul.Event
toWatchObject(r.Body, &events)
ep.inChan <- events
// set status to OK
}
func (n *NotifEngine) sendCustom(messages notifier.Messages) {
for _, notifCmd := range consulClient.CustomNotifiers() {
data, err := json.Marshal(&messages)
if err != nil {
log.Println("Unable to read messages: ", err)
return
}
input := bytes.NewReader(data)
output := new(bytes.Buffer)
cmd := exec.Command(notifCmd)
cmd.Stdin = input
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Run(); err != nil {
log.Println("error running notifier: ", err)
} else {
log.Println(">>> notification sent to:", notifCmd)
}
log.Println(output)
}
}
func (logNotifier *LogNotifier) Notify(alerts Messages) bool {
logrus.Println("logging messages...")
logDir := path.Dir(logNotifier.LogFile)
err := os.MkdirAll(logDir, os.ModePerm)
if err != nil {
logrus.Printf("unable to create directory for logfile: %v\n", err)
return false
}
file, err := os.OpenFile(logNotifier.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
logrus.Printf("unable to write to logfile: %v\n", err)
return false
}
logger := log.New(file, "[consul-notifier] ", log.LstdFlags)
for _, alert := range alerts {
logger.Printf("Node=%s, Service=%s, Check=%s, Status=%s\n", alert.Node, alert.Service, alert.Check, alert.Status)
}
logrus.Println("Notifications logged.")
return true
}
func (c *CheckProcessor) checkHandler(w http.ResponseWriter, r *http.Request) {
consulClient.LoadConfig()
if c.firstRun {
log.Println("Now watching for health changes.")
c.firstRun = false
w.WriteHeader(200)
return
}
if !consulClient.ChecksEnabled() {
log.Println("Checks handling disabled. Checks ignored.")
w.WriteHeader(200)
return
}
if len(c.inChan) == 1 {
<-c.inChan
}
var checks []consul.Check
toWatchObject(r.Body, &checks)
c.inChan <- checks
w.WriteHeader(200)
}
func NewClient(address, dc, aclToken string) (*ConsulAlertClient, error) {
config := consulapi.DefaultConfig()
config.Address = address
config.Datacenter = dc
config.Token = aclToken
api, _ := consulapi.NewClient(config)
alertConfig := DefaultAlertConfig()
client := &ConsulAlertClient{
api: api,
config: alertConfig,
}
log.Println("Checking consul agent connection...")
if _, err := client.api.Status().Leader(); err != nil {
return nil, err
}
client.LoadConfig()
client.UpdateCheckData()
return client, nil
}
func (c *CheckProcessor) notify(alerts []consul.Check) {
messages := make([]notifier.Message, len(alerts))
for i, alert := range alerts {
messages[i] = notifier.Message{
Node: alert.Node,
ServiceId: alert.ServiceID,
Service: alert.ServiceName,
CheckId: alert.CheckID,
Check: alert.Name,
Status: alert.Status,
Output: alert.Output,
Notes: alert.Notes,
Timestamp: time.Now(),
}
}
if len(messages) == 0 {
log.Println("Nothing to notify.")
return
}
c.notifEngine.queueMessages(messages)
}
func (pd *PagerDutyNotifier) Notify(messages Messages) bool {
client := gopherduty.NewClient(pd.ServiceKey)
result := true
for _, message := range messages {
incidentKey := message.Node
if message.ServiceId != "" {
incidentKey += ":" + message.ServiceId
}
incidentKey += ":" + message.CheckId
var response *gopherduty.PagerDutyResponse
switch {
case message.IsPassing():
description := incidentKey + " is now HEALTHY"
response = client.Resolve(incidentKey, description, message)
case message.IsWarning():
description := incidentKey + " is UNSTABLE"
response = client.Trigger(incidentKey, description, pd.ClientName, pd.ClientUrl, message)
case message.IsCritical():
description := incidentKey + " is CRITICAL"
response = client.Trigger(incidentKey, description, pd.ClientName, pd.ClientUrl, message)
}
if response.HasErrors() {
for _, err := range response.Errors {
log.Printf("Error sending %s notification to pagerduty: %s\n", incidentKey, err)
}
result = false
}
}
log.Println("PagerDuty notification complete")
return result
}
func (c *ConsulAlertClient) LoadConfig() {
if kvPairs, _, err := c.api.KV().List("consul-alerts/config", nil); err == nil {
config := c.config
for _, kvPair := range kvPairs {
key := kvPair.Key
val := kvPair.Value
var valErr error
switch key {
// checks config
case "consul-alerts/config/checks/enabled":
valErr = loadCustomValue(&config.Checks.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/checks/change-threshold":
valErr = loadCustomValue(&config.Checks.ChangeThreshold, val, ConfigTypeInt)
// events config
case "consul-alerts/config/events/enabled":
valErr = loadCustomValue(&config.Events.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/events/handlers":
valErr = loadCustomValue(&config.Events.Handlers, val, ConfigTypeStrArray)
// notifiers config
case "consul-alerts/config/notifiers/custom":
valErr = loadCustomValue(&config.Notifiers.Custom, val, ConfigTypeStrArray)
// email notifier config
case "consul-alerts/config/notifiers/email/cluster-name":
valErr = loadCustomValue(&config.Notifiers.Email.ClusterName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/template":
valErr = loadCustomValue(&config.Notifiers.Email.Template, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/enabled":
valErr = loadCustomValue(&config.Notifiers.Email.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/email/password":
valErr = loadCustomValue(&config.Notifiers.Email.Password, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/port":
valErr = loadCustomValue(&config.Notifiers.Email.Port, val, ConfigTypeInt)
case "consul-alerts/config/notifiers/email/receivers":
valErr = loadCustomValue(&config.Notifiers.Email.Receivers, val, ConfigTypeStrArray)
case "consul-alerts/config/notifiers/email/sender-alias":
valErr = loadCustomValue(&config.Notifiers.Email.SenderAlias, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/sender-email":
valErr = loadCustomValue(&config.Notifiers.Email.SenderEmail, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/url":
valErr = loadCustomValue(&config.Notifiers.Email.Url, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/username":
valErr = loadCustomValue(&config.Notifiers.Email.Username, val, ConfigTypeString)
// log notifier config
case "consul-alerts/config/notifiers/log/enabled":
valErr = loadCustomValue(&config.Notifiers.Log.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/log/path":
valErr = loadCustomValue(&config.Notifiers.Log.Path, val, ConfigTypeString)
// influxdb notifier config
case "consul-alerts/config/notifiers/influxdb/enabled":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/influxdb/host":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Host, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/username":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Username, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/password":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Password, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/database":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Database, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/series-name":
valErr = loadCustomValue(&config.Notifiers.Influxdb.SeriesName, val, ConfigTypeString)
// slack notfier config
case "consul-alerts/config/notifiers/slack/enabled":
valErr = loadCustomValue(&config.Notifiers.Slack.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/slack/cluster-name":
valErr = loadCustomValue(&config.Notifiers.Slack.ClusterName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/url":
valErr = loadCustomValue(&config.Notifiers.Slack.Url, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/channel":
valErr = loadCustomValue(&config.Notifiers.Slack.Channel, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/username":
valErr = loadCustomValue(&config.Notifiers.Slack.Username, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/icon-url":
valErr = loadCustomValue(&config.Notifiers.Slack.IconUrl, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/icon-emoji":
valErr = loadCustomValue(&config.Notifiers.Slack.IconEmoji, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/detailed":
valErr = loadCustomValue(&config.Notifiers.Slack.Detailed, val, ConfigTypeBool)
// pager-duty notfier config
case "consul-alerts/config/notifiers/pagerduty/enabled":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/pagerduty/service-key":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.ServiceKey, val, ConfigTypeString)
case "consul-alerts/config/notifiers/pagerduty/client-name":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.ClientName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/pagerduty/client-url":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.ClientUrl, val, ConfigTypeString)
// hipchat notfier config
case "consul-alerts/config/notifiers/hipchat/enabled":
valErr = loadCustomValue(&config.Notifiers.HipChat.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/hipchat/cluster-name":
valErr = loadCustomValue(&config.Notifiers.HipChat.ClusterName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/hipchat/room-id":
valErr = loadCustomValue(&config.Notifiers.HipChat.RoomId, val, ConfigTypeString)
case "consul-alerts/config/notifiers/hipchat/auth-token":
valErr = loadCustomValue(&config.Notifiers.HipChat.AuthToken, val, ConfigTypeString)
case "consul-alerts/config/notifiers/hipchat/base-url":
valErr = loadCustomValue(&config.Notifiers.HipChat.BaseURL, val, ConfigTypeString)
}
if valErr != nil {
log.Printf(`unable to load custom value for "%s". Using default instead. Error: %s`, key, valErr.Error())
}
}
} else {
log.Println("Unable to load custom config, using default instead:", err)
}
}
func (n *NotifEngine) queueMessages(messages notifier.Messages) {
n.inChan <- messages
log.Println("messages sent for notification")
}