func (ep *EventProcessor) handleEvents(events []consul.Event) { for _, event := range events { log.Println("----------------------------------------") log.Printf("Processing event %s:\n", event.ID) log.Println("----------------------------------------") eventHandlers := consulClient.EventHandlers(event.Name) for _, eventHandler := range eventHandlers { data, err := json.Marshal(&event) if err != nil { log.Println("Unable to read event: ", event) // then what? } input := bytes.NewReader(data) output := new(bytes.Buffer) cmd := exec.Command(eventHandler) cmd.Stdin = input cmd.Stdout = output cmd.Stderr = output if err := cmd.Run(); err != nil { log.Println("error running handler: ", err) } else { log.Printf(">>> \n%s -> %s:\n %s\n", event.ID, eventHandler, output) } } log.Printf("Event Processed.\n\n") } }
func (c *CheckProcessor) handleChecks(checks []consul.Check) { consulClient.LoadConfig() retryCount := 0 for !hasLeader() { if retryCount >= 6 { return } log.Println("There is current no consul-alerts leader... waiting for one.") time.Sleep(5 * time.Second) retryCount++ } if !c.leaderElection.leader { log.Println("Currently not the leader. Ignoring checks.") return } log.Println("Running health check.") changeThreshold := consulClient.CheckChangeThreshold() for elapsed := 0; elapsed < changeThreshold; elapsed += 10 { consulClient.UpdateCheckData() time.Sleep(10 * time.Second) } consulClient.UpdateCheckData() log.Println("Processing health checks for notification.") alerts := consulClient.NewAlerts() if len(alerts) > 0 { c.notify(alerts) } }
func (influxdb *InfluxdbNotifier) Notify(messages Messages) bool { config := &client.ClientConfig{ Host: influxdb.Host, Username: influxdb.Username, Password: influxdb.Password, Database: influxdb.Database, } influxdbClient, err := client.New(config) if err != nil { log.Println("unable to access influxdb. can't send notification. ", err) return false } seriesList := influxdb.toSeries(messages) err = influxdbClient.WriteSeries(seriesList) if err != nil { log.Println("unable to send notifications: ", err) return false } log.Println("influxdb notification sent.") return true }
func (slack *SlackNotifier) postToSlack() bool { data, err := json.Marshal(slack) if err != nil { log.Println("Unable to marshal slack payload:", err) return false } log.Debugf("struct = %+v, json = %s", slack, string(data)) b := bytes.NewBuffer(data) if res, err := http.Post(slack.Url, "application/json", b); err != nil { log.Println("Unable to send data to slack:", err) return false } else { defer res.Body.Close() statusCode := res.StatusCode if statusCode != 200 { body, _ := ioutil.ReadAll(res.Body) log.Println("Unable to notify slack:", string(body)) return false } else { log.Println("Slack notification sent.") return true } } }
func runWatcher(address, datacenter, watchType string) { consulAlert := os.Args[0] cmd := exec.Command( "consul", "watch", "-http-addr", address, "-datacenter", datacenter, "-type", watchType, consulAlert, "watch", watchType) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { var exitCode int switch err.(type) { case *exec.ExitError: exitError, _ := err.(*exec.ExitError) status, _ := exitError.Sys().(syscall.WaitStatus) exitCode = status.ExitStatus() log.Println("Shutting down watcher --> Exit Code: ", exitCode) case *exec.Error: exitCode = 1 log.Println("Shutting down watcher --> Something went wrong running consul watch: ", err.Error()) default: exitCode = 127 log.Println("Shutting down watcher --> Unknown error: ", err.Error()) } os.Exit(exitCode) } else { log.Printf("Execution complete.") } }
func toWatchObject(reader io.Reader, v interface{}) { data, err := ioutil.ReadAll(reader) if err != nil { log.Println("stdin read error: ", err) // todo: what to do when can't read? } err = json.Unmarshal(data, v) if err != nil { log.Println("json unmarshall error: ", err) // todo: what if we can't serialise? } }
func (emailNotifier *EmailNotifier) Notify(alerts Messages) bool { overAllStatus, pass, warn, fail := alerts.Summary() nodeMap := mapByNodes(alerts) e := EmailData{ ClusterName: emailNotifier.ClusterName, SystemStatus: overAllStatus, FailCount: fail, WarnCount: warn, PassCount: pass, Nodes: nodeMap, } var tmpl *template.Template var err error if emailNotifier.Template == "" { tmpl, err = template.New("base").Parse(defaultTemplate) } else { tmpl, err = template.ParseFiles(emailNotifier.Template) } if err != nil { log.Println("Template error, unable to send email notification: ", err) return false } var body bytes.Buffer if err := tmpl.Execute(&body, e); err != nil { log.Println("Template error, unable to send email notification: ", err) return false } msg := "" msg += fmt.Sprintf("From: \"%s\" <%s>\n", emailNotifier.SenderAlias, emailNotifier.SenderEmail) msg += fmt.Sprintf("Subject: %s is %s\n", emailNotifier.ClusterName, overAllStatus) msg += "MIME-version: 1.0;\nContent-Type: text/html; charset=\"UTF-8\";\n\n" msg += body.String() addr := fmt.Sprintf("%s:%d", emailNotifier.Url, emailNotifier.Port) auth := smtp.PlainAuth("", emailNotifier.Username, emailNotifier.Password, emailNotifier.Url) if err := smtp.SendMail(addr, auth, emailNotifier.SenderEmail, emailNotifier.Receivers, []byte(msg)); err != nil { log.Println("Unable to send notification:", err) return false } log.Println("Email notification sent.") return true }
func healthHandler(w http.ResponseWriter, r *http.Request) { node := r.URL.Query().Get("node") service := r.URL.Query().Get("service") check := r.URL.Query().Get("check") log.Println(node, service, check) status, output := consulClient.CheckStatus(node, service, check) var code int switch status { case "passing": code = 200 case "warning", "critical": code = 503 default: status = "unknown" code = 404 } log.Printf("health status check result for node=%s,service=%s,check=%s: %d", node, service, check, code) var result string if output == "" { result = "" } else { result = fmt.Sprintf("output: %s\n", output) } body := fmt.Sprintf("status: %s\n%s", status, result) w.WriteHeader(code) w.Write([]byte(body)) }
func (ep *EventProcessor) eventHandler(w http.ResponseWriter, r *http.Request) { consulClient.LoadConfig() if ep.firstRun { log.Println("Now watching for events.") ep.firstRun = false // set status to OK return } if !consulClient.EventsEnabled() { log.Println("Event handling disabled. Event ignored.") // set to OK? return } var events []consul.Event toWatchObject(r.Body, &events) ep.inChan <- events // set status to OK }
func (n *NotifEngine) sendCustom(messages notifier.Messages) { for _, notifCmd := range consulClient.CustomNotifiers() { data, err := json.Marshal(&messages) if err != nil { log.Println("Unable to read messages: ", err) return } input := bytes.NewReader(data) output := new(bytes.Buffer) cmd := exec.Command(notifCmd) cmd.Stdin = input cmd.Stdout = output cmd.Stderr = output if err := cmd.Run(); err != nil { log.Println("error running notifier: ", err) } else { log.Println(">>> notification sent to:", notifCmd) } log.Println(output) } }
func (logNotifier *LogNotifier) Notify(alerts Messages) bool { logrus.Println("logging messages...") logDir := path.Dir(logNotifier.LogFile) err := os.MkdirAll(logDir, os.ModePerm) if err != nil { logrus.Printf("unable to create directory for logfile: %v\n", err) return false } file, err := os.OpenFile(logNotifier.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666) if err != nil { logrus.Printf("unable to write to logfile: %v\n", err) return false } logger := log.New(file, "[consul-notifier] ", log.LstdFlags) for _, alert := range alerts { logger.Printf("Node=%s, Service=%s, Check=%s, Status=%s\n", alert.Node, alert.Service, alert.Check, alert.Status) } logrus.Println("Notifications logged.") return true }
func (c *CheckProcessor) checkHandler(w http.ResponseWriter, r *http.Request) { consulClient.LoadConfig() if c.firstRun { log.Println("Now watching for health changes.") c.firstRun = false w.WriteHeader(200) return } if !consulClient.ChecksEnabled() { log.Println("Checks handling disabled. Checks ignored.") w.WriteHeader(200) return } if len(c.inChan) == 1 { <-c.inChan } var checks []consul.Check toWatchObject(r.Body, &checks) c.inChan <- checks w.WriteHeader(200) }
func NewClient(address, dc, aclToken string) (*ConsulAlertClient, error) { config := consulapi.DefaultConfig() config.Address = address config.Datacenter = dc config.Token = aclToken api, _ := consulapi.NewClient(config) alertConfig := DefaultAlertConfig() client := &ConsulAlertClient{ api: api, config: alertConfig, } log.Println("Checking consul agent connection...") if _, err := client.api.Status().Leader(); err != nil { return nil, err } client.LoadConfig() client.UpdateCheckData() return client, nil }
func (c *CheckProcessor) notify(alerts []consul.Check) { messages := make([]notifier.Message, len(alerts)) for i, alert := range alerts { messages[i] = notifier.Message{ Node: alert.Node, ServiceId: alert.ServiceID, Service: alert.ServiceName, CheckId: alert.CheckID, Check: alert.Name, Status: alert.Status, Output: alert.Output, Notes: alert.Notes, Timestamp: time.Now(), } } if len(messages) == 0 { log.Println("Nothing to notify.") return } c.notifEngine.queueMessages(messages) }
func (pd *PagerDutyNotifier) Notify(messages Messages) bool { client := gopherduty.NewClient(pd.ServiceKey) result := true for _, message := range messages { incidentKey := message.Node if message.ServiceId != "" { incidentKey += ":" + message.ServiceId } incidentKey += ":" + message.CheckId var response *gopherduty.PagerDutyResponse switch { case message.IsPassing(): description := incidentKey + " is now HEALTHY" response = client.Resolve(incidentKey, description, message) case message.IsWarning(): description := incidentKey + " is UNSTABLE" response = client.Trigger(incidentKey, description, pd.ClientName, pd.ClientUrl, message) case message.IsCritical(): description := incidentKey + " is CRITICAL" response = client.Trigger(incidentKey, description, pd.ClientName, pd.ClientUrl, message) } if response.HasErrors() { for _, err := range response.Errors { log.Printf("Error sending %s notification to pagerduty: %s\n", incidentKey, err) } result = false } } log.Println("PagerDuty notification complete") return result }
func (c *ConsulAlertClient) LoadConfig() { if kvPairs, _, err := c.api.KV().List("consul-alerts/config", nil); err == nil { config := c.config for _, kvPair := range kvPairs { key := kvPair.Key val := kvPair.Value var valErr error switch key { // checks config case "consul-alerts/config/checks/enabled": valErr = loadCustomValue(&config.Checks.Enabled, val, ConfigTypeBool) case "consul-alerts/config/checks/change-threshold": valErr = loadCustomValue(&config.Checks.ChangeThreshold, val, ConfigTypeInt) // events config case "consul-alerts/config/events/enabled": valErr = loadCustomValue(&config.Events.Enabled, val, ConfigTypeBool) case "consul-alerts/config/events/handlers": valErr = loadCustomValue(&config.Events.Handlers, val, ConfigTypeStrArray) // notifiers config case "consul-alerts/config/notifiers/custom": valErr = loadCustomValue(&config.Notifiers.Custom, val, ConfigTypeStrArray) // email notifier config case "consul-alerts/config/notifiers/email/cluster-name": valErr = loadCustomValue(&config.Notifiers.Email.ClusterName, val, ConfigTypeString) case "consul-alerts/config/notifiers/email/template": valErr = loadCustomValue(&config.Notifiers.Email.Template, val, ConfigTypeString) case "consul-alerts/config/notifiers/email/enabled": valErr = loadCustomValue(&config.Notifiers.Email.Enabled, val, ConfigTypeBool) case "consul-alerts/config/notifiers/email/password": valErr = loadCustomValue(&config.Notifiers.Email.Password, val, ConfigTypeString) case "consul-alerts/config/notifiers/email/port": valErr = loadCustomValue(&config.Notifiers.Email.Port, val, ConfigTypeInt) case "consul-alerts/config/notifiers/email/receivers": valErr = loadCustomValue(&config.Notifiers.Email.Receivers, val, ConfigTypeStrArray) case "consul-alerts/config/notifiers/email/sender-alias": valErr = loadCustomValue(&config.Notifiers.Email.SenderAlias, val, ConfigTypeString) case "consul-alerts/config/notifiers/email/sender-email": valErr = loadCustomValue(&config.Notifiers.Email.SenderEmail, val, ConfigTypeString) case "consul-alerts/config/notifiers/email/url": valErr = loadCustomValue(&config.Notifiers.Email.Url, val, ConfigTypeString) case "consul-alerts/config/notifiers/email/username": valErr = loadCustomValue(&config.Notifiers.Email.Username, val, ConfigTypeString) // log notifier config case "consul-alerts/config/notifiers/log/enabled": valErr = loadCustomValue(&config.Notifiers.Log.Enabled, val, ConfigTypeBool) case "consul-alerts/config/notifiers/log/path": valErr = loadCustomValue(&config.Notifiers.Log.Path, val, ConfigTypeString) // influxdb notifier config case "consul-alerts/config/notifiers/influxdb/enabled": valErr = loadCustomValue(&config.Notifiers.Influxdb.Enabled, val, ConfigTypeBool) case "consul-alerts/config/notifiers/influxdb/host": valErr = loadCustomValue(&config.Notifiers.Influxdb.Host, val, ConfigTypeString) case "consul-alerts/config/notifiers/influxdb/username": valErr = loadCustomValue(&config.Notifiers.Influxdb.Username, val, ConfigTypeString) case "consul-alerts/config/notifiers/influxdb/password": valErr = loadCustomValue(&config.Notifiers.Influxdb.Password, val, ConfigTypeString) case "consul-alerts/config/notifiers/influxdb/database": valErr = loadCustomValue(&config.Notifiers.Influxdb.Database, val, ConfigTypeString) case "consul-alerts/config/notifiers/influxdb/series-name": valErr = loadCustomValue(&config.Notifiers.Influxdb.SeriesName, val, ConfigTypeString) // slack notfier config case "consul-alerts/config/notifiers/slack/enabled": valErr = loadCustomValue(&config.Notifiers.Slack.Enabled, val, ConfigTypeBool) case "consul-alerts/config/notifiers/slack/cluster-name": valErr = loadCustomValue(&config.Notifiers.Slack.ClusterName, val, ConfigTypeString) case "consul-alerts/config/notifiers/slack/url": valErr = loadCustomValue(&config.Notifiers.Slack.Url, val, ConfigTypeString) case "consul-alerts/config/notifiers/slack/channel": valErr = loadCustomValue(&config.Notifiers.Slack.Channel, val, ConfigTypeString) case "consul-alerts/config/notifiers/slack/username": valErr = loadCustomValue(&config.Notifiers.Slack.Username, val, ConfigTypeString) case "consul-alerts/config/notifiers/slack/icon-url": valErr = loadCustomValue(&config.Notifiers.Slack.IconUrl, val, ConfigTypeString) case "consul-alerts/config/notifiers/slack/icon-emoji": valErr = loadCustomValue(&config.Notifiers.Slack.IconEmoji, val, ConfigTypeString) case "consul-alerts/config/notifiers/slack/detailed": valErr = loadCustomValue(&config.Notifiers.Slack.Detailed, val, ConfigTypeBool) // pager-duty notfier config case "consul-alerts/config/notifiers/pagerduty/enabled": valErr = loadCustomValue(&config.Notifiers.PagerDuty.Enabled, val, ConfigTypeBool) case "consul-alerts/config/notifiers/pagerduty/service-key": valErr = loadCustomValue(&config.Notifiers.PagerDuty.ServiceKey, val, ConfigTypeString) case "consul-alerts/config/notifiers/pagerduty/client-name": valErr = loadCustomValue(&config.Notifiers.PagerDuty.ClientName, val, ConfigTypeString) case "consul-alerts/config/notifiers/pagerduty/client-url": valErr = loadCustomValue(&config.Notifiers.PagerDuty.ClientUrl, val, ConfigTypeString) // hipchat notfier config case "consul-alerts/config/notifiers/hipchat/enabled": valErr = loadCustomValue(&config.Notifiers.HipChat.Enabled, val, ConfigTypeBool) case "consul-alerts/config/notifiers/hipchat/cluster-name": valErr = loadCustomValue(&config.Notifiers.HipChat.ClusterName, val, ConfigTypeString) case "consul-alerts/config/notifiers/hipchat/room-id": valErr = loadCustomValue(&config.Notifiers.HipChat.RoomId, val, ConfigTypeString) case "consul-alerts/config/notifiers/hipchat/auth-token": valErr = loadCustomValue(&config.Notifiers.HipChat.AuthToken, val, ConfigTypeString) case "consul-alerts/config/notifiers/hipchat/base-url": valErr = loadCustomValue(&config.Notifiers.HipChat.BaseURL, val, ConfigTypeString) } if valErr != nil { log.Printf(`unable to load custom value for "%s". Using default instead. Error: %s`, key, valErr.Error()) } } } else { log.Println("Unable to load custom config, using default instead:", err) } }
func (n *NotifEngine) queueMessages(messages notifier.Messages) { n.inChan <- messages log.Println("messages sent for notification") }