func (ep *EventProcessor) handleEvents(events []consul.Event) {
for _, event := range events {
log.Println("----------------------------------------")
log.Printf("Processing event %s:\n", event.ID)
log.Println("----------------------------------------")
eventHandlers := consulClient.EventHandlers(event.Name)
for _, eventHandler := range eventHandlers {
data, err := json.Marshal(&event)
if err != nil {
log.Println("Unable to read event: ", event)
// then what?
}
input := bytes.NewReader(data)
output := new(bytes.Buffer)
cmd := exec.Command(eventHandler)
cmd.Stdin = input
cmd.Stdout = output
cmd.Stderr = output
if err := cmd.Run(); err != nil {
log.Println("error running handler: ", err)
} else {
log.Printf(">>> \n%s -> %s:\n %s\n", event.ID, eventHandler, output)
}
}
log.Printf("Event Processed.\n\n")
}
}
func (c *ConsulAlertClient) registerHealthCheck(key string, health *Check) {
log.Printf(
"Registering new health check: node=%s, service=%s, check=%s, status=%s",
health.Node,
health.ServiceName,
health.Name,
health.Status,
)
var newStatus Status
if health.Status == "passing" {
newStatus = Status{
Current: health.Status,
CurrentTimestamp: time.Now(),
HealthCheck: health,
}
} else {
newStatus = Status{
Pending: health.Status,
PendingTimestamp: time.Now(),
HealthCheck: health,
}
}
statusData, _ := json.Marshal(newStatus)
c.api.KV().Put(&consulapi.KVPair{Key: key, Value: statusData}, nil)
}
func (c *ConsulAlertClient) UpdateCheckData() {
healthApi := c.api.Health()
kvApi := c.api.KV()
healths, _, _ := healthApi.State("any", nil)
for _, health := range healths {
node := health.Node
service := health.ServiceID
check := health.CheckID
if service == "" {
service = "_"
}
key := fmt.Sprintf("consul-alerts/checks/%s/%s/%s", node, service, check)
status, _, _ := kvApi.Get(key, nil)
existing := status != nil
localHealth := Check(*health)
if c.IsBlacklisted(&localHealth) {
log.Printf("%s:%s:%s is blacklisted.", node, service, check)
continue
}
if !existing {
c.registerHealthCheck(key, &localHealth)
} else {
c.updateHealthCheck(key, &localHealth)
}
}
}
func healthHandler(w http.ResponseWriter, r *http.Request) {
node := r.URL.Query().Get("node")
service := r.URL.Query().Get("service")
check := r.URL.Query().Get("check")
log.Println(node, service, check)
status, output := consulClient.CheckStatus(node, service, check)
var code int
switch status {
case "passing":
code = 200
case "warning", "critical":
code = 503
default:
status = "unknown"
code = 404
}
log.Printf("health status check result for node=%s,service=%s,check=%s: %d", node, service, check, code)
var result string
if output == "" {
result = ""
} else {
result = fmt.Sprintf("output: %s\n", output)
}
body := fmt.Sprintf("status: %s\n%s", status, result)
w.WriteHeader(code)
w.Write([]byte(body))
}
func runWatcher(address, datacenter, watchType string) {
consulAlert := os.Args[0]
cmd := exec.Command(
"consul", "watch",
"-http-addr", address,
"-datacenter", datacenter,
"-type", watchType,
consulAlert, "watch", watchType)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
var exitCode int
switch err.(type) {
case *exec.ExitError:
exitError, _ := err.(*exec.ExitError)
status, _ := exitError.Sys().(syscall.WaitStatus)
exitCode = status.ExitStatus()
log.Println("Shutting down watcher --> Exit Code: ", exitCode)
case *exec.Error:
exitCode = 1
log.Println("Shutting down watcher --> Something went wrong running consul watch: ", err.Error())
default:
exitCode = 127
log.Println("Shutting down watcher --> Unknown error: ", err.Error())
}
os.Exit(exitCode)
} else {
log.Printf("Execution complete.")
}
}
func (notifier *HipChatNotifier) Notify(messages Messages) bool {
overallStatus, pass, warn, fail := messages.Summary()
text := fmt.Sprintf(header, notifier.ClusterName, overallStatus, fail, warn, pass)
for _, message := range messages {
text += fmt.Sprintf("\n%s:%s:%s is %s.", message.Node, message.Service, message.Check, message.Status)
text += fmt.Sprintf("\n%s", message.Output)
}
level := "green"
if fail > 0 {
level = "red"
} else if warn > 0 {
level = "yellow"
}
client := hipchat.NewClient(notifier.AuthToken)
if notifier.BaseURL != "" {
url, err := url.Parse(notifier.BaseURL)
if err != nil {
log.Printf("Error parsing hipchat base url: %s\n", err)
}
client.BaseURL = url
}
notifRq := &hipchat.NotificationRequest{
Message: text,
Color: level,
Notify: true,
}
resp, err := client.Room.Notification(notifier.RoomId, notifRq)
if err != nil {
log.Printf("Error sending notification to hipchat: %s\n", err)
log.Printf("Server returns %+v\n", resp)
return false
}
return true
}
func (logNotifier *LogNotifier) Notify(alerts Messages) bool {
logrus.Println("logging messages...")
logDir := path.Dir(logNotifier.LogFile)
err := os.MkdirAll(logDir, os.ModePerm)
if err != nil {
logrus.Printf("unable to create directory for logfile: %v\n", err)
return false
}
file, err := os.OpenFile(logNotifier.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
if err != nil {
logrus.Printf("unable to write to logfile: %v\n", err)
return false
}
logger := log.New(file, "[consul-notifier] ", log.LstdFlags)
for _, alert := range alerts {
logger.Printf("Node=%s, Service=%s, Check=%s, Status=%s\n", alert.Node, alert.Service, alert.Check, alert.Status)
}
logrus.Println("Notifications logged.")
return true
}
func (pd *PagerDutyNotifier) Notify(messages Messages) bool {
client := gopherduty.NewClient(pd.ServiceKey)
result := true
for _, message := range messages {
incidentKey := message.Node
if message.ServiceId != "" {
incidentKey += ":" + message.ServiceId
}
incidentKey += ":" + message.CheckId
var response *gopherduty.PagerDutyResponse
switch {
case message.IsPassing():
description := incidentKey + " is now HEALTHY"
response = client.Resolve(incidentKey, description, message)
case message.IsWarning():
description := incidentKey + " is UNSTABLE"
response = client.Trigger(incidentKey, description, pd.ClientName, pd.ClientUrl, message)
case message.IsCritical():
description := incidentKey + " is CRITICAL"
response = client.Trigger(incidentKey, description, pd.ClientName, pd.ClientUrl, message)
}
if response.HasErrors() {
for _, err := range response.Errors {
log.Printf("Error sending %s notification to pagerduty: %s\n", incidentKey, err)
}
result = false
}
}
log.Println("PagerDuty notification complete")
return result
}
func (c *ConsulAlertClient) LoadConfig() {
if kvPairs, _, err := c.api.KV().List("consul-alerts/config", nil); err == nil {
config := c.config
for _, kvPair := range kvPairs {
key := kvPair.Key
val := kvPair.Value
var valErr error
switch key {
// checks config
case "consul-alerts/config/checks/enabled":
valErr = loadCustomValue(&config.Checks.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/checks/change-threshold":
valErr = loadCustomValue(&config.Checks.ChangeThreshold, val, ConfigTypeInt)
// events config
case "consul-alerts/config/events/enabled":
valErr = loadCustomValue(&config.Events.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/events/handlers":
valErr = loadCustomValue(&config.Events.Handlers, val, ConfigTypeStrArray)
// notifiers config
case "consul-alerts/config/notifiers/custom":
valErr = loadCustomValue(&config.Notifiers.Custom, val, ConfigTypeStrArray)
// email notifier config
case "consul-alerts/config/notifiers/email/cluster-name":
valErr = loadCustomValue(&config.Notifiers.Email.ClusterName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/template":
valErr = loadCustomValue(&config.Notifiers.Email.Template, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/enabled":
valErr = loadCustomValue(&config.Notifiers.Email.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/email/password":
valErr = loadCustomValue(&config.Notifiers.Email.Password, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/port":
valErr = loadCustomValue(&config.Notifiers.Email.Port, val, ConfigTypeInt)
case "consul-alerts/config/notifiers/email/receivers":
valErr = loadCustomValue(&config.Notifiers.Email.Receivers, val, ConfigTypeStrArray)
case "consul-alerts/config/notifiers/email/sender-alias":
valErr = loadCustomValue(&config.Notifiers.Email.SenderAlias, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/sender-email":
valErr = loadCustomValue(&config.Notifiers.Email.SenderEmail, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/url":
valErr = loadCustomValue(&config.Notifiers.Email.Url, val, ConfigTypeString)
case "consul-alerts/config/notifiers/email/username":
valErr = loadCustomValue(&config.Notifiers.Email.Username, val, ConfigTypeString)
// log notifier config
case "consul-alerts/config/notifiers/log/enabled":
valErr = loadCustomValue(&config.Notifiers.Log.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/log/path":
valErr = loadCustomValue(&config.Notifiers.Log.Path, val, ConfigTypeString)
// influxdb notifier config
case "consul-alerts/config/notifiers/influxdb/enabled":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/influxdb/host":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Host, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/username":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Username, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/password":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Password, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/database":
valErr = loadCustomValue(&config.Notifiers.Influxdb.Database, val, ConfigTypeString)
case "consul-alerts/config/notifiers/influxdb/series-name":
valErr = loadCustomValue(&config.Notifiers.Influxdb.SeriesName, val, ConfigTypeString)
// slack notfier config
case "consul-alerts/config/notifiers/slack/enabled":
valErr = loadCustomValue(&config.Notifiers.Slack.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/slack/cluster-name":
valErr = loadCustomValue(&config.Notifiers.Slack.ClusterName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/url":
valErr = loadCustomValue(&config.Notifiers.Slack.Url, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/channel":
valErr = loadCustomValue(&config.Notifiers.Slack.Channel, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/username":
valErr = loadCustomValue(&config.Notifiers.Slack.Username, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/icon-url":
valErr = loadCustomValue(&config.Notifiers.Slack.IconUrl, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/icon-emoji":
valErr = loadCustomValue(&config.Notifiers.Slack.IconEmoji, val, ConfigTypeString)
case "consul-alerts/config/notifiers/slack/detailed":
valErr = loadCustomValue(&config.Notifiers.Slack.Detailed, val, ConfigTypeBool)
// pager-duty notfier config
case "consul-alerts/config/notifiers/pagerduty/enabled":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/pagerduty/service-key":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.ServiceKey, val, ConfigTypeString)
case "consul-alerts/config/notifiers/pagerduty/client-name":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.ClientName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/pagerduty/client-url":
valErr = loadCustomValue(&config.Notifiers.PagerDuty.ClientUrl, val, ConfigTypeString)
// hipchat notfier config
case "consul-alerts/config/notifiers/hipchat/enabled":
valErr = loadCustomValue(&config.Notifiers.HipChat.Enabled, val, ConfigTypeBool)
case "consul-alerts/config/notifiers/hipchat/cluster-name":
valErr = loadCustomValue(&config.Notifiers.HipChat.ClusterName, val, ConfigTypeString)
case "consul-alerts/config/notifiers/hipchat/room-id":
valErr = loadCustomValue(&config.Notifiers.HipChat.RoomId, val, ConfigTypeString)
case "consul-alerts/config/notifiers/hipchat/auth-token":
valErr = loadCustomValue(&config.Notifiers.HipChat.AuthToken, val, ConfigTypeString)
case "consul-alerts/config/notifiers/hipchat/base-url":
valErr = loadCustomValue(&config.Notifiers.HipChat.BaseURL, val, ConfigTypeString)
}
if valErr != nil {
log.Printf(`unable to load custom value for "%s". Using default instead. Error: %s`, key, valErr.Error())
}
}
} else {
log.Println("Unable to load custom config, using default instead:", err)
}
}
func (c *ConsulAlertClient) updateHealthCheck(key string, health *Check) {
kvpair, _, _ := c.api.KV().Get(key, nil)
val := kvpair.Value
var storedStatus Status
json.Unmarshal(val, &storedStatus)
// no status change if the stored status and latest status is the same
noStatusChange := storedStatus.Current == health.Status
// new pending status if it's a new status and it's not the same as the pending status
newPendingStatus := storedStatus.Current != health.Status && storedStatus.Pending != health.Status
// status is still pending for change. will change if it reaches threshold
stillPendingStatus := storedStatus.Current != health.Status && storedStatus.Pending == health.Status
switch {
case noStatusChange:
if storedStatus.Pending != "" {
storedStatus.Pending = ""
storedStatus.PendingTimestamp = time.Time{}
log.Printf(
"%s:%s:%s is now back to %s.",
health.Node,
health.ServiceName,
health.Name,
storedStatus.Current,
)
}
case newPendingStatus:
storedStatus.Pending = health.Status
storedStatus.PendingTimestamp = time.Now()
log.Printf(
"%s:%s:%s is now pending status change from %s to %s.",
health.Node,
health.ServiceName,
health.Name,
storedStatus.Current,
storedStatus.Pending,
)
case stillPendingStatus:
duration := time.Since(storedStatus.PendingTimestamp)
if int(duration.Seconds()) >= c.config.Checks.ChangeThreshold {
log.Printf(
"%s:%s:%s has changed status from %s to %s.",
health.Node,
health.ServiceName,
health.Name,
storedStatus.Current,
storedStatus.Pending,
)
storedStatus.Current = storedStatus.Pending
storedStatus.CurrentTimestamp = time.Now()
storedStatus.Pending = ""
storedStatus.PendingTimestamp = time.Time{}
storedStatus.ForNotification = true
} else {
log.Printf(
"%s:%s:%s is pending status change from %s to %s for %s.",
health.Node,
health.ServiceName,
health.Name,
storedStatus.Current,
storedStatus.Pending,
duration,
)
}
}
storedStatus.HealthCheck = health
data, _ := json.Marshal(storedStatus)
c.api.KV().Put(&consulapi.KVPair{Key: key, Value: data}, nil)
}
