func daemonMode(arguments map[string]interface{}) {
// Define options before setting in either config file or on command line
loglevelString := ""
consulAclToken := ""
consulAddr := ""
consulDc := ""
watchChecks := false
watchEvents := false
addr := ""
var confData map[string]interface{}
// This exists check only works for arguments with no default. arguments with defaults will always exist.
// Because of this the current code overrides command line flags with config file options if set.
if configFile, exists := arguments["--config-file"].(string); exists {
file, err := ioutil.ReadFile(configFile)
if err != nil {
log.Error(err)
}
err = json.Unmarshal(file, &confData)
if err != nil {
log.Error(err)
}
log.Debug("Config data: ", confData)
}
if confData["log-level"] != nil {
loglevelString = confData["log-level"].(string)
} else {
loglevelString = arguments["--log-level"].(string)
}
if confData["consul-acl-token"] != nil {
consulAclToken = confData["consul-acl-token"].(string)
} else {
consulAclToken = arguments["--consul-acl-token"].(string)
}
if confData["consul-addr"] != nil {
consulAddr = confData["consul-addr"].(string)
} else {
consulAddr = arguments["--consul-addr"].(string)
}
if confData["consul-dc"] != nil {
consulDc = confData["consul-dc"].(string)
} else {
consulDc = arguments["--consul-dc"].(string)
}
if confData["alert-addr"] != nil {
addr = confData["alert-addr"].(string)
} else {
addr = arguments["--alert-addr"].(string)
}
if confData["watch-checks"] != nil {
watchChecks = confData["watch-checks"].(bool)
} else {
watchChecks = arguments["--watch-checks"].(bool)
}
if confData["watch-events"] != nil {
watchEvents = confData["watch-events"].(bool)
} else {
watchEvents = arguments["--watch-events"].(bool)
}
if loglevelString != "" {
loglevel, err := log.ParseLevel(loglevelString)
if err == nil {
log.SetLevel(loglevel)
} else {
log.Println("Log level not set:", err)
}
}
url := fmt.Sprintf("http://%s/v1/info", addr)
resp, err := http.Get(url)
if err == nil && resp.StatusCode == 201 {
version := resp.Header.Get("version")
resp.Body.Close()
log.Printf("consul-alert daemon already running version: %s", version)
os.Exit(1)
}
consulClient, err = consul.NewClient(consulAddr, consulDc, consulAclToken)
if err != nil {
log.Println("Cluster has no leader or is unreacheable.", err)
os.Exit(3)
}
hostname, _ := os.Hostname()
log.Println("Consul ACL Token:", consulAclToken)
log.Println("Consul Alerts daemon started")
log.Println("Consul Alerts Host:", hostname)
log.Println("Consul Agent:", consulAddr)
log.Println("Consul Datacenter:", consulDc)
leaderCandidate := startLeaderElection(consulAddr, consulDc, consulAclToken)
notifEngine := startNotifEngine()
ep := startEventProcessor()
cp := startCheckProcessor(leaderCandidate, notifEngine)
http.HandleFunc("/v1/info", infoHandler)
http.HandleFunc("/v1/process/events", ep.eventHandler)
http.HandleFunc("/v1/process/checks", cp.checkHandler)
http.HandleFunc("/v1/health/wildcard", healthWildcardHandler)
http.HandleFunc("/v1/health", healthHandler)
go startAPI(addr)
log.Println("Started Consul-Alerts API")
if watchChecks {
go runWatcher(consulAddr, consulDc, addr, loglevelString, "checks")
}
if watchEvents {
go runWatcher(consulAddr, consulDc, addr, loglevelString, "event")
}
ch := make(chan os.Signal)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, os.Interrupt)
<-ch
cleanup(notifEngine, cp, ep, leaderCandidate)
}
// Notify sends messages to the endpoint notifier
func (vo *VictorOpsNotifier) Notify(messages Messages) bool {
endpoint := fmt.Sprintf(apiEndpointTemplate, vo.APIKey, vo.RoutingKey)
ok := true
for _, message := range messages {
entityID := fmt.Sprintf("%s:", message.Node)
entityDisplayName := entityID
// This might be a node level check without an explicit service
if message.ServiceId == "" {
entityID += message.CheckId
entityDisplayName += message.Check
} else {
entityID += message.ServiceId
entityDisplayName += message.Service
}
var messageType string
switch {
case message.IsCritical():
messageType = "CRITICAL"
case message.IsWarning():
messageType = "WARNING"
case message.IsPassing():
messageType = "RECOVERY"
default:
log.Warn(fmt.Sprintf("Message with status %s was neither critical, warning, nor passing, reporting to VictorOps as INFO", message.Status))
messageType = "INFO"
}
// VictorOps automatically displays the entity display name in notifications and page SMSs / emails,
// so for brevity we don't repeat it in the "StateMessage" field
stateMessage := fmt.Sprintf("%s: %s\n%s", messageType, message.Notes, message.Output)
event := VictorOpsEvent{
MessageType: messageType,
EntityID: entityID,
Timestamp: uint32(message.Timestamp.Unix()),
StateMessage: stateMessage,
MonitoringTool: monitoringToolName,
EntityDisplayName: entityDisplayName,
HostName: message.Node,
MonitorName: message.Check,
ConsulNode: message.Node,
ConsulService: message.Service,
ConsulServiceID: message.ServiceId,
ConsulCheck: message.Check,
ConsulCheckID: message.CheckId,
}
eventJSON, jsonError := json.Marshal(event)
if jsonError != nil {
ok = false
log.Error("Error JSON-ifying VictorOps alert. ", jsonError)
continue
}
response, httpError := http.Post(endpoint, "application/json", bytes.NewBuffer(eventJSON))
if httpError != nil {
ok = false
log.Error("Error hitting VictorOps API. ", httpError)
continue
}
if response.StatusCode != 200 {
ok = false
log.Error(fmt.Sprintf("Expected VictorOps endpoint to return 200, but it returned %d"), response.StatusCode)
continue
}
}
log.Println("VictorOps notification sent.")
return ok
}