// spawnHostExpirationWarnings is a notificationBuilder to build any necessary // warnings about hosts that will be expiring soon (but haven't expired yet) func spawnHostExpirationWarnings(settings *evergreen.Settings) ([]notification, error) { evergreen.Logger.Logf(slogger.INFO, "Building spawned host expiration"+ " warnings...") // sanity check, since the thresholds are supplied in code if len(spawnWarningThresholds) == 0 { evergreen.Logger.Logf(slogger.WARN, "there are no currently set warning"+ " thresholds for spawned hosts - users will not receive emails"+ " warning them of imminent host expiration") return nil, nil } // assumed to be the first warning threshold (the least recent one) firstWarningThreshold := spawnWarningThresholds[len(spawnWarningThresholds)-1] // find all spawned hosts that have passed at least one warning threshold now := time.Now() thresholdTime := now.Add(firstWarningThreshold) hosts, err := host.Find(host.ByExpiringBetween(now, thresholdTime)) if err != nil { return nil, fmt.Errorf("error finding spawned hosts that will be"+ " expiring soon: %v", err) } // the eventual list of warning notifications to be sent warnings := []notification{} for _, h := range hosts { // figure out the most recent expiration notification threshold the host // has crossed threshold := lastWarningThresholdCrossed(&h) // for keying into the host's notifications map thresholdKey := strconv.Itoa(int(threshold.Minutes())) // if a notification has already been sent for the threshold for this // host, skip it if h.Notifications[thresholdKey] { continue } evergreen.Logger.Logf(slogger.INFO, "Warning needs to be sent for threshold"+ " '%v' for host %v", thresholdKey, h.Id) // fetch information about the user we are notifying userToNotify, err := user.FindOne(user.ById(h.StartedBy)) if err != nil { return nil, fmt.Errorf("error finding user to notify by Id %v: %v", h.StartedBy, err) } // if we didn't find a user (in the case of testing) set the timezone to "" // to avoid triggering a nil pointer exception timezone := "" if userToNotify != nil { timezone = userToNotify.Settings.Timezone } var expirationTimeFormatted string // use our fetched information to load proper time zone to notify the user with // (if time zone is empty, defaults to UTC) loc, err := time.LoadLocation(timezone) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error loading timezone for email format with user_id %v: %v", userToNotify.Id, err) expirationTimeFormatted = h.ExpirationTime.Format(time.RFC1123) } else { expirationTimeFormatted = h.ExpirationTime.In(loc).Format(time.RFC1123) } // we need to send a notification for the threshold for this host hostNotification := notification{ recipient: h.StartedBy, subject: fmt.Sprintf("%v host termination reminder", h.Distro.Id), message: fmt.Sprintf("Your %v host with id %v will be terminated"+ " at %v, in %v minutes. Visit %v to extend its lifetime.", h.Distro.Id, h.Id, expirationTimeFormatted, h.ExpirationTime.Sub(time.Now()), settings.Ui.Url+"/ui/spawn"), threshold: thresholdKey, host: h, callback: func(h host.Host, thresholdKey string) error { return h.SetExpirationNotification(thresholdKey) }, } // add it to the list warnings = append(warnings, hostNotification) } evergreen.Logger.Logf(slogger.INFO, "Built %v warnings about imminently"+ " expiring hosts", len(warnings)) return warnings, nil }
// run all monitoring functions func RunAllMonitoring(settings *evergreen.Settings) error { // load in all of the distros distros, err := distro.Find(db.Q{}) if err != nil { return fmt.Errorf("error finding distros: %v", err) } // fetch the project refs, which we will use to get all of the projects projectRefs, err := model.FindAllProjectRefs() if err != nil { return fmt.Errorf("error loading in project refs: %v", err) } // turn the project refs into a map of the project id -> project projects := map[string]model.Project{} for _, ref := range projectRefs { // only monitor projects that are enabled if !ref.Enabled { continue } project, err := model.FindProject("", &ref) // continue on error to stop the whole monitoring process from // being held up if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error finding project %v: %v", ref.Identifier, err) continue } if project == nil { evergreen.Logger.Logf(slogger.ERROR, "no project entry found for"+ " ref %v", ref.Identifier) continue } projects[project.Identifier] = *project } // initialize the task monitor taskMonitor := &TaskMonitor{ flaggingFuncs: defaultTaskFlaggingFuncs, } // clean up any necessary tasks errs := taskMonitor.CleanupTasks(projects) for _, err := range errs { evergreen.Logger.Logf(slogger.ERROR, "Error cleaning up tasks: %v", err) } // initialize the host monitor hostMonitor := &HostMonitor{ flaggingFuncs: defaultHostFlaggingFuncs, monitoringFuncs: defaultHostMonitoringFuncs, } // clean up any necessary hosts errs = hostMonitor.CleanupHosts(distros, settings) for _, err := range errs { evergreen.Logger.Logf(slogger.ERROR, "Error cleaning up hosts: %v", err) } // run monitoring checks errs = hostMonitor.RunMonitoringChecks(settings) for _, err := range errs { evergreen.Logger.Logf(slogger.ERROR, "Error running host monitoring checks: %v", err) } // initialize the notifier notifier := &Notifier{ notificationBuilders: defaultNotificationBuilders, } // send notifications errs = notifier.Notify(settings) for _, err := range errs { evergreen.Logger.Logf(slogger.ERROR, "Error sending notifications: %v", err) } // Do alerts for spawnhosts - collect all hosts expiring in the next 12 hours. // The trigger logic will filter out any hosts that aren't in a notification window, or have // already have alerts sent. now := time.Now() thresholdTime := now.Add(12 * time.Hour) expiringSoonHosts, err := host.Find(host.ByExpiringBetween(now, thresholdTime)) if err != nil { return err } for _, h := range expiringSoonHosts { err := alerts.RunSpawnWarningTriggers(&h) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error queueing alert: %v", err) } } return nil }
// spawnHostExpirationWarnings is a notificationBuilder to build any necessary // warnings about hosts that will be expiring soon (but haven't expired yet) func spawnHostExpirationWarnings(settings *evergreen.Settings) ([]notification, error) { evergreen.Logger.Logf(slogger.INFO, "Building spawned host expiration"+ " warnings...") // sanity check, since the thresholds are supplied in code if len(spawnWarningThresholds) == 0 { evergreen.Logger.Logf(slogger.WARN, "there are no currently set warning"+ " thresholds for spawned hosts - users will not receive emails"+ " warning them of imminent host expiration") return nil, nil } // assumed to be the first warning threshold (the least recent one) firstWarningThreshold := spawnWarningThresholds[len(spawnWarningThresholds)-1] // find all spawned hosts that have passed at least one warning threshold now := time.Now() thresholdTime := now.Add(firstWarningThreshold) hosts, err := host.Find(host.ByExpiringBetween(now, thresholdTime)) if err != nil { return nil, fmt.Errorf("error finding spawned hosts that will be"+ " expiring soon: %v", err) } // the eventual list of warning notifications to be sent warnings := []notification{} for _, h := range hosts { // figure out the most recent expiration notification threshold the host // has crossed threshold := lastWarningThresholdCrossed(&h) // for keying into the host's notifications map thresholdKey := strconv.Itoa(int(threshold.Minutes())) // if a notification has already been sent for the threshold for this // host, skip it if h.Notifications[thresholdKey] { continue } evergreen.Logger.Logf(slogger.INFO, "Warning needs to be sent for threshold"+ " '%v' for host %v", thresholdKey, h.Id) // we need to send a notification for the threshold for this host hostNotification := notification{ recipient: h.StartedBy, subject: fmt.Sprintf("%v host termination reminder", h.Distro.Id), message: fmt.Sprintf("Your %v host with id %v will be terminated"+ " at %v, in %v minutes. Visit %v to extend its lifetime.", h.Distro.Id, h.Id, h.ExpirationTime.Format(time.RFC850), h.ExpirationTime.Sub(time.Now()), settings.Ui.Url+"/ui/spawn"), threshold: thresholdKey, host: h, callback: func(h host.Host, thresholdKey string) error { return h.SetExpirationNotification(thresholdKey) }, } // add it to the list warnings = append(warnings, hostNotification) } evergreen.Logger.Logf(slogger.INFO, "Built %v warnings about imminently"+ " expiring hosts", len(warnings)) return warnings, nil }