// spawnHostExpirationWarnings is a notificationBuilder to build any necessary
// warnings about hosts that will be expiring soon (but haven't expired yet)
func spawnHostExpirationWarnings(settings *evergreen.Settings) ([]notification,
error) {
evergreen.Logger.Logf(slogger.INFO, "Building spawned host expiration"+
" warnings...")
// sanity check, since the thresholds are supplied in code
if len(spawnWarningThresholds) == 0 {
evergreen.Logger.Logf(slogger.WARN, "there are no currently set warning"+
" thresholds for spawned hosts - users will not receive emails"+
" warning them of imminent host expiration")
return nil, nil
}
// assumed to be the first warning threshold (the least recent one)
firstWarningThreshold :=
spawnWarningThresholds[len(spawnWarningThresholds)-1]
// find all spawned hosts that have passed at least one warning threshold
now := time.Now()
thresholdTime := now.Add(firstWarningThreshold)
hosts, err := host.Find(host.ByExpiringBetween(now, thresholdTime))
if err != nil {
return nil, fmt.Errorf("error finding spawned hosts that will be"+
" expiring soon: %v", err)
}
// the eventual list of warning notifications to be sent
warnings := []notification{}
for _, h := range hosts {
// figure out the most recent expiration notification threshold the host
// has crossed
threshold := lastWarningThresholdCrossed(&h)
// for keying into the host's notifications map
thresholdKey := strconv.Itoa(int(threshold.Minutes()))
// if a notification has already been sent for the threshold for this
// host, skip it
if h.Notifications[thresholdKey] {
continue
}
evergreen.Logger.Logf(slogger.INFO, "Warning needs to be sent for threshold"+
" '%v' for host %v", thresholdKey, h.Id)
// fetch information about the user we are notifying
userToNotify, err := user.FindOne(user.ById(h.StartedBy))
if err != nil {
return nil, fmt.Errorf("error finding user to notify by Id %v: %v", h.StartedBy, err)
}
// if we didn't find a user (in the case of testing) set the timezone to ""
// to avoid triggering a nil pointer exception
timezone := ""
if userToNotify != nil {
timezone = userToNotify.Settings.Timezone
}
var expirationTimeFormatted string
// use our fetched information to load proper time zone to notify the user with
// (if time zone is empty, defaults to UTC)
loc, err := time.LoadLocation(timezone)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error loading timezone for email format with user_id %v: %v", userToNotify.Id, err)
expirationTimeFormatted = h.ExpirationTime.Format(time.RFC1123)
} else {
expirationTimeFormatted = h.ExpirationTime.In(loc).Format(time.RFC1123)
}
// we need to send a notification for the threshold for this host
hostNotification := notification{
recipient: h.StartedBy,
subject: fmt.Sprintf("%v host termination reminder", h.Distro.Id),
message: fmt.Sprintf("Your %v host with id %v will be terminated"+
" at %v, in %v minutes. Visit %v to extend its lifetime.",
h.Distro.Id, h.Id,
expirationTimeFormatted,
h.ExpirationTime.Sub(time.Now()),
settings.Ui.Url+"/ui/spawn"),
threshold: thresholdKey,
host: h,
callback: func(h host.Host, thresholdKey string) error {
return h.SetExpirationNotification(thresholdKey)
},
}
// add it to the list
warnings = append(warnings, hostNotification)
}
evergreen.Logger.Logf(slogger.INFO, "Built %v warnings about imminently"+
" expiring hosts", len(warnings))
return warnings, nil
}
// run all monitoring functions
func RunAllMonitoring(settings *evergreen.Settings) error {
// load in all of the distros
distros, err := distro.Find(db.Q{})
if err != nil {
return fmt.Errorf("error finding distros: %v", err)
}
// fetch the project refs, which we will use to get all of the projects
projectRefs, err := model.FindAllProjectRefs()
if err != nil {
return fmt.Errorf("error loading in project refs: %v", err)
}
// turn the project refs into a map of the project id -> project
projects := map[string]model.Project{}
for _, ref := range projectRefs {
// only monitor projects that are enabled
if !ref.Enabled {
continue
}
project, err := model.FindProject("", &ref)
// continue on error to stop the whole monitoring process from
// being held up
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error finding project %v: %v",
ref.Identifier, err)
continue
}
if project == nil {
evergreen.Logger.Logf(slogger.ERROR, "no project entry found for"+
" ref %v", ref.Identifier)
continue
}
projects[project.Identifier] = *project
}
// initialize the task monitor
taskMonitor := &TaskMonitor{
flaggingFuncs: defaultTaskFlaggingFuncs,
}
// clean up any necessary tasks
errs := taskMonitor.CleanupTasks(projects)
for _, err := range errs {
evergreen.Logger.Logf(slogger.ERROR, "Error cleaning up tasks: %v", err)
}
// initialize the host monitor
hostMonitor := &HostMonitor{
flaggingFuncs: defaultHostFlaggingFuncs,
monitoringFuncs: defaultHostMonitoringFuncs,
}
// clean up any necessary hosts
errs = hostMonitor.CleanupHosts(distros, settings)
for _, err := range errs {
evergreen.Logger.Logf(slogger.ERROR, "Error cleaning up hosts: %v", err)
}
// run monitoring checks
errs = hostMonitor.RunMonitoringChecks(settings)
for _, err := range errs {
evergreen.Logger.Logf(slogger.ERROR, "Error running host monitoring checks: %v", err)
}
// initialize the notifier
notifier := &Notifier{
notificationBuilders: defaultNotificationBuilders,
}
// send notifications
errs = notifier.Notify(settings)
for _, err := range errs {
evergreen.Logger.Logf(slogger.ERROR, "Error sending notifications: %v", err)
}
// Do alerts for spawnhosts - collect all hosts expiring in the next 12 hours.
// The trigger logic will filter out any hosts that aren't in a notification window, or have
// already have alerts sent.
now := time.Now()
thresholdTime := now.Add(12 * time.Hour)
expiringSoonHosts, err := host.Find(host.ByExpiringBetween(now, thresholdTime))
if err != nil {
return err
}
for _, h := range expiringSoonHosts {
err := alerts.RunSpawnWarningTriggers(&h)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error queueing alert: %v", err)
}
}
return nil
}
// spawnHostExpirationWarnings is a notificationBuilder to build any necessary
// warnings about hosts that will be expiring soon (but haven't expired yet)
func spawnHostExpirationWarnings(settings *evergreen.Settings) ([]notification,
error) {
evergreen.Logger.Logf(slogger.INFO, "Building spawned host expiration"+
" warnings...")
// sanity check, since the thresholds are supplied in code
if len(spawnWarningThresholds) == 0 {
evergreen.Logger.Logf(slogger.WARN, "there are no currently set warning"+
" thresholds for spawned hosts - users will not receive emails"+
" warning them of imminent host expiration")
return nil, nil
}
// assumed to be the first warning threshold (the least recent one)
firstWarningThreshold :=
spawnWarningThresholds[len(spawnWarningThresholds)-1]
// find all spawned hosts that have passed at least one warning threshold
now := time.Now()
thresholdTime := now.Add(firstWarningThreshold)
hosts, err := host.Find(host.ByExpiringBetween(now, thresholdTime))
if err != nil {
return nil, fmt.Errorf("error finding spawned hosts that will be"+
" expiring soon: %v", err)
}
// the eventual list of warning notifications to be sent
warnings := []notification{}
for _, h := range hosts {
// figure out the most recent expiration notification threshold the host
// has crossed
threshold := lastWarningThresholdCrossed(&h)
// for keying into the host's notifications map
thresholdKey := strconv.Itoa(int(threshold.Minutes()))
// if a notification has already been sent for the threshold for this
// host, skip it
if h.Notifications[thresholdKey] {
continue
}
evergreen.Logger.Logf(slogger.INFO, "Warning needs to be sent for threshold"+
" '%v' for host %v", thresholdKey, h.Id)
// we need to send a notification for the threshold for this host
hostNotification := notification{
recipient: h.StartedBy,
subject: fmt.Sprintf("%v host termination reminder", h.Distro.Id),
message: fmt.Sprintf("Your %v host with id %v will be terminated"+
" at %v, in %v minutes. Visit %v to extend its lifetime.",
h.Distro.Id, h.Id,
h.ExpirationTime.Format(time.RFC850),
h.ExpirationTime.Sub(time.Now()),
settings.Ui.Url+"/ui/spawn"),
threshold: thresholdKey,
host: h,
callback: func(h host.Host, thresholdKey string) error {
return h.SetExpirationNotification(thresholdKey)
},
}
// add it to the list
warnings = append(warnings, hostNotification)
}
evergreen.Logger.Logf(slogger.INFO, "Built %v warnings about imminently"+
" expiring hosts", len(warnings))
return warnings, nil
}
