// startRunners starts a goroutine for each runner exposed via Runners. It
// returns a channel on which all runners listen on, for when to terminate.
func startRunners(wg *sync.WaitGroup, s *evergreen.Settings) chan bool {
c := make(chan bool)
duration := time.Duration(runInterval) * time.Second
for _, r := range Runners {
wg.Add(1)
// start each runner in its own goroutine
go func(r ProcessRunner, s *evergreen.Settings, terminateChan chan bool) {
defer wg.Done()
evergreen.Logger.Logf(slogger.INFO, "Starting %v", r.Name())
loop := true
for loop {
if err := r.Run(s); err != nil {
subject := fmt.Sprintf("%v failure", r.Name())
evergreen.Logger.Logf(slogger.ERROR, "%v", err)
if err = notify.NotifyAdmins(subject, err.Error(), s); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error sending email: %v", err)
}
}
select {
case <-time.NewTimer(duration).C:
case loop = <-terminateChan:
}
}
evergreen.Logger.Logf(slogger.INFO, "Cleanly terminated %v", r.Name())
}(r, s, c)
}
return c
}
// helper to terminate a single host
func terminateHost(host *host.Host, settings *evergreen.Settings) error {
// convert the host to a cloud host
cloudHost, err := providers.GetCloudHost(host, settings)
if err != nil {
return fmt.Errorf("error getting cloud host for %v: %v", host.Id, err)
}
// run teardown script if we have one, sending notifications if things go awry
if host.Distro.Teardown != "" && host.Provisioned {
evergreen.Logger.Logf(slogger.ERROR, "Running teardown script for host %v", host.Id)
if err := runHostTeardown(host, cloudHost); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error running teardown script for %v: %v", host.Id, err)
subj := fmt.Sprintf("%v Error running teardown for host %v",
notify.TeardownFailurePreface, host.Id)
if err := notify.NotifyAdmins(subj, err.Error(), settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
}
}
// terminate the instance
if err := cloudHost.TerminateInstance(); err != nil {
return fmt.Errorf("error terminating host %v: %v", host.Id, err)
}
return nil
}
// sendFailureNotification sends a notification to the MCI Team when the
// repotracker is unable to fetch revisions from a given project ref
func (repoTracker *RepoTracker) sendFailureNotification(lastRevision string,
err error) {
// Send a notification to the MCI team
projectRef := repoTracker.ProjectRef
settings := repoTracker.Settings
subject := fmt.Sprintf(notify.RepotrackerFailurePreface,
projectRef.Identifier, lastRevision)
url := fmt.Sprintf("%v/%v/%v/commits/%v", thirdparty.GithubBase,
projectRef.Owner, projectRef.Repo, projectRef.Branch)
message := fmt.Sprintf("Could not find last known revision '%v' "+
"within the most recent %v revisions at %v: %v", lastRevision,
settings.RepoTracker.MaxRepoRevisionsToSearch, url, err)
nErr := notify.NotifyAdmins(subject, message, settings)
if nErr != nil {
evergreen.Logger.Logf(slogger.ERROR, "error sending email: %v", nErr)
}
}
// setupReadyHosts runs the distro setup script of all hosts that are up and reachable.
func (init *HostInit) setupReadyHosts() error {
// set SSH timeout duration
if timeoutSecs := init.Settings.HostInit.SSHTimeoutSeconds; timeoutSecs <= 0 {
evergreen.Logger.Logf(slogger.WARN, "SSH timeout set to %vs (<= 0s) using %vs instead", timeoutSecs, SSHTimeoutSeconds)
} else {
SSHTimeoutSeconds = timeoutSecs
}
// find all hosts in the uninitialized state
uninitializedHosts, err := host.Find(host.IsUninitialized)
if err != nil {
return fmt.Errorf("error fetching uninitialized hosts: %v", err)
}
evergreen.Logger.Logf(slogger.DEBUG, "There are %v uninitialized hosts",
len(uninitializedHosts))
// used for making sure we don't exit before a setup script is done
wg := &sync.WaitGroup{}
for _, h := range uninitializedHosts {
// check whether or not the host is ready for its setup script to be run
ready, err := init.IsHostReady(&h)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error checking host %v for readiness: %v",
h.Id, err)
continue
}
// if the host isn't ready (for instance, it might not be up yet), skip it
if !ready {
evergreen.Logger.Logf(slogger.DEBUG, "Host %v not ready for setup", h.Id)
continue
}
evergreen.Logger.Logf(slogger.INFO, "Running setup script for host %v", h.Id)
// kick off the setup, in its own goroutine, so pending setups don't have
// to wait for it to finish
wg.Add(1)
go func(h host.Host) {
if err := init.ProvisionHost(&h); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error provisioning host %v: %v",
h.Id, err)
// notify the admins of the failure
subject := fmt.Sprintf("%v Evergreen provisioning failure on %v",
notify.ProvisionFailurePreface, h.Distro.Id)
hostLink := fmt.Sprintf("%v/host/%v", init.Settings.Ui.Url, h.Id)
message := fmt.Sprintf("Provisioning failed on %v host -- %v: see %v",
h.Distro.Id, h.Id, hostLink)
if err := notify.NotifyAdmins(subject, message, init.Settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
}
wg.Done()
}(h)
}
// let all setup routines finish
wg.Wait()
return nil
}
func (as *APIServer) hostReady(w http.ResponseWriter, r *http.Request) {
hostObj, err := getHostFromRequest(r)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, err.Error())
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// if the host failed
setupSuccess := mux.Vars(r)["status"]
if setupSuccess == evergreen.HostStatusFailed {
evergreen.Logger.Logf(slogger.INFO, "Initializing host %v failed", hostObj.Id)
// send notification to the Evergreen team about this provisioning failure
subject := fmt.Sprintf("%v Evergreen provisioning failure on %v", notify.ProvisionFailurePreface, hostObj.Distro.Id)
hostLink := fmt.Sprintf("%v/host/%v", as.Settings.Ui.Url, hostObj.Id)
message := fmt.Sprintf("Provisioning failed on %v host -- %v (%v). %v",
hostObj.Distro.Id, hostObj.Id, hostObj.Host, hostLink)
if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
// get/store setup logs
setupLog, err := ioutil.ReadAll(r.Body)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
event.LogProvisionFailed(hostObj.Id, string(setupLog))
err = hostObj.SetUnprovisioned()
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
as.WriteJSON(w, http.StatusOK, fmt.Sprintf("Initializing host %v failed", hostObj.Id))
return
}
cloudManager, err := providers.GetCloudManager(hostObj.Provider, &as.Settings)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
subject := fmt.Sprintf("%v Evergreen provisioning completion failure on %v",
notify.ProvisionFailurePreface, hostObj.Distro.Id)
message := fmt.Sprintf("Failed to get cloud manager for host %v with provider %v: %v",
hostObj.Id, hostObj.Provider, err)
if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
return
}
dns, err := cloudManager.GetDNSName(hostObj)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
// mark host as provisioned
if err := hostObj.MarkAsProvisioned(); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
evergreen.Logger.Logf(slogger.INFO, "Successfully marked host “%v” with dns “%v” as provisioned", hostObj.Id, dns)
}
func (as *APIServer) spawnHostReady(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
instanceId := vars["instance_id"]
status := vars["status"]
// mark the host itself as provisioned
host, err := host.FindOne(host.ById(instanceId))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if host == nil {
http.Error(w, "host not found", http.StatusNotFound)
return
}
if status == evergreen.HostStatusSuccess {
if err := host.SetRunning(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error marking host id %v as %v: %v", instanceId, evergreen.HostStatusSuccess, err)
}
} else {
alerts.RunHostProvisionFailTriggers(host)
if err = host.SetDecommissioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error marking host %v for user %v as decommissioned: %v", host.Host, host.StartedBy, err)
}
evergreen.Logger.Logf(slogger.INFO, "Decommissioned %v for user %v because provisioning failed", host.Host, host.StartedBy)
// send notification to the Evergreen team about this provisioning failure
subject := fmt.Sprintf("%v Spawn provisioning failure on %v", notify.ProvisionFailurePreface, host.Distro.Id)
message := fmt.Sprintf("Provisioning failed on %v host %v for user %v", host.Distro.Id, host.Host, host.StartedBy)
if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
// get/store setup logs
setupLog, err := ioutil.ReadAll(r.Body)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, fmt.Sprintf("error reading request: %v", err))
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
event.LogProvisionFailed(instanceId, string(setupLog))
}
message := fmt.Sprintf(`
Host with id %v spawned.
The host's dns name is %v.
To ssh in: ssh -i <your private key> %v@%v`,
host.Id, host.Host, host.User, host.Host)
if status == evergreen.HostStatusFailed {
message += fmt.Sprintf("\nUnfortunately, the host's setup script did not run fully - check the setup.log " +
"file in the machine's home directory to see more details")
}
err = notify.TrySendNotificationToUser(host.StartedBy, "Your host is ready", message, notify.ConstructMailer(as.Settings.Notify))
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
as.WriteJSON(w, http.StatusOK, spawnResponse{HostInfo: *host})
}