// Provision the host, and update the database accordingly.
func (init *HostInit) ProvisionHost(h *host.Host) error {
// run the setup script
evergreen.Logger.Logf(slogger.INFO, "Setting up host %v", h.Id)
output, err := init.setupHost(h)
// deal with any errors that occured while running the setup
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error running setup script: %v", err)
// another hostinit process beat us there
if err == ErrHostAlreadyInitializing {
evergreen.Logger.Logf(slogger.DEBUG, "Attempted to initialize already initializing host %v", h.Id)
return nil
}
alerts.RunHostProvisionFailTriggers(h)
event.LogProvisionFailed(h.Id, output)
// setup script failed, mark the host's provisioning as failed
if err := h.SetUnprovisioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "unprovisioning host %v failed: %v", h.Id, err)
}
return fmt.Errorf("error initializing host %v: %v", h.Id, err)
}
evergreen.Logger.Logf(slogger.INFO, "Setup complete for host %v", h.Id)
if h.ProvisionOptions != nil &&
h.ProvisionOptions.LoadCLI &&
h.ProvisionOptions.OwnerId != "" {
evergreen.Logger.Logf(slogger.INFO, "Uploading client binary to host %v", h.Id)
lcr, err := init.LoadClient(h)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Failed to load client binary onto host %v: %v", h.Id, err)
} else if err == nil && len(h.ProvisionOptions.TaskId) > 0 {
evergreen.Logger.Logf(slogger.INFO, "Fetching data for task %v onto host %v", h.ProvisionOptions.TaskId, h.Id)
err = init.fetchRemoteTaskData(h.ProvisionOptions.TaskId, lcr.BinaryPath, lcr.ConfigPath, h)
evergreen.Logger.Logf(slogger.ERROR, "Failed to fetch data onto host %v: %v", h.Id, err)
}
}
// the setup was successful. update the host accordingly in the database
if err := h.MarkAsProvisioned(); err != nil {
return fmt.Errorf("error marking host %v as provisioned: %v", err)
}
evergreen.Logger.Logf(slogger.INFO, "Host %v successfully provisioned", h.Id)
return nil
}
// Provision the host, and update the database accordingly.
func (init *HostInit) ProvisionHost(h *host.Host) error {
// run the setup script
output, err := init.setupHost(h)
// deal with any errors that occured while running the setup
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error running setup script: %v", err)
// another hostinit process beat us there
if err == ErrHostAlreadyInitializing {
evergreen.Logger.Logf(slogger.DEBUG,
"Attempted to initialize already initializing host %v", h.Id)
return nil
}
// log the provisioning failure
setupLog := ""
if output != nil {
setupLog = string(output)
}
alerts.RunHostProvisionFailTriggers(h)
event.LogProvisionFailed(h.Id, setupLog)
// setup script failed, mark the host's provisioning as failed
if err := h.SetUnprovisioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "unprovisioning host %v failed: %v", h.Id, err)
}
return fmt.Errorf("error initializing host %v: %v", h.Id, err)
}
// the setup was successful. update the host accordingly in the database
if err := h.MarkAsProvisioned(); err != nil {
return fmt.Errorf("error marking host %v as provisioned: %v", err)
}
evergreen.Logger.Logf(slogger.INFO, "Host %v successfully provisioned", h.Id)
return nil
}
func (as *APIServer) hostReady(w http.ResponseWriter, r *http.Request) {
hostObj, err := getHostFromRequest(r)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, err.Error())
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// if the host failed
setupSuccess := mux.Vars(r)["status"]
if setupSuccess == evergreen.HostStatusFailed {
evergreen.Logger.Logf(slogger.INFO, "Initializing host %v failed", hostObj.Id)
// send notification to the Evergreen team about this provisioning failure
subject := fmt.Sprintf("%v Evergreen provisioning failure on %v", notify.ProvisionFailurePreface, hostObj.Distro.Id)
hostLink := fmt.Sprintf("%v/host/%v", as.Settings.Ui.Url, hostObj.Id)
message := fmt.Sprintf("Provisioning failed on %v host -- %v (%v). %v",
hostObj.Distro.Id, hostObj.Id, hostObj.Host, hostLink)
if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
// get/store setup logs
setupLog, err := ioutil.ReadAll(r.Body)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
event.LogProvisionFailed(hostObj.Id, string(setupLog))
err = hostObj.SetUnprovisioned()
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
as.WriteJSON(w, http.StatusOK, fmt.Sprintf("Initializing host %v failed", hostObj.Id))
return
}
cloudManager, err := providers.GetCloudManager(hostObj.Provider, &as.Settings)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
subject := fmt.Sprintf("%v Evergreen provisioning completion failure on %v",
notify.ProvisionFailurePreface, hostObj.Distro.Id)
message := fmt.Sprintf("Failed to get cloud manager for host %v with provider %v: %v",
hostObj.Id, hostObj.Provider, err)
if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
return
}
dns, err := cloudManager.GetDNSName(hostObj)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
// mark host as provisioned
if err := hostObj.MarkAsProvisioned(); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
evergreen.Logger.Logf(slogger.INFO, "Successfully marked host “%v” with dns “%v” as provisioned", hostObj.Id, dns)
}
func (as *APIServer) spawnHostReady(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
instanceId := vars["instance_id"]
status := vars["status"]
// mark the host itself as provisioned
host, err := host.FindOne(host.ById(instanceId))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if host == nil {
http.Error(w, "host not found", http.StatusNotFound)
return
}
if status == evergreen.HostStatusSuccess {
if err := host.SetRunning(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error marking host id %v as %v: %v", instanceId, evergreen.HostStatusSuccess, err)
}
} else {
alerts.RunHostProvisionFailTriggers(host)
if err = host.SetDecommissioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error marking host %v for user %v as decommissioned: %v", host.Host, host.StartedBy, err)
}
evergreen.Logger.Logf(slogger.INFO, "Decommissioned %v for user %v because provisioning failed", host.Host, host.StartedBy)
// send notification to the Evergreen team about this provisioning failure
subject := fmt.Sprintf("%v Spawn provisioning failure on %v", notify.ProvisionFailurePreface, host.Distro.Id)
message := fmt.Sprintf("Provisioning failed on %v host %v for user %v", host.Distro.Id, host.Host, host.StartedBy)
if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
// get/store setup logs
setupLog, err := ioutil.ReadAll(r.Body)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, fmt.Sprintf("error reading request: %v", err))
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
event.LogProvisionFailed(instanceId, string(setupLog))
}
message := fmt.Sprintf(`
Host with id %v spawned.
The host's dns name is %v.
To ssh in: ssh -i <your private key> %v@%v`,
host.Id, host.Host, host.User, host.Host)
if status == evergreen.HostStatusFailed {
message += fmt.Sprintf("\nUnfortunately, the host's setup script did not run fully - check the setup.log " +
"file in the machine's home directory to see more details")
}
err = notify.TrySendNotificationToUser(host.StartedBy, "Your host is ready", message, notify.ConstructMailer(as.Settings.Notify))
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err)
}
as.WriteJSON(w, http.StatusOK, spawnResponse{HostInfo: *host})
}
