// setupHost runs the specified setup script for an individual host. Returns // the output from running the script remotely, as well as any error that // occurs. If the script exits with a non-zero exit code, the error will be non-nil. func (init *HostInit) setupHost(targetHost *host.Host) (string, error) { // fetch the appropriate cloud provider for the host cloudMgr, err := providers.GetCloudManager(targetHost.Provider, init.Settings) if err != nil { return "", fmt.Errorf("failed to get cloud manager for host %v with provider %v: %v", targetHost.Id, targetHost.Provider, err) } // mark the host as initializing if err := targetHost.SetInitializing(); err != nil { if err == mgo.ErrNotFound { return "", ErrHostAlreadyInitializing } else { return "", fmt.Errorf("database error: %v", err) } } /* TESTING ONLY setupDebugSSHTunnel(path_to_ssh_key, targetHost.User, targetHost.Host) */ // run the function scheduled for when the host is up err = cloudMgr.OnUp(targetHost) if err != nil { // if this fails it is probably due to an API hiccup, so we keep going. evergreen.Logger.Logf(slogger.WARN, "OnUp callback failed for host '%v': '%v'", targetHost.Id, err) } cloudHost, err := providers.GetCloudHost(targetHost, init.Settings) if err != nil { return "", fmt.Errorf("failed to get cloud host for %v: %v", targetHost.Id, err) } sshOptions, err := cloudHost.GetSSHOptions() if err != nil { return "", fmt.Errorf("error getting ssh options for host %v: %v", targetHost.Id, err) } if targetHost.Distro.Teardown != "" { err = init.copyScript(targetHost, teardownScriptName, targetHost.Distro.Teardown) if err != nil { return "", fmt.Errorf("error copying script %v to host %v: %v", teardownScriptName, targetHost.Id, err) } } if targetHost.Distro.Setup != "" { err = init.copyScript(targetHost, setupScriptName, targetHost.Distro.Setup) if err != nil { return "", fmt.Errorf("error copying script %v to host %v: %v", setupScriptName, targetHost.Id, err) } logs, err := hostutil.RunRemoteScript(targetHost, setupScriptName, sshOptions) if err != nil { return logs, fmt.Errorf("error running setup script over ssh: %v", err) } return logs, nil } return "", nil }
// Call out to the embedded CloudManager to spawn hosts. Takes in a map of // distro -> number of hosts to spawn for the distro. // Returns a map of distro -> hosts spawned, and an error if one occurs. func (s *Scheduler) spawnHosts(newHostsNeeded map[string]int) ( map[string][]host.Host, error) { // loop over the distros, spawning up the appropriate number of hosts // for each distro hostsSpawnedPerDistro := make(map[string][]host.Host) for distroId, numHostsToSpawn := range newHostsNeeded { if numHostsToSpawn == 0 { continue } hostsSpawnedPerDistro[distroId] = make([]host.Host, 0, numHostsToSpawn) for i := 0; i < numHostsToSpawn; i++ { d, err := distro.FindOne(distro.ById(distroId)) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Failed to find distro '%v': %v", distroId, err) } allDistroHosts, err := host.Find(host.ByDistroId(distroId)) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error getting hosts for distro %v: %v", distroId, err) continue } if len(allDistroHosts) >= d.PoolSize { evergreen.Logger.Logf(slogger.ERROR, "Already at max (%v) hosts for distro '%v'", distroId, d.PoolSize) continue } cloudManager, err := providers.GetCloudManager(d.Provider, s.Settings) if err != nil { evergreen.Logger.Errorf(slogger.ERROR, "Error getting cloud manager for distro: %v", err) continue } hostOptions := cloud.HostOptions{ UserName: evergreen.User, UserHost: false, } newHost, err := cloudManager.SpawnInstance(d, hostOptions) if err != nil { evergreen.Logger.Errorf(slogger.ERROR, "Error spawning instance: %v,", err) continue } hostsSpawnedPerDistro[distroId] = append(hostsSpawnedPerDistro[distroId], *newHost) } // if none were spawned successfully if len(hostsSpawnedPerDistro[distroId]) == 0 { delete(hostsSpawnedPerDistro, distroId) } } return hostsSpawnedPerDistro, nil }
// IsHostReady returns whether or not the specified host is ready for its setup script // to be run. func (init *HostInit) IsHostReady(host *host.Host) (bool, error) { // fetch the appropriate cloud provider for the host cloudMgr, err := providers.GetCloudManager(host.Distro.Provider, init.Settings) if err != nil { return false, fmt.Errorf("failed to get cloud manager for provider %v: %v", host.Distro.Provider, err) } // ask for the instance's status hostStatus, err := cloudMgr.GetInstanceStatus(host) if err != nil { return false, fmt.Errorf("error checking instance status of host %v: %v", host.Id, err) } // if the host isn't up yet, we can't do anything if hostStatus != cloud.StatusRunning { return false, nil } // set the host's dns name, if it is not set if host.Host == "" { // get the DNS name for the host hostDNS, err := cloudMgr.GetDNSName(host) if err != nil { return false, fmt.Errorf("error checking DNS name for host %v: %v", host.Id, err) } // sanity check for the host DNS name if hostDNS == "" { return false, fmt.Errorf("instance %v is running but not returning a DNS name", host.Id) } // update the host's DNS name if err := host.SetDNSName(hostDNS); err != nil { return false, fmt.Errorf("error setting DNS name for host %v: %v", host.Id, err) } } // check if the host is reachable via SSH cloudHost, err := providers.GetCloudHost(host, init.Settings) if err != nil { return false, fmt.Errorf("failed to get cloud host for %v: %v", host.Id, err) } reachable, err := cloudHost.IsSSHReachable() if err != nil { return false, fmt.Errorf("error checking if host %v is reachable: %v", host.Id, err) } // at this point, we can run the setup if the host is reachable return reachable, nil }
// flagIdleHosts is a hostFlaggingFunc to get all hosts which have spent too // long without running a task func flagIdleHosts(d []distro.Distro, s *evergreen.Settings) ([]host.Host, error) { // will ultimately contain all of the hosts determined to be idle idleHosts := []host.Host{} // fetch all hosts not currently running a task freeHosts, err := host.Find(host.IsFree) if err != nil { return nil, fmt.Errorf("error finding free hosts: %v", err) } // go through the hosts, and see if they have idled long enough to // be terminated for _, host := range freeHosts { // ask the host how long it has been idle idleTime := host.IdleTime() // get a cloud manager for the host cloudManager, err := providers.GetCloudManager(host.Provider, s) if err != nil { return nil, fmt.Errorf("error getting cloud manager for host %v: %v", host.Id, err) } // if the host is not dynamically spun up (and can thus be terminated), // skip it canTerminate, err := hostCanBeTerminated(host, s) if err != nil { return nil, fmt.Errorf("error checking if host %v can be terminated: %v", host.Id, err) } if !canTerminate { continue } // ask how long until the next payment for the host tilNextPayment := cloudManager.TimeTilNextPayment(&host) // current determinants for idle: // idle for at least 15 minutes and // less than 5 minutes til next payment if idleTime >= 15*time.Minute && tilNextPayment <= 5*time.Minute { idleHosts = append(idleHosts, host) } } return idleHosts, nil }
// helper to check if a host can be terminated func hostCanBeTerminated(h host.Host, s *evergreen.Settings) (bool, error) { // get a cloud manager for the host cloudManager, err := providers.GetCloudManager(h.Provider, s) if err != nil { return false, fmt.Errorf("error getting cloud manager for host %v: %v", h.Id, err) } // if the host is not part of a spawnable distro, then it was not // dynamically spun up and as such cannot be terminated canSpawn, err := cloudManager.CanSpawn() if err != nil { return false, fmt.Errorf("error checking if cloud manager for host %v supports spawning: %v", h.Id, err) } return canSpawn, nil }
// numNewHostsForDistro determine how many new hosts should be spun up for an // individual distro func (self *DeficitBasedHostAllocator) numNewHostsForDistro( hostAllocatorData *HostAllocatorData, distro distro.Distro, settings *evergreen.Settings) int { cloudManager, err := providers.GetCloudManager(distro.Provider, settings) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Couldn't get cloud manager for distro %v with provider %v: %v", distro.Id, distro.Provider, err) return 0 } can, err := cloudManager.CanSpawn() if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Couldn't check if cloud provider %v is spawnable: %v", distro.Provider, err) return 0 } if !can { return 0 } existingDistroHosts := hostAllocatorData.existingDistroHosts[distro.Id] runnableDistroTasks := hostAllocatorData. taskQueueItems[distro.Id] freeHosts := make([]host.Host, 0, len(existingDistroHosts)) for _, existingDistroHost := range existingDistroHosts { if existingDistroHost.RunningTask == "" { freeHosts = append(freeHosts, existingDistroHost) } } numNewHosts := util.Min( // the deficit of available hosts vs. tasks to be run len(runnableDistroTasks)-len(freeHosts), // the maximum number of new hosts we're allowed to spin up distro.PoolSize-len(existingDistroHosts), ) // cap to zero as lower bound if numNewHosts < 0 { numNewHosts = 0 } return numNewHosts }
// updateTaskCost determines a task's cost based on the host it ran on. Hosts that // are unable to calculate their own costs will not set a task's Cost field. Errors // are logged but not returned, since any number of API failures could happen and // we shouldn't sacrifice a task's status for them. func (as *APIServer) updateTaskCost(t *task.Task, h *host.Host, finishTime time.Time) { manager, err := providers.GetCloudManager(h.Provider, &as.Settings) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error loading provider for host %v cost calculation: %v ", t.HostId, err) return } if calc, ok := manager.(cloud.CloudCostCalculator); ok { evergreen.Logger.Logf(slogger.INFO, "Calculating cost for task %v", t.Id) cost, err := calc.CostForDuration(h, t.StartTime, finishTime) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error calculating cost for task %v: %v ", t.Id, err) return } if err := t.SetCost(cost); err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error updating cost for task %v: %v ", t.Id, err) return } } }
// numNewHostsForDistro determine how many new hosts should be spun up for an // individual distro. func (self *DurationBasedHostAllocator) numNewHostsForDistro( hostAllocatorData *HostAllocatorData, distro distro.Distro, tasksAccountedFor map[string]bool, distroScheduleData map[string]DistroScheduleData, settings *evergreen.Settings) (numNewHosts int, err error) { projectTaskDurations := hostAllocatorData.projectTaskDurations existingDistroHosts := hostAllocatorData.existingDistroHosts[distro.Id] taskQueueItems := hostAllocatorData.taskQueueItems[distro.Id] taskRunDistros := hostAllocatorData.taskRunDistros // determine how many free hosts we have numFreeHosts := 0 for _, existingDistroHost := range existingDistroHosts { if existingDistroHost.RunningTask == "" { numFreeHosts += 1 } } // determine the total remaining running time of all // tasks currently running on the hosts for this distro runningTasksDuration, err := computeRunningTasksDuration( existingDistroHosts, projectTaskDurations) if err != nil { return numNewHosts, err } // construct the data needed by computeScheduledTasksDuration scheduledDistroTasksData := &ScheduledDistroTasksData{ taskQueueItems: taskQueueItems, tasksAccountedFor: tasksAccountedFor, taskRunDistros: taskRunDistros, currentDistroId: distro.Id, } // determine the total expected running time of all scheduled // tasks for this distro scheduledTasksDuration, sharedTasksDuration := computeScheduledTasksDuration(scheduledDistroTasksData) // find the number of new hosts needed based on the total estimated // duration for all outstanding and in-flight tasks for this distro durationBasedNumNewHosts := computeDurationBasedNumNewHosts( scheduledTasksDuration, runningTasksDuration, float64(len(existingDistroHosts)), MaxDurationPerDistroHost) // revise the new host estimate based on the cap of the number of new hosts // and the number of free hosts numNewHosts = numNewDistroHosts(distro.PoolSize, len(existingDistroHosts), numFreeHosts, durationBasedNumNewHosts, len(taskQueueItems)) // create an entry for this distro in the scheduling map distroScheduleData[distro.Id] = DistroScheduleData{ nominalNumNewHosts: numNewHosts, numFreeHosts: numFreeHosts, poolSize: distro.PoolSize, taskQueueLength: len(taskQueueItems), sharedTasksDuration: sharedTasksDuration, runningTasksDuration: runningTasksDuration, numExistingHosts: len(existingDistroHosts), totalTasksDuration: scheduledTasksDuration + runningTasksDuration, } cloudManager, err := providers.GetCloudManager(distro.Provider, settings) if err != nil { return 0, evergreen.Logger.Errorf(slogger.ERROR, "Couldn't get cloud manager for %v (%v): %v", distro.Provider, distro.Id, err) } can, err := cloudManager.CanSpawn() if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error checking if '%v' provider can spawn hosts: %v", distro.Provider, err) return 0, nil } if !can { return 0, nil } // revise the nominal number of new hosts if needed numNewHosts = orderedScheduleNumNewHosts(distroScheduleData, distro.Id, MaxDurationPerDistroHost, SharedTasksAllocationProportion) evergreen.Logger.Logf(slogger.INFO, "Spawning %v additional hosts for %v - "+ "currently at %v existing hosts (%v free)", numNewHosts, distro.Id, len(existingDistroHosts), numFreeHosts) evergreen.Logger.Logf(slogger.INFO, "Total estimated time to process all '%v' "+ "scheduled tasks is %v; %v running tasks at %v, %v pending tasks at "+ "%v (shared tasks duration map: %v)", distro.Id, time.Duration(scheduledTasksDuration+runningTasksDuration)*time.Second, len(existingDistroHosts)-numFreeHosts, time.Duration(runningTasksDuration)*time.Second, len(taskQueueItems), time.Duration(scheduledTasksDuration)*time.Second, sharedTasksDuration) return numNewHosts, nil }
// setupHost runs the specified setup script for an individual host. Returns // the output from running the script remotely, as well as any error that // occurs. If the script exits with a non-zero exit code, the error will be non-nil. func (init *HostInit) setupHost(targetHost *host.Host) ([]byte, error) { // fetch the appropriate cloud provider for the host cloudMgr, err := providers.GetCloudManager(targetHost.Provider, init.Settings) if err != nil { return nil, fmt.Errorf("failed to get cloud manager for host %v with provider %v: %v", targetHost.Id, targetHost.Provider, err) } // mark the host as initializing if err := targetHost.SetInitializing(); err != nil { if err == mgo.ErrNotFound { return nil, ErrHostAlreadyInitializing } else { return nil, fmt.Errorf("database error: %v", err) } } // run the function scheduled for when the host is up err = cloudMgr.OnUp(targetHost) if err != nil { // if this fails it is probably due to an API hiccup, so we keep going. evergreen.Logger.Logf(slogger.WARN, "OnUp callback failed for host '%v': '%v'", targetHost.Id, err) } // run the remote setup script as sudo, if appropriate sudoStr := "" if targetHost.Distro.SetupAsSudo { sudoStr = "sudo " } // parse the hostname into the user, host and port hostInfo, err := util.ParseSSHInfo(targetHost.Host) if err != nil { return nil, err } user := targetHost.Distro.User if hostInfo.User != "" { user = hostInfo.User } // create a temp file for the setup script fileName := "setup.sh" file, err := ioutil.TempFile("", fileName) if err != nil { return nil, fmt.Errorf("error creating setup script: %v", err) } defer func() { file.Close() os.Remove(file.Name()) }() // build the setup script setup, err := init.buildSetupScript(targetHost) if err != nil { return nil, fmt.Errorf("error building setup script for host %v: %v", targetHost.Id, err) } // write the setup script to the file if _, err := file.Write([]byte(setup)); err != nil { return nil, fmt.Errorf("error writing remote setup script: %v", err) } cloudHost, err := providers.GetCloudHost(targetHost, init.Settings) if err != nil { return nil, fmt.Errorf("Failed to get cloud host for %v: %v", targetHost.Id, err) } sshOptions, err := cloudHost.GetSSHOptions() if err != nil { return nil, fmt.Errorf("Error getting ssh options for host %v: %v", targetHost.Id, err) } // copy setup script over to the remote machine var scpSetupCmdStderr bytes.Buffer scpSetupCmd := &command.ScpCommand{ Source: file.Name(), Dest: fileName, Stdout: &scpSetupCmdStderr, Stderr: &scpSetupCmdStderr, RemoteHostName: hostInfo.Hostname, User: user, Options: append([]string{"-P", hostInfo.Port}, sshOptions...), } // run the command to scp the setup script with a timeout err = util.RunFunctionWithTimeout( scpSetupCmd.Run, SCPTimeout, ) if err != nil { if err == util.ErrTimedOut { scpSetupCmd.Stop() return nil, fmt.Errorf("scp-ing setup script timed out") } return nil, fmt.Errorf("error (%v) copying setup script to remote "+ "machine: %v", err, scpSetupCmdStderr.String()) } // run command to ssh into remote machine and execute setup script var sshSetupCmdStderr bytes.Buffer runSetupCmd := &command.RemoteCommand{ CmdString: sudoStr + "sh " + fileName, Stdout: &sshSetupCmdStderr, Stderr: &sshSetupCmdStderr, RemoteHostName: hostInfo.Hostname, User: user, Options: []string{"-p", hostInfo.Port}, Background: false, } // only force creation of a tty if sudo if targetHost.Distro.SetupAsSudo { runSetupCmd.Options = []string{"-t", "-t", "-p", hostInfo.Port} } runSetupCmd.Options = append(runSetupCmd.Options, sshOptions...) // run the ssh command with given timeout err = util.RunFunctionWithTimeout( runSetupCmd.Run, time.Duration(SSHTimeoutSeconds)*time.Second, ) return sshSetupCmdStderr.Bytes(), err }
// CreateHost spawns a host with the given options. func (sm Spawn) CreateHost(so Options) (*host.Host, error) { // load in the appropriate distro d, err := distro.FindOne(distro.ById(so.Distro)) if err != nil { return nil, err } // get the appropriate cloud manager cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings) if err != nil { return nil, err } // spawn the host h, err := cloudManager.SpawnInstance(d, so.UserName, true) if err != nil { return nil, err } // set the expiration time for the host expireTime := h.CreationTime.Add(DefaultExpiration) err = h.SetExpirationTime(expireTime) if err != nil { return h, evergreen.Logger.Errorf(slogger.ERROR, "error setting expiration on host %v: %v", h.Id, err) } // set the user data, if applicable if so.UserData != "" { err = h.SetUserData(so.UserData) if err != nil { return h, evergreen.Logger.Errorf(slogger.ERROR, "Failed setting userData on host %v: %v", h.Id, err) } } // create a hostinit to take care of setting up the host init := &hostinit.HostInit{ Settings: sm.settings, } // for making sure the host doesn't take too long to spawn startTime := time.Now() // spin until the host is ready for its setup script to be run for { // make sure we haven't been spinning for too long if time.Now().Sub(startTime) > 15*time.Minute { if err := h.SetDecommissioned(); err != nil { evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err) } return nil, fmt.Errorf("host took too long to come up") } time.Sleep(5000 * time.Millisecond) evergreen.Logger.Logf(slogger.INFO, "Checking if host %v is up and ready", h.Id) // see if the host is ready for its setup script to be run ready, err := init.IsHostReady(h) if err != nil { if err := h.SetDecommissioned(); err != nil { evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err) } return nil, fmt.Errorf("error checking on host %v; decommissioning to save resources: %v", h.Id, err) } // if the host is ready, move on to running the setup script if ready { break } } evergreen.Logger.Logf(slogger.INFO, "Host %v is ready for its setup script to be run", h.Id) // add any extra user-specified data into the setup script if h.Distro.UserData.File != "" { userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n", strings.Replace(so.UserData, "\"", "\\\"", -1), h.Distro.UserData.File) // prepend the setup script to add the userdata file if strings.HasPrefix(h.Distro.Setup, "#!") { firstLF := strings.Index(h.Distro.Setup, "\n") h.Distro.Setup = h.Distro.Setup[0:firstLF+1] + userDataCmd + h.Distro.Setup[firstLF+1:] } else { h.Distro.Setup = userDataCmd + h.Distro.Setup } } // modify the setup script to add the user's public key h.Distro.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n", so.PublicKey, h.Distro.User) // replace expansions in the script exp := command.NewExpansions(init.Settings.Expansions) h.Distro.Setup, err = exp.ExpandString(h.Distro.Setup) if err != nil { return nil, fmt.Errorf("expansions error: %v", err) } // provision the host err = init.ProvisionHost(h) if err != nil { return nil, fmt.Errorf("error provisioning host %v: %v", h.Id, err) } return h, nil }
func (as *APIServer) hostReady(w http.ResponseWriter, r *http.Request) { hostObj, err := getHostFromRequest(r) if err != nil { evergreen.Logger.Errorf(slogger.ERROR, err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) return } // if the host failed setupSuccess := mux.Vars(r)["status"] if setupSuccess == evergreen.HostStatusFailed { evergreen.Logger.Logf(slogger.INFO, "Initializing host %v failed", hostObj.Id) // send notification to the Evergreen team about this provisioning failure subject := fmt.Sprintf("%v Evergreen provisioning failure on %v", notify.ProvisionFailurePreface, hostObj.Distro.Id) hostLink := fmt.Sprintf("%v/host/%v", as.Settings.Ui.Url, hostObj.Id) message := fmt.Sprintf("Provisioning failed on %v host -- %v (%v). %v", hostObj.Distro.Id, hostObj.Id, hostObj.Host, hostLink) if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil { evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err) } // get/store setup logs setupLog, err := ioutil.ReadAll(r.Body) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } event.LogProvisionFailed(hostObj.Id, string(setupLog)) err = hostObj.SetUnprovisioned() if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } as.WriteJSON(w, http.StatusOK, fmt.Sprintf("Initializing host %v failed", hostObj.Id)) return } cloudManager, err := providers.GetCloudManager(hostObj.Provider, &as.Settings) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) subject := fmt.Sprintf("%v Evergreen provisioning completion failure on %v", notify.ProvisionFailurePreface, hostObj.Distro.Id) message := fmt.Sprintf("Failed to get cloud manager for host %v with provider %v: %v", hostObj.Id, hostObj.Provider, err) if err = notify.NotifyAdmins(subject, message, &as.Settings); err != nil { evergreen.Logger.Errorf(slogger.ERROR, "Error sending email: %v", err) } return } dns, err := cloudManager.GetDNSName(hostObj) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } // mark host as provisioned if err := hostObj.MarkAsProvisioned(); err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } evergreen.Logger.Logf(slogger.INFO, "Successfully marked host “%v” with dns “%v” as provisioned", hostObj.Id, dns) }
// ensureHasRequiredFields check that the distro configuration has all the required fields func ensureHasRequiredFields(d *distro.Distro, s *evergreen.Settings) []ValidationError { errs := []ValidationError{} if d.Id == "" { errs = append(errs, ValidationError{ Message: fmt.Sprintf("distro '%v' cannot be blank", distro.IdKey), Level: Error, }) } if d.Arch == "" { errs = append(errs, ValidationError{ Message: fmt.Sprintf("distro '%v' cannot be blank", distro.ArchKey), Level: Error, }) } if d.User == "" { errs = append(errs, ValidationError{ Message: fmt.Sprintf("distro '%v' cannot be blank", distro.UserKey), Level: Error, }) } if d.WorkDir == "" { errs = append(errs, ValidationError{ Message: fmt.Sprintf("distro '%v' cannot be blank", distro.WorkDirKey), Level: Error, }) } if d.SSHKey == "" && d.Provider != static.ProviderName { errs = append(errs, ValidationError{ Message: fmt.Sprintf("distro '%v' cannot be blank", distro.SSHKeyKey), Level: Error, }) } if d.Provider == "" { errs = append(errs, ValidationError{ Message: fmt.Sprintf("distro '%v' cannot be blank", distro.ProviderKey), Level: Error, }) return errs } mgr, err := providers.GetCloudManager(d.Provider, s) if err != nil { errs = append(errs, ValidationError{ Message: err.Error(), Level: Error, }) return errs } settings := mgr.GetSettings() if err = mapstructure.Decode(d.ProviderSettings, settings); err != nil { errs = append(errs, ValidationError{ Message: fmt.Sprintf("distro '%v' decode error: %v", distro.ProviderSettingsKey, err), Level: Error, }) return errs } if err := settings.Validate(); err != nil { errs = append(errs, ValidationError{Error, err.Error()}) } return errs }
// CreateHost spawns a host with the given options. func (sm Spawn) CreateHost(so Options, owner *user.DBUser) error { // load in the appropriate distro d, err := distro.FindOne(distro.ById(so.Distro)) if err != nil { return err } // add any extra user-specified data into the setup script if d.UserData.File != "" { userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n", strings.Replace(so.UserData, "\"", "\\\"", -1), d.UserData.File) // prepend the setup script to add the userdata file if strings.HasPrefix(d.Setup, "#!") { firstLF := strings.Index(d.Setup, "\n") d.Setup = d.Setup[0:firstLF+1] + userDataCmd + d.Setup[firstLF+1:] } else { d.Setup = userDataCmd + d.Setup } } // modify the setup script to add the user's public key d.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n", so.PublicKey, d.User) // replace expansions in the script exp := command.NewExpansions(sm.settings.Expansions) d.Setup, err = exp.ExpandString(d.Setup) if err != nil { return fmt.Errorf("expansions error: %v", err) } // fake out replacing spot instances with on-demand equivalents if d.Provider == ec2.SpotProviderName { d.Provider = ec2.OnDemandProviderName } // get the appropriate cloud manager cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings) if err != nil { return err } // spawn the host provisionOptions := &host.ProvisionOptions{ LoadCLI: true, TaskId: so.TaskId, OwnerId: owner.Id, } expiration := DefaultExpiration hostOptions := cloud.HostOptions{ ProvisionOptions: provisionOptions, UserName: so.UserName, ExpirationDuration: &expiration, UserData: so.UserData, UserHost: true, } _, err = cloudManager.SpawnInstance(d, hostOptions) if err != nil { return err } return nil }