// Call out to the embedded CloudManager to spawn hosts. Takes in a map of
// distro -> number of hosts to spawn for the distro.
// Returns a map of distro -> hosts spawned, and an error if one occurs.
func (s *Scheduler) spawnHosts(newHostsNeeded map[string]int) (
map[string][]host.Host, error) {
// loop over the distros, spawning up the appropriate number of hosts
// for each distro
hostsSpawnedPerDistro := make(map[string][]host.Host)
for distroId, numHostsToSpawn := range newHostsNeeded {
if numHostsToSpawn == 0 {
continue
}
hostsSpawnedPerDistro[distroId] = make([]host.Host, 0, numHostsToSpawn)
for i := 0; i < numHostsToSpawn; i++ {
d, err := distro.FindOne(distro.ById(distroId))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Failed to find distro '%v': %v", distroId, err)
}
allDistroHosts, err := host.Find(host.ByDistroId(distroId))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error getting hosts for distro %v: %v", distroId, err)
continue
}
if len(allDistroHosts) >= d.PoolSize {
evergreen.Logger.Logf(slogger.ERROR, "Already at max (%v) hosts for distro '%v'",
distroId,
d.PoolSize)
continue
}
cloudManager, err := providers.GetCloudManager(d.Provider, s.Settings)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error getting cloud manager for distro: %v", err)
continue
}
hostOptions := cloud.HostOptions{
UserName: evergreen.User,
UserHost: false,
}
newHost, err := cloudManager.SpawnInstance(d, hostOptions)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error spawning instance: %v,",
err)
continue
}
hostsSpawnedPerDistro[distroId] =
append(hostsSpawnedPerDistro[distroId], *newHost)
}
// if none were spawned successfully
if len(hostsSpawnedPerDistro[distroId]) == 0 {
delete(hostsSpawnedPerDistro, distroId)
}
}
return hostsSpawnedPerDistro, nil
}
// flagExcessHosts is a hostFlaggingFunc to get all hosts that push their
// distros over the specified max hosts
func flagExcessHosts(distros []distro.Distro, s *evergreen.Settings) ([]host.Host, error) {
// will ultimately contain all the hosts that can be terminated
excessHosts := []host.Host{}
// figure out the excess hosts for each distro
for _, d := range distros {
// fetch any hosts for the distro that count towards max hosts
allHostsForDistro, err := host.Find(host.ByDistroId(d.Id))
if err != nil {
return nil, fmt.Errorf("error fetching hosts for distro %v: %v", d.Id, err)
}
// if there are more than the specified max hosts, then terminate
// some, if they are not running tasks
numExcessHosts := len(allHostsForDistro) - d.PoolSize
if numExcessHosts > 0 {
// track how many hosts for the distro are terminated
counter := 0
for _, host := range allHostsForDistro {
// if the host is not dynamically spun up (and can
// thus be terminated), skip it
canTerminate, err := hostCanBeTerminated(host, s)
if err != nil {
return nil, fmt.Errorf("error checking if host %v can be terminated: %v", host.Id, err)
}
if !canTerminate {
continue
}
// if the host is not running a task, it can be
// safely terminated
if host.RunningTask == "" {
excessHosts = append(excessHosts, host)
counter++
}
// break if we've marked enough to be terminated
if counter == numExcessHosts {
break
}
}
evergreen.Logger.Logf(slogger.INFO, "Found %v excess hosts for distro %v", counter, d.Id)
}
}
return excessHosts, nil
}