func (uis *UIServer) removeDistro(w http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["distro_id"]
u := MustHaveUser(r)
d, err := distro.FindOne(distro.ById(id))
if err != nil {
message := fmt.Sprintf("error finding distro: %v", err)
PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
http.Error(w, message, http.StatusInternalServerError)
return
}
if err = distro.Remove(id); err != nil {
message := fmt.Sprintf("error removing distro '%v': %v", id, err)
PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
http.Error(w, message, http.StatusInternalServerError)
return
}
event.LogDistroRemoved(id, u.Username(), d)
PushFlash(uis.CookieStore, r, w, NewSuccessFlash(fmt.Sprintf("Distro %v successfully removed.", id)))
uis.WriteJSON(w, http.StatusOK, "distro successfully removed")
}
func (uis *UIServer) spawnPage(w http.ResponseWriter, r *http.Request) {
flashes := PopFlashes(uis.CookieStore, r, w)
projCtx := MustHaveProjectContext(r)
var spawnDistro *distro.Distro
var spawnTask *task.Task
var err error
if len(r.FormValue("distro_id")) > 0 {
spawnDistro, err = distro.FindOne(distro.ById(r.FormValue("distro_id")))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError,
fmt.Errorf("Error finding distro %v: %v", r.FormValue("distro_id"), err))
return
}
}
if len(r.FormValue("task_id")) > 0 {
spawnTask, err = task.FindOne(task.ById(r.FormValue("task_id")))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError,
fmt.Errorf("Error finding task %v: %v", r.FormValue("task_id"), err))
return
}
}
uis.WriteHTML(w, http.StatusOK, struct {
ProjectData projectContext
User *user.DBUser
Flashes []interface{}
Distro *distro.Distro
Task *task.Task
MaxHostsPerUser int
}{projCtx, GetUser(r), flashes, spawnDistro, spawnTask, spawn.MaxPerUser}, "base", "spawned_hosts.html", "base_angular.html", "menu.html")
}
// Call out to the embedded CloudManager to spawn hosts. Takes in a map of
// distro -> number of hosts to spawn for the distro.
// Returns a map of distro -> hosts spawned, and an error if one occurs.
func (s *Scheduler) spawnHosts(newHostsNeeded map[string]int) (
map[string][]host.Host, error) {
// loop over the distros, spawning up the appropriate number of hosts
// for each distro
hostsSpawnedPerDistro := make(map[string][]host.Host)
for distroId, numHostsToSpawn := range newHostsNeeded {
if numHostsToSpawn == 0 {
continue
}
hostsSpawnedPerDistro[distroId] = make([]host.Host, 0, numHostsToSpawn)
for i := 0; i < numHostsToSpawn; i++ {
d, err := distro.FindOne(distro.ById(distroId))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Failed to find distro '%v': %v", distroId, err)
}
allDistroHosts, err := host.Find(host.ByDistroId(distroId))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error getting hosts for distro %v: %v", distroId, err)
continue
}
if len(allDistroHosts) >= d.PoolSize {
evergreen.Logger.Logf(slogger.ERROR, "Already at max (%v) hosts for distro '%v'",
distroId,
d.PoolSize)
continue
}
cloudManager, err := providers.GetCloudManager(d.Provider, s.Settings)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error getting cloud manager for distro: %v", err)
continue
}
hostOptions := cloud.HostOptions{
UserName: evergreen.User,
UserHost: false,
}
newHost, err := cloudManager.SpawnInstance(d, hostOptions)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error spawning instance: %v,",
err)
continue
}
hostsSpawnedPerDistro[distroId] =
append(hostsSpawnedPerDistro[distroId], *newHost)
}
// if none were spawned successfully
if len(hostsSpawnedPerDistro[distroId]) == 0 {
delete(hostsSpawnedPerDistro, distroId)
}
}
return hostsSpawnedPerDistro, nil
}
func (uis *UIServer) modifyDistro(w http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["distro_id"]
u := MustHaveUser(r)
b, err := ioutil.ReadAll(r.Body)
if err != nil {
message := fmt.Sprintf("error reading request: %v", err)
PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
http.Error(w, message, http.StatusBadRequest)
return
}
defer r.Body.Close()
oldDistro, err := distro.FindOne(distro.ById(id))
if err != nil {
message := fmt.Sprintf("error finding distro: %v", err)
PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
http.Error(w, message, http.StatusInternalServerError)
return
}
newDistro := *oldDistro
// attempt to unmarshal data into distros field for type validation
if err = json.Unmarshal(b, &newDistro); err != nil {
message := fmt.Sprintf("error unmarshaling request: %v", err)
PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
http.Error(w, message, http.StatusBadRequest)
return
}
// check that the resulting distro is valid
vErrs := validator.CheckDistro(&newDistro, &uis.Settings, false)
if len(vErrs) != 0 {
for _, e := range vErrs {
PushFlash(uis.CookieStore, r, w, NewErrorFlash(e.Error()))
}
uis.WriteJSON(w, http.StatusBadRequest, vErrs)
return
}
if err = newDistro.Update(); err != nil {
message := fmt.Sprintf("error updating distro: %v", err)
PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
http.Error(w, message, http.StatusBadRequest)
return
}
event.LogDistroModified(id, u.Username(), newDistro)
PushFlash(uis.CookieStore, r, w, NewSuccessFlash(fmt.Sprintf("Distro %v successfully updated.", id)))
uis.WriteJSON(w, http.StatusOK, "distro successfully updated")
}
// AverageStatistics uses an agg pipeline that creates buckets given a time frame and finds the average scheduled ->
// start time for that time frame.
// One thing to note is that the average time is in milliseconds, not nanoseconds and must be converted.
func AverageStatistics(distroId string, bounds FrameBounds) (AvgBuckets, error) {
// error out if the distro does not exist
_, err := distro.FindOne(distro.ById(distroId))
if err != nil {
return nil, err
}
intBucketSize := util.FromNanoseconds(bounds.BucketSize)
buckets := AvgBuckets{}
pipeline := []bson.M{
// find all tasks that have started within the time frame for a given distro and only valid statuses.
{"$match": bson.M{
task.StartTimeKey: bson.M{
"$gte": bounds.StartTime,
"$lte": bounds.EndTime,
},
// only need tasks that have already started or those that have finished,
// not looking for tasks that have been scheduled but not started.
task.StatusKey: bson.M{
"$in": []string{evergreen.TaskStarted,
evergreen.TaskFailed, evergreen.TaskSucceeded},
},
task.DistroIdKey: distroId,
}},
// project the difference in scheduled -> start, as well as the bucket
{"$project": bson.M{
"diff": bson.M{
"$subtract": []interface{}{"$" + task.StartTimeKey, "$" + task.ScheduledTimeKey},
},
"b": bson.M{
"$floor": bson.M{
"$divide": []interface{}{
bson.M{"$subtract": []interface{}{"$" + task.StartTimeKey, bounds.StartTime}},
intBucketSize},
},
},
}},
{"$group": bson.M{
"_id": "$b",
"a": bson.M{"$avg": "$diff"},
"n": bson.M{"$sum": 1},
}},
{"$sort": bson.M{
"_id": 1,
}},
}
if err := db.Aggregate(task.Collection, pipeline, &buckets); err != nil {
return nil, err
}
return convertBucketsToNanoseconds(buckets, bounds), nil
}
func (uis *UIServer) getDistro(w http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["distro_id"]
d, err := distro.FindOne(distro.ById(id))
if err != nil {
message := fmt.Sprintf("error fetching distro '%v': %v", id, err)
PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
http.Error(w, message, http.StatusInternalServerError)
return
}
uis.WriteJSON(w, http.StatusOK, d)
}
// executableSubPath returns the directory containing the compiled agents.
func executableSubPath(id string) (string, error) {
// get the full distro info, so we can figure out the architecture
d, err := distro.FindOne(distro.ById(id))
if err != nil {
return "", fmt.Errorf("error finding distro %v: %v", id, err)
}
mainName := "main"
if strings.HasPrefix(d.Arch, "windows") {
mainName = "main.exe"
}
return filepath.Join("snapshot", d.Arch, mainName), nil
}
// Validate returns an instance of BadOptionsErr if the SpawnOptions object contains invalid
// data, SpawnLimitErr if the user is already at the spawned host limit, or some other untyped
// instance of Error if something fails during validation.
func (sm Spawn) Validate(so Options) error {
d, err := distro.FindOne(distro.ById(so.Distro))
if err != nil {
return BadOptionsErr{fmt.Sprintf("Invalid dist %v", so.Distro)}
}
if !d.SpawnAllowed {
return BadOptionsErr{fmt.Sprintf("Spawning not allowed for dist %v", so.Distro)}
}
// if the user already has too many active spawned hosts, deny the request
activeSpawnedHosts, err := host.Find(host.ByUserWithRunningStatus(so.UserName))
if err != nil {
return fmt.Errorf("Error occurred finding user's current hosts: %v", err)
}
if len(activeSpawnedHosts) >= MaxPerUser {
return SpawnLimitErr
}
// validate public key
rsa := "ssh-rsa"
dss := "ssh-dss"
isRSA := strings.HasPrefix(so.PublicKey, rsa)
isDSS := strings.HasPrefix(so.PublicKey, dss)
if !isRSA && !isDSS {
return BadOptionsErr{"key does not start with ssh-rsa or ssh-dss"}
}
sections := strings.Split(so.PublicKey, " ")
if len(sections) < 2 {
keyType := rsa
if sections[0] == dss {
keyType = dss
}
return BadOptionsErr{fmt.Sprintf("missing space after '%v'", keyType)}
}
// check for valid base64
if _, err = base64.StdEncoding.DecodeString(sections[1]); err != nil {
return BadOptionsErr{"key contains invalid base64 string"}
}
if d.UserData.File != "" {
if strings.TrimSpace(so.UserData) == "" {
return BadOptionsErr{}
}
var err error
switch d.UserData.Validate {
case distro.UserDataFormatFormURLEncoded:
_, err = url.ParseQuery(so.UserData)
case distro.UserDataFormatJSON:
var out map[string]interface{}
err = json.Unmarshal([]byte(so.UserData), &out)
case distro.UserDataFormatYAML:
var out map[string]interface{}
err = yaml.Unmarshal([]byte(so.UserData), &out)
}
if err != nil {
return BadOptionsErr{fmt.Sprintf("invalid %v: %v", d.UserData.Validate, err)}
}
}
return nil
}
// CreateHost spawns a host with the given options.
func (sm Spawn) CreateHost(so Options) (*host.Host, error) {
// load in the appropriate distro
d, err := distro.FindOne(distro.ById(so.Distro))
if err != nil {
return nil, err
}
// get the appropriate cloud manager
cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings)
if err != nil {
return nil, err
}
// spawn the host
h, err := cloudManager.SpawnInstance(d, so.UserName, true)
if err != nil {
return nil, err
}
// set the expiration time for the host
expireTime := h.CreationTime.Add(DefaultExpiration)
err = h.SetExpirationTime(expireTime)
if err != nil {
return h, evergreen.Logger.Errorf(slogger.ERROR,
"error setting expiration on host %v: %v", h.Id, err)
}
// set the user data, if applicable
if so.UserData != "" {
err = h.SetUserData(so.UserData)
if err != nil {
return h, evergreen.Logger.Errorf(slogger.ERROR,
"Failed setting userData on host %v: %v", h.Id, err)
}
}
// create a hostinit to take care of setting up the host
init := &hostinit.HostInit{
Settings: sm.settings,
}
// for making sure the host doesn't take too long to spawn
startTime := time.Now()
// spin until the host is ready for its setup script to be run
for {
// make sure we haven't been spinning for too long
if time.Now().Sub(startTime) > 15*time.Minute {
if err := h.SetDecommissioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err)
}
return nil, fmt.Errorf("host took too long to come up")
}
time.Sleep(5000 * time.Millisecond)
evergreen.Logger.Logf(slogger.INFO, "Checking if host %v is up and ready", h.Id)
// see if the host is ready for its setup script to be run
ready, err := init.IsHostReady(h)
if err != nil {
if err := h.SetDecommissioned(); err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err)
}
return nil, fmt.Errorf("error checking on host %v; decommissioning to save resources: %v",
h.Id, err)
}
// if the host is ready, move on to running the setup script
if ready {
break
}
}
evergreen.Logger.Logf(slogger.INFO, "Host %v is ready for its setup script to be run", h.Id)
// add any extra user-specified data into the setup script
if h.Distro.UserData.File != "" {
userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n",
strings.Replace(so.UserData, "\"", "\\\"", -1), h.Distro.UserData.File)
// prepend the setup script to add the userdata file
if strings.HasPrefix(h.Distro.Setup, "#!") {
firstLF := strings.Index(h.Distro.Setup, "\n")
h.Distro.Setup = h.Distro.Setup[0:firstLF+1] + userDataCmd + h.Distro.Setup[firstLF+1:]
} else {
h.Distro.Setup = userDataCmd + h.Distro.Setup
}
}
// modify the setup script to add the user's public key
h.Distro.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n",
so.PublicKey, h.Distro.User)
// replace expansions in the script
exp := command.NewExpansions(init.Settings.Expansions)
h.Distro.Setup, err = exp.ExpandString(h.Distro.Setup)
if err != nil {
return nil, fmt.Errorf("expansions error: %v", err)
}
// provision the host
err = init.ProvisionHost(h)
if err != nil {
return nil, fmt.Errorf("error provisioning host %v: %v", h.Id, err)
}
return h, nil
}
// CreateHost spawns a host with the given options.
func (sm Spawn) CreateHost(so Options, owner *user.DBUser) error {
// load in the appropriate distro
d, err := distro.FindOne(distro.ById(so.Distro))
if err != nil {
return err
}
// add any extra user-specified data into the setup script
if d.UserData.File != "" {
userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n",
strings.Replace(so.UserData, "\"", "\\\"", -1), d.UserData.File)
// prepend the setup script to add the userdata file
if strings.HasPrefix(d.Setup, "#!") {
firstLF := strings.Index(d.Setup, "\n")
d.Setup = d.Setup[0:firstLF+1] + userDataCmd + d.Setup[firstLF+1:]
} else {
d.Setup = userDataCmd + d.Setup
}
}
// modify the setup script to add the user's public key
d.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n", so.PublicKey, d.User)
// replace expansions in the script
exp := command.NewExpansions(sm.settings.Expansions)
d.Setup, err = exp.ExpandString(d.Setup)
if err != nil {
return fmt.Errorf("expansions error: %v", err)
}
// fake out replacing spot instances with on-demand equivalents
if d.Provider == ec2.SpotProviderName {
d.Provider = ec2.OnDemandProviderName
}
// get the appropriate cloud manager
cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings)
if err != nil {
return err
}
// spawn the host
provisionOptions := &host.ProvisionOptions{
LoadCLI: true,
TaskId: so.TaskId,
OwnerId: owner.Id,
}
expiration := DefaultExpiration
hostOptions := cloud.HostOptions{
ProvisionOptions: provisionOptions,
UserName: so.UserName,
ExpirationDuration: &expiration,
UserData: so.UserData,
UserHost: true,
}
_, err = cloudManager.SpawnInstance(d, hostOptions)
if err != nil {
return err
}
return nil
}