Beispiel #1
0
func (uis *UIServer) removeDistro(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["distro_id"]

	u := MustHaveUser(r)

	d, err := distro.FindOne(distro.ById(id))
	if err != nil {
		message := fmt.Sprintf("error finding distro: %v", err)
		PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
		http.Error(w, message, http.StatusInternalServerError)
		return
	}

	if err = distro.Remove(id); err != nil {
		message := fmt.Sprintf("error removing distro '%v': %v", id, err)
		PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
		http.Error(w, message, http.StatusInternalServerError)
		return
	}

	event.LogDistroRemoved(id, u.Username(), d)

	PushFlash(uis.CookieStore, r, w, NewSuccessFlash(fmt.Sprintf("Distro %v successfully removed.", id)))
	uis.WriteJSON(w, http.StatusOK, "distro successfully removed")
}
Beispiel #2
0
func (uis *UIServer) spawnPage(w http.ResponseWriter, r *http.Request) {
	flashes := PopFlashes(uis.CookieStore, r, w)
	projCtx := MustHaveProjectContext(r)

	var spawnDistro *distro.Distro
	var spawnTask *task.Task
	var err error
	if len(r.FormValue("distro_id")) > 0 {
		spawnDistro, err = distro.FindOne(distro.ById(r.FormValue("distro_id")))
		if err != nil {
			uis.LoggedError(w, r, http.StatusInternalServerError,
				fmt.Errorf("Error finding distro %v: %v", r.FormValue("distro_id"), err))
			return
		}
	}
	if len(r.FormValue("task_id")) > 0 {
		spawnTask, err = task.FindOne(task.ById(r.FormValue("task_id")))
		if err != nil {
			uis.LoggedError(w, r, http.StatusInternalServerError,
				fmt.Errorf("Error finding task %v: %v", r.FormValue("task_id"), err))
			return
		}
	}

	uis.WriteHTML(w, http.StatusOK, struct {
		ProjectData     projectContext
		User            *user.DBUser
		Flashes         []interface{}
		Distro          *distro.Distro
		Task            *task.Task
		MaxHostsPerUser int
	}{projCtx, GetUser(r), flashes, spawnDistro, spawnTask, spawn.MaxPerUser}, "base", "spawned_hosts.html", "base_angular.html", "menu.html")
}
Beispiel #3
0
// Call out to the embedded CloudManager to spawn hosts.  Takes in a map of
// distro -> number of hosts to spawn for the distro.
// Returns a map of distro -> hosts spawned, and an error if one occurs.
func (s *Scheduler) spawnHosts(newHostsNeeded map[string]int) (
	map[string][]host.Host, error) {

	// loop over the distros, spawning up the appropriate number of hosts
	// for each distro
	hostsSpawnedPerDistro := make(map[string][]host.Host)
	for distroId, numHostsToSpawn := range newHostsNeeded {

		if numHostsToSpawn == 0 {
			continue
		}

		hostsSpawnedPerDistro[distroId] = make([]host.Host, 0, numHostsToSpawn)
		for i := 0; i < numHostsToSpawn; i++ {
			d, err := distro.FindOne(distro.ById(distroId))
			if err != nil {
				evergreen.Logger.Logf(slogger.ERROR, "Failed to find distro '%v': %v", distroId, err)
			}

			allDistroHosts, err := host.Find(host.ByDistroId(distroId))
			if err != nil {
				evergreen.Logger.Logf(slogger.ERROR, "Error getting hosts for distro %v: %v", distroId, err)
				continue
			}

			if len(allDistroHosts) >= d.PoolSize {
				evergreen.Logger.Logf(slogger.ERROR, "Already at max (%v) hosts for distro '%v'",
					distroId,
					d.PoolSize)
				continue
			}

			cloudManager, err := providers.GetCloudManager(d.Provider, s.Settings)
			if err != nil {
				evergreen.Logger.Errorf(slogger.ERROR, "Error getting cloud manager for distro: %v", err)
				continue
			}

			hostOptions := cloud.HostOptions{
				UserName: evergreen.User,
				UserHost: false,
			}
			newHost, err := cloudManager.SpawnInstance(d, hostOptions)
			if err != nil {
				evergreen.Logger.Errorf(slogger.ERROR, "Error spawning instance: %v,",
					err)
				continue
			}
			hostsSpawnedPerDistro[distroId] =
				append(hostsSpawnedPerDistro[distroId], *newHost)

		}
		// if none were spawned successfully
		if len(hostsSpawnedPerDistro[distroId]) == 0 {
			delete(hostsSpawnedPerDistro, distroId)
		}
	}
	return hostsSpawnedPerDistro, nil
}
Beispiel #4
0
func (uis *UIServer) modifyDistro(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["distro_id"]

	u := MustHaveUser(r)

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		message := fmt.Sprintf("error reading request: %v", err)
		PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
		http.Error(w, message, http.StatusBadRequest)
		return
	}
	defer r.Body.Close()

	oldDistro, err := distro.FindOne(distro.ById(id))
	if err != nil {
		message := fmt.Sprintf("error finding distro: %v", err)
		PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
		http.Error(w, message, http.StatusInternalServerError)
		return
	}

	newDistro := *oldDistro

	// attempt to unmarshal data into distros field for type validation
	if err = json.Unmarshal(b, &newDistro); err != nil {
		message := fmt.Sprintf("error unmarshaling request: %v", err)
		PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
		http.Error(w, message, http.StatusBadRequest)
		return
	}

	// check that the resulting distro is valid
	vErrs := validator.CheckDistro(&newDistro, &uis.Settings, false)
	if len(vErrs) != 0 {
		for _, e := range vErrs {
			PushFlash(uis.CookieStore, r, w, NewErrorFlash(e.Error()))
		}
		uis.WriteJSON(w, http.StatusBadRequest, vErrs)
		return
	}

	if err = newDistro.Update(); err != nil {
		message := fmt.Sprintf("error updating distro: %v", err)
		PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
		http.Error(w, message, http.StatusBadRequest)
		return
	}

	event.LogDistroModified(id, u.Username(), newDistro)

	PushFlash(uis.CookieStore, r, w, NewSuccessFlash(fmt.Sprintf("Distro %v successfully updated.", id)))
	uis.WriteJSON(w, http.StatusOK, "distro successfully updated")
}
Beispiel #5
0
// AverageStatistics uses an agg pipeline that creates buckets given a time frame and finds the average scheduled ->
// start time for that time frame.
// One thing to note is that the average time is in milliseconds, not nanoseconds and must be converted.
func AverageStatistics(distroId string, bounds FrameBounds) (AvgBuckets, error) {

	// error out if the distro does not exist
	_, err := distro.FindOne(distro.ById(distroId))
	if err != nil {
		return nil, err
	}
	intBucketSize := util.FromNanoseconds(bounds.BucketSize)
	buckets := AvgBuckets{}
	pipeline := []bson.M{
		// find all tasks that have started within the time frame for a given distro and only valid statuses.
		{"$match": bson.M{
			task.StartTimeKey: bson.M{
				"$gte": bounds.StartTime,
				"$lte": bounds.EndTime,
			},
			// only need tasks that have already started or those that have finished,
			// not looking for tasks that have been scheduled but not started.
			task.StatusKey: bson.M{
				"$in": []string{evergreen.TaskStarted,
					evergreen.TaskFailed, evergreen.TaskSucceeded},
			},
			task.DistroIdKey: distroId,
		}},
		// project the difference in scheduled -> start, as well as the bucket
		{"$project": bson.M{
			"diff": bson.M{
				"$subtract": []interface{}{"$" + task.StartTimeKey, "$" + task.ScheduledTimeKey},
			},
			"b": bson.M{
				"$floor": bson.M{
					"$divide": []interface{}{
						bson.M{"$subtract": []interface{}{"$" + task.StartTimeKey, bounds.StartTime}},
						intBucketSize},
				},
			},
		}},
		{"$group": bson.M{
			"_id": "$b",
			"a":   bson.M{"$avg": "$diff"},
			"n":   bson.M{"$sum": 1},
		}},

		{"$sort": bson.M{
			"_id": 1,
		}},
	}

	if err := db.Aggregate(task.Collection, pipeline, &buckets); err != nil {
		return nil, err
	}
	return convertBucketsToNanoseconds(buckets, bounds), nil
}
Beispiel #6
0
func (uis *UIServer) getDistro(w http.ResponseWriter, r *http.Request) {
	id := mux.Vars(r)["distro_id"]

	d, err := distro.FindOne(distro.ById(id))
	if err != nil {
		message := fmt.Sprintf("error fetching distro '%v': %v", id, err)
		PushFlash(uis.CookieStore, r, w, NewErrorFlash(message))
		http.Error(w, message, http.StatusInternalServerError)
		return
	}

	uis.WriteJSON(w, http.StatusOK, d)
}
Beispiel #7
0
// executableSubPath returns the directory containing the compiled agents.
func executableSubPath(id string) (string, error) {

	// get the full distro info, so we can figure out the architecture
	d, err := distro.FindOne(distro.ById(id))
	if err != nil {
		return "", fmt.Errorf("error finding distro %v: %v", id, err)
	}

	mainName := "main"
	if strings.HasPrefix(d.Arch, "windows") {
		mainName = "main.exe"
	}

	return filepath.Join("snapshot", d.Arch, mainName), nil
}
Beispiel #8
0
// Validate returns an instance of BadOptionsErr if the SpawnOptions object contains invalid
// data, SpawnLimitErr if the user is already at the spawned host limit, or some other untyped
// instance of Error if something fails during validation.
func (sm Spawn) Validate(so Options) error {
	d, err := distro.FindOne(distro.ById(so.Distro))
	if err != nil {
		return BadOptionsErr{fmt.Sprintf("Invalid dist %v", so.Distro)}
	}

	if !d.SpawnAllowed {
		return BadOptionsErr{fmt.Sprintf("Spawning not allowed for dist %v", so.Distro)}
	}

	// if the user already has too many active spawned hosts, deny the request
	activeSpawnedHosts, err := host.Find(host.ByUserWithRunningStatus(so.UserName))
	if err != nil {
		return fmt.Errorf("Error occurred finding user's current hosts: %v", err)
	}

	if len(activeSpawnedHosts) >= MaxPerUser {
		return SpawnLimitErr
	}

	// validate public key
	rsa := "ssh-rsa"
	dss := "ssh-dss"
	isRSA := strings.HasPrefix(so.PublicKey, rsa)
	isDSS := strings.HasPrefix(so.PublicKey, dss)
	if !isRSA && !isDSS {
		return BadOptionsErr{"key does not start with ssh-rsa or ssh-dss"}
	}

	sections := strings.Split(so.PublicKey, " ")
	if len(sections) < 2 {
		keyType := rsa
		if sections[0] == dss {
			keyType = dss
		}
		return BadOptionsErr{fmt.Sprintf("missing space after '%v'", keyType)}
	}

	// check for valid base64
	if _, err = base64.StdEncoding.DecodeString(sections[1]); err != nil {
		return BadOptionsErr{"key contains invalid base64 string"}
	}

	if d.UserData.File != "" {
		if strings.TrimSpace(so.UserData) == "" {
			return BadOptionsErr{}
		}

		var err error
		switch d.UserData.Validate {
		case distro.UserDataFormatFormURLEncoded:
			_, err = url.ParseQuery(so.UserData)
		case distro.UserDataFormatJSON:
			var out map[string]interface{}
			err = json.Unmarshal([]byte(so.UserData), &out)
		case distro.UserDataFormatYAML:
			var out map[string]interface{}
			err = yaml.Unmarshal([]byte(so.UserData), &out)
		}

		if err != nil {
			return BadOptionsErr{fmt.Sprintf("invalid %v: %v", d.UserData.Validate, err)}
		}
	}
	return nil
}
Beispiel #9
0
// CreateHost spawns a host with the given options.
func (sm Spawn) CreateHost(so Options) (*host.Host, error) {

	// load in the appropriate distro
	d, err := distro.FindOne(distro.ById(so.Distro))
	if err != nil {
		return nil, err
	}

	// get the appropriate cloud manager
	cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings)
	if err != nil {
		return nil, err
	}

	// spawn the host
	h, err := cloudManager.SpawnInstance(d, so.UserName, true)
	if err != nil {
		return nil, err
	}

	// set the expiration time for the host
	expireTime := h.CreationTime.Add(DefaultExpiration)
	err = h.SetExpirationTime(expireTime)
	if err != nil {
		return h, evergreen.Logger.Errorf(slogger.ERROR,
			"error setting expiration on host %v: %v", h.Id, err)
	}

	// set the user data, if applicable
	if so.UserData != "" {
		err = h.SetUserData(so.UserData)
		if err != nil {
			return h, evergreen.Logger.Errorf(slogger.ERROR,
				"Failed setting userData on host %v: %v", h.Id, err)
		}
	}

	// create a hostinit to take care of setting up the host
	init := &hostinit.HostInit{
		Settings: sm.settings,
	}

	// for making sure the host doesn't take too long to spawn
	startTime := time.Now()

	// spin until the host is ready for its setup script to be run
	for {

		// make sure we haven't been spinning for too long
		if time.Now().Sub(startTime) > 15*time.Minute {
			if err := h.SetDecommissioned(); err != nil {
				evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err)
			}
			return nil, fmt.Errorf("host took too long to come up")
		}

		time.Sleep(5000 * time.Millisecond)

		evergreen.Logger.Logf(slogger.INFO, "Checking if host %v is up and ready", h.Id)

		// see if the host is ready for its setup script to be run
		ready, err := init.IsHostReady(h)
		if err != nil {
			if err := h.SetDecommissioned(); err != nil {
				evergreen.Logger.Logf(slogger.ERROR, "error decommissioning host %v: %v", h.Id, err)
			}
			return nil, fmt.Errorf("error checking on host %v; decommissioning to save resources: %v",
				h.Id, err)
		}

		// if the host is ready, move on to running the setup script
		if ready {
			break
		}

	}

	evergreen.Logger.Logf(slogger.INFO, "Host %v is ready for its setup script to be run", h.Id)

	// add any extra user-specified data into the setup script
	if h.Distro.UserData.File != "" {
		userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n",
			strings.Replace(so.UserData, "\"", "\\\"", -1), h.Distro.UserData.File)
		// prepend the setup script to add the userdata file
		if strings.HasPrefix(h.Distro.Setup, "#!") {
			firstLF := strings.Index(h.Distro.Setup, "\n")
			h.Distro.Setup = h.Distro.Setup[0:firstLF+1] + userDataCmd + h.Distro.Setup[firstLF+1:]
		} else {
			h.Distro.Setup = userDataCmd + h.Distro.Setup
		}
	}

	// modify the setup script to add the user's public key
	h.Distro.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n",
		so.PublicKey, h.Distro.User)

	// replace expansions in the script
	exp := command.NewExpansions(init.Settings.Expansions)
	h.Distro.Setup, err = exp.ExpandString(h.Distro.Setup)
	if err != nil {
		return nil, fmt.Errorf("expansions error: %v", err)
	}

	// provision the host
	err = init.ProvisionHost(h)
	if err != nil {
		return nil, fmt.Errorf("error provisioning host %v: %v", h.Id, err)
	}

	return h, nil
}
Beispiel #10
0
// CreateHost spawns a host with the given options.
func (sm Spawn) CreateHost(so Options, owner *user.DBUser) error {

	// load in the appropriate distro
	d, err := distro.FindOne(distro.ById(so.Distro))
	if err != nil {
		return err
	}
	// add any extra user-specified data into the setup script
	if d.UserData.File != "" {
		userDataCmd := fmt.Sprintf("echo \"%v\" > %v\n",
			strings.Replace(so.UserData, "\"", "\\\"", -1), d.UserData.File)
		// prepend the setup script to add the userdata file
		if strings.HasPrefix(d.Setup, "#!") {
			firstLF := strings.Index(d.Setup, "\n")
			d.Setup = d.Setup[0:firstLF+1] + userDataCmd + d.Setup[firstLF+1:]
		} else {
			d.Setup = userDataCmd + d.Setup
		}
	}

	// modify the setup script to add the user's public key
	d.Setup += fmt.Sprintf("\necho \"\n%v\" >> ~%v/.ssh/authorized_keys\n", so.PublicKey, d.User)

	// replace expansions in the script
	exp := command.NewExpansions(sm.settings.Expansions)
	d.Setup, err = exp.ExpandString(d.Setup)
	if err != nil {
		return fmt.Errorf("expansions error: %v", err)
	}

	// fake out replacing spot instances with on-demand equivalents
	if d.Provider == ec2.SpotProviderName {
		d.Provider = ec2.OnDemandProviderName
	}

	// get the appropriate cloud manager
	cloudManager, err := providers.GetCloudManager(d.Provider, sm.settings)
	if err != nil {
		return err
	}

	// spawn the host
	provisionOptions := &host.ProvisionOptions{
		LoadCLI: true,
		TaskId:  so.TaskId,
		OwnerId: owner.Id,
	}
	expiration := DefaultExpiration
	hostOptions := cloud.HostOptions{
		ProvisionOptions:   provisionOptions,
		UserName:           so.UserName,
		ExpirationDuration: &expiration,
		UserData:           so.UserData,
		UserHost:           true,
	}

	_, err = cloudManager.SpawnInstance(d, hostOptions)
	if err != nil {
		return err
	}

	return nil
}