Пример #1
0
func TestFinalizePatch(t *testing.T) {
	testutil.ConfigureIntegrationTest(t, patchTestConfig, "TestFinalizePatch")

	Convey("With FinalizePatch on a project and commit event generated from GetPatchedProject path",
		t, func() {
			configPatch := resetPatchSetup(t, configFilePath)
			Convey("a patched config should drive version creation", func() {
				project, err := GetPatchedProject(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				yamlBytes, err := yaml.Marshal(project)
				So(err, ShouldBeNil)
				configPatch.PatchedConfig = string(yamlBytes)
				version, err := model.FinalizePatch(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				So(version, ShouldNotBeNil)
				// ensure the relevant builds/tasks were created
				builds, err := build.Find(build.All)
				So(err, ShouldBeNil)
				So(len(builds), ShouldEqual, 1)
				So(len(builds[0].Tasks), ShouldEqual, 2)
				tasks, err := task.Find(task.All)
				So(err, ShouldBeNil)
				So(len(tasks), ShouldEqual, 2)
			})

			Convey("a patch that does not include the remote config should not "+
				"drive version creation", func() {
				patchedConfigFile := "fakeInPatchSoNotPatched"
				configPatch := resetPatchSetup(t, patchedConfigFile)
				project, err := GetPatchedProject(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				yamlBytes, err := yaml.Marshal(project)
				So(err, ShouldBeNil)
				configPatch.PatchedConfig = string(yamlBytes)
				version, err := model.FinalizePatch(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				So(version, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(version, ShouldNotBeNil)

				// ensure the relevant builds/tasks were created
				builds, err := build.Find(build.All)
				So(err, ShouldBeNil)
				So(len(builds), ShouldEqual, 1)
				So(len(builds[0].Tasks), ShouldEqual, 1)
				tasks, err := task.Find(task.All)
				So(err, ShouldBeNil)
				So(len(tasks), ShouldEqual, 1)
			})

			Reset(func() {
				db.Clear(distro.Collection)
			})
		})
}
Пример #2
0
// Deactivate any previously activated but undispatched
// tasks for the same build variant + display name + project combination
// as the task.
func DeactivatePreviousTasks(taskId, caller string) error {
	t, err := task.FindOne(task.ById(taskId))
	if err != nil {
		return err
	}
	statuses := []string{evergreen.TaskUndispatched}
	allTasks, err := task.Find(task.ByActivatedBeforeRevisionWithStatuses(t.RevisionOrderNumber, statuses, t.BuildVariant,
		t.DisplayName, t.Project))
	if err != nil {
		return err
	}
	for _, t := range allTasks {
		err = SetActiveState(t.Id, caller, false)
		if err != nil {
			return err
		}
		event.LogTaskDeactivated(t.Id, caller)
		// update the cached version of the task, in its build document to be deactivated
		if err = build.SetCachedTaskActivated(t.BuildId, t.Id, false); err != nil {
			return err
		}
	}

	return nil
}
Пример #3
0
// flagTimedOutHeartbeats is a taskFlaggingFunc to flag any tasks whose
// heartbeats have timed out
func flagTimedOutHeartbeats() ([]doomedTaskWrapper, error) {

	evergreen.Logger.Logf(slogger.INFO, "Finding tasks with timed-out heartbeats...")

	// fetch any running tasks whose last heartbeat was too long in the past
	threshold := time.Now().Add(-HeartbeatTimeoutThreshold)

	tasks, err := task.Find(task.ByRunningLastHeartbeat(threshold))
	if err != nil {
		return nil, fmt.Errorf("error finding tasks with timed-out"+
			" heartbeats: %v", err)
	}

	// convert to be returned
	wrappers := make([]doomedTaskWrapper, 0, len(tasks))

	for _, task := range tasks {
		wrappers = append(wrappers, doomedTaskWrapper{task, HeartbeatTimeout})
	}

	evergreen.Logger.Logf(slogger.INFO, "Found %v tasks whose heartbeats timed out",
		len(wrappers))

	return wrappers, nil
}
Пример #4
0
// FindRunnableTasks finds all tasks that are ready to be run.
// This works by fetching all undispatched tasks from the database,
// and filtering out any whose dependencies are not met.
func (self *DBTaskFinder) FindRunnableTasks() ([]task.Task, error) {

	// find all of the undispatched tasks
	undispatchedTasks, err := task.Find(task.IsUndispatched)
	if err != nil {
		return nil, err
	}

	// filter out any tasks whose dependencies are not met
	runnableTasks := make([]task.Task, 0, len(undispatchedTasks))
	dependencyCaches := make(map[string]task.Task)
	for _, task := range undispatchedTasks {
		depsMet, err := task.DependenciesMet(dependencyCaches)
		if err != nil {
			evergreen.Logger.Logf(slogger.ERROR, "Error checking dependencies for"+
				" task %v: %v", task.Id, err)
			continue
		}
		if depsMet {
			runnableTasks = append(runnableTasks, task)
		}
	}

	return runnableTasks, nil
}
Пример #5
0
// RefreshTasksCache updates a build document so that the tasks cache reflects the correct current
// state of the tasks it represents.
func RefreshTasksCache(buildId string) error {
	tasks, err := task.Find(task.ByBuildId(buildId).WithFields(task.IdKey, task.DisplayNameKey, task.StatusKey,
		task.DetailsKey, task.StartTimeKey, task.TimeTakenKey, task.ActivatedKey, task.DependsOnKey))
	if err != nil {
		return err
	}
	cache := CreateTasksCache(tasks)
	return build.SetTasksCache(buildId, cache)
}
Пример #6
0
func TestCreateTaskBuckets(t *testing.T) {
	testutil.HandleTestingErr(db.ClearCollections(task.Collection), t, "couldnt reset host")
	Convey("With a starting time and a minute bucket size and inserting tasks with different start and finish", t, func() {
		now := time.Now()
		bucketSize := time.Duration(10) * time.Second

		// -20 -> 20
		beforeStartHost := task.Task{Id: "beforeStartTask", StartTime: now.Add(time.Duration(-20) * time.Second), FinishTime: now.Add(time.Duration(20) * time.Second), Status: evergreen.TaskSucceeded}
		So(beforeStartHost.Insert(), ShouldBeNil)

		// 80 -> 120
		afterEndHost := task.Task{Id: "afterStartTask", StartTime: now.Add(time.Duration(80) * time.Second), FinishTime: now.Add(time.Duration(120) * time.Second), Status: evergreen.TaskFailed}
		So(afterEndHost.Insert(), ShouldBeNil)

		// 20 -> 40: shouldnt be added
		h1 := task.Task{Id: "h1", StartTime: now.Add(time.Duration(20) * time.Second), FinishTime: now.Add(time.Duration(40) * time.Second), Status: evergreen.TaskUndispatched}
		So(h1.Insert(), ShouldBeNil)

		// 10 -> 80
		h2 := task.Task{Id: "h2", StartTime: now.Add(time.Duration(10) * time.Second), FinishTime: now.Add(time.Duration(80) * time.Second), Status: evergreen.TaskSucceeded}
		So(h2.Insert(), ShouldBeNil)

		// 20 -> shouldnt be added
		neverEnding := task.Task{Id: "neverEnding", StartTime: now.Add(time.Duration(20) * time.Second), Status: evergreen.TaskSucceeded}
		So(neverEnding.Insert(), ShouldBeNil)

		// 5 -> 7
		sameBucket := task.Task{Id: "sameBucket", StartTime: now.Add(time.Duration(5) * time.Second), FinishTime: now.Add(time.Duration(7) * time.Second), Status: evergreen.TaskFailed}
		So(sameBucket.Insert(), ShouldBeNil)

		// 5 -> 30
		h4 := task.Task{Id: "h4", StartTime: now.Add(time.Duration(5) * time.Second), FinishTime: now.Add(time.Duration(30) * time.Second), Status: evergreen.TaskFailed}
		So(h4.Insert(), ShouldBeNil)

		endTime := now.Add(time.Duration(40) * time.Second)
		frameBounds := FrameBounds{
			StartTime:     now,
			EndTime:       endTime,
			NumberBuckets: 4,
			BucketSize:    bucketSize,
		}
		Convey("for four buckets of 10 seconds", func() {
			tasks, err := task.Find(task.ByTimeRun(now, endTime))
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 4)

			buckets, errors := CreateTaskBuckets(tasks, []task.Task{}, frameBounds)
			So(errors, ShouldBeEmpty)
			So(len(buckets), ShouldEqual, 4)
			So(int(buckets[0].TotalTime.Seconds()), ShouldEqual, 17)
			So(int(math.Ceil(buckets[1].TotalTime.Seconds())), ShouldEqual, 30)
			So(int(math.Ceil(buckets[2].TotalTime.Seconds())), ShouldEqual, 20)
		})

	})
}
Пример #7
0
// updateMakespans
func updateMakespans(b *build.Build) error {
	// find all tasks associated with the build
	tasks, err := task.Find(task.ByBuildId(b.Id))
	if err != nil {
		return err
	}

	depPath := FindPredictedMakespan(tasks)
	return b.UpdateMakespans(depPath.TotalTime, CalculateActualMakespan(tasks))
}
Пример #8
0
func TestDeletingBuild(t *testing.T) {

	Convey("With a build", t, func() {

		testutil.HandleTestingErr(db.Clear(build.Collection), t, "Error clearing"+
			" '%v' collection", build.Collection)

		b := &build.Build{
			Id: "build",
		}
		So(b.Insert(), ShouldBeNil)

		Convey("deleting it should remove it and all its associated"+
			" tasks from the database", func() {

			testutil.HandleTestingErr(db.ClearCollections(task.Collection), t, "Error"+
				" clearing '%v' collection", task.Collection)

			// insert two tasks that are part of the build, and one that isn't
			matchingTaskOne := &task.Task{
				Id:      "matchingOne",
				BuildId: b.Id,
			}
			So(matchingTaskOne.Insert(), ShouldBeNil)

			matchingTaskTwo := &task.Task{
				Id:      "matchingTwo",
				BuildId: b.Id,
			}
			So(matchingTaskTwo.Insert(), ShouldBeNil)

			nonMatchingTask := &task.Task{
				Id:      "nonMatching",
				BuildId: "blech",
			}
			So(nonMatchingTask.Insert(), ShouldBeNil)

			// delete the build, make sure only it and its tasks are deleted

			So(DeleteBuild(b.Id), ShouldBeNil)

			b, err := build.FindOne(build.ById(b.Id))
			So(err, ShouldBeNil)
			So(b, ShouldBeNil)

			matchingTasks, err := task.Find(task.ByBuildId("build"))
			So(err, ShouldBeNil)
			So(len(matchingTasks), ShouldEqual, 0)

			nonMatchingTask, err = task.FindOne(task.ById(nonMatchingTask.Id))
			So(err, ShouldBeNil)
			So(nonMatchingTask, ShouldNotBeNil)
		})
	})
}
Пример #9
0
// addRecDeps recursively finds all dependencies of tasks and adds them to tasks and uiDeps.
// done is a hashtable of task IDs whose dependencies we have found.
// TODO EVG-614: delete this function once Task.DependsOn includes all recursive dependencies.
func addRecDeps(tasks map[string]task.Task, uiDeps map[string]uiDep, done map[string]bool) error {
	curTask := make(map[string]bool)
	depIds := make([]string, 0)
	for _, t := range tasks {
		if _, ok := done[t.Id]; !ok {
			for _, dep := range t.DependsOn {
				depIds = append(depIds, dep.TaskId)
			}
			curTask[t.Id] = true
		}
	}

	if len(depIds) == 0 {
		return nil
	}

	deps, err := task.Find(task.ByIds(depIds).WithFields(task.DisplayNameKey, task.StatusKey, task.ActivatedKey,
		task.BuildVariantKey, task.DetailsKey, task.DependsOnKey))

	if err != nil {
		return err
	}

	for _, dep := range deps {
		tasks[dep.Id] = dep
	}

	for _, t := range tasks {
		if _, ok := curTask[t.Id]; ok {
			for _, dep := range t.DependsOn {
				if uid, ok := uiDeps[dep.TaskId]; !ok ||
					// only replace if the current uiDep is not strict and not recursive
					(uid.RequiredStatus == model.AllStatuses && !uid.Recursive) {
					depTask := tasks[dep.TaskId]
					uiDeps[depTask.Id] = uiDep{
						Id:             depTask.Id,
						Name:           depTask.DisplayName,
						Status:         depTask.Status,
						RequiredStatus: dep.Status,
						Activated:      depTask.Activated,
						BuildVariant:   depTask.BuildVariant,
						Details:        depTask.Details,
						Recursive:      true,
					}
				}
			}
			done[t.Id] = true
		}
	}

	return addRecDeps(tasks, uiDeps, done)
}
Пример #10
0
// This is used to pull recently finished tasks
func getRecentlyFinishedTasks(notificationKey *NotificationKey) (tasks []task.Task, err error) {
	if cachedProjectRecords[notificationKey.String()] == nil {

		tasks, err = task.Find(task.ByRecentlyFinished(lastProjectNotificationTime[notificationKey.Project],
			notificationKey.Project, notificationKey.NotificationRequester))
		if err != nil {
			return nil, err
		}
		cachedProjectRecords[notificationKey.String()] = tasks
		return tasks, err
	}
	return cachedProjectRecords[notificationKey.String()].([]task.Task), nil
}
Пример #11
0
// getTaskDependencies returns the uiDeps for the task and its status (either its original status,
// "blocked", or "pending")
func getTaskDependencies(t *task.Task) ([]uiDep, string, error) {
	depIds := []string{}
	for _, dep := range t.DependsOn {
		depIds = append(depIds, dep.TaskId)
	}
	dependencies, err := task.Find(task.ByIds(depIds).WithFields(task.DisplayNameKey, task.StatusKey,
		task.ActivatedKey, task.BuildVariantKey, task.DetailsKey, task.DependsOnKey))
	if err != nil {
		return nil, "", err
	}

	idToUiDep := make(map[string]uiDep)
	// match each task with its dependency requirements
	for _, depTask := range dependencies {
		for _, dep := range t.DependsOn {
			if dep.TaskId == depTask.Id {
				idToUiDep[depTask.Id] = uiDep{
					Id:             depTask.Id,
					Name:           depTask.DisplayName,
					Status:         depTask.Status,
					RequiredStatus: dep.Status,
					Activated:      depTask.Activated,
					BuildVariant:   depTask.BuildVariant,
					Details:        depTask.Details,
					//TODO EVG-614: add "Recursive: dep.Recursive," once Task.DependsOn includes all recursive dependencies
				}
			}
		}
	}

	idToDep := make(map[string]task.Task)
	for _, dep := range dependencies {
		idToDep[dep.Id] = dep
	}

	// TODO EVG 614: delete this section once Task.DependsOn includes all recursive dependencies
	err = addRecDeps(idToDep, idToUiDep, make(map[string]bool))
	if err != nil {
		return nil, "", err
	}

	// set the status for each of the uiDeps as "blocked" or "pending" if appropriate
	// and get the status for task
	status := setBlockedOrPending(*t, idToDep, idToUiDep)

	uiDeps := make([]uiDep, 0, len(idToUiDep))
	for _, dep := range idToUiDep {
		uiDeps = append(uiDeps, dep)
	}
	return uiDeps, status, nil
}
// computeRunningTasksDuration returns the estimated time to completion of all
// currently running tasks for a given distro given its hosts
func computeRunningTasksDuration(existingDistroHosts []host.Host,
	taskDurations model.ProjectTaskDurations) (runningTasksDuration float64,
	err error) {

	runningTaskIds := []string{}

	for _, existingDistroHost := range existingDistroHosts {
		if existingDistroHost.RunningTask != "" {
			runningTaskIds = append(runningTaskIds,
				existingDistroHost.RunningTask)
		}
	}

	// if this distro's hosts are all free, return immediately
	if len(runningTaskIds) == 0 {
		return
	}

	runningTasksMap := make(map[string]task.Task)
	runningTasks, err := task.Find(task.ByIds(runningTaskIds))
	if err != nil {
		return runningTasksDuration, err
	}

	// build a map of task id => task
	for _, runningTask := range runningTasks {
		runningTasksMap[runningTask.Id] = runningTask
	}

	// compute the total time to completion for running tasks
	for _, runningTaskId := range runningTaskIds {
		runningTask, ok := runningTasksMap[runningTaskId]
		if !ok {
			return runningTasksDuration, fmt.Errorf("Unable to find running "+
				"task with _id %v", runningTaskId)
		}
		expectedDuration := model.GetTaskExpectedDuration(runningTask,
			taskDurations)
		elapsedTime := time.Now().Sub(runningTask.StartTime)
		if elapsedTime > expectedDuration {
			// probably an outlier; or an unknown data point
			continue
		}
		runningTasksDuration += expectedDuration.Seconds() -
			elapsedTime.Seconds()
	}
	return
}
Пример #13
0
// Given a task name and a slice of versions, return the appropriate sibling
// groups of tasks.  They will be sorted by ascending revision order number,
// unless reverseOrder is true, in which case they will be sorted
// descending.
func getTaskDrawerItems(displayName string, variant string, reverseOrder bool, versions []version.Version) ([]taskDrawerItem, error) {

	orderNumbers := make([]int, 0, len(versions))
	for _, v := range versions {
		orderNumbers = append(orderNumbers, v.RevisionOrderNumber)
	}

	revisionSort := task.RevisionOrderNumberKey
	if reverseOrder {
		revisionSort = "-" + revisionSort
	}

	tasks, err := task.Find(task.ByOrderNumbersForNameAndVariant(orderNumbers, displayName, variant).Sort([]string{revisionSort}))

	if err != nil {
		return nil, fmt.Errorf("error getting sibling tasks: %v", err)
	}
	return createSiblingTaskGroups(tasks, versions), nil
}
Пример #14
0
// getUiTaskCache takes a build object and returns a slice of
// uiTask objects suitable for front-end
func getUiTaskCache(build *build.Build) ([]uiTask, error) {
	tasks, err := task.Find(task.ByBuildId(build.Id))
	if err != nil {
		return nil, err
	}

	idToTask := make(map[string]task.Task)
	for _, task := range tasks {
		idToTask[task.Id] = task
	}

	// Insert the tasks in the same order as the task cache
	uiTasks := make([]uiTask, 0, len(build.Tasks))
	for _, taskCache := range build.Tasks {
		taskAsUI := uiTask{Task: idToTask[taskCache.Id]}
		uiTasks = append(uiTasks, taskAsUI)
	}
	return uiTasks, nil
}
Пример #15
0
// getTaskForVariant finds a task by name and variant and finds
// the document in the json collection associated with that task's id.
func getTaskForVariant(w http.ResponseWriter, r *http.Request) {
	t := plugin.GetTask(r)
	if t == nil {
		http.Error(w, "task not found", http.StatusNotFound)
		return
	}
	name := mux.Vars(r)["name"]
	taskName := mux.Vars(r)["task_name"]
	variantId := mux.Vars(r)["variant"]
	// Find the task for the other variant, if it exists
	ts, err := task.Find(db.Query(bson.M{task.VersionKey: t.Version, task.BuildVariantKey: variantId,
		task.DisplayNameKey: taskName}).Limit(1))
	if err != nil {
		if err == mgo.ErrNotFound {
			plugin.WriteJSON(w, http.StatusNotFound, nil)
			return
		}
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if len(ts) == 0 {
		plugin.WriteJSON(w, http.StatusNotFound, nil)
		return
	}
	otherVariantTask := ts[0]

	var jsonForTask TaskJSON
	err = db.FindOneQ(collection, db.Query(bson.M{TaskIdKey: otherVariantTask.Id, NameKey: name}), &jsonForTask)
	if err != nil {
		if err == mgo.ErrNotFound {
			plugin.WriteJSON(w, http.StatusNotFound, nil)
			return
		}
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	if len(r.FormValue("full")) != 0 { // if specified, include the json data's container as well
		plugin.WriteJSON(w, http.StatusOK, jsonForTask)
		return
	}
	plugin.WriteJSON(w, http.StatusOK, jsonForTask.Data)
}
Пример #16
0
// CreateAllHostUtilizationBuckets aggregates each bucket by creating a time frame given the number of days back
// and the granularity wanted (ie. days, minutes, seconds, hours) all in seconds. It returns a list of Host utilization
// information for each bucket.
func CreateAllHostUtilizationBuckets(daysBack, granularity int) ([]HostUtilizationBucket, error) {
	bounds := CalculateBounds(daysBack, granularity)
	// find non-static hosts
	dynamicHosts, err := host.Find(host.ByDynamicWithinTime(bounds.StartTime, bounds.EndTime))
	if err != nil {
		return nil, err
	}
	// find static hosts
	staticHosts, err := host.Find(host.AllStatic)
	if err != nil {
		return nil, err
	}

	dynamicBuckets, _ := CreateHostBuckets(dynamicHosts, bounds)
	staticBuckets, _ := CreateHostBuckets(staticHosts, bounds)

	tasks, err := task.Find(task.ByTimeRun(bounds.StartTime, bounds.EndTime).WithFields(task.StartTimeKey, task.FinishTimeKey, task.HostIdKey))
	if err != nil {
		return nil, err
	}

	oldTasks, err := task.FindOld(task.ByTimeRun(bounds.StartTime, bounds.EndTime))
	if err != nil {
		return nil, err
	}

	taskBuckets, _ := CreateTaskBuckets(tasks, oldTasks, bounds)
	bucketData := []HostUtilizationBucket{}
	for i, staticBucket := range staticBuckets {
		b := HostUtilizationBucket{
			StaticHost:  staticBucket.TotalTime,
			DynamicHost: dynamicBuckets[i].TotalTime,
			Task:        taskBuckets[i].TotalTime,
			StartTime:   bounds.StartTime.Add(time.Duration(i) * bounds.BucketSize),
			EndTime:     bounds.StartTime.Add(time.Duration(i+1) * bounds.BucketSize),
		}
		bucketData = append(bucketData, b)

	}
	return bucketData, nil
}
Пример #17
0
// RestartBuild restarts completed tasks associated with a given buildId.
// If abortInProgress is true, it also sets the abort flag on any in-progress tasks.
func RestartBuild(buildId string, taskIds []string, abortInProgress bool, caller string) error {
	// restart all the 'not in-progress' tasks for the build
	allTasks, err := task.Find(task.ByIdsBuildAndStatus(taskIds, buildId, task.CompletedStatuses))
	if err != nil && err != mgo.ErrNotFound {
		return err
	}

	for _, t := range allTasks {
		if t.DispatchTime != util.ZeroTime {
			err = resetTask(t.Id)
			if err != nil {
				return fmt.Errorf("Restarting build %v failed, could not task.reset on task: %v",
					buildId, t.Id, err)
			}
		}
	}

	if abortInProgress {
		// abort in-progress tasks in this build
		_, err = task.UpdateAll(
			bson.M{
				task.BuildIdKey: buildId,
				task.StatusKey: bson.M{
					"$in": evergreen.AbortableStatuses,
				},
			},
			bson.M{
				"$set": bson.M{
					task.AbortedKey: true,
				},
			},
		)
		if err != nil {
			return err
		}
	}

	return build.UpdateActivation(buildId, true, caller)
}
Пример #18
0
// Given a patch version and set of variant/task pairs, creates any tasks that don't exist yet,
// within the set of already existing builds.
func AddNewTasksForPatch(p *patch.Patch, patchVersion *version.Version, project *Project, pairs TVPairSet) error {
	builds, err := build.Find(build.ByIds(patchVersion.BuildIds).WithFields(build.IdKey, build.BuildVariantKey))
	if err != nil {
		return err
	}

	for _, b := range builds {
		// Find the set of task names that already exist for the given build
		tasksInBuild, err := task.Find(task.ByBuildId(b.Id).WithFields(task.DisplayNameKey))
		if err != nil {
			return err
		}
		// build an index to keep track of which tasks already exist
		existingTasksIndex := map[string]bool{}
		for _, t := range tasksInBuild {
			existingTasksIndex[t.DisplayName] = true
		}
		// if the patch is activated, treat the build as activated
		b.Activated = p.Activated

		// build a list of tasks that haven't been created yet for the given variant, but have
		// a record in the TVPairSet indicating that it should exist
		tasksToAdd := []string{}
		for _, taskname := range pairs.TaskNames(b.BuildVariant) {
			if _, ok := existingTasksIndex[taskname]; ok {
				continue
			}
			tasksToAdd = append(tasksToAdd, taskname)
		}
		if len(tasksToAdd) == 0 { // no tasks to add, so we do nothing.
			continue
		}
		// Add the new set of tasks to the build.
		if _, err = AddTasksToBuild(&b, project, patchVersion, tasksToAdd); err != nil {
			return err
		}
	}
	return nil
}
Пример #19
0
// GetFailedTests returns a mapping of task id to a slice of failed tasks
// extracted from a pipeline of aggregated tasks
func (self *taskHistoryIterator) GetFailedTests(aggregatedTasks *mgo.Pipe) (map[string][]task.TestResult, error) {
	// get the ids of the failed task
	var failedTaskIds []string
	var taskHistory TaskHistory
	iter := aggregatedTasks.Iter()
	for {
		if iter.Next(&taskHistory) {
			for _, task := range taskHistory.Tasks {
				if task.Status == evergreen.TaskFailed {
					failedTaskIds = append(failedTaskIds, task.Id)
				}
			}
		} else {
			break
		}
	}

	if err := iter.Err(); err != nil {
		return nil, err
	}

	// find all the relevant failed tests
	failedTestsMap := make(map[string][]task.TestResult)
	tasks, err := task.Find(task.ByIds(failedTaskIds).WithFields(task.IdKey, task.TestResultsKey))
	if err != nil {
		return nil, err
	}

	// create the mapping of the task id to the list of failed tasks
	for _, task := range tasks {
		for _, test := range task.TestResults {
			if test.Status == evergreen.TestFailedStatus {
				failedTestsMap[task.Id] = append(failedTestsMap[task.Id], test)
			}
		}
	}
	return failedTestsMap, nil
}
Пример #20
0
func TestBuildSetPriority(t *testing.T) {

	Convey("With a build", t, func() {

		testutil.HandleTestingErr(db.ClearCollections(build.Collection, task.Collection), t,
			"Error clearing test collection")

		b := &build.Build{
			Id: "build",
		}
		So(b.Insert(), ShouldBeNil)

		taskOne := &task.Task{Id: "taskOne", BuildId: b.Id}
		So(taskOne.Insert(), ShouldBeNil)

		taskTwo := &task.Task{Id: "taskTwo", BuildId: b.Id}
		So(taskTwo.Insert(), ShouldBeNil)

		taskThree := &task.Task{Id: "taskThree", BuildId: b.Id}
		So(taskThree.Insert(), ShouldBeNil)

		Convey("setting its priority should update the priority"+
			" of all its tasks in the database", func() {

			So(SetBuildPriority(b.Id, 42), ShouldBeNil)

			tasks, err := task.Find(task.ByBuildId(b.Id))
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 3)
			So(tasks[0].Priority, ShouldEqual, 42)
			So(tasks[1].Priority, ShouldEqual, 42)
			So(tasks[2].Priority, ShouldEqual, 42)
		})

	})

}
Пример #21
0
func TestCLIFunctions(t *testing.T) {
	testutil.ConfigureIntegrationTest(t, testConfig, "TestCLIFunctions")

	Convey("with API test server running", t, func() {
		// create a test API server
		testServer, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)

		// create a test user
		So(db.Clear(user.Collection), ShouldBeNil)
		So(db.Clear(patch.Collection), ShouldBeNil)
		So(db.Clear(model.ProjectRefCollection), ShouldBeNil)
		So((&user.DBUser{Id: "testuser", APIKey: "testapikey", EmailAddress: "*****@*****.**"}).Insert(), ShouldBeNil)
		localConfBytes, err := ioutil.ReadFile("testdata/sample.yml")
		So(err, ShouldBeNil)

		projectRef := &model.ProjectRef{
			Identifier:  "sample",
			Owner:       "evergreen-ci",
			Repo:        "sample",
			RepoKind:    "github",
			Branch:      "master",
			RemotePath:  "evergreen.yml",
			LocalConfig: string(localConfBytes),
			Enabled:     true,
			BatchTime:   180,
		}
		So(projectRef.Insert(), ShouldBeNil)

		// create a settings file for the command line client
		settings := Settings{
			APIServerHost: testServer.URL + "/api",
			UIServerHost:  "http://dev-evg.mongodb.com",
			APIKey:        "testapikey",
			User:          "******",
		}
		settingsFile, err := ioutil.TempFile("", "settings")
		So(err, ShouldBeNil)
		settingsBytes, err := yaml.Marshal(settings)
		So(err, ShouldBeNil)
		_, err = settingsFile.Write(settingsBytes)
		So(err, ShouldBeNil)
		settingsFile.Close()
		t.Log("Wrote settings file to ", settingsFile.Name())

		ac, _, err := getAPIClient(&Options{settingsFile.Name()})
		So(err, ShouldBeNil)

		Convey("check that creating a patch works", func() {
			Convey("user should start with no patches present", func() {
				patches, err := ac.GetPatches()
				So(err, ShouldBeNil)
				So(len(patches), ShouldEqual, 0)
			})

			Convey("Creating a simple patch should be successful", func() {
				patchSub := patchSubmission{"sample",
					testPatch,
					"sample patch",
					"3c7bfeb82d492dc453e7431be664539c35b5db4b",
					"all",
					[]string{"all"},
					false}

				newPatch, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)

				Convey("Newly created patch should be fetchable via API", func() {
					patches, err := ac.GetPatches()
					So(err, ShouldBeNil)
					So(len(patches), ShouldEqual, 1)
				})

				Convey("Adding a module to the patch should work", func() {
					err = ac.UpdatePatchModule(newPatch.Id.Hex(), "render-module", testPatch, "1e5232709595db427893826ce19289461cba3f75")
					So(err, ShouldBeNil)
					patches, err := ac.GetPatches()
					So(err, ShouldBeNil)
					So(patches[0].Patches[0].ModuleName, ShouldEqual, "")
					So(patches[0].Patches[1].ModuleName, ShouldEqual, "render-module")
					Convey("Removing the module from the patch should work", func() {
						So(ac.DeletePatchModule(newPatch.Id.Hex(), "render-module"), ShouldBeNil)
						patches, err := ac.GetPatches()
						So(err, ShouldBeNil)
						So(len(patches[0].Patches), ShouldEqual, 1)
						Convey("Finalizing the patch should work", func() {
							// First double check that the patch starts with no "version" field
							So(patches[0].Version, ShouldEqual, "")
							So(ac.FinalizePatch(newPatch.Id.Hex()), ShouldBeNil)
							patches, err := ac.GetPatches()
							So(err, ShouldBeNil)
							// After finalizing, the patch should now have a version populated
							So(patches[0].Version, ShouldNotEqual, "")
							Convey("Cancelling the patch should work", func() {
								So(ac.CancelPatch(newPatch.Id.Hex()), ShouldBeNil)
								patches, err := ac.GetPatches()
								So(err, ShouldBeNil)
								// After cancelling, tasks in the version should be deactivated
								tasks, err := task.Find(task.ByVersion(patches[0].Version))
								So(err, ShouldBeNil)
								for _, t := range tasks {
									So(t.Activated, ShouldBeFalse)
								}
							})
						})
					})
				})
			})

			Convey("Creating a complex patch should be successful", func() {
				patchSub := patchSubmission{"sample",
					testPatch,
					"sample patch #2",
					"3c7bfeb82d492dc453e7431be664539c35b5db4b",
					"osx-108",
					[]string{"failing_test"},
					false}

				_, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)

				Convey("Newly created patch should be fetchable via API", func() {
					patches, err := ac.GetPatches()
					Println(patches)
					So(err, ShouldBeNil)
					So(len(patches), ShouldEqual, 1)
					So(len(patches[0].BuildVariants), ShouldEqual, 1)
					So(patches[0].BuildVariants[0], ShouldEqual, "osx-108")
					So(len(patches[0].Tasks), ShouldEqual, 2)
					So(patches[0].Tasks, ShouldContain, "failing_test")
					Convey("and have expanded dependencies", func() {
						So(patches[0].Tasks, ShouldContain, "compile")
					})
				})
			})
			Convey("Listing variants or tasks for a project should list all variants", func() {
				tasks, err := ac.ListTasks("sample")
				So(err, ShouldBeNil)
				So(tasks, ShouldNotBeEmpty)
				So(len(tasks), ShouldEqual, 4)
			})
			Convey("Listing variants for a project should list all variants", func() {

				variants, err := ac.ListVariants("sample")
				So(err, ShouldBeNil)
				So(variants, ShouldNotBeEmpty)
				So(len(variants), ShouldEqual, 2)
			})

		})
	})
}
Пример #22
0
// RestartVersion restarts completed tasks associated with a given versionId.
// If abortInProgress is true, it also sets the abort flag on any in-progress tasks.
func RestartVersion(versionId string, taskIds []string, abortInProgress bool, caller string) error {
	// restart all the 'not in-progress' tasks for the version
	allTasks, err := task.Find(task.ByDispatchedWithIdsVersionAndStatus(taskIds, versionId, task.CompletedStatuses))

	if err != nil && err != mgo.ErrNotFound {
		return err
	}

	// archive all the tasks
	for _, t := range allTasks {
		if err := t.Archive(); err != nil {
			return fmt.Errorf("failed to archive task: %v", err)
		}
	}

	// Set all the task fields to indicate restarted
	err = task.ResetTasks(taskIds)
	if err != nil {
		return err
	}

	// TODO figure out a way to coalesce updates for task cache for the same build, so we
	// only need to do one update per-build instead of one per-task here.
	// Doesn't seem to be possible as-is because $ can only apply to one array element matched per
	// document.
	buildIdSet := map[string]bool{}
	for _, t := range allTasks {
		buildIdSet[t.BuildId] = true
		err = build.ResetCachedTask(t.BuildId, t.Id)
		if err != nil {
			return err
		}
	}

	// reset the build statuses, once per build
	buildIdList := make([]string, 0, len(buildIdSet))
	for k, _ := range buildIdSet {
		buildIdList = append(buildIdList, k)
	}

	// Set the build status for all the builds containing the tasks that we touched
	_, err = build.UpdateAllBuilds(
		bson.M{build.IdKey: bson.M{"$in": buildIdList}},
		bson.M{"$set": bson.M{build.StatusKey: evergreen.BuildStarted}},
	)

	if abortInProgress {
		// abort in-progress tasks in this build
		_, err = task.UpdateAll(
			bson.M{
				task.VersionKey: versionId,
				task.IdKey:      bson.M{"$in": taskIds},
				task.StatusKey:  bson.M{"$in": evergreen.AbortableStatuses},
			},
			bson.M{"$set": bson.M{task.AbortedKey: true}},
		)
		if err != nil {
			return err
		}
	}

	// update activation for all the builds
	for _, b := range buildIdList {
		err := build.UpdateActivation(b, true, caller)
		if err != nil {
			return err
		}
	}
	return nil

}
Пример #23
0
// Fetch versions until 'numVersionElements' elements are created, including
// elements consisting of multiple versions rolled-up into one.
// The skip value indicates how many versions back in time should be skipped
// before starting to fetch versions, the project indicates which project the
// returned versions should be a part of.
func getVersionsAndVariants(skip, numVersionElements int, project *model.Project) (versionVariantData, error) {
	// the final array of versions to return
	finalVersions := []waterfallVersion{}

	// keep track of the build variants we see
	bvSet := map[string]bool{}

	waterfallRows := map[string]waterfallRow{}

	// build variant mappings - used so we can store the display name as
	// the build variant field of a build
	buildVariantMappings := project.GetVariantMappings()

	// keep track of the last rolled-up version, so inactive versions can
	// be added
	var lastRolledUpVersion *waterfallVersion = nil

	// loop until we have enough from the db
	for len(finalVersions) < numVersionElements {

		// fetch the versions and associated builds
		versionsFromDB, buildsByVersion, err :=
			fetchVersionsAndAssociatedBuilds(project, skip, numVersionElements)

		if err != nil {
			return versionVariantData{}, fmt.Errorf("error fetching versions and builds:"+
				" %v", err)
		}

		// if we've reached the beginning of all versions
		if len(versionsFromDB) == 0 {
			break
		}

		// to fetch started tasks and failed tests for providing additional context
		// in a tooltip
		failedAndStartedTaskIds := []string{}

		// update the amount skipped
		skip += len(versionsFromDB)

		// create the necessary versions, rolling up inactive ones
		for _, versionFromDB := range versionsFromDB {

			// if we have hit enough versions, break out
			if len(finalVersions) == numVersionElements {
				break
			}

			// the builds for the version
			buildsInVersion := buildsByVersion[versionFromDB.Id]

			// see if there are any active tasks in the version
			versionActive := anyActiveTasks(buildsInVersion)

			// add any represented build variants to the set and initialize rows
			for _, b := range buildsInVersion {
				bvSet[b.BuildVariant] = true

				buildVariant := waterfallBuildVariant{
					Id:          b.BuildVariant,
					DisplayName: buildVariantMappings[b.BuildVariant],
				}

				if buildVariant.DisplayName == "" {
					buildVariant.DisplayName = b.BuildVariant +
						" (removed)"
				}

				if _, ok := waterfallRows[b.BuildVariant]; !ok {
					waterfallRows[b.BuildVariant] = waterfallRow{
						Builds:       map[string]waterfallBuild{},
						BuildVariant: buildVariant,
					}
				}

			}

			// if it is inactive, roll up the version and don't create any
			// builds for it
			if !versionActive {
				if lastRolledUpVersion == nil {
					lastRolledUpVersion = &waterfallVersion{RolledUp: true, RevisionOrderNumber: versionFromDB.RevisionOrderNumber}
				}

				// add the version metadata into the last rolled-up version
				lastRolledUpVersion.Ids = append(lastRolledUpVersion.Ids,
					versionFromDB.Id)
				lastRolledUpVersion.Authors = append(lastRolledUpVersion.Authors,
					versionFromDB.Author)
				lastRolledUpVersion.Errors = append(
					lastRolledUpVersion.Errors, waterfallVersionError{versionFromDB.Errors})
				lastRolledUpVersion.Warnings = append(
					lastRolledUpVersion.Warnings, waterfallVersionError{versionFromDB.Warnings})
				lastRolledUpVersion.Messages = append(
					lastRolledUpVersion.Messages, versionFromDB.Message)
				lastRolledUpVersion.Ignoreds = append(
					lastRolledUpVersion.Ignoreds, versionFromDB.Ignored)
				lastRolledUpVersion.CreateTimes = append(
					lastRolledUpVersion.CreateTimes, versionFromDB.CreateTime)
				lastRolledUpVersion.Revisions = append(
					lastRolledUpVersion.Revisions, versionFromDB.Revision)

				// move on to the next version
				continue
			}

			// add a pending rolled-up version, if it exists
			if lastRolledUpVersion != nil {
				finalVersions = append(finalVersions, *lastRolledUpVersion)
				lastRolledUpVersion = nil
			}

			// if we have hit enough versions, break out
			if len(finalVersions) == numVersionElements {
				break
			}

			// if the version can not be rolled up, create a fully fledged
			// version for it
			activeVersion := waterfallVersion{
				Ids:                 []string{versionFromDB.Id},
				Messages:            []string{versionFromDB.Message},
				Authors:             []string{versionFromDB.Author},
				CreateTimes:         []time.Time{versionFromDB.CreateTime},
				Revisions:           []string{versionFromDB.Revision},
				Errors:              []waterfallVersionError{{versionFromDB.Errors}},
				Warnings:            []waterfallVersionError{{versionFromDB.Warnings}},
				Ignoreds:            []bool{versionFromDB.Ignored},
				RevisionOrderNumber: versionFromDB.RevisionOrderNumber,
			}

			// add the builds to the waterfall row
			for _, b := range buildsInVersion {
				currentRow := waterfallRows[b.BuildVariant]
				buildForWaterfall := waterfallBuild{
					Id:      b.Id,
					Version: versionFromDB.Id,
				}

				tasks, statusCount := createWaterfallTasks(b.Tasks)
				buildForWaterfall.Tasks = tasks
				buildForWaterfall.TaskStatusCount = statusCount
				currentRow.Builds[versionFromDB.Id] = buildForWaterfall
				waterfallRows[b.BuildVariant] = currentRow
				for _, task := range buildForWaterfall.Tasks {
					if task.Status == evergreen.TaskFailed || task.Status == evergreen.TaskStarted {
						failedAndStartedTaskIds = append(failedAndStartedTaskIds, task.Id)
					}
				}
			}

			// add the version
			finalVersions = append(finalVersions, activeVersion)

		}

		failedAndStartedTasks, err := task.Find(task.ByIds(failedAndStartedTaskIds))
		if err != nil {
			return versionVariantData{}, fmt.Errorf("error fetching failed tasks:"+
				" %v", err)

		}
		addFailedAndStartedTests(waterfallRows, failedAndStartedTasks)
	}

	// if the last version was rolled-up, add it
	if lastRolledUpVersion != nil {
		finalVersions = append(finalVersions, *lastRolledUpVersion)
	}

	// create the list of display names for the build variants represented
	buildVariants := waterfallBuildVariants{}
	for name := range bvSet {
		displayName := buildVariantMappings[name]
		if displayName == "" {
			displayName = name + " (removed)"
		}
		buildVariants = append(buildVariants, waterfallBuildVariant{Id: name, DisplayName: displayName})
	}

	return versionVariantData{
		Rows:          waterfallRows,
		Versions:      finalVersions,
		BuildVariants: buildVariants,
	}, nil

}
Пример #24
0
func TestCLIFunctions(t *testing.T) {
	testutil.ConfigureIntegrationTest(t, testConfig, "TestCLIFunctions")

	Convey("with API test server running", t, func() {
		testSetup := setupCLITestHarness()

		ac, _, _, err := getAPIClients(&Options{testSetup.settingsFilePath})
		So(err, ShouldBeNil)

		Convey("check that creating a patch works", func() {
			Convey("user should start with no patches present", func() {
				patches, err := ac.GetPatches(0)
				So(err, ShouldBeNil)
				So(len(patches), ShouldEqual, 0)
			})

			Convey("Creating a simple patch should be successful", func() {
				patchSub := patchSubmission{"sample",
					testPatch,
					"sample patch",
					"3c7bfeb82d492dc453e7431be664539c35b5db4b",
					"all",
					[]string{"all"},
					false}

				newPatch, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)

				Convey("Newly created patch should be fetchable via API", func() {
					patches, err := ac.GetPatches(0)
					So(err, ShouldBeNil)
					So(len(patches), ShouldEqual, 1)
				})

				Convey("Adding a module to the patch should work", func() {
					err = ac.UpdatePatchModule(newPatch.Id.Hex(), "render-module", testPatch, "1e5232709595db427893826ce19289461cba3f75")
					So(err, ShouldBeNil)
					patches, err := ac.GetPatches(0)
					So(err, ShouldBeNil)
					So(patches[0].Patches[0].ModuleName, ShouldEqual, "")
					So(patches[0].Patches[1].ModuleName, ShouldEqual, "render-module")
					Convey("Removing the module from the patch should work", func() {
						So(ac.DeletePatchModule(newPatch.Id.Hex(), "render-module"), ShouldBeNil)
						patches, err := ac.GetPatches(0)
						So(err, ShouldBeNil)
						So(len(patches[0].Patches), ShouldEqual, 1)
						Convey("Finalizing the patch should work", func() {
							// First double check that the patch starts with no "version" field
							So(patches[0].Version, ShouldEqual, "")
							So(ac.FinalizePatch(newPatch.Id.Hex()), ShouldBeNil)
							patches, err := ac.GetPatches(0)
							So(err, ShouldBeNil)
							// After finalizing, the patch should now have a version populated
							So(patches[0].Version, ShouldNotEqual, "")
							Convey("Cancelling the patch should work", func() {
								So(ac.CancelPatch(newPatch.Id.Hex()), ShouldBeNil)
								patches, err := ac.GetPatches(0)
								So(err, ShouldBeNil)
								// After cancelling, tasks in the version should be deactivated
								tasks, err := task.Find(task.ByVersion(patches[0].Version))
								So(err, ShouldBeNil)
								for _, t := range tasks {
									So(t.Activated, ShouldBeFalse)
								}
							})
						})
					})
				})
			})

			Convey("Creating a patch without variants should be successful", func() {
				patchSub := patchSubmission{
					"sample",
					testPatch,
					"sample patch",
					"3c7bfeb82d492dc453e7431be664539c35b5db4b",
					"all",
					[]string{},
					false,
				}
				_, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)
			})

			Convey("Creating a complex patch should be successful", func() {
				patchSub := patchSubmission{"sample",
					testPatch,
					"sample patch #2",
					"3c7bfeb82d492dc453e7431be664539c35b5db4b",
					"osx-108",
					[]string{"failing_test"},
					false}

				_, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)

				Convey("Newly created patch should be fetchable via API", func() {
					patches, err := ac.GetPatches(1)
					So(err, ShouldBeNil)
					So(len(patches), ShouldEqual, 1)
					So(len(patches[0].BuildVariants), ShouldEqual, 1)
					So(patches[0].BuildVariants[0], ShouldEqual, "osx-108")
					So(len(patches[0].Tasks), ShouldEqual, 2)
					So(patches[0].Tasks, ShouldContain, "failing_test")
					Convey("and have expanded dependencies", func() {
						So(patches[0].Tasks, ShouldContain, "compile")
					})

					Convey("putting the patch again", func() {
						_, err := ac.PutPatch(patchSub)
						So(err, ShouldBeNil)
						Convey("GetPatches where n=1 should return 1 patch", func() {
							patches, err := ac.GetPatches(1)
							So(err, ShouldBeNil)
							So(len(patches), ShouldEqual, 1)
						})
						Convey("GetPatches where n=2 should return 2 patches", func() {
							patches, err := ac.GetPatches(2)
							So(err, ShouldBeNil)
							So(len(patches), ShouldEqual, 2)
						})
					})
				})
			})

			Convey("Creating an empty patch should not error out anything", func() {
				patchSub := patchSubmission{
					projectId:   "sample",
					patchData:   emptyPatch,
					description: "sample patch",
					base:        "3c7bfeb82d492dc453e7431be664539c35b5db4b",
					variants:    "all",
					tasks:       []string{"all"},
					finalize:    false}

				newPatch, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)

				Convey("Newly created patch should be fetchable via API", func() {
					patches, err := ac.GetPatches(0)
					So(err, ShouldBeNil)
					So(len(patches), ShouldEqual, 1)
				})

				Convey("Adding a module to the patch should still work as designed even with empty patch", func() {
					err = ac.UpdatePatchModule(newPatch.Id.Hex(), "render-module", emptyPatch, "1e5232709595db427893826ce19289461cba3f75")
					So(err, ShouldBeNil)
					patches, err := ac.GetPatches(0)
					So(err, ShouldBeNil)
					So(patches[0].Patches[0].ModuleName, ShouldEqual, "")
					So(patches[0].Patches[1].ModuleName, ShouldEqual, "render-module")
					Convey("Removing the module from the patch should work as designed even with empty patch", func() {
						So(ac.DeletePatchModule(newPatch.Id.Hex(), "render-module"), ShouldBeNil)
						patches, err := ac.GetPatches(0)
						So(err, ShouldBeNil)
						So(len(patches[0].Patches), ShouldEqual, 1)
						Convey("Finalizing the patch should start with no version field and then be populated", func() {
							So(patches[0].Version, ShouldEqual, "")
							So(ac.FinalizePatch(newPatch.Id.Hex()), ShouldBeNil)
							patches, err := ac.GetPatches(0)
							So(err, ShouldBeNil)
							So(patches[0].Version, ShouldNotEqual, "")
							Convey("Cancelling the patch should work and the version should be deactivated", func() {
								So(ac.CancelPatch(newPatch.Id.Hex()), ShouldBeNil)
								patches, err := ac.GetPatches(0)
								So(err, ShouldBeNil)
								tasks, err := task.Find(task.ByVersion(patches[0].Version))
								So(err, ShouldBeNil)
								for _, t := range tasks {
									So(t.Activated, ShouldBeFalse)
								}
							})
						})
					})
				})
			})
			Convey("Listing variants or tasks for a project should list all variants", func() {
				tasks, err := ac.ListTasks("sample")
				So(err, ShouldBeNil)
				So(tasks, ShouldNotBeEmpty)
				So(len(tasks), ShouldEqual, 4)
			})
			Convey("Listing variants for a project should list all variants", func() {

				variants, err := ac.ListVariants("sample")
				So(err, ShouldBeNil)
				So(variants, ShouldNotBeEmpty)
				So(len(variants), ShouldEqual, 2)
			})
			Convey("Creating a patch using 'all' as variants should schedule all variants", func() {
				patchSub := patchSubmission{"sample",
					testPatch,
					"sample patch #2",
					"3c7bfeb82d492dc453e7431be664539c35b5db4b",
					"all",
					[]string{"failing_test"},
					false}

				_, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)

				Convey("Newly created patch should be fetchable via API", func() {
					patches, err := ac.GetPatches(1)
					So(err, ShouldBeNil)
					So(len(patches), ShouldEqual, 1)
					So(len(patches[0].BuildVariants), ShouldEqual, 2)
					So(patches[0].BuildVariants, ShouldContain, "osx-108")
					So(patches[0].BuildVariants, ShouldContain, "ubuntu")
					So(len(patches[0].Tasks), ShouldEqual, 2)
					So(patches[0].Tasks, ShouldContain, "failing_test")
					Convey("and have expanded dependencies", func() {
						So(patches[0].Tasks, ShouldContain, "compile")
					})
				})
			})

			Convey("Creating a patch using 'all' as tasks should schedule all tasks", func() {
				patchSub := patchSubmission{"sample",
					testPatch,
					"sample patch #2",
					"3c7bfeb82d492dc453e7431be664539c35b5db4b",
					"osx-108",
					[]string{"all"},
					false}

				_, err := ac.PutPatch(patchSub)
				So(err, ShouldBeNil)

				Convey("Newly created patch should be fetchable via API", func() {
					patches, err := ac.GetPatches(1)
					So(err, ShouldBeNil)
					So(len(patches), ShouldEqual, 1)
					So(len(patches[0].BuildVariants), ShouldEqual, 1)
					So(patches[0].BuildVariants[0], ShouldEqual, "osx-108")
					So(len(patches[0].Tasks), ShouldEqual, 4)
					So(patches[0].Tasks, ShouldContain, "compile")
					So(patches[0].Tasks, ShouldContain, "passing_test")
					So(patches[0].Tasks, ShouldContain, "failing_test")
					So(patches[0].Tasks, ShouldContain, "timeout_test")
				})
			})

		})
	})
}
Пример #25
0
// UpdateBuildStatusForTask finds all the builds for a task and updates the
// status of the build based on the task's status.
func UpdateBuildAndVersionStatusForTask(taskId string) error {
	// retrieve the task by the task id
	t, err := task.FindOne(task.ById(taskId))
	if err != nil {
		return err
	}

	finishTime := time.Now()
	// get all of the tasks in the same build
	b, err := build.FindOne(build.ById(t.BuildId))
	if err != nil {
		return err
	}

	buildTasks, err := task.Find(task.ByBuildId(b.Id))
	if err != nil {
		return err
	}

	pushTaskExists := false
	for _, t := range buildTasks {
		if t.DisplayName == evergreen.PushStage {
			pushTaskExists = true
		}
	}

	failedTask := false
	pushSuccess := true
	pushCompleted := false
	finishedTasks := 0

	// update the build's status based on tasks for this build
	for _, t := range buildTasks {
		if task.IsFinished(t) {
			finishedTasks += 1
			// if it was a compile task, mark the build status accordingly
			if t.DisplayName == evergreen.CompileStage {
				if t.Status != evergreen.TaskSucceeded {
					failedTask = true
					finishedTasks = -1
					err = b.MarkFinished(evergreen.BuildFailed, finishTime)
					if err != nil {
						evergreen.Logger.Errorf(slogger.ERROR, "Error marking build as finished: %v", err)
						return err
					}
					break
				}
			} else if t.DisplayName == evergreen.PushStage {
				pushCompleted = true
				// if it's a finished push, check if it was successful
				if t.Status != evergreen.TaskSucceeded {
					err = b.UpdateStatus(evergreen.BuildFailed)
					if err != nil {
						evergreen.Logger.Errorf(slogger.ERROR, "Error updating build status: %v", err)
						return err
					}
					pushSuccess = false
				}
			} else {
				// update the build's status when a test task isn't successful
				if t.Status != evergreen.TaskSucceeded {
					err = b.UpdateStatus(evergreen.BuildFailed)
					if err != nil {
						evergreen.Logger.Errorf(slogger.ERROR, "Error updating build status: %v", err)
						return err
					}
					failedTask = true
				}
			}
		}
	}

	// if there are no failed tasks, mark the build as started
	if !failedTask {
		err = b.UpdateStatus(evergreen.BuildStarted)
		if err != nil {
			evergreen.Logger.Errorf(slogger.ERROR, "Error updating build status: %v", err)
			return err
		}
	}
	// if a compile task didn't fail, then the
	// build is only finished when both the compile
	// and test tasks are completed or when those are
	// both completed in addition to a push (a push
	// does not occur if there's a failed task)
	if finishedTasks >= len(buildTasks)-1 {
		if !failedTask {
			if pushTaskExists { // this build has a push task associated with it.
				if pushCompleted && pushSuccess { // the push succeeded, so mark the build as succeeded.
					err = b.MarkFinished(evergreen.BuildSucceeded, finishTime)
					if err != nil {
						evergreen.Logger.Errorf(slogger.ERROR, "Error marking build as finished: %v", err)
						return err
					}
				} else if pushCompleted && !pushSuccess { // the push failed, mark build failed.
					err = b.MarkFinished(evergreen.BuildFailed, finishTime)
					if err != nil {
						evergreen.Logger.Errorf(slogger.ERROR, "Error marking build as finished: %v", err)
						return err
					}
				} else {
					//This build does have a "push" task, but it hasn't finished yet
					//So do nothing, since we don't know the status yet.
				}
				if err = MarkVersionCompleted(b.Version, finishTime); err != nil {
					evergreen.Logger.Errorf(slogger.ERROR, "Error marking version as finished: %v", err)
					return err
				}
			} else { // this build has no push task. so go ahead and mark it success/failure.
				if err = b.MarkFinished(evergreen.BuildSucceeded, finishTime); err != nil {
					evergreen.Logger.Errorf(slogger.ERROR, "Error marking build as finished: %v", err)
					return err
				}
				if b.Requester == evergreen.PatchVersionRequester {
					if err = TryMarkPatchBuildFinished(b, finishTime); err != nil {
						evergreen.Logger.Errorf(slogger.ERROR, "Error marking patch as finished: %v", err)
						return err
					}
				}
				if err = MarkVersionCompleted(b.Version, finishTime); err != nil {
					evergreen.Logger.Errorf(slogger.ERROR, "Error marking version as finished: %v", err)
					return err
				}
			}
		} else {
			// some task failed
			if err = b.MarkFinished(evergreen.BuildFailed, finishTime); err != nil {
				evergreen.Logger.Errorf(slogger.ERROR, "Error marking build as finished: %v", err)
				return err
			}
			if b.Requester == evergreen.PatchVersionRequester {
				if err = TryMarkPatchBuildFinished(b, finishTime); err != nil {
					evergreen.Logger.Errorf(slogger.ERROR, "Error marking patch as finished: %v", err)
					return err
				}
			}
			if err = MarkVersionCompleted(b.Version, finishTime); err != nil {
				evergreen.Logger.Errorf(slogger.ERROR, "Error marking version as finished: %v", err)
				return err
			}
		}

		// update the build's makespan information if the task has finished
		err = updateMakespans(b)
		if err != nil {
			evergreen.Logger.Errorf(slogger.ERROR, "Error updating makespan information: %v", err)
			return err
		}
	}

	// this is helpful for when we restart a compile task
	if finishedTasks == 0 {
		err = b.UpdateStatus(evergreen.BuildCreated)
		if err != nil {
			evergreen.Logger.Errorf(slogger.ERROR, "Error updating build status: %v", err)
			return err
		}
	}

	return nil
}
Пример #26
0
func (uis *UIServer) allTaskQueues(w http.ResponseWriter, r *http.Request) {
	projCtx := MustHaveProjectContext(r)

	taskQueues, err := model.FindAllTaskQueues()
	if err != nil {
		uis.LoggedError(w, r, http.StatusInternalServerError,
			fmt.Errorf("Error finding task queues: %v", err))
		return
	}

	// cached map of version id to relevant patch
	cachedPatches := map[string]*patch.Patch{}

	// convert the task queues to the ui versions
	uiTaskQueues := []uiTaskQueue{}
	for _, tQ := range taskQueues {
		asUI := uiTaskQueue{
			Distro: tQ.Distro,
			Queue:  []uiTaskQueueItem{},
		}

		if len(tQ.Queue) == 0 {
			uiTaskQueues = append(uiTaskQueues, asUI)
			continue
		}

		// convert the individual task queue items
		taskIds := []string{}
		for _, item := range tQ.Queue {

			// cache the ids, for fetching the tasks from the db
			taskIds = append(taskIds, item.Id)
			queueItemAsUI := uiTaskQueueItem{
				Id:                  item.Id,
				DisplayName:         item.DisplayName,
				BuildVariant:        item.BuildVariant,
				RevisionOrderNumber: item.RevisionOrderNumber,
				Requester:           item.Requester,
				Revision:            item.Revision,
				Project:             item.Project,
			}
			asUI.Queue = append(asUI.Queue, queueItemAsUI)
		}

		// find all the relevant tasks
		tasks, err := task.Find(task.ByIds(taskIds).WithFields(task.VersionKey, task.BuildIdKey))
		if err != nil {
			msg := fmt.Sprintf("Error finding tasks: %v", err)
			evergreen.Logger.Errorf(slogger.ERROR, msg)
			http.Error(w, msg, http.StatusInternalServerError)
			return
		}

		// store all of the version and build ids in the relevant task queue
		// items
		for _, task := range tasks {
			// this sucks, but it's because we're not guaranteed the order out
			// of the db
			for idx, queueItemAsUI := range asUI.Queue {
				if queueItemAsUI.Id == task.Id {
					queueItemAsUI.Version = task.Version
					queueItemAsUI.Build = task.BuildId
					asUI.Queue[idx] = queueItemAsUI
				}
			}
		}

		// add all of the necessary patch info into the relevant task queue
		// items
		for idx, queueItemAsUI := range asUI.Queue {
			if queueItemAsUI.Requester == evergreen.PatchVersionRequester {
				// fetch the patch, if necessary
				var p *patch.Patch
				var ok bool
				if p, ok = cachedPatches[queueItemAsUI.Version]; ok {
					queueItemAsUI.User = p.Author
					asUI.Queue[idx] = queueItemAsUI
				} else {
					p, err = patch.FindOne(
						patch.ByVersion(queueItemAsUI.Version).WithFields(patch.AuthorKey),
					)
					if err != nil {
						msg := fmt.Sprintf("Error finding patch: %v", err)
						evergreen.Logger.Errorf(slogger.ERROR, msg)
						http.Error(w, msg, http.StatusInternalServerError)
						return
					}
					if p == nil {
						msg := fmt.Sprintf("Couldn't find patch for version %v", queueItemAsUI.Version)
						evergreen.Logger.Errorf(slogger.ERROR, msg)
						http.Error(w, msg, http.StatusInternalServerError)
						return
					}
					cachedPatches[queueItemAsUI.Version] = p
				}
				queueItemAsUI.User = p.Author
				asUI.Queue[idx] = queueItemAsUI

			}
		}

		uiTaskQueues = append(uiTaskQueues, asUI)

	}

	// add other useful statistics to view alongside queue
	idleHosts, err := host.Find(host.IsIdle)
	if err != nil {
		msg := fmt.Sprintf("Error finding idle hosts: %v", err)
		evergreen.Logger.Errorf(slogger.ERROR, msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	activeHosts, err := host.Find(host.IsLive)
	if err != nil {
		msg := fmt.Sprintf("Error finding active hosts: %v", err)
		evergreen.Logger.Errorf(slogger.ERROR, msg)
		http.Error(w, msg, http.StatusInternalServerError)
		return
	}
	idleStaticHostsCount := 0
	for _, host := range idleHosts {
		if host.Provider == evergreen.HostTypeStatic {
			idleStaticHostsCount++
		}
	}
	activeStaticHostsCount := 0
	for _, host := range activeHosts {
		if host.Provider == evergreen.HostTypeStatic {
			activeStaticHostsCount++
		}
	}
	hostStats := uiHostStatistics{
		ActiveHosts:       len(activeHosts),
		ActiveStaticHosts: activeStaticHostsCount,
		IdleHosts:         len(idleHosts),
		IdleStaticHosts:   idleStaticHostsCount,
	}

	uis.WriteHTML(w, http.StatusOK, struct {
		ProjectData projectContext
		User        *user.DBUser
		Flashes     []interface{}
		Data        uiResourceInfo
	}{projCtx, GetUser(r), []interface{}{}, uiResourceInfo{uiTaskQueues, hostStats}},
		"base", "task_queues.html", "base_angular.html", "menu.html")
}
Пример #27
0
func TestBuildMarkAborted(t *testing.T) {
	Convey("With a build", t, func() {

		testutil.HandleTestingErr(db.ClearCollections(build.Collection, task.Collection, version.Collection), t,
			"Error clearing test collection")

		v := &version.Version{
			Id: "v",
			BuildVariants: []version.BuildStatus{
				{
					BuildVariant: "bv",
					Activated:    true,
				},
			},
		}

		So(v.Insert(), ShouldBeNil)

		b := &build.Build{
			Id:           "build",
			Activated:    true,
			BuildVariant: "bv",
			Version:      "v",
		}
		So(b.Insert(), ShouldBeNil)

		Convey("when marking it as aborted", func() {

			Convey("it should be deactivated", func() {
				So(AbortBuild(b.Id, evergreen.DefaultTaskActivator), ShouldBeNil)
				b, err := build.FindOne(build.ById(b.Id))
				So(err, ShouldBeNil)
				So(b.Activated, ShouldBeFalse)
			})

			Convey("all abortable tasks for it should be aborted", func() {

				// insert two abortable tasks and one non-abortable task
				// for the correct build, and one abortable task for
				// a different build

				abortableOne := &task.Task{
					Id:      "abortableOne",
					BuildId: b.Id,
					Status:  evergreen.TaskStarted,
				}
				So(abortableOne.Insert(), ShouldBeNil)

				abortableTwo := &task.Task{
					Id:      "abortableTwo",
					BuildId: b.Id,
					Status:  evergreen.TaskDispatched,
				}
				So(abortableTwo.Insert(), ShouldBeNil)

				notAbortable := &task.Task{
					Id:      "notAbortable",
					BuildId: b.Id,
					Status:  evergreen.TaskSucceeded,
				}
				So(notAbortable.Insert(), ShouldBeNil)

				wrongBuildId := &task.Task{
					Id:      "wrongBuildId",
					BuildId: "blech",
					Status:  evergreen.TaskStarted,
				}
				So(wrongBuildId.Insert(), ShouldBeNil)

				// aborting the build should mark only the two abortable tasks
				// with the correct build id as aborted

				So(AbortBuild(b.Id, evergreen.DefaultTaskActivator), ShouldBeNil)

				abortedTasks, err := task.Find(task.ByAborted(true))
				So(err, ShouldBeNil)
				So(len(abortedTasks), ShouldEqual, 2)
				So(taskIdInSlice(abortedTasks, abortableOne.Id), ShouldBeTrue)
				So(taskIdInSlice(abortedTasks, abortableTwo.Id), ShouldBeTrue)
			})
		})
	})
}
Пример #28
0
func (uis *UIServer) taskHistoryPickaxe(w http.ResponseWriter, r *http.Request) {
	projCtx := MustHaveProjectContext(r)

	if projCtx.Project == nil {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}

	taskName := mux.Vars(r)["task_name"]

	highOrder, err := strconv.ParseInt(r.FormValue("high"), 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing high: `%s`", err.Error()), http.StatusBadRequest)
		return
	}
	lowOrder, err := strconv.ParseInt(r.FormValue("low"), 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing low: `%s`", err.Error()), http.StatusBadRequest)
		return
	}

	filter := struct {
		BuildVariants []string          `json:"buildVariants"`
		Tests         map[string]string `json:"tests"`
	}{}

	err = json.Unmarshal([]byte(r.FormValue("filter")), &filter)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error in filter: %v", err.Error()), http.StatusBadRequest)
		return
	}
	buildVariants := projCtx.Project.GetVariantsWithTask(taskName)

	onlyMatchingTasks := (r.FormValue("only_matching_tasks") == "true")

	// If there are no build variants, use all of them for the given task name.
	// Need this because without the build_variant specified, no amount of hinting
	// will get sort to use the proper index
	query := bson.M{
		"build_variant": bson.M{
			"$in": buildVariants,
		},
		"display_name": taskName,
		"order": bson.M{
			"$gte": lowOrder,
			"$lte": highOrder,
		},
		"branch": projCtx.Project.Identifier,
	}

	// If there are build variants, use them instead
	if len(filter.BuildVariants) > 0 {
		query["build_variant"] = bson.M{
			"$in": filter.BuildVariants,
		}
	}

	// If there are tests to filter by, create a big $elemMatch $or in the
	// projection to make sure we only get the tests we care about.
	elemMatchOr := make([]bson.M, 0)
	for test, result := range filter.Tests {
		regexp := fmt.Sprintf(testMatchRegex, test, test)
		if result == "ran" {
			// Special case: if asking for tasks where the test ran, don't care
			// about the test status
			elemMatchOr = append(elemMatchOr, bson.M{
				"test_file": bson.RegEx{regexp, ""},
			})
		} else {
			elemMatchOr = append(elemMatchOr, bson.M{
				"test_file": bson.RegEx{regexp, ""},
				"status":    result,
			})
		}
	}

	elemMatch := bson.M{"$or": elemMatchOr}

	// Special case: if only one test filter, don't need to use a $or
	if 1 == len(elemMatchOr) {
		elemMatch = elemMatchOr[0]
	}

	projection := bson.M{
		"_id":           1,
		"status":        1,
		"activated":     1,
		"time_taken":    1,
		"build_variant": 1,
	}

	if len(elemMatchOr) > 0 {
		projection["test_results"] = bson.M{
			"$elemMatch": elemMatch,
		}

		// If we only care about matching tasks, put the elemMatch in the query too
		if onlyMatchingTasks {
			query["test_results"] = bson.M{
				"$elemMatch": elemMatch,
			}
		}
	}

	last, err := task.Find(db.Query(query).Project(projection))

	if err != nil {
		http.Error(w, fmt.Sprintf("Error querying tasks: `%s`", err.Error()), http.StatusInternalServerError)
		return
	}

	uis.WriteJSON(w, http.StatusOK, last)
}
Пример #29
0
func TestBuildSetActivated(t *testing.T) {
	Convey("With a build", t, func() {

		testutil.HandleTestingErr(db.ClearCollections(build.Collection, task.Collection), t,
			"Error clearing test collection")

		Convey("when changing the activated status of the build to true", func() {
			Convey("the activated status of the build and all undispatched"+
				" tasks that are part of it should be set", func() {

				user := "******"

				b := &build.Build{
					Id:           "build",
					Activated:    true,
					BuildVariant: "bv",
				}
				So(b.Insert(), ShouldBeNil)

				// insert three tasks, with only one of them undispatched and
				// belonging to the correct build

				wrongBuildId := &task.Task{
					Id:        "wrongBuildId",
					BuildId:   "blech",
					Status:    evergreen.TaskUndispatched,
					Activated: true,
				}
				So(wrongBuildId.Insert(), ShouldBeNil)

				wrongStatus := &task.Task{
					Id:        "wrongStatus",
					BuildId:   b.Id,
					Status:    evergreen.TaskDispatched,
					Activated: true,
				}
				So(wrongStatus.Insert(), ShouldBeNil)

				matching := &task.Task{
					Id:        "matching",
					BuildId:   b.Id,
					Status:    evergreen.TaskUndispatched,
					Activated: true,
				}

				So(matching.Insert(), ShouldBeNil)

				differentUser := &task.Task{
					Id:          "differentUser",
					BuildId:     b.Id,
					Status:      evergreen.TaskUndispatched,
					Activated:   true,
					ActivatedBy: user,
				}

				So(differentUser.Insert(), ShouldBeNil)

				So(SetBuildActivation(b.Id, false, evergreen.DefaultTaskActivator), ShouldBeNil)
				// the build should have been updated in the db
				b, err := build.FindOne(build.ById(b.Id))
				So(err, ShouldBeNil)
				So(b.Activated, ShouldBeFalse)
				So(b.ActivatedBy, ShouldEqual, evergreen.DefaultTaskActivator)

				// only the matching task should have been updated that has not been set by a user
				deactivatedTasks, err := task.Find(task.ByActivation(false))
				So(err, ShouldBeNil)
				So(len(deactivatedTasks), ShouldEqual, 1)
				So(deactivatedTasks[0].Id, ShouldEqual, matching.Id)

				// task with the different user activating should be activated with that user
				differentUserTask, err := task.FindOne(task.ById(differentUser.Id))
				So(err, ShouldBeNil)
				So(differentUserTask.Activated, ShouldBeTrue)
				So(differentUserTask.ActivatedBy, ShouldEqual, user)

			})

			Convey("all of the undispatched task caches within the build"+
				" should be updated, both in memory and in the"+
				" database", func() {

				b := &build.Build{
					Id:           "build",
					Activated:    true,
					BuildVariant: "foo",
					Tasks: []build.TaskCache{
						{
							Id:        "tc1",
							Status:    evergreen.TaskUndispatched,
							Activated: true,
						},
						{
							Id:        "tc2",
							Status:    evergreen.TaskDispatched,
							Activated: true,
						},
						{
							Id:        "tc3",
							Status:    evergreen.TaskUndispatched,
							Activated: true,
						},
						{
							Id:        "tc4",
							Status:    evergreen.TaskUndispatched,
							Activated: true,
						},
					},
				}
				So(b.Insert(), ShouldBeNil)

				t1 := &task.Task{Id: "tc1", DisplayName: "tc1", BuildId: b.Id, Status: evergreen.TaskUndispatched, Activated: true}
				t2 := &task.Task{Id: "tc2", DisplayName: "tc2", BuildId: b.Id, Status: evergreen.TaskDispatched, Activated: true}
				t3 := &task.Task{Id: "tc3", DisplayName: "tc3", BuildId: b.Id, Status: evergreen.TaskUndispatched, Activated: true}
				t4 := &task.Task{Id: "tc4", DisplayName: "tc4", BuildId: b.Id, Status: evergreen.TaskUndispatched, Activated: true, ActivatedBy: "anotherUser"}
				So(t1.Insert(), ShouldBeNil)
				So(t2.Insert(), ShouldBeNil)
				So(t3.Insert(), ShouldBeNil)
				So(t4.Insert(), ShouldBeNil)

				So(SetBuildActivation(b.Id, false, evergreen.DefaultTaskActivator), ShouldBeNil)
				// refresh from the database and check again
				b, err := build.FindOne(build.ById(b.Id))
				So(err, ShouldBeNil)
				So(b.Activated, ShouldBeFalse)
				So(b.Tasks[0].Activated, ShouldBeFalse)
				So(b.Tasks[1].Activated, ShouldBeTrue)
				So(b.Tasks[2].Activated, ShouldBeFalse)
				So(b.Tasks[3].Activated, ShouldBeTrue)
			})

			Convey("if a build is activated by a user it should not be able to be deactivated by evergreen", func() {
				user := "******"

				b := &build.Build{
					Id:           "anotherBuild",
					Activated:    true,
					BuildVariant: "bv",
				}

				So(b.Insert(), ShouldBeNil)

				matching := &task.Task{
					Id:        "matching",
					BuildId:   b.Id,
					Status:    evergreen.TaskUndispatched,
					Activated: false,
				}
				So(matching.Insert(), ShouldBeNil)

				matching2 := &task.Task{
					Id:        "matching2",
					BuildId:   b.Id,
					Status:    evergreen.TaskUndispatched,
					Activated: false,
				}
				So(matching2.Insert(), ShouldBeNil)

				// have a user set the build activation to true
				So(SetBuildActivation(b.Id, true, user), ShouldBeNil)

				// task with the different user activating should be activated with that user
				task1, err := task.FindOne(task.ById(matching.Id))
				So(err, ShouldBeNil)
				So(task1.Activated, ShouldBeTrue)
				So(task1.ActivatedBy, ShouldEqual, user)

				// task with the different user activating should be activated with that user
				task2, err := task.FindOne(task.ById(matching2.Id))
				So(err, ShouldBeNil)
				So(task2.Activated, ShouldBeTrue)
				So(task2.ActivatedBy, ShouldEqual, user)

				// refresh from the database and check again
				b, err = build.FindOne(build.ById(b.Id))
				So(b.Activated, ShouldBeTrue)
				So(b.ActivatedBy, ShouldEqual, user)

				// deactivate the task from evergreen and nothing should be deactivated.
				So(SetBuildActivation(b.Id, false, evergreen.DefaultTaskActivator), ShouldBeNil)

				// refresh from the database and check again
				b, err = build.FindOne(build.ById(b.Id))
				So(b.Activated, ShouldBeTrue)
				So(b.ActivatedBy, ShouldEqual, user)

				// task with the different user activating should be activated with that user
				task1, err = task.FindOne(task.ById(matching.Id))
				So(err, ShouldBeNil)
				So(task1.Activated, ShouldBeTrue)
				So(task1.ActivatedBy, ShouldEqual, user)

				// task with the different user activating should be activated with that user
				task2, err = task.FindOne(task.ById(matching2.Id))
				So(err, ShouldBeNil)
				So(task2.Activated, ShouldBeTrue)
				So(task2.ActivatedBy, ShouldEqual, user)

			})
		})

	})
}
Пример #30
0
func TestCreateBuildFromVersion(t *testing.T) {

	Convey("When creating a build from a version", t, func() {

		testutil.HandleTestingErr(db.ClearCollections(build.Collection, task.Collection), t,
			"Error clearing test collection")

		// the mock build variant we'll be using. runs all three tasks
		buildVar1 := BuildVariant{
			Name:        "buildVar",
			DisplayName: "Build Variant",
			Tasks: []BuildVariantTask{
				{Name: "taskA"}, {Name: "taskB"}, {Name: "taskC"}, {Name: "taskD"},
			},
		}
		buildVar2 := BuildVariant{
			Name:        "buildVar2",
			DisplayName: "Build Variant 2",
			Tasks: []BuildVariantTask{
				{Name: "taskA"}, {Name: "taskB"}, {Name: "taskC"}, {Name: "taskE"},
			},
		}
		buildVar3 := BuildVariant{
			Name:        "buildVar3",
			DisplayName: "Build Variant 3",
			Tasks: []BuildVariantTask{
				{
					// wait for the first BV's taskA to complete
					Name:      "taskA",
					DependsOn: []TaskDependency{{Name: "taskA", Variant: "buildVar"}},
				},
			},
		}

		project := &Project{
			Tasks: []ProjectTask{
				{
					Name:      "taskA",
					Priority:  5,
					Tags:      []string{"tag1", "tag2"},
					DependsOn: []TaskDependency{},
				},
				{
					Name:      "taskB",
					Tags:      []string{"tag1", "tag2"},
					DependsOn: []TaskDependency{{Name: "taskA", Variant: "buildVar"}},
				},
				{
					Name: "taskC",
					Tags: []string{"tag1", "tag2"},
					DependsOn: []TaskDependency{
						{Name: "taskA"},
						{Name: "taskB"},
					},
				},
				{
					Name:      "taskD",
					Tags:      []string{"tag1", "tag2"},
					DependsOn: []TaskDependency{{Name: AllDependencies}},
				},
				{
					Name: "taskE",
					Tags: []string{"tag1", "tag2"},
					DependsOn: []TaskDependency{
						{
							Name:    AllDependencies,
							Variant: AllVariants,
						},
					},
				},
			},
			BuildVariants: []BuildVariant{buildVar1, buildVar2, buildVar3},
		}

		// the mock version we'll be using
		v := &version.Version{
			Id:                  "versionId",
			CreateTime:          time.Now(),
			Revision:            "foobar",
			RevisionOrderNumber: 500,
			Requester:           evergreen.RepotrackerVersionRequester,
			BuildVariants: []version.BuildStatus{
				{
					BuildVariant: buildVar1.Name,
					Activated:    false,
				},
				{
					BuildVariant: buildVar2.Name,
					Activated:    false,
				},
				{
					BuildVariant: buildVar3.Name,
					Activated:    false,
				},
			},
		}

		tt := BuildTaskIdTable(project, v)

		Convey("the task id table should be well-formed", func() {
			So(tt.GetId("buildVar", "taskA"), ShouldNotEqual, "")
			So(tt.GetId("buildVar", "taskB"), ShouldNotEqual, "")
			So(tt.GetId("buildVar", "taskC"), ShouldNotEqual, "")
			So(tt.GetId("buildVar", "taskD"), ShouldNotEqual, "")
			So(tt.GetId("buildVar2", "taskA"), ShouldNotEqual, "")
			So(tt.GetId("buildVar2", "taskB"), ShouldNotEqual, "")
			So(tt.GetId("buildVar2", "taskC"), ShouldNotEqual, "")
			So(tt.GetId("buildVar2", "taskE"), ShouldNotEqual, "")
			So(tt.GetId("buildVar3", "taskA"), ShouldNotEqual, "")

			Convey(`and incorrect GetId() calls should return ""`, func() {
				So(tt.GetId("buildVar", "taskF"), ShouldEqual, "")
				So(tt.GetId("buildVar2", "taskD"), ShouldEqual, "")
				So(tt.GetId("buildVar7", "taskA"), ShouldEqual, "")
			})
		})

		Convey("if a non-existent build variant is passed in, an error should be returned", func() {

			buildId, err := CreateBuildFromVersion(project, v, tt, "blecch", false, []string{})
			So(err, ShouldNotBeNil)
			So(buildId, ShouldEqual, "")

		})

		Convey("if no task names are passed in to be used, all of the default"+
			" tasks for the build variant should be created", func() {

			buildId, err := CreateBuildFromVersion(project, v, tt, buildVar1.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId, ShouldNotEqual, "")
			buildId2, err := CreateBuildFromVersion(project, v, tt, buildVar2.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId2, ShouldNotEqual, "")

			// find the tasks, make sure they were all created
			tasks, err := task.Find(task.All)
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 8)
			So(len(tasks[0].Tags), ShouldEqual, 2)

		})

		Convey("if a non-empty list of task names is passed in, only the"+
			" specified tasks should be created", func() {

			buildId, err := CreateBuildFromVersion(project, v, tt, buildVar1.Name, false,
				[]string{"taskA", "taskB"})
			So(err, ShouldBeNil)
			So(buildId, ShouldNotEqual, "")

			// find the tasks, make sure they were all created
			tasks, err := task.Find(task.All)
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 2)

		})

		Convey("the build should contain task caches that correspond exactly"+
			" to the tasks created", func() {

			buildId, err := CreateBuildFromVersion(project, v, tt, buildVar1.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId, ShouldNotEqual, "")

			// find the tasks, make sure they were all created
			tasks, err := task.Find(task.All)
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 4)

			// find the build from the db
			b, err := build.FindOne(build.ById(buildId))
			So(err, ShouldBeNil)
			So(len(b.Tasks), ShouldEqual, 4)

			// make sure the task caches are correct.  they should also appear
			// in the same order that they appear in the project file
			So(b.Tasks[0].Id, ShouldNotEqual, "")
			So(b.Tasks[0].DisplayName, ShouldEqual, "taskA")
			So(b.Tasks[0].Status, ShouldEqual, evergreen.TaskUndispatched)
			So(b.Tasks[1].Id, ShouldNotEqual, "")
			So(b.Tasks[1].DisplayName, ShouldEqual, "taskB")
			So(b.Tasks[1].Status, ShouldEqual, evergreen.TaskUndispatched)
			So(b.Tasks[2].Id, ShouldNotEqual, "")
			So(b.Tasks[2].DisplayName, ShouldEqual, "taskC")
			So(b.Tasks[2].Status, ShouldEqual, evergreen.TaskUndispatched)
			So(b.Tasks[3].Id, ShouldNotEqual, "")
			So(b.Tasks[3].DisplayName, ShouldEqual, "taskD")
			So(b.Tasks[3].Status, ShouldEqual, evergreen.TaskUndispatched)

		})

		Convey("all of the tasks created should have the dependencies"+
			"and priorities specified in the project", func() {

			buildId, err := CreateBuildFromVersion(project, v, tt, buildVar1.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId, ShouldNotEqual, "")
			buildId2, err := CreateBuildFromVersion(project, v, tt, buildVar2.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId2, ShouldNotEqual, "")
			buildId3, err := CreateBuildFromVersion(project, v, tt, buildVar3.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId3, ShouldNotEqual, "")

			// find the tasks, make sure they were all created
			tasks, err := task.Find(task.All.Sort([]string{task.DisplayNameKey, task.BuildVariantKey}))
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 9)

			// taskA
			So(len(tasks[0].DependsOn), ShouldEqual, 0)
			So(len(tasks[1].DependsOn), ShouldEqual, 0)
			So(len(tasks[2].DependsOn), ShouldEqual, 1)
			So(tasks[0].Priority, ShouldEqual, 5)
			So(tasks[1].Priority, ShouldEqual, 5)
			So(tasks[2].DependsOn, ShouldResemble,
				[]task.Dependency{{tasks[0].Id, evergreen.TaskSucceeded}})

			// taskB
			So(tasks[3].DependsOn, ShouldResemble,
				[]task.Dependency{{tasks[0].Id, evergreen.TaskSucceeded}})
			So(tasks[4].DependsOn, ShouldResemble,
				[]task.Dependency{{tasks[0].Id, evergreen.TaskSucceeded}}) //cross-variant
			So(tasks[3].Priority, ShouldEqual, 0)
			So(tasks[4].Priority, ShouldEqual, 0) //default priority

			// taskC
			So(tasks[5].DependsOn, ShouldResemble,
				[]task.Dependency{
					{tasks[0].Id, evergreen.TaskSucceeded},
					{tasks[3].Id, evergreen.TaskSucceeded}})
			So(tasks[6].DependsOn, ShouldResemble,
				[]task.Dependency{
					{tasks[1].Id, evergreen.TaskSucceeded},
					{tasks[4].Id, evergreen.TaskSucceeded}})
			So(tasks[7].DependsOn, ShouldResemble,
				[]task.Dependency{
					{tasks[0].Id, evergreen.TaskSucceeded},
					{tasks[3].Id, evergreen.TaskSucceeded},
					{tasks[5].Id, evergreen.TaskSucceeded}})
			So(tasks[8].DisplayName, ShouldEqual, "taskE")
			So(len(tasks[8].DependsOn), ShouldEqual, 8)
		})

		Convey("all of the build's essential fields should be set"+
			" correctly", func() {

			buildId, err := CreateBuildFromVersion(project, v, tt, buildVar1.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId, ShouldNotEqual, "")

			// find the build from the db
			b, err := build.FindOne(build.ById(buildId))
			So(err, ShouldBeNil)

			// verify all the fields are set appropriately
			So(len(b.Tasks), ShouldEqual, 4)
			So(b.CreateTime.Truncate(time.Second), ShouldResemble,
				v.CreateTime.Truncate(time.Second))
			So(b.PushTime.Truncate(time.Second), ShouldResemble,
				v.CreateTime.Truncate(time.Second))
			So(b.Activated, ShouldEqual, v.BuildVariants[0].Activated)
			So(b.Project, ShouldEqual, project.Identifier)
			So(b.Revision, ShouldEqual, v.Revision)
			So(b.Status, ShouldEqual, evergreen.BuildCreated)
			So(b.BuildVariant, ShouldEqual, buildVar1.Name)
			So(b.Version, ShouldEqual, v.Id)
			So(b.DisplayName, ShouldEqual, buildVar1.DisplayName)
			So(b.RevisionOrderNumber, ShouldEqual, v.RevisionOrderNumber)
			So(b.Requester, ShouldEqual, v.Requester)

		})

		Convey("all of the tasks' essential fields should be set"+
			" correctly", func() {

			buildId, err := CreateBuildFromVersion(project, v, tt, buildVar1.Name, false, nil)
			So(err, ShouldBeNil)
			So(buildId, ShouldNotEqual, "")

			// find the build from the db
			b, err := build.FindOne(build.ById(buildId))
			So(err, ShouldBeNil)

			// find the tasks, make sure they were all created
			tasks, err := task.Find(task.All.Sort([]string{task.DisplayNameKey}))
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 4)

			So(tasks[0].Id, ShouldNotEqual, "")
			So(tasks[0].Secret, ShouldNotEqual, "")
			So(tasks[0].DisplayName, ShouldEqual, "taskA")
			So(tasks[0].BuildId, ShouldEqual, buildId)
			So(tasks[0].DistroId, ShouldEqual, "")
			So(tasks[0].BuildVariant, ShouldEqual, buildVar1.Name)
			So(tasks[0].CreateTime.Truncate(time.Second), ShouldResemble,
				b.CreateTime.Truncate(time.Second))
			So(tasks[0].PushTime.Truncate(time.Second), ShouldResemble,
				b.PushTime.Truncate(time.Second))
			So(tasks[0].Status, ShouldEqual, evergreen.TaskUndispatched)
			So(tasks[0].Activated, ShouldEqual, b.Activated)
			So(tasks[0].RevisionOrderNumber, ShouldEqual, b.RevisionOrderNumber)
			So(tasks[0].Requester, ShouldEqual, b.Requester)
			So(tasks[0].Version, ShouldEqual, v.Id)
			So(tasks[0].Revision, ShouldEqual, v.Revision)
			So(tasks[0].Project, ShouldEqual, project.Identifier)

			So(tasks[1].Id, ShouldNotEqual, "")
			So(tasks[1].Secret, ShouldNotEqual, "")
			So(tasks[1].DisplayName, ShouldEqual, "taskB")
			So(tasks[1].BuildId, ShouldEqual, buildId)
			So(tasks[1].DistroId, ShouldEqual, "")
			So(tasks[1].BuildVariant, ShouldEqual, buildVar1.Name)
			So(tasks[1].CreateTime.Truncate(time.Second), ShouldResemble,
				b.CreateTime.Truncate(time.Second))
			So(tasks[1].PushTime.Truncate(time.Second), ShouldResemble,
				b.PushTime.Truncate(time.Second))
			So(tasks[1].Status, ShouldEqual, evergreen.TaskUndispatched)
			So(tasks[1].Activated, ShouldEqual, b.Activated)
			So(tasks[1].RevisionOrderNumber, ShouldEqual, b.RevisionOrderNumber)
			So(tasks[1].Requester, ShouldEqual, b.Requester)
			So(tasks[1].Version, ShouldEqual, v.Id)
			So(tasks[1].Revision, ShouldEqual, v.Revision)
			So(tasks[1].Project, ShouldEqual, project.Identifier)

			So(tasks[2].Id, ShouldNotEqual, "")
			So(tasks[2].Secret, ShouldNotEqual, "")
			So(tasks[2].DisplayName, ShouldEqual, "taskC")
			So(tasks[2].BuildId, ShouldEqual, buildId)
			So(tasks[2].DistroId, ShouldEqual, "")
			So(tasks[2].BuildVariant, ShouldEqual, buildVar1.Name)
			So(tasks[2].CreateTime.Truncate(time.Second), ShouldResemble,
				b.CreateTime.Truncate(time.Second))
			So(tasks[2].PushTime.Truncate(time.Second), ShouldResemble,
				b.PushTime.Truncate(time.Second))
			So(tasks[2].Status, ShouldEqual, evergreen.TaskUndispatched)
			So(tasks[2].Activated, ShouldEqual, b.Activated)
			So(tasks[2].RevisionOrderNumber, ShouldEqual, b.RevisionOrderNumber)
			So(tasks[2].Requester, ShouldEqual, b.Requester)
			So(tasks[2].Version, ShouldEqual, v.Id)
			So(tasks[2].Revision, ShouldEqual, v.Revision)
			So(tasks[2].Project, ShouldEqual, project.Identifier)

			So(tasks[3].Id, ShouldNotEqual, "")
			So(tasks[3].Secret, ShouldNotEqual, "")
			So(tasks[3].DisplayName, ShouldEqual, "taskD")
			So(tasks[3].BuildId, ShouldEqual, buildId)
			So(tasks[3].DistroId, ShouldEqual, "")
			So(tasks[3].BuildVariant, ShouldEqual, buildVar1.Name)
			So(tasks[3].CreateTime.Truncate(time.Second), ShouldResemble,
				b.CreateTime.Truncate(time.Second))
			So(tasks[3].PushTime.Truncate(time.Second), ShouldResemble,
				b.PushTime.Truncate(time.Second))
			So(tasks[3].Status, ShouldEqual, evergreen.TaskUndispatched)
			So(tasks[3].Activated, ShouldEqual, b.Activated)
			So(tasks[3].RevisionOrderNumber, ShouldEqual, b.RevisionOrderNumber)
			So(tasks[3].Requester, ShouldEqual, b.Requester)
			So(tasks[3].Version, ShouldEqual, v.Id)
			So(tasks[3].Revision, ShouldEqual, v.Revision)
			So(tasks[3].Project, ShouldEqual, project.Identifier)
		})

		Convey("if the activated flag is set, the build and all its tasks should be activated",
			func() {

				buildId, err := CreateBuildFromVersion(project, v, tt, buildVar1.Name, true, nil)
				So(err, ShouldBeNil)
				So(buildId, ShouldNotEqual, "")

				// find the build from the db
				build, err := build.FindOne(build.ById(buildId))
				So(err, ShouldBeNil)
				So(build.Activated, ShouldBeTrue)

				// find the tasks, make sure they were all created
				tasks, err := task.Find(task.All.Sort([]string{task.DisplayNameKey}))
				So(err, ShouldBeNil)
				So(len(tasks), ShouldEqual, 4)

				So(tasks[0].Id, ShouldNotEqual, "")
				So(tasks[0].Secret, ShouldNotEqual, "")
				So(tasks[0].DisplayName, ShouldEqual, "taskA")
				So(tasks[0].BuildId, ShouldEqual, buildId)
				So(tasks[0].DistroId, ShouldEqual, "")
				So(tasks[0].BuildVariant, ShouldEqual, buildVar1.Name)
				So(tasks[0].CreateTime.Truncate(time.Second), ShouldResemble,
					build.CreateTime.Truncate(time.Second))
				So(tasks[0].PushTime.Truncate(time.Second), ShouldResemble,
					build.PushTime.Truncate(time.Second))
				So(tasks[0].Status, ShouldEqual, evergreen.TaskUndispatched)
				So(tasks[0].Activated, ShouldEqual, build.Activated)
				So(tasks[0].RevisionOrderNumber, ShouldEqual, build.RevisionOrderNumber)
				So(tasks[0].Requester, ShouldEqual, build.Requester)
				So(tasks[0].Version, ShouldEqual, v.Id)
				So(tasks[0].Revision, ShouldEqual, v.Revision)
				So(tasks[0].Project, ShouldEqual, project.Identifier)

				So(tasks[1].Id, ShouldNotEqual, "")
				So(tasks[1].Secret, ShouldNotEqual, "")
				So(tasks[1].DisplayName, ShouldEqual, "taskB")
				So(tasks[1].BuildId, ShouldEqual, buildId)
				So(tasks[1].DistroId, ShouldEqual, "")
				So(tasks[1].BuildVariant, ShouldEqual, buildVar1.Name)
				So(tasks[1].CreateTime.Truncate(time.Second), ShouldResemble,
					build.CreateTime.Truncate(time.Second))
				So(tasks[1].PushTime.Truncate(time.Second), ShouldResemble,
					build.PushTime.Truncate(time.Second))
				So(tasks[1].Status, ShouldEqual, evergreen.TaskUndispatched)
				So(tasks[1].Activated, ShouldEqual, build.Activated)
				So(tasks[1].RevisionOrderNumber, ShouldEqual, build.RevisionOrderNumber)
				So(tasks[1].Requester, ShouldEqual, build.Requester)
				So(tasks[1].Version, ShouldEqual, v.Id)
				So(tasks[1].Revision, ShouldEqual, v.Revision)
				So(tasks[1].Project, ShouldEqual, project.Identifier)

				So(tasks[2].Id, ShouldNotEqual, "")
				So(tasks[2].Secret, ShouldNotEqual, "")
				So(tasks[2].DisplayName, ShouldEqual, "taskC")
				So(tasks[2].BuildId, ShouldEqual, buildId)
				So(tasks[2].DistroId, ShouldEqual, "")
				So(tasks[2].BuildVariant, ShouldEqual, buildVar1.Name)
				So(tasks[2].CreateTime.Truncate(time.Second), ShouldResemble,
					build.CreateTime.Truncate(time.Second))
				So(tasks[2].PushTime.Truncate(time.Second), ShouldResemble,
					build.PushTime.Truncate(time.Second))
				So(tasks[2].Status, ShouldEqual, evergreen.TaskUndispatched)
				So(tasks[2].Activated, ShouldEqual, build.Activated)
				So(tasks[2].RevisionOrderNumber, ShouldEqual, build.RevisionOrderNumber)
				So(tasks[2].Requester, ShouldEqual, build.Requester)
				So(tasks[2].Version, ShouldEqual, v.Id)
				So(tasks[2].Revision, ShouldEqual, v.Revision)
				So(tasks[2].Project, ShouldEqual, project.Identifier)

				So(tasks[3].Id, ShouldNotEqual, "")
				So(tasks[3].Secret, ShouldNotEqual, "")
				So(tasks[3].DisplayName, ShouldEqual, "taskD")
				So(tasks[3].BuildId, ShouldEqual, buildId)
				So(tasks[3].DistroId, ShouldEqual, "")
				So(tasks[3].BuildVariant, ShouldEqual, buildVar1.Name)
				So(tasks[3].CreateTime.Truncate(time.Second), ShouldResemble,
					build.CreateTime.Truncate(time.Second))
				So(tasks[3].PushTime.Truncate(time.Second), ShouldResemble,
					build.PushTime.Truncate(time.Second))
				So(tasks[3].Status, ShouldEqual, evergreen.TaskUndispatched)
				So(tasks[3].Activated, ShouldEqual, build.Activated)
				So(tasks[3].RevisionOrderNumber, ShouldEqual, build.RevisionOrderNumber)
				So(tasks[3].Requester, ShouldEqual, build.Requester)
				So(tasks[3].Version, ShouldEqual, v.Id)
				So(tasks[3].Revision, ShouldEqual, v.Revision)
				So(tasks[3].Project, ShouldEqual, project.Identifier)
			})

	})
}