Exemplo n.º 1
0
// Given a task name and a slice of versions, return the appropriate sibling
// groups of tasks.  They will be sorted by ascending revision order number,
// unless reverseOrder is true, in which case they will be sorted
// descending.
func getTaskDrawerItems(displayName string, variant string, reverseOrder bool, versions []version.Version) ([]taskDrawerItem, error) {

	orderNumbers := make([]int, 0, len(versions))
	for _, v := range versions {
		orderNumbers = append(orderNumbers, v.RevisionOrderNumber)
	}
	revisionCriteria := bson.M{"$in": orderNumbers}

	revisionSort := model.TaskRevisionOrderNumberKey
	if reverseOrder {
		revisionSort = "-" + revisionSort
	}

	tasks, err := model.FindAllTasks(
		bson.M{
			model.TaskRevisionOrderNumberKey: revisionCriteria,
			model.TaskDisplayNameKey:         displayName,
			model.TaskBuildVariantKey:        variant,
		},
		db.NoProjection,
		[]string{
			revisionSort,
		},
		db.NoSkip,
		db.NoLimit,
	)

	if err != nil {
		return nil, fmt.Errorf("error getting sibling tasks: %v", err)
	}
	return createSiblingTaskGroups(tasks, versions), nil
}
Exemplo n.º 2
0
func (uis *UIServer) taskDependencies(w http.ResponseWriter, r *http.Request) {
	projCtx := MustHaveProjectContext(r)

	if projCtx.Task == nil {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}

	dependencies, err := model.FindAllTasks(
		bson.M{
			"_id": bson.M{"$in": projCtx.Task.DependsOn},
		},
		bson.M{
			"display_name":   1,
			"status":         1,
			"activated":      1,
			"status_details": 1,
		}, []string{}, 0, 0)

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	uis.WriteJSON(w, http.StatusOK, dependencies)
}
Exemplo n.º 3
0
func (uis *UIServer) taskDependencies(w http.ResponseWriter, r *http.Request) {
	projCtx := MustHaveProjectContext(r)

	if projCtx.Task == nil {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}

	depIds := []string{}
	for _, dep := range projCtx.Task.DependsOn {
		depIds = append(depIds, dep.TaskId)
	}
	dependencies, err := model.FindAllTasks(
		bson.M{
			"_id": bson.M{"$in": depIds},
		},
		bson.M{
			"display_name":     1,
			"status":           1,
			"activated":        1,
			"status_details":   1,
			"build_variant":    1,
			"task_end_details": 1,
		}, []string{}, 0, 0)

	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	type uiDep struct {
		Id             string                  `json:"id"`
		Name           string                  `json:"display_name"`
		Status         string                  `json:"status"`
		RequiredStatus string                  `json:"required"`
		Activated      bool                    `json:"activated"`
		BuildVariant   string                  `json:"build_variant"`
		Details        apimodels.TaskEndDetail `json:"task_end_details"`
	}
	uiDeps := []uiDep{}
	// match each task with its dependency requirements
	for _, depTask := range dependencies {
		for _, dep := range projCtx.Task.DependsOn {
			if dep.TaskId == depTask.Id {
				uiDeps = append(uiDeps, uiDep{
					Id:             depTask.Id,
					Name:           depTask.DisplayName,
					Status:         depTask.Status,
					RequiredStatus: dep.Status,
					Activated:      depTask.Activated,
					BuildVariant:   depTask.BuildVariant,
					Details:        depTask.Details,
				})
			}
		}
	}
	uis.WriteJSON(w, http.StatusOK, uiDeps)
}
Exemplo n.º 4
0
func (uis *UIServer) taskHistoryPickaxe(w http.ResponseWriter, r *http.Request) {
	projCtx := MustHaveProjectContext(r)

	if projCtx.Project == nil {
		http.Error(w, "not found", http.StatusNotFound)
		return
	}

	taskName := mux.Vars(r)["task_name"]

	highOrder, err := strconv.ParseInt(r.FormValue("high"), 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing high: `%s`", err.Error()), http.StatusBadRequest)
		return
	}
	lowOrder, err := strconv.ParseInt(r.FormValue("low"), 10, 64)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error parsing low: `%s`", err.Error()), http.StatusBadRequest)
		return
	}

	filter := struct {
		BuildVariants []string          `json:"buildVariants"`
		Tests         map[string]string `json:"tests"`
	}{}

	err = json.Unmarshal([]byte(r.FormValue("filter")), &filter)
	if err != nil {
		http.Error(w, fmt.Sprintf("Error in filter: %v", err.Error()), http.StatusBadRequest)
		return
	}
	buildVariants := projCtx.Project.GetVariantsWithTask(taskName)

	onlyMatchingTasks := (r.FormValue("only_matching_tasks") == "true")

	// If there are no build variants, use all of them for the given task name.
	// Need this because without the build_variant specified, no amount of hinting
	// will get sort to use the proper index
	query := bson.M{
		"build_variant": bson.M{
			"$in": buildVariants,
		},
		"display_name": taskName,
		"order": bson.M{
			"$gte": lowOrder,
			"$lte": highOrder,
		},
		"branch": projCtx.Project.Identifier,
	}

	// If there are build variants, use them instead
	if len(filter.BuildVariants) > 0 {
		query["build_variant"] = bson.M{
			"$in": filter.BuildVariants,
		}
	}

	// If there are tests to filter by, create a big $elemMatch $or in the
	// projection to make sure we only get the tests we care about.
	elemMatchOr := make([]bson.M, 0)
	for test, result := range filter.Tests {
		regexp := fmt.Sprintf(testMatchRegex, test, test)
		if result == "ran" {
			// Special case: if asking for tasks where the test ran, don't care
			// about the test status
			elemMatchOr = append(elemMatchOr, bson.M{
				"test_file": bson.RegEx{regexp, ""},
			})
		} else {
			elemMatchOr = append(elemMatchOr, bson.M{
				"test_file": bson.RegEx{regexp, ""},
				"status":    result,
			})
		}
	}

	elemMatch := bson.M{"$or": elemMatchOr}

	// Special case: if only one test filter, don't need to use a $or
	if 1 == len(elemMatchOr) {
		elemMatch = elemMatchOr[0]
	}

	projection := bson.M{
		"_id":           1,
		"status":        1,
		"activated":     1,
		"time_taken":    1,
		"build_variant": 1,
	}

	if len(elemMatchOr) > 0 {
		projection["test_results"] = bson.M{
			"$elemMatch": elemMatch,
		}

		// If we only care about matching tasks, put the elemMatch in the query too
		if onlyMatchingTasks {
			query["test_results"] = bson.M{
				"$elemMatch": elemMatch,
			}
		}
	}

	last, err := model.FindAllTasks(query, projection, []string{}, 0, 0)

	if err != nil {
		http.Error(w, fmt.Sprintf("Error querying tasks: `%s`", err.Error()), http.StatusInternalServerError)
		return
	}

	uis.WriteJSON(w, http.StatusOK, last)
}
Exemplo n.º 5
0
func TestFinalizePatch(t *testing.T) {
	testutil.ConfigureIntegrationTest(t, patchTestConfig, "TestFinalizePatch")

	Convey("With FinalizePatch on a project and commit event generated from GetPatchedProject path",
		t, func() {
			configPatch := resetPatchSetup(t, configFilePath)
			Convey("a patched config should drive version creation", func() {
				project, err := GetPatchedProject(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				yamlBytes, err := yaml.Marshal(project)
				So(err, ShouldBeNil)
				configPatch.PatchedConfig = string(yamlBytes)
				version, err := model.FinalizePatch(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				So(version, ShouldNotBeNil)
				// ensure the relevant builds/tasks were created
				builds, err := build.Find(build.All)
				So(err, ShouldBeNil)
				So(len(builds), ShouldEqual, 1)
				So(len(builds[0].Tasks), ShouldEqual, 2)
				tasks, err := model.FindAllTasks(bson.M{},
					db.NoProjection,
					db.NoSort,
					db.NoSkip,
					db.NoLimit,
				)
				So(err, ShouldBeNil)
				So(len(tasks), ShouldEqual, 2)
			})

			Convey("a patch that does not include the remote config should not "+
				"drive version creation", func() {
				patchedConfigFile := "fakeInPatchSoNotPatched"
				configPatch := resetPatchSetup(t, patchedConfigFile)
				project, err := GetPatchedProject(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				yamlBytes, err := yaml.Marshal(project)
				So(err, ShouldBeNil)
				configPatch.PatchedConfig = string(yamlBytes)
				version, err := model.FinalizePatch(configPatch, patchTestConfig)
				So(err, ShouldBeNil)
				So(version, ShouldNotBeNil)
				So(err, ShouldBeNil)
				So(version, ShouldNotBeNil)

				// ensure the relevant builds/tasks were created
				builds, err := build.Find(build.All)
				So(err, ShouldBeNil)
				So(len(builds), ShouldEqual, 1)
				So(len(builds[0].Tasks), ShouldEqual, 1)
				tasks, err := model.FindAllTasks(bson.M{},
					db.NoProjection,
					db.NoSort,
					db.NoSkip,
					db.NoLimit,
				)
				So(err, ShouldBeNil)
				So(len(tasks), ShouldEqual, 1)
			})

			Reset(func() {
				db.Clear(distro.Collection)
			})
		})
}
Exemplo n.º 6
0
// addRecDeps recursively finds all dependencies of tasks and adds them to tasks and uiDeps.
// done is a hashtable of task IDs whose dependencies we have found.
// TODO EVG-614: delete this function once Task.DependsOn includes all recursive dependencies.
func addRecDeps(tasks map[string]model.Task, uiDeps map[string]uiDep, done map[string]bool) error {
	curTask := make(map[string]bool)
	depIds := make([]string, 0)
	for _, task := range tasks {
		if _, ok := done[task.Id]; !ok {
			for _, dep := range task.DependsOn {
				depIds = append(depIds, dep.TaskId)
			}
			curTask[task.Id] = true
		}
	}

	if len(depIds) == 0 {
		return nil
	}

	deps, err := model.FindAllTasks(
		bson.M{
			"_id": bson.M{"$in": depIds},
		},
		bson.M{
			"display_name":     1,
			"status":           1,
			"activated":        1,
			"status_details":   1,
			"build_variant":    1,
			"task_end_details": 1,
			"depends_on":       1,
		}, []string{}, 0, 0)

	if err != nil {
		return err
	}

	for _, dep := range deps {
		tasks[dep.Id] = dep
	}

	for _, task := range tasks {
		if _, ok := curTask[task.Id]; ok {
			for _, dep := range task.DependsOn {
				if uid, ok := uiDeps[dep.TaskId]; !ok ||
					// only replace if the current uiDep is not strict and not recursive
					(uid.RequiredStatus == model.AllStatuses && !uid.Recursive) {
					depTask := tasks[dep.TaskId]
					uiDeps[depTask.Id] = uiDep{
						Id:             depTask.Id,
						Name:           depTask.DisplayName,
						Status:         depTask.Status,
						RequiredStatus: dep.Status,
						Activated:      depTask.Activated,
						BuildVariant:   depTask.BuildVariant,
						Details:        depTask.Details,
						Recursive:      true,
					}
				}
			}
			done[task.Id] = true
		}
	}

	return addRecDeps(tasks, uiDeps, done)
}
Exemplo n.º 7
0
// getTaskDependencies returns the uiDeps for the task and its status (either its original status,
// "blocked", or "pending")
func getTaskDependencies(task *model.Task) ([]uiDep, string, error) {
	depIds := []string{}
	for _, dep := range task.DependsOn {
		depIds = append(depIds, dep.TaskId)
	}
	dependencies, err := model.FindAllTasks(
		bson.M{
			"_id": bson.M{"$in": depIds},
		},
		bson.M{
			"display_name":     1,
			"status":           1,
			"activated":        1,
			"status_details":   1,
			"build_variant":    1,
			"task_end_details": 1,
			"depends_on":       1,
		}, []string{}, 0, 0)

	if err != nil {
		return nil, "", err
	}

	idToUiDep := make(map[string]uiDep)
	// match each task with its dependency requirements
	for _, depTask := range dependencies {
		for _, dep := range task.DependsOn {
			if dep.TaskId == depTask.Id {
				idToUiDep[depTask.Id] = uiDep{
					Id:             depTask.Id,
					Name:           depTask.DisplayName,
					Status:         depTask.Status,
					RequiredStatus: dep.Status,
					Activated:      depTask.Activated,
					BuildVariant:   depTask.BuildVariant,
					Details:        depTask.Details,
					//TODO EVG-614: add "Recursive: dep.Recursive," once Task.DependsOn includes all recursive dependencies
				}
			}
		}
	}

	idToDep := make(map[string]model.Task)
	for _, dep := range dependencies {
		idToDep[dep.Id] = dep
	}

	// TODO EVG 614: delete this section once Task.DependsOn includes all recursive dependencies
	err = addRecDeps(idToDep, idToUiDep, make(map[string]bool))
	if err != nil {
		return nil, "", err
	}

	// set the status for each of the uiDeps as "blocked" or "pending" if appropriate
	// and get the status for task
	status := setBlockedOrPending(*task, idToDep, idToUiDep)

	uiDeps := make([]uiDep, 0, len(idToUiDep))
	for _, dep := range idToUiDep {
		uiDeps = append(uiDeps, dep)
	}
	return uiDeps, status, nil
}
Exemplo n.º 8
0
func TestFinalize(t *testing.T) {
	testutil.ConfigureIntegrationTest(t, patchTestConfig, "TestFinalize")

	Convey("With calling ValidateAndFinalize with a config and remote configuration "+
		"path", t, func() {
		testutil.HandleTestingErr(db.ClearCollections(
			model.ProjectRefCollection,
			patch.Collection,
			version.Collection,
			build.Collection,
			model.TasksCollection),
			t, "Error clearing test collection")

		Convey("a patched config should drive version creation", func() {
			configFilePath := "testing/mci.yml"
			// insert distros to be used
			distros := []distro.Distro{
				distro.Distro{Id: "d1"},
				distro.Distro{Id: "d2"},
			}

			for _, d := range distros {
				So(d.Insert(), ShouldBeNil)
			}

			projectRef := &model.ProjectRef{
				Identifier: patchedProject,
				RemotePath: configFilePath,
				Owner:      patchOwner,
				Repo:       patchRepo,
				Branch:     patchBranch,
			}
			err := projectRef.Insert()
			testutil.HandleTestingErr(err, t, "Couldn't insert test project ref: "+
				"%v", err)
			fileBytes, err := ioutil.ReadFile(patchFile)
			So(err, ShouldBeNil)

			// this patch adds a new task to the existing build
			configPatch := &patch.Patch{
				Id:            "52549c143122",
				Project:       patchedProject,
				BuildVariants: []string{"all"},
				Githash:       patchedRevision,
				Patches: []patch.ModulePatch{
					patch.ModulePatch{
						Githash: "revision",
						PatchSet: patch.PatchSet{
							Patch: fmt.Sprintf(string(fileBytes), configFilePath,
								configFilePath, configFilePath, configFilePath),
							Summary: []thirdparty.Summary{
								thirdparty.Summary{
									Name:      configFilePath,
									Additions: 4,
									Deletions: 80,
								},
								thirdparty.Summary{
									Name:      "random.txt",
									Additions: 6,
									Deletions: 0,
								},
							},
						},
					},
				},
			}
			err = configPatch.Insert()
			testutil.HandleTestingErr(err, t, "Couldn't insert test patch: %v", err)
			version, err := ValidateAndFinalize(configPatch, patchTestConfig)
			So(err, ShouldBeNil)
			So(version, ShouldNotBeNil)
			// ensure the relevant builds/tasks were created
			builds, err := build.Find(build.All)
			So(err, ShouldBeNil)
			So(len(builds), ShouldEqual, 1)
			So(len(builds[0].Tasks), ShouldEqual, 2)
			tasks, err := model.FindAllTasks(bson.M{},
				db.NoProjection,
				db.NoSort,
				db.NoSkip,
				db.NoLimit,
			)
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 2)
		})

		Convey("a patch that does not include the remote config should not "+
			"drive version creation", func() {
			configFilePath := "testing/mci.yml"
			projectRef := &model.ProjectRef{
				Identifier: patchedProject,
				RemotePath: configFilePath,
				Owner:      patchOwner,
				Repo:       patchRepo,
				Branch:     patchBranch,
			}
			err := projectRef.Insert()
			testutil.HandleTestingErr(err, t, "Couldn't insert test project ref: "+
				"%v", err)
			patchedConfigFile := "fakeInPatchSoNotPatched"
			fileBytes, err := ioutil.ReadFile(patchFile)
			So(err, ShouldBeNil)

			// insert distros to be used
			distros := []distro.Distro{
				distro.Distro{Id: "d1"},
				distro.Distro{Id: "d2"},
			}

			for _, d := range distros {
				So(d.Insert(), ShouldBeNil)
			}

			// this patch adds a new task to the existing build
			configPatch := &patch.Patch{
				Id:            "52549c143122",
				Project:       patchedProject,
				BuildVariants: []string{"all"},
				Githash:       patchedRevision,
				Patches: []patch.ModulePatch{
					patch.ModulePatch{
						Githash: "revision",
						PatchSet: patch.PatchSet{
							Patch: fmt.Sprintf(string(fileBytes), patchedConfigFile,
								patchedConfigFile, patchedConfigFile, patchedConfigFile),
							Summary: []thirdparty.Summary{
								thirdparty.Summary{
									Name:      configFilePath,
									Additions: 4,
									Deletions: 80,
								},
								thirdparty.Summary{
									Name:      patchedProject,
									Additions: 6,
									Deletions: 0,
								},
							},
						},
					},
				},
			}
			err = configPatch.Insert()
			testutil.HandleTestingErr(err, t, "Couldn't insert test patch: %v", err)
			version, err := ValidateAndFinalize(configPatch, patchTestConfig)
			So(err, ShouldBeNil)
			So(version, ShouldNotBeNil)

			// ensure the relevant builds/tasks were created
			builds, err := build.Find(build.All)
			So(err, ShouldBeNil)
			So(len(builds), ShouldEqual, 1)
			So(len(builds[0].Tasks), ShouldEqual, 1)
			tasks, err := model.FindAllTasks(bson.M{},
				db.NoProjection,
				db.NoSort,
				db.NoSkip,
				db.NoLimit,
			)
			So(err, ShouldBeNil)
			So(len(tasks), ShouldEqual, 1)
		})

		Reset(func() {
			db.Clear(distro.Collection)
		})
	})
}