// Given a task name and a slice of versions, return the appropriate sibling
// groups of tasks. They will be sorted by ascending revision order number,
// unless reverseOrder is true, in which case they will be sorted
// descending.
func getTaskDrawerItems(displayName string, variant string, reverseOrder bool, versions []version.Version) ([]taskDrawerItem, error) {
orderNumbers := make([]int, 0, len(versions))
for _, v := range versions {
orderNumbers = append(orderNumbers, v.RevisionOrderNumber)
}
revisionCriteria := bson.M{"$in": orderNumbers}
revisionSort := model.TaskRevisionOrderNumberKey
if reverseOrder {
revisionSort = "-" + revisionSort
}
tasks, err := model.FindAllTasks(
bson.M{
model.TaskRevisionOrderNumberKey: revisionCriteria,
model.TaskDisplayNameKey: displayName,
model.TaskBuildVariantKey: variant,
},
db.NoProjection,
[]string{
revisionSort,
},
db.NoSkip,
db.NoLimit,
)
if err != nil {
return nil, fmt.Errorf("error getting sibling tasks: %v", err)
}
return createSiblingTaskGroups(tasks, versions), nil
}
func (uis *UIServer) taskDependencies(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
if projCtx.Task == nil {
http.Error(w, "not found", http.StatusNotFound)
return
}
dependencies, err := model.FindAllTasks(
bson.M{
"_id": bson.M{"$in": projCtx.Task.DependsOn},
},
bson.M{
"display_name": 1,
"status": 1,
"activated": 1,
"status_details": 1,
}, []string{}, 0, 0)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
uis.WriteJSON(w, http.StatusOK, dependencies)
}
func (uis *UIServer) taskDependencies(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
if projCtx.Task == nil {
http.Error(w, "not found", http.StatusNotFound)
return
}
depIds := []string{}
for _, dep := range projCtx.Task.DependsOn {
depIds = append(depIds, dep.TaskId)
}
dependencies, err := model.FindAllTasks(
bson.M{
"_id": bson.M{"$in": depIds},
},
bson.M{
"display_name": 1,
"status": 1,
"activated": 1,
"status_details": 1,
"build_variant": 1,
"task_end_details": 1,
}, []string{}, 0, 0)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
type uiDep struct {
Id string `json:"id"`
Name string `json:"display_name"`
Status string `json:"status"`
RequiredStatus string `json:"required"`
Activated bool `json:"activated"`
BuildVariant string `json:"build_variant"`
Details apimodels.TaskEndDetail `json:"task_end_details"`
}
uiDeps := []uiDep{}
// match each task with its dependency requirements
for _, depTask := range dependencies {
for _, dep := range projCtx.Task.DependsOn {
if dep.TaskId == depTask.Id {
uiDeps = append(uiDeps, uiDep{
Id: depTask.Id,
Name: depTask.DisplayName,
Status: depTask.Status,
RequiredStatus: dep.Status,
Activated: depTask.Activated,
BuildVariant: depTask.BuildVariant,
Details: depTask.Details,
})
}
}
}
uis.WriteJSON(w, http.StatusOK, uiDeps)
}
func (uis *UIServer) taskHistoryPickaxe(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
if projCtx.Project == nil {
http.Error(w, "not found", http.StatusNotFound)
return
}
taskName := mux.Vars(r)["task_name"]
highOrder, err := strconv.ParseInt(r.FormValue("high"), 10, 64)
if err != nil {
http.Error(w, fmt.Sprintf("Error parsing high: `%s`", err.Error()), http.StatusBadRequest)
return
}
lowOrder, err := strconv.ParseInt(r.FormValue("low"), 10, 64)
if err != nil {
http.Error(w, fmt.Sprintf("Error parsing low: `%s`", err.Error()), http.StatusBadRequest)
return
}
filter := struct {
BuildVariants []string `json:"buildVariants"`
Tests map[string]string `json:"tests"`
}{}
err = json.Unmarshal([]byte(r.FormValue("filter")), &filter)
if err != nil {
http.Error(w, fmt.Sprintf("Error in filter: %v", err.Error()), http.StatusBadRequest)
return
}
buildVariants := projCtx.Project.GetVariantsWithTask(taskName)
onlyMatchingTasks := (r.FormValue("only_matching_tasks") == "true")
// If there are no build variants, use all of them for the given task name.
// Need this because without the build_variant specified, no amount of hinting
// will get sort to use the proper index
query := bson.M{
"build_variant": bson.M{
"$in": buildVariants,
},
"display_name": taskName,
"order": bson.M{
"$gte": lowOrder,
"$lte": highOrder,
},
"branch": projCtx.Project.Identifier,
}
// If there are build variants, use them instead
if len(filter.BuildVariants) > 0 {
query["build_variant"] = bson.M{
"$in": filter.BuildVariants,
}
}
// If there are tests to filter by, create a big $elemMatch $or in the
// projection to make sure we only get the tests we care about.
elemMatchOr := make([]bson.M, 0)
for test, result := range filter.Tests {
regexp := fmt.Sprintf(testMatchRegex, test, test)
if result == "ran" {
// Special case: if asking for tasks where the test ran, don't care
// about the test status
elemMatchOr = append(elemMatchOr, bson.M{
"test_file": bson.RegEx{regexp, ""},
})
} else {
elemMatchOr = append(elemMatchOr, bson.M{
"test_file": bson.RegEx{regexp, ""},
"status": result,
})
}
}
elemMatch := bson.M{"$or": elemMatchOr}
// Special case: if only one test filter, don't need to use a $or
if 1 == len(elemMatchOr) {
elemMatch = elemMatchOr[0]
}
projection := bson.M{
"_id": 1,
"status": 1,
"activated": 1,
"time_taken": 1,
"build_variant": 1,
}
if len(elemMatchOr) > 0 {
projection["test_results"] = bson.M{
"$elemMatch": elemMatch,
}
// If we only care about matching tasks, put the elemMatch in the query too
if onlyMatchingTasks {
query["test_results"] = bson.M{
"$elemMatch": elemMatch,
}
}
}
last, err := model.FindAllTasks(query, projection, []string{}, 0, 0)
if err != nil {
http.Error(w, fmt.Sprintf("Error querying tasks: `%s`", err.Error()), http.StatusInternalServerError)
return
}
uis.WriteJSON(w, http.StatusOK, last)
}
func TestFinalizePatch(t *testing.T) {
testutil.ConfigureIntegrationTest(t, patchTestConfig, "TestFinalizePatch")
Convey("With FinalizePatch on a project and commit event generated from GetPatchedProject path",
t, func() {
configPatch := resetPatchSetup(t, configFilePath)
Convey("a patched config should drive version creation", func() {
project, err := GetPatchedProject(configPatch, patchTestConfig)
So(err, ShouldBeNil)
yamlBytes, err := yaml.Marshal(project)
So(err, ShouldBeNil)
configPatch.PatchedConfig = string(yamlBytes)
version, err := model.FinalizePatch(configPatch, patchTestConfig)
So(err, ShouldBeNil)
So(version, ShouldNotBeNil)
// ensure the relevant builds/tasks were created
builds, err := build.Find(build.All)
So(err, ShouldBeNil)
So(len(builds), ShouldEqual, 1)
So(len(builds[0].Tasks), ShouldEqual, 2)
tasks, err := model.FindAllTasks(bson.M{},
db.NoProjection,
db.NoSort,
db.NoSkip,
db.NoLimit,
)
So(err, ShouldBeNil)
So(len(tasks), ShouldEqual, 2)
})
Convey("a patch that does not include the remote config should not "+
"drive version creation", func() {
patchedConfigFile := "fakeInPatchSoNotPatched"
configPatch := resetPatchSetup(t, patchedConfigFile)
project, err := GetPatchedProject(configPatch, patchTestConfig)
So(err, ShouldBeNil)
yamlBytes, err := yaml.Marshal(project)
So(err, ShouldBeNil)
configPatch.PatchedConfig = string(yamlBytes)
version, err := model.FinalizePatch(configPatch, patchTestConfig)
So(err, ShouldBeNil)
So(version, ShouldNotBeNil)
So(err, ShouldBeNil)
So(version, ShouldNotBeNil)
// ensure the relevant builds/tasks were created
builds, err := build.Find(build.All)
So(err, ShouldBeNil)
So(len(builds), ShouldEqual, 1)
So(len(builds[0].Tasks), ShouldEqual, 1)
tasks, err := model.FindAllTasks(bson.M{},
db.NoProjection,
db.NoSort,
db.NoSkip,
db.NoLimit,
)
So(err, ShouldBeNil)
So(len(tasks), ShouldEqual, 1)
})
Reset(func() {
db.Clear(distro.Collection)
})
})
}
// addRecDeps recursively finds all dependencies of tasks and adds them to tasks and uiDeps.
// done is a hashtable of task IDs whose dependencies we have found.
// TODO EVG-614: delete this function once Task.DependsOn includes all recursive dependencies.
func addRecDeps(tasks map[string]model.Task, uiDeps map[string]uiDep, done map[string]bool) error {
curTask := make(map[string]bool)
depIds := make([]string, 0)
for _, task := range tasks {
if _, ok := done[task.Id]; !ok {
for _, dep := range task.DependsOn {
depIds = append(depIds, dep.TaskId)
}
curTask[task.Id] = true
}
}
if len(depIds) == 0 {
return nil
}
deps, err := model.FindAllTasks(
bson.M{
"_id": bson.M{"$in": depIds},
},
bson.M{
"display_name": 1,
"status": 1,
"activated": 1,
"status_details": 1,
"build_variant": 1,
"task_end_details": 1,
"depends_on": 1,
}, []string{}, 0, 0)
if err != nil {
return err
}
for _, dep := range deps {
tasks[dep.Id] = dep
}
for _, task := range tasks {
if _, ok := curTask[task.Id]; ok {
for _, dep := range task.DependsOn {
if uid, ok := uiDeps[dep.TaskId]; !ok ||
// only replace if the current uiDep is not strict and not recursive
(uid.RequiredStatus == model.AllStatuses && !uid.Recursive) {
depTask := tasks[dep.TaskId]
uiDeps[depTask.Id] = uiDep{
Id: depTask.Id,
Name: depTask.DisplayName,
Status: depTask.Status,
RequiredStatus: dep.Status,
Activated: depTask.Activated,
BuildVariant: depTask.BuildVariant,
Details: depTask.Details,
Recursive: true,
}
}
}
done[task.Id] = true
}
}
return addRecDeps(tasks, uiDeps, done)
}
// getTaskDependencies returns the uiDeps for the task and its status (either its original status,
// "blocked", or "pending")
func getTaskDependencies(task *model.Task) ([]uiDep, string, error) {
depIds := []string{}
for _, dep := range task.DependsOn {
depIds = append(depIds, dep.TaskId)
}
dependencies, err := model.FindAllTasks(
bson.M{
"_id": bson.M{"$in": depIds},
},
bson.M{
"display_name": 1,
"status": 1,
"activated": 1,
"status_details": 1,
"build_variant": 1,
"task_end_details": 1,
"depends_on": 1,
}, []string{}, 0, 0)
if err != nil {
return nil, "", err
}
idToUiDep := make(map[string]uiDep)
// match each task with its dependency requirements
for _, depTask := range dependencies {
for _, dep := range task.DependsOn {
if dep.TaskId == depTask.Id {
idToUiDep[depTask.Id] = uiDep{
Id: depTask.Id,
Name: depTask.DisplayName,
Status: depTask.Status,
RequiredStatus: dep.Status,
Activated: depTask.Activated,
BuildVariant: depTask.BuildVariant,
Details: depTask.Details,
//TODO EVG-614: add "Recursive: dep.Recursive," once Task.DependsOn includes all recursive dependencies
}
}
}
}
idToDep := make(map[string]model.Task)
for _, dep := range dependencies {
idToDep[dep.Id] = dep
}
// TODO EVG 614: delete this section once Task.DependsOn includes all recursive dependencies
err = addRecDeps(idToDep, idToUiDep, make(map[string]bool))
if err != nil {
return nil, "", err
}
// set the status for each of the uiDeps as "blocked" or "pending" if appropriate
// and get the status for task
status := setBlockedOrPending(*task, idToDep, idToUiDep)
uiDeps := make([]uiDep, 0, len(idToUiDep))
for _, dep := range idToUiDep {
uiDeps = append(uiDeps, dep)
}
return uiDeps, status, nil
}
func TestFinalize(t *testing.T) {
testutil.ConfigureIntegrationTest(t, patchTestConfig, "TestFinalize")
Convey("With calling ValidateAndFinalize with a config and remote configuration "+
"path", t, func() {
testutil.HandleTestingErr(db.ClearCollections(
model.ProjectRefCollection,
patch.Collection,
version.Collection,
build.Collection,
model.TasksCollection),
t, "Error clearing test collection")
Convey("a patched config should drive version creation", func() {
configFilePath := "testing/mci.yml"
// insert distros to be used
distros := []distro.Distro{
distro.Distro{Id: "d1"},
distro.Distro{Id: "d2"},
}
for _, d := range distros {
So(d.Insert(), ShouldBeNil)
}
projectRef := &model.ProjectRef{
Identifier: patchedProject,
RemotePath: configFilePath,
Owner: patchOwner,
Repo: patchRepo,
Branch: patchBranch,
}
err := projectRef.Insert()
testutil.HandleTestingErr(err, t, "Couldn't insert test project ref: "+
"%v", err)
fileBytes, err := ioutil.ReadFile(patchFile)
So(err, ShouldBeNil)
// this patch adds a new task to the existing build
configPatch := &patch.Patch{
Id: "52549c143122",
Project: patchedProject,
BuildVariants: []string{"all"},
Githash: patchedRevision,
Patches: []patch.ModulePatch{
patch.ModulePatch{
Githash: "revision",
PatchSet: patch.PatchSet{
Patch: fmt.Sprintf(string(fileBytes), configFilePath,
configFilePath, configFilePath, configFilePath),
Summary: []thirdparty.Summary{
thirdparty.Summary{
Name: configFilePath,
Additions: 4,
Deletions: 80,
},
thirdparty.Summary{
Name: "random.txt",
Additions: 6,
Deletions: 0,
},
},
},
},
},
}
err = configPatch.Insert()
testutil.HandleTestingErr(err, t, "Couldn't insert test patch: %v", err)
version, err := ValidateAndFinalize(configPatch, patchTestConfig)
So(err, ShouldBeNil)
So(version, ShouldNotBeNil)
// ensure the relevant builds/tasks were created
builds, err := build.Find(build.All)
So(err, ShouldBeNil)
So(len(builds), ShouldEqual, 1)
So(len(builds[0].Tasks), ShouldEqual, 2)
tasks, err := model.FindAllTasks(bson.M{},
db.NoProjection,
db.NoSort,
db.NoSkip,
db.NoLimit,
)
So(err, ShouldBeNil)
So(len(tasks), ShouldEqual, 2)
})
Convey("a patch that does not include the remote config should not "+
"drive version creation", func() {
configFilePath := "testing/mci.yml"
projectRef := &model.ProjectRef{
Identifier: patchedProject,
RemotePath: configFilePath,
Owner: patchOwner,
Repo: patchRepo,
Branch: patchBranch,
}
err := projectRef.Insert()
testutil.HandleTestingErr(err, t, "Couldn't insert test project ref: "+
"%v", err)
patchedConfigFile := "fakeInPatchSoNotPatched"
fileBytes, err := ioutil.ReadFile(patchFile)
So(err, ShouldBeNil)
// insert distros to be used
distros := []distro.Distro{
distro.Distro{Id: "d1"},
distro.Distro{Id: "d2"},
}
for _, d := range distros {
So(d.Insert(), ShouldBeNil)
}
// this patch adds a new task to the existing build
configPatch := &patch.Patch{
Id: "52549c143122",
Project: patchedProject,
BuildVariants: []string{"all"},
Githash: patchedRevision,
Patches: []patch.ModulePatch{
patch.ModulePatch{
Githash: "revision",
PatchSet: patch.PatchSet{
Patch: fmt.Sprintf(string(fileBytes), patchedConfigFile,
patchedConfigFile, patchedConfigFile, patchedConfigFile),
Summary: []thirdparty.Summary{
thirdparty.Summary{
Name: configFilePath,
Additions: 4,
Deletions: 80,
},
thirdparty.Summary{
Name: patchedProject,
Additions: 6,
Deletions: 0,
},
},
},
},
},
}
err = configPatch.Insert()
testutil.HandleTestingErr(err, t, "Couldn't insert test patch: %v", err)
version, err := ValidateAndFinalize(configPatch, patchTestConfig)
So(err, ShouldBeNil)
So(version, ShouldNotBeNil)
// ensure the relevant builds/tasks were created
builds, err := build.Find(build.All)
So(err, ShouldBeNil)
So(len(builds), ShouldEqual, 1)
So(len(builds[0].Tasks), ShouldEqual, 1)
tasks, err := model.FindAllTasks(bson.M{},
db.NoProjection,
db.NoSort,
db.NoSkip,
db.NoLimit,
)
So(err, ShouldBeNil)
So(len(tasks), ShouldEqual, 1)
})
Reset(func() {
db.Clear(distro.Collection)
})
})
}