// FindMinimumQueuePositionForTask finds the position of a task in the many task queues
// where its position is the lowest. It returns an error if the aggregation it runs fails.
func FindMinimumQueuePositionForTask(taskId string) (int, error) {
var results []struct {
Index int `bson:"index"`
}
queueItemIdKey := fmt.Sprintf("%v.%v", TaskQueueQueueKey, TaskQueueItemIdKey)
// NOTE: this aggregation requires 3.2+ because of its use of
// $unwind's 'path' and 'includeArrayIndex'
pipeline := []bson.M{
{"$match": bson.M{
queueItemIdKey: taskId}},
{"$unwind": bson.M{
"path": fmt.Sprintf("$%s", TaskQueueQueueKey),
"includeArrayIndex": "index"}},
{"$match": bson.M{
queueItemIdKey: taskId}},
{"$sort": bson.M{
"index": 1}},
{"$limit": 1},
}
err := db.Aggregate(TaskQueuesCollection, pipeline, &results)
if len(results) == 0 {
return -1, err
}
return (results[0].Index + 1), err
}
// ExpectedTaskDuration takes a given project and buildvariant and computes
// the average duration - grouped by task display name - for tasks that have
// completed within a given threshold as determined by the window
func ExpectedTaskDuration(project, buildvariant string, window time.Duration) (map[string]time.Duration, error) {
pipeline := []bson.M{
{
"$match": bson.M{
BuildVariantKey: buildvariant,
ProjectKey: project,
StatusKey: bson.M{
"$in": []string{evergreen.TaskSucceeded, evergreen.TaskFailed},
},
DetailsKey + "." + TaskEndDetailTimedOut: bson.M{
"$ne": true,
},
FinishTimeKey: bson.M{
"$gte": time.Now().Add(-window),
},
StartTimeKey: bson.M{
// make sure all documents have a valid start time so we don't
// return tasks with runtimes of multiple years
"$gt": util.ZeroTime,
},
},
},
{
"$project": bson.M{
DisplayNameKey: 1,
TimeTakenKey: 1,
IdKey: 0,
},
},
{
"$group": bson.M{
"_id": fmt.Sprintf("$%v", DisplayNameKey),
"exp_dur": bson.M{
"$avg": fmt.Sprintf("$%v", TimeTakenKey),
},
},
},
}
// anonymous struct for unmarshalling result bson
var results []struct {
DisplayName string `bson:"_id"`
ExpectedDuration int64 `bson:"exp_dur"`
}
err := db.Aggregate(Collection, pipeline, &results)
if err != nil {
return nil, fmt.Errorf("error aggregating task average duration: %v", err)
}
expDurations := make(map[string]time.Duration)
for _, result := range results {
expDuration := time.Duration(result.ExpectedDuration) * time.Nanosecond
expDurations[result.DisplayName] = expDuration
}
return expDurations, nil
}
// FetchCells returns a Grid of Cells - grouped by variant and display name.
// current is the most recent version and from which to fetch prior Cells
// going back as far as depth versions.
func FetchCells(current version.Version, depth int) (Grid, error) {
cells := Grid{}
pipeline := []bson.M{
// Stage 1: Get all builds from the current version going back
// as far as depth versions.
{"$match": bson.M{
build.RequesterKey: evergreen.RepotrackerVersionRequester,
build.RevisionOrderNumberKey: bson.M{
"$lte": current.RevisionOrderNumber,
"$gte": (current.RevisionOrderNumber - depth),
},
build.ProjectKey: current.Identifier,
}},
// Stage 2: Sort the builds by the most recently completed.
{"$sort": bson.M{
build.RevisionOrderNumberKey: -1,
}},
// Stage 3: Project only the relevant fields.
{"$project": bson.M{
build.TasksKey: 1,
build.RevisionKey: 1,
"v": "$" + build.BuildVariantKey,
}},
// Stage 4: Flatten the task cache for easier grouping.
{"$unwind": "$tasks"},
// Stage 5: Rewrite and project out only the relevant fields.
{"$project": bson.M{
"_id": 0,
"v": 1,
"r": "$" + build.RevisionKey,
"d": "$" + build.TasksKey + "." + build.TaskCacheDisplayNameKey,
"st": "$" + build.TasksKey + "." + build.TaskCacheStatusKey,
"ed": "$" + build.TasksKey + "." + build.TaskCacheStatusDetailsKey,
"id": "$" + build.TasksKey + "." + build.TaskCacheIdKey,
}},
// Stage 6: Group the tasks by variant and display name. For each group,
// add the history - all prior versioned tasks along with their status,
// id, and revision identifier.
{"$group": bson.M{
"_id": bson.M{
"v": "$v",
"d": "$d",
},
"h": bson.M{
"$push": bson.M{
"s": "$st",
"e": "$ed",
"d": "$id",
"v": "$r",
},
},
}},
}
return cells, db.Aggregate(build.Collection, pipeline, &cells)
}
// AverageStatistics uses an agg pipeline that creates buckets given a time frame and finds the average scheduled ->
// start time for that time frame.
// One thing to note is that the average time is in milliseconds, not nanoseconds and must be converted.
func AverageStatistics(distroId string, bounds FrameBounds) (AvgBuckets, error) {
// error out if the distro does not exist
_, err := distro.FindOne(distro.ById(distroId))
if err != nil {
return nil, err
}
intBucketSize := util.FromNanoseconds(bounds.BucketSize)
buckets := AvgBuckets{}
pipeline := []bson.M{
// find all tasks that have started within the time frame for a given distro and only valid statuses.
{"$match": bson.M{
task.StartTimeKey: bson.M{
"$gte": bounds.StartTime,
"$lte": bounds.EndTime,
},
// only need tasks that have already started or those that have finished,
// not looking for tasks that have been scheduled but not started.
task.StatusKey: bson.M{
"$in": []string{evergreen.TaskStarted,
evergreen.TaskFailed, evergreen.TaskSucceeded},
},
task.DistroIdKey: distroId,
}},
// project the difference in scheduled -> start, as well as the bucket
{"$project": bson.M{
"diff": bson.M{
"$subtract": []interface{}{"$" + task.StartTimeKey, "$" + task.ScheduledTimeKey},
},
"b": bson.M{
"$floor": bson.M{
"$divide": []interface{}{
bson.M{"$subtract": []interface{}{"$" + task.StartTimeKey, bounds.StartTime}},
intBucketSize},
},
},
}},
{"$group": bson.M{
"_id": "$b",
"a": bson.M{"$avg": "$diff"},
"n": bson.M{"$sum": 1},
}},
{"$sort": bson.M{
"_id": 1,
}},
}
if err := db.Aggregate(task.Collection, pipeline, &buckets); err != nil {
return nil, err
}
return convertBucketsToNanoseconds(buckets, bounds), nil
}
// FetchRevisionOrderFailures returns the most recent test failures
// grouped by revision - looking as far back as depth revisions
func FetchRevisionOrderFailures(current version.Version, depth int) (RevisionFailures, error) {
pipeline := []bson.M{
// Stage 1: Get the most recent completed tasks - looking back as far as
// depth versions - on this project.
{"$match": bson.M{
task.RevisionOrderNumberKey: bson.M{
"$lte": current.RevisionOrderNumber,
"$gte": (current.RevisionOrderNumber - depth),
},
task.ProjectKey: current.Identifier,
task.RequesterKey: evergreen.RepotrackerVersionRequester,
}},
// Stage 2: Project only relevant fields.
{"$project": bson.M{
task.DisplayNameKey: 1,
task.RevisionKey: 1,
task.BuildVariantKey: 1,
task.IdKey: 1,
"l": "$" + task.TestResultsKey,
}},
// Stage 3: Flatten out the test results
{"$unwind": "$l"},
// Stage 4: Take only failed test results
{"$match": bson.M{
"l." + task.TestResultStatusKey: evergreen.TestFailedStatus,
}},
// Stage 5: Project only relevant fields including just the test file key
{"$project": bson.M{
task.RevisionKey: 1,
task.BuildVariantKey: 1,
task.DisplayNameKey: 1,
task.IdKey: 1,
"f": "$l." + task.TestResultTestFileKey,
}},
// Stage 6: Group by revision. For each one include the
// variant name, task name, task id and test name
{"$group": bson.M{
"_id": "$" + task.RevisionKey,
"a": bson.M{
"$push": bson.M{
"n": "$" + task.BuildVariantKey,
"i": "$f",
"t": "$" + task.DisplayNameKey,
"tid": "$" + task.IdKey,
},
},
}},
}
taskFailures := RevisionFailures{}
return taskFailures, db.Aggregate(task.Collection, pipeline, &taskFailures)
}
// getTags finds TaskJSONs that have tags
func getTags(w http.ResponseWriter, r *http.Request) {
t, err := task.FindOne(task.ById(mux.Vars(r)["task_id"]))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
tags := []struct {
Tag string `bson:"_id" json:"tag"`
}{}
err = db.Aggregate(collection, []bson.M{
{"$match": bson.M{ProjectIdKey: t.Project, TagKey: bson.M{"$exists": true, "$ne": ""}}},
{"$project": bson.M{TagKey: 1}}, bson.M{"$group": bson.M{"_id": "$tag"}},
}, &tags)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, tags)
}
// AverageTaskTimeDifference takes two field names (such that field2 happened
// after field1), a field to group on, and a cutoff time.
// It returns the average duration between fields 1 and 2, grouped by
// the groupBy field, including only task documents where both time
// fields happened after the given cutoff time. This information is returned
// as a map from groupBy_field -> avg_time_difference
//
// NOTE: THIS FUNCTION DOES NOT SANITIZE INPUT!
// BAD THINGS CAN HAPPEN IF NON-TIME FIELDNAMES ARE PASSED IN
// OR IF A FIELD OF NON-STRING TYPE IS SUPPLIED FOR groupBy!
func AverageTaskTimeDifference(field1 string, field2 string,
groupByField string, cutoff time.Time) (map[string]time.Duration, error) {
// This pipeline returns the average time difference between
// two time fields, grouped by a given field of "string" type.
// It assumes field2 happened later than field1.
// Time difference returned in milliseconds.
pipeline := []bson.M{
{"$match": bson.M{
field1: bson.M{"$gt": cutoff},
field2: bson.M{"$gt": cutoff}}},
{"$group": bson.M{
"_id": "$" + groupByField,
"avg_time": bson.M{
"$avg": bson.M{
"$subtract": []string{"$" + field2, "$" + field1},
},
},
}},
}
// anonymous struct for unmarshalling result bson
// NOTE: This means we can only group by string fields currently
var results []struct {
GroupId string `bson:"_id"`
AverageTime int64 `bson:"avg_time"`
}
err := db.Aggregate(TasksCollection, pipeline, &results)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error aggregating task times by [%v, %v]: %v", field1, field2, err)
return nil, err
}
avgTimes := make(map[string]time.Duration)
for _, res := range results {
avgTimes[res.GroupId] = time.Duration(res.AverageTime) * time.Millisecond
}
return avgTimes, nil
}
// FetchFailures returns the most recent test failures that have occurred at or
// before the current version - looking back as far as depth versions.
func FetchFailures(current version.Version, depth int) (Failures, error) {
pipeline := []bson.M{
// Stage 1: Get the most recent completed tasks - looking back as far as
// depth versions - on this project.
{"$match": bson.M{
model.TaskRevisionOrderNumberKey: bson.M{
"$lte": current.RevisionOrderNumber,
"$gte": (current.RevisionOrderNumber - depth),
},
model.TaskProjectKey: current.Project,
model.TaskRequesterKey: evergreen.RepotrackerVersionRequester,
model.TaskStatusKey: bson.M{
"$in": []string{
evergreen.TaskFailed,
evergreen.TaskSucceeded,
},
},
}},
// Stage 2: Sort the tasks by the most recently completed.
{"$sort": bson.M{
model.TaskRevisionOrderNumberKey: -1,
}},
// Stage 3: Project only relevant fields.
{"$project": bson.M{
model.TaskDisplayNameKey: 1,
model.TaskBuildVariantKey: 1,
model.TaskTestResultsKey: 1,
model.TaskIdKey: 1,
}},
// Stage 4: Group these tasks by display name and buildvariant -
// this returns the most recently completed grouped by task display name
// and by variant. We take only the first test results (adding its task
// id) for each task/variant group.
{"$group": bson.M{
"_id": bson.M{
"t": "$" + model.TaskDisplayNameKey,
"v": "$" + model.TaskBuildVariantKey,
},
"l": bson.M{
"$first": "$" + model.TaskTestResultsKey,
},
"tid": bson.M{
"$first": "$" + model.TaskIdKey,
},
}},
// Stage 5: For each group, filter out those task/variant combinations
// that don't have at least one test failure in them.
{"$match": bson.M{
"l." + model.TestResultStatusKey: evergreen.TestFailedStatus,
}},
// Stage 6: Rewrite each task/variant combination from the _id into the
// top-level. Project only the test name and status for all tests, and
// add a 'status' literal string to each group. This sets up the
// documents for redacting in next stage.
{"$project": bson.M{
"tid": 1,
"t": "$_id.t",
"v": "$_id.v",
"l." + model.TestResultStatusKey: 1,
"l." + model.TestResultTestFileKey: 1,
"status": bson.M{
"$literal": evergreen.TestFailedStatus,
},
}},
// Stage 7: While each test result contains at least one failed test,
// some other tests may have passed. Prune individual tests that did
// not fail.
{"$redact": bson.M{
"$cond": bson.M{
"if": bson.M{
"$eq": []string{
"$status", evergreen.TestFailedStatus,
},
},
"then": "$$DESCEND",
"else": "$$PRUNE",
},
}},
// Stage 8: We no longer need the status fields so project only fields
// we want to return.
{"$project": bson.M{
"f": "$l." + model.TestResultTestFileKey,
"_id": 0,
"tid": 1,
"t": 1,
"v": 1,
}},
// Stage 9: Flatten each failing test so we can group them by all the
// variants on which they are failing.
{"$unwind": "$f"},
// Stage 10: Group individual test failure. For each, add the variants
// it's failing on (and the accompanying task id) and include task's
// display name.
{"$group": bson.M{
"_id": bson.M{
"t": "$f",
"d": "$t",
},
"a": bson.M{
"$push": bson.M{
"n": "$v",
"i": "$tid",
},
},
}},
}
failures := Failures{}
return failures, db.Aggregate(model.TasksCollection, pipeline, &failures)
}
// Returns a JSON response with the status of the specified version
// grouped on the tasks. The keys of the object are the task names,
// with each key in the nested object representing a particular build
// variant.
func (restapi *restAPI) getVersionStatusByTask(versionId string, w http.ResponseWriter, r *http.Request) {
id := "_id"
taskName := "task_name"
statuses := "statuses"
pipeline := []bson.M{
// 1. Find only builds corresponding to the specified version
{
"$match": bson.M{
build.VersionKey: versionId,
},
},
// 2. Loop through each task run on a particular build variant
{
"$unwind": fmt.Sprintf("$%v", build.TasksKey),
},
// 3. Group on the task name and construct a new document containing
// all of the relevant info about the task status
{
"$group": bson.M{
id: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheDisplayNameKey),
statuses: bson.M{
"$push": bson.M{
build.BuildVariantKey: fmt.Sprintf("$%v", build.BuildVariantKey),
build.TaskCacheIdKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheIdKey),
build.TaskCacheStatusKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheStatusKey),
build.TaskCacheStartTimeKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheStartTimeKey),
build.TaskCacheTimeTakenKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheTimeTakenKey),
build.TaskCacheActivatedKey: fmt.Sprintf("$%v.%v", build.TasksKey, build.TaskCacheActivatedKey),
},
},
},
},
// 4. Rename the "_id" field to "task_name"
{
"$project": bson.M{
id: 0,
taskName: fmt.Sprintf("$%v", id),
statuses: 1,
},
},
}
// Anonymous struct used to unmarshal output from the aggregation pipeline
var tasks []struct {
DisplayName string `bson:"task_name"`
Statuses []struct {
BuildVariant string `bson:"build_variant"`
// Use an anonyous field to make the semantics of inlining
build.TaskCache `bson:",inline"`
} `bson:"statuses"`
}
err := db.Aggregate(build.Collection, pipeline, &tasks)
if err != nil {
msg := fmt.Sprintf("Error finding status for version '%v'", versionId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
result := versionStatusByTaskContent{
Id: versionId,
Tasks: make(map[string]versionStatusByTask, len(tasks)),
}
for _, task := range tasks {
statuses := make(versionStatusByTask, len(task.Statuses))
for _, task := range task.Statuses {
status := versionStatus{
Id: task.Id,
Status: task.Status,
TimeTaken: task.TimeTaken,
}
statuses[task.BuildVariant] = status
}
result.Tasks[task.DisplayName] = statuses
}
restapi.WriteJSON(w, http.StatusOK, result)
return
}
// getTasksForLatestVersion sends back the TaskJSON data associated with the latest version.
func getTasksForLatestVersion(w http.ResponseWriter, r *http.Request) {
project := mux.Vars(r)["project_id"]
name := mux.Vars(r)["name"]
skip, err := util.GetIntValue(r, "skip", 0)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if skip < 0 {
http.Error(w, "negative skip", http.StatusBadRequest)
return
}
pipeline := []bson.M{
// match on name and project
{"$match": bson.M{
NameKey: name,
ProjectIdKey: project,
}},
// sort on the revision number
{"$sort": bson.M{
RevisionOrderNumberKey: -1,
}},
// group by version id
{"$group": bson.M{
"_id": bson.M{
"r": "$" + RevisionOrderNumberKey,
"vid": "$" + VersionIdKey,
},
"t": bson.M{
"$push": bson.M{
"d": "$" + DataKey,
"t_id": "$" + TaskIdKey,
"tn": "$" + TaskNameKey,
"var": "$" + VariantKey,
},
},
}},
// sort on the _id
{"$sort": bson.M{
"_id.r": -1,
}},
{"$skip": skip},
{"$limit": 2},
}
tasksForVersions := []TasksForVersion{}
err = db.Aggregate(collection, pipeline, &tasksForVersions)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(tasksForVersions) == 0 {
http.Error(w, "no tasks found", http.StatusNotFound)
return
}
// we default have another revision
lastRevision := false
currentVersion := tasksForVersions[0]
// if there is only one version, then we are at the last revision.
if len(tasksForVersions) < 2 {
lastRevision = true
}
// get the version commit info
v, err := version.FindOne(version.ById(currentVersion.Id.VersionId))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if v == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
commitInfo := CommitInfo{
Author: v.Author,
Message: v.Message,
CreateTime: v.CreateTime,
Revision: v.Revision,
VersionId: v.Id,
}
data := VersionData{currentVersion.Tasks, commitInfo, lastRevision}
plugin.WriteJSON(w, http.StatusOK, data)
}
func (hwp *JSONPlugin) GetUIHandler() http.Handler {
r := mux.NewRouter()
r.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
plugin.WriteJSON(w, http.StatusOK, "1")
})
r.HandleFunc("/task/{task_id}/{name}/", func(w http.ResponseWriter, r *http.Request) {
var jsonForTask TaskJSON
err := db.FindOneQ(collection, db.Query(bson.M{"task_id": mux.Vars(r)["task_id"], "name": mux.Vars(r)["name"]}), &jsonForTask)
if err != nil {
if err != mgo.ErrNotFound {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Error(w, "{}", http.StatusNotFound)
return
}
plugin.WriteJSON(w, http.StatusOK, jsonForTask)
})
r.HandleFunc("/task/{task_id}/{name}/tags", func(w http.ResponseWriter, r *http.Request) {
t, err := model.FindTask(mux.Vars(r)["task_id"])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
tags := []struct {
Tag string `bson:"_id" json:"tag"`
}{}
err = db.Aggregate(collection, []bson.M{
{"$match": bson.M{"project_id": t.Project, "tag": bson.M{"$exists": true, "$ne": ""}}},
{"$project": bson.M{"tag": 1}}, bson.M{"$group": bson.M{"_id": "$tag"}},
}, &tags)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, tags)
})
r.HandleFunc("/task/{task_id}/{name}/tag", func(w http.ResponseWriter, r *http.Request) {
t, err := model.FindTask(mux.Vars(r)["task_id"])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if t == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
if r.Method == "DELETE" {
if _, err = db.UpdateAll(collection,
bson.M{"version_id": t.Version, "name": mux.Vars(r)["name"]},
bson.M{"$unset": bson.M{"tag": 1}}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, "")
}
inTag := struct {
Tag string `json:"tag"`
}{}
err = util.ReadJSONInto(r.Body, &inTag)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(inTag.Tag) == 0 {
http.Error(w, "tag must not be blank", http.StatusBadRequest)
return
}
_, err = db.UpdateAll(collection,
bson.M{"version_id": t.Version, "name": mux.Vars(r)["name"]},
bson.M{"$set": bson.M{"tag": inTag.Tag}})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, "")
})
r.HandleFunc("/tag/{project_id}/{tag}/{variant}/{task_name}/{name}", func(w http.ResponseWriter, r *http.Request) {
var jsonForTask TaskJSON
err := db.FindOneQ(collection,
db.Query(bson.M{"project_id": mux.Vars(r)["project_id"],
"tag": mux.Vars(r)["tag"],
"variant": mux.Vars(r)["variant"],
"task_name": mux.Vars(r)["task_name"],
"name": mux.Vars(r)["name"],
}), &jsonForTask)
if err != nil {
if err != mgo.ErrNotFound {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Error(w, "{}", http.StatusNotFound)
return
}
plugin.WriteJSON(w, http.StatusOK, jsonForTask)
})
r.HandleFunc("/commit/{project_id}/{revision}/{variant}/{task_name}/{name}", func(w http.ResponseWriter, r *http.Request) {
var jsonForTask TaskJSON
err := db.FindOneQ(collection,
db.Query(bson.M{"project_id": mux.Vars(r)["project_id"],
"revision": bson.RegEx{"^" + regexp.QuoteMeta(mux.Vars(r)["revision"]), "i"},
"variant": mux.Vars(r)["variant"],
"task_name": mux.Vars(r)["task_name"],
"name": mux.Vars(r)["name"],
"is_patch": false,
}), &jsonForTask)
if err != nil {
if err != mgo.ErrNotFound {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Error(w, "{}", http.StatusNotFound)
return
}
plugin.WriteJSON(w, http.StatusOK, jsonForTask)
})
r.HandleFunc("/history/{task_id}/{name}", func(w http.ResponseWriter, r *http.Request) {
t, err := model.FindTask(mux.Vars(r)["task_id"])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if t == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
before := []TaskJSON{}
jsonQuery := db.Query(bson.M{
"project_id": t.Project,
"variant": t.BuildVariant,
"order": bson.M{"$lte": t.RevisionOrderNumber},
"task_name": t.DisplayName,
"is_patch": false,
"name": mux.Vars(r)["name"]})
jsonQuery = jsonQuery.Sort([]string{"-order"}).Limit(100)
err = db.FindAllQ(collection, jsonQuery, &before)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//reverse order of "before" because we had to sort it backwards to apply the limit correctly:
for i, j := 0, len(before)-1; i < j; i, j = i+1, j-1 {
before[i], before[j] = before[j], before[i]
}
after := []TaskJSON{}
jsonAfterQuery := db.Query(bson.M{
"project_id": t.Project,
"variant": t.BuildVariant,
"order": bson.M{"$gt": t.RevisionOrderNumber},
"task_name": t.DisplayName,
"is_patch": false,
"name": mux.Vars(r)["name"]}).Sort([]string{"order"}).Limit(100)
err = db.FindAllQ(collection, jsonAfterQuery, &after)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//concatenate before + after
before = append(before, after...)
plugin.WriteJSON(w, http.StatusOK, before)
})
return r
}
// Returns versions and tasks grouped by gitspec, newest first (sorted by order number desc)
func (self *buildVariantHistoryIterator) GetItems(beforeCommit *version.Version, numRevisions int) ([]bson.M, []version.Version, error) {
var versionQuery db.Q
if beforeCommit != nil {
versionQuery = db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.RevisionOrderNumberKey: bson.M{"$lt": beforeCommit.RevisionOrderNumber},
version.IdentifierKey: self.ProjectName,
version.BuildVariantsKey: bson.M{
"$elemMatch": bson.M{
version.BuildStatusVariantKey: self.BuildVariantInVersion,
},
},
})
} else {
versionQuery = db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.IdentifierKey: self.ProjectName,
version.BuildVariantsKey: bson.M{
"$elemMatch": bson.M{
version.BuildStatusVariantKey: self.BuildVariantInVersion,
},
},
})
}
versionQuery = versionQuery.WithFields(
version.IdKey,
version.RevisionOrderNumberKey,
version.RevisionKey,
version.MessageKey,
version.CreateTimeKey,
).Sort([]string{"-" + version.RevisionOrderNumberKey}).Limit(numRevisions)
//Get the next numCommits
versions, err := version.Find(versionQuery)
if err != nil {
return nil, nil, err
}
if len(versions) == 0 {
return nil, []version.Version{}, nil
}
//versionEndBoundary is the *earliest* version which should be included in results
versionEndBoundary := versions[len(versions)-1]
matchFilter := bson.M{
task.RequesterKey: evergreen.RepotrackerVersionRequester,
task.BuildVariantKey: self.BuildVariantInTask,
task.ProjectKey: self.ProjectName,
}
if beforeCommit != nil {
matchFilter[task.RevisionOrderNumberKey] = bson.M{
"$gte": versionEndBoundary.RevisionOrderNumber,
"$lt": beforeCommit.RevisionOrderNumber,
}
} else {
matchFilter[task.RevisionOrderNumberKey] = bson.M{
"$gte": versionEndBoundary.RevisionOrderNumber,
}
}
pipeline := []bson.M{
{"$match": matchFilter},
{"$sort": bson.D{{task.RevisionOrderNumberKey, 1}}},
{
"$group": bson.M{
"_id": "$" + task.RevisionKey,
"order": bson.M{"$first": "$" + task.RevisionOrderNumberKey},
"tasks": bson.M{
"$push": bson.M{
"_id": "$" + task.IdKey,
"status": "$" + task.StatusKey,
"task_end_details": "$" + task.DetailsKey,
"activated": "$" + task.ActivatedKey,
"time_taken": "$" + task.TimeTakenKey,
"display_name": "$" + task.DisplayNameKey,
},
},
},
},
{"$sort": bson.M{task.RevisionOrderNumberKey: -1, task.DisplayNameKey: 1}},
}
var output []bson.M
if err = db.Aggregate(task.Collection, pipeline, &output); err != nil {
return nil, nil, err
}
return output, versions, nil
}
// Given a project name and a list of build variants, return the latest version
// on which all the given build variants succeeded. Gives up after 100 versions.
func FindLastPassingVersionForBuildVariants(project *Project, buildVariantNames []string) (*version.Version, error) {
if len(buildVariantNames) == 0 {
return nil, fmt.Errorf("No build variants specified!")
}
// Get latest commit order number for this project
latestVersion, err := version.FindOne(db.Query(
version.ByMostRecentForRequester(project.Identifier, evergreen.RepotrackerVersionRequester).
WithFields(version.RevisionOrderNumberKey)))
if err != nil {
return nil, fmt.Errorf("Error getting latest version: %v", err)
}
if latestVersion == nil {
return nil, nil
}
mostRecentRevisionOrderNumber := latestVersion.RevisionOrderNumber
// Earliest commit order number to consider
leastRecentRevisionOrderNumber := mostRecentRevisionOrderNumber - StaleVersionCutoff
if leastRecentRevisionOrderNumber < 0 {
leastRecentRevisionOrderNumber = 0
}
pipeline := []bson.M{
// Limit ourselves to builds for non-stale versions and the given project
// and build variants
{
"$match": bson.M{
build.ProjectKey: project.Identifier,
build.RevisionOrderNumberKey: bson.M{"$gte": leastRecentRevisionOrderNumber},
build.BuildVariantKey: bson.M{"$in": buildVariantNames},
build.StatusKey: evergreen.BuildSucceeded,
},
},
// Sum up the number of builds that succeeded for each commit order number
{
"$group": bson.M{
"_id": fmt.Sprintf("$%v", build.RevisionOrderNumberKey),
"numSucceeded": bson.M{
"$sum": 1,
},
},
},
// Find builds that succeeded on all of the requested build variants
{
"$match": bson.M{"numSucceeded": len(buildVariantNames)},
},
// Order by commit order number, descending
{
"$sort": bson.M{"_id": -1},
},
// Get the highest commit order number where builds succeeded on all the
// requested build variants
{
"$limit": 1,
},
}
var result []bson.M
err = db.Aggregate(build.Collection, pipeline, &result)
if err != nil {
return nil, fmt.Errorf("Aggregation failed: %v", err)
}
if len(result) == 0 {
return nil, nil
}
// Get the version corresponding to the resulting commit order number
v, err := version.FindOne(
db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.IdentifierKey: project.Identifier,
version.RevisionOrderNumberKey: result[0]["_id"],
}))
if err != nil {
return nil, err
}
if v == nil {
return nil, fmt.Errorf("Couldn't find version with id `%v` after "+
"successful aggregation.", result[0]["_id"])
}
return v, nil
}