// populatePatch loads a patch into the project context, using patchId if provided.
// If patchId is blank, will try to infer the patch ID from the version already loaded
// into context, if available.
func (pc *projectContext) populatePatch(patchId string) error {
var err error
if len(patchId) > 0 {
// The patch is explicitly identified in the URL, so fetch it
if !patch.IsValidId(patchId) {
return fmt.Errorf("patch id '%v' is not an object id", patchId)
}
pc.Patch, err = patch.FindOne(patch.ById(patch.NewId(patchId)).Project(patch.ExcludePatchDiff))
} else if pc.Version != nil {
// patch isn't in URL but the version in context has one, get it
pc.Patch, err = patch.FindOne(patch.ByVersion(pc.Version.Id).Project(patch.ExcludePatchDiff))
}
if err != nil {
return err
}
// If there's a finalized patch loaded into context but not a version, load the version
// associated with the patch as the context's version.
if pc.Version == nil && pc.Patch != nil && pc.Patch.Version != "" {
pc.Version, err = version.FindOne(version.ById(pc.Patch.Version))
if err != nil {
return err
}
}
return nil
}
// Returns a JSON response with the marshalled output of the version
// specified in the request.
func (restapi restAPI) getVersionInfo(w http.ResponseWriter, r *http.Request) {
versionId := mux.Vars(r)["version_id"]
srcVersion, err := version.FindOne(version.ById(versionId))
if err != nil || srcVersion == nil {
msg := fmt.Sprintf("Error finding version '%v'", versionId)
statusCode := http.StatusNotFound
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
statusCode = http.StatusInternalServerError
}
restapi.WriteJSON(w, statusCode, responseError{Message: msg})
return
}
destVersion := &restVersion{}
// Copy the contents from the database into our local version type
err = angier.TransferByFieldNames(srcVersion, destVersion)
if err != nil {
msg := fmt.Sprintf("Error finding version '%v'", versionId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
for _, buildStatus := range srcVersion.BuildVariants {
destVersion.BuildVariants = append(destVersion.BuildVariants, buildStatus.BuildVariant)
evergreen.Logger.Logf(slogger.ERROR, "adding BuildVariant %v", buildStatus.BuildVariant)
}
restapi.WriteJSON(w, http.StatusOK, destVersion)
return
}
// TryMarkPatchBuildFinished attempts to mark a patch as finished if all
// the builds for the patch are finished as well
func TryMarkPatchBuildFinished(b *build.Build, finishTime time.Time) error {
v, err := version.FindOne(version.ById(b.Version))
if err != nil {
return err
}
if v == nil {
return fmt.Errorf("Cannot find version for build %v with version %v", b.Id, b.Version)
}
// ensure all builds for this patch are finished as well
builds, err := build.Find(build.ByIds(v.BuildIds).WithFields(build.StatusKey))
if err != nil {
return err
}
patchCompleted := true
status := evergreen.PatchSucceeded
for _, build := range builds {
if !build.IsFinished() {
patchCompleted = false
}
if build.Status != evergreen.BuildSucceeded {
status = evergreen.PatchFailed
}
}
// nothing to do if the patch isn't completed
if !patchCompleted {
return nil
}
return patch.TryMarkFinished(v.Id, finishTime, status)
}
// getTasksForLatestVersion sends back the TaskJSON data associated with the latest version.
func getTasksForLatestVersion(w http.ResponseWriter, r *http.Request) {
name := mux.Vars(r)["name"]
var jsonTask TaskJSON
projects := []string{}
err := util.ReadJSONInto(r.Body, &projects)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
versionData := []VersionData{}
for _, project := range projects {
err := db.FindOneQ(collection, db.Query(bson.M{NameKey: name,
ProjectIdKey: project}).Sort([]string{"-" + RevisionOrderNumberKey}).WithFields(VersionIdKey), &jsonTask)
if err != nil {
if err != mgo.ErrNotFound {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Error(w, "{}", http.StatusNotFound)
return
}
if jsonTask.VersionId == "" {
http.Error(w, "{}", http.StatusNotFound)
}
jsonTasks, err := findTasksForVersion(jsonTask.VersionId, name)
if jsonTasks == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// get the version commit info
v, err := version.FindOne(version.ById(jsonTask.VersionId))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if v == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
commitInfo := CommitInfo{
Author: v.Author,
Message: v.Message,
CreateTime: v.CreateTime,
Revision: v.Revision,
VersionId: jsonTask.VersionId,
}
versionData = append(versionData, VersionData{jsonTasks, commitInfo})
}
plugin.WriteJSON(w, http.StatusOK, versionData)
}
func TestUpdateBuildStatusForTask(t *testing.T) {
Convey("With two tasks and a build", t, func() {
testutil.HandleTestingErr(db.ClearCollections(task.Collection, build.Collection, version.Collection), t,
"Error clearing task and build collections")
displayName := "testName"
b := &build.Build{
Id: "buildtest",
Status: evergreen.BuildStarted,
Version: "abc",
}
v := &version.Version{
Id: b.Version,
Status: evergreen.VersionStarted,
}
testTask := task.Task{
Id: "testone",
DisplayName: displayName,
Activated: false,
BuildId: b.Id,
Project: "sample",
Status: evergreen.TaskFailed,
}
anotherTask := task.Task{
Id: "two",
DisplayName: displayName,
Activated: true,
BuildId: b.Id,
Project: "sample",
Status: evergreen.TaskFailed,
}
b.Tasks = []build.TaskCache{
{
Id: testTask.Id,
Status: evergreen.TaskSucceeded,
},
{
Id: anotherTask.Id,
Status: evergreen.TaskFailed,
},
}
So(b.Insert(), ShouldBeNil)
So(testTask.Insert(), ShouldBeNil)
So(anotherTask.Insert(), ShouldBeNil)
So(v.Insert(), ShouldBeNil)
Convey("updating the build for a task should update the build's status and the version's status", func() {
So(UpdateBuildAndVersionStatusForTask(testTask.Id), ShouldBeNil)
b, err := build.FindOne(build.ById(b.Id))
So(err, ShouldBeNil)
So(b.Status, ShouldEqual, evergreen.BuildFailed)
v, err := version.FindOne(version.ById(v.Id))
So(v.Status, ShouldEqual, evergreen.VersionFailed)
})
})
}
// prependConfigToVersion modifies the project config with the given id
func prependConfigToVersion(t *testing.T, versionId, configData string) {
v, err := version.FindOne(version.ById(versionId))
testutil.HandleTestingErr(err, t, "failed to load version")
if v == nil {
err = fmt.Errorf("could not find version to update")
testutil.HandleTestingErr(err, t, "failed to find version")
}
v.Config = configData + v.Config
testutil.HandleTestingErr(dbutil.ClearCollections(version.Collection), t, "couldnt reset version")
testutil.HandleTestingErr(v.Insert(), t, "failed to insert version")
}
func TestMarkStart(t *testing.T) {
Convey("With a task, build and version", t, func() {
testutil.HandleTestingErr(db.ClearCollections(task.Collection, build.Collection, version.Collection), t,
"Error clearing task and build collections")
displayName := "testName"
b := &build.Build{
Id: "buildtest",
Status: evergreen.BuildCreated,
Version: "abc",
}
v := &version.Version{
Id: b.Version,
Status: evergreen.VersionCreated,
}
testTask := task.Task{
Id: "testTask",
DisplayName: displayName,
Activated: true,
BuildId: b.Id,
Project: "sample",
Status: evergreen.TaskUndispatched,
Version: b.Version,
}
b.Tasks = []build.TaskCache{
{
Id: testTask.Id,
Status: evergreen.TaskUndispatched,
},
}
So(b.Insert(), ShouldBeNil)
So(testTask.Insert(), ShouldBeNil)
So(v.Insert(), ShouldBeNil)
Convey("when calling MarkStart, the task, version and build should be updated", func() {
So(MarkStart(testTask.Id), ShouldBeNil)
testTask, err := task.FindOne(task.ById(testTask.Id))
So(err, ShouldBeNil)
So(testTask.Status, ShouldEqual, evergreen.TaskStarted)
b, err := build.FindOne(build.ById(b.Id))
So(err, ShouldBeNil)
So(b.Status, ShouldEqual, evergreen.BuildStarted)
So(b.Tasks, ShouldNotBeNil)
So(len(b.Tasks), ShouldEqual, 1)
So(b.Tasks[0].Status, ShouldEqual, evergreen.TaskStarted)
v, err := version.FindOne(version.ById(v.Id))
So(err, ShouldBeNil)
So(v.Status, ShouldEqual, evergreen.VersionStarted)
})
})
}
func (as *APIServer) GetVersion(w http.ResponseWriter, r *http.Request) {
task := MustHaveTask(r)
// Get the version for this task, so we can get its config data
v, err := version.FindOne(version.ById(task.Version))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if v == nil {
http.Error(w, "version not found", http.StatusNotFound)
return
}
as.WriteJSON(w, http.StatusOK, v)
}
// populateTaskBuildVersion takes a task, build, and version ID and populates a projectContext
// with as many of the task, build, and version documents as possible.
// If any of the provided IDs is blank, they will be inferred from the more selective ones.
// Returns the project ID of the data found, which may be blank if the IDs are empty.
func (pc *projectContext) populateTaskBuildVersion(taskId, buildId, versionId string) (string, error) {
projectId := ""
var err error
// Fetch task if there's a task ID present; if we find one, populate build/version IDs from it
if len(taskId) > 0 {
pc.Task, err = task.FindOne(task.ById(taskId))
if err != nil {
return "", err
}
if pc.Task != nil {
// override build and version ID with the ones this task belongs to
buildId = pc.Task.BuildId
versionId = pc.Task.Version
projectId = pc.Task.Project
}
}
// Fetch build if there's a build ID present; if we find one, populate version ID from it
if len(buildId) > 0 {
pc.Build, err = build.FindOne(build.ById(buildId))
if err != nil {
return "", err
}
if pc.Build != nil {
versionId = pc.Build.Version
projectId = pc.Build.Project
}
}
if len(versionId) > 0 {
pc.Version, err = version.FindOne(version.ById(versionId))
if err != nil {
return "", err
}
if pc.Version != nil {
projectId = pc.Version.Identifier
}
}
return projectId, nil
}
func (self *TaskNotificationHandler) constructChangeInfo(allTasks []task.Task,
key *NotificationKey) ([]ChangeInfo, error) {
changeInfoSlice := make([]ChangeInfo, 0)
for _, task := range allTasks {
// add blamelist information for each task
v, err := version.FindOne(version.ById(task.Version))
if err != nil {
return changeInfoSlice, err
}
if v == nil {
return changeInfoSlice, fmt.Errorf("No version found for task %v with version id %v",
task.Id, task.Version)
}
changeInfo := constructChangeInfo(v, key)
changeInfo.Pushtime = task.PushTime.Format(time.RFC850)
changeInfoSlice = append(changeInfoSlice, *changeInfo)
}
return changeInfoSlice, nil
}
func (self *BuildNotificationHandler) constructChangeInfo(allBuilds []build.Build,
key *NotificationKey) ([]ChangeInfo, error) {
changeInfoSlice := make([]ChangeInfo, 0)
for _, build := range allBuilds {
// add blamelist information for each build
v, err := version.FindOne(version.ById(build.Version))
if err != nil {
return changeInfoSlice, err
}
if v == nil {
err = fmt.Errorf("No version found for build %v with version id %v",
build.Id, build.Version)
return changeInfoSlice, err
}
changeInfo := constructChangeInfo(v, key)
changeInfo.Pushtime = build.PushTime.Format(time.RFC850)
changeInfoSlice = append(changeInfoSlice, *changeInfo)
}
return changeInfoSlice, nil
}
// Modifies part of the version specified in the request, and returns a
// JSON response with the marshalled output of its new state.
func (restapi restAPI) modifyVersionInfo(w http.ResponseWriter, r *http.Request) {
versionId := mux.Vars(r)["version_id"]
var input struct {
Activated *bool `json:"activated"`
}
json.NewDecoder(r.Body).Decode(&input)
v, err := version.FindOne(version.ById(versionId))
if err != nil || v == nil {
msg := fmt.Sprintf("Error finding version '%v'", versionId)
statusCode := http.StatusNotFound
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
statusCode = http.StatusInternalServerError
}
restapi.WriteJSON(w, statusCode, responseError{Message: msg})
return
}
if input.Activated != nil {
if err = model.SetVersionActivation(v.Id, *input.Activated); err != nil {
state := "inactive"
if *input.Activated {
state = "active"
}
msg := fmt.Sprintf("Error marking version '%v' as %v", versionId, state)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
}
restapi.getVersionInfo(w, r)
}
// Takes in a version id and a map of "key -> buildvariant" (where "key" is of
// type "versionBuildVariant") and updates the map with an entry for the
// buildvariants associated with "versionStr"
func (self *Scheduler) updateVersionBuildVarMap(versionStr string,
versionBuildVarMap map[versionBuildVariant]model.BuildVariant) (err error) {
version, err := version.FindOne(version.ById(versionStr))
if err != nil {
return
}
if version == nil {
return fmt.Errorf("nil version returned for version id '%v'", versionStr)
}
project := &model.Project{}
err = model.LoadProjectInto([]byte(version.Config), version.Identifier, project)
if err != nil {
return fmt.Errorf("unable to load project config for version %v: "+
"%v", versionStr, err)
}
// create buildvariant map (for accessing purposes)
for _, buildVariant := range project.BuildVariants {
key := versionBuildVariant{versionStr, buildVariant.Name}
versionBuildVarMap[key] = buildVariant
}
return
}
func (uis *UIServer) buildHistory(w http.ResponseWriter, r *http.Request) {
buildId := mux.Vars(r)["build_id"]
before, err := getIntValue(r, "before", 3)
if err != nil {
http.Error(w, fmt.Sprintf("invalid param 'before': %v", r.FormValue("before")), http.StatusBadRequest)
return
}
after, err := getIntValue(r, "after", 3)
if err != nil {
http.Error(w, fmt.Sprintf("invalid param 'after': %v", r.FormValue("after")), http.StatusBadRequest)
return
}
builds, err := getBuildVariantHistory(buildId, before, after)
if err != nil {
http.Error(w, fmt.Sprintf("error getting build history: %v", err), http.StatusInternalServerError)
return
}
history := &struct {
Builds []*uiBuild `json:"builds"`
LastSuccess *uiBuild `json:"lastSuccess"`
}{}
history.Builds = make([]*uiBuild, len(builds))
for i := 0; i < len(builds); i++ {
v, err := version.FindOne(version.ById(builds[i].Version))
if err != nil {
http.Error(w, fmt.Sprintf("error getting version for build %v: %v", builds[i].Id, err), http.StatusInternalServerError)
return
}
if v == nil {
http.Error(w, fmt.Sprintf("no version found for build %v", builds[i].Id), http.StatusNotFound)
return
}
history.Builds[i] = &uiBuild{
Build: builds[i],
CurrentTime: time.Now().UnixNano(),
Elapsed: time.Now().Sub(builds[i].StartTime),
RepoOwner: v.Owner,
Repo: v.Repo,
Version: *v,
}
}
lastSuccess, err := getBuildVariantHistoryLastSuccess(buildId)
if err == nil && lastSuccess != nil {
v, err := version.FindOne(version.ById(lastSuccess.Version))
if err != nil {
http.Error(
w, fmt.Sprintf("error getting last successful build version: %v", err),
http.StatusInternalServerError)
return
}
if v == nil {
http.Error(w, fmt.Sprintf("no version '%v' found", lastSuccess.Version), http.StatusNotFound)
return
}
history.LastSuccess = &uiBuild{
Build: *lastSuccess,
CurrentTime: time.Now().UnixNano(),
Elapsed: time.Now().Sub(lastSuccess.StartTime),
RepoOwner: v.Owner,
Repo: v.Repo,
Version: *v,
}
}
uis.WriteJSON(w, http.StatusOK, history)
}
// loadAlertContext fetches details from the database for all documents that are associated with the
// AlertRequest. For example, it populates the task/build/version/project using the
// task/build/version/project ids in the alert request document.
func (qp *QueueProcessor) loadAlertContext(a *alert.AlertRequest) (*AlertContext, error) {
aCtx := &AlertContext{AlertRequest: a}
aCtx.Settings = qp.config
taskId, projectId, buildId, versionId := a.TaskId, a.ProjectId, a.BuildId, a.VersionId
patchId := a.PatchId
var err error
if len(a.HostId) > 0 {
aCtx.Host, err = host.FindOne(host.ById(a.HostId))
if err != nil {
return nil, err
}
}
// Fetch task if there's a task ID present; if we find one, populate build/version IDs from it
if len(taskId) > 0 {
aCtx.Task, err = model.FindTask(taskId)
if err != nil {
return nil, err
}
if aCtx.Task != nil && aCtx.Task.Execution != a.Execution {
oldTaskId := fmt.Sprintf("%s_%v", taskId, a.Execution)
aCtx.Task, err = model.FindOneOldTask(bson.M{"_id": oldTaskId}, db.NoProjection, db.NoSort)
if err != nil {
return nil, err
}
}
if aCtx.Task != nil {
// override build and version ID with the ones this task belongs to
buildId = aCtx.Task.BuildId
versionId = aCtx.Task.Version
projectId = aCtx.Task.Project
aCtx.FailedTests = []model.TestResult{}
for _, test := range aCtx.Task.TestResults {
if test.Status == "fail" {
aCtx.FailedTests = append(aCtx.FailedTests, test)
}
}
}
}
// Fetch build if there's a build ID present; if we find one, populate version ID from it
if len(buildId) > 0 {
aCtx.Build, err = build.FindOne(build.ById(buildId))
if err != nil {
return nil, err
}
if aCtx.Build != nil {
versionId = aCtx.Build.Version
projectId = aCtx.Build.Project
}
}
if len(versionId) > 0 {
aCtx.Version, err = version.FindOne(version.ById(versionId))
if err != nil {
return nil, err
}
if aCtx.Version != nil {
projectId = aCtx.Version.Identifier
}
}
if len(patchId) > 0 {
if !patch.IsValidId(patchId) {
return nil, fmt.Errorf("patch id '%v' is not an object id", patchId)
}
aCtx.Patch, err = patch.FindOne(patch.ById(patch.NewId(patchId)).Project(patch.ExcludePatchDiff))
} else if aCtx.Version != nil {
// patch isn't in URL but the version in context has one, get it
aCtx.Patch, err = patch.FindOne(patch.ByVersion(aCtx.Version.Id).Project(patch.ExcludePatchDiff))
}
// If there's a finalized patch loaded into context but not a version, load the version
// associated with the patch as the context's version.
if aCtx.Version == nil && aCtx.Patch != nil && aCtx.Patch.Version != "" {
aCtx.Version, err = version.FindOne(version.ById(aCtx.Patch.Version).WithoutFields(version.ConfigKey))
if err != nil {
return nil, err
}
}
if len(projectId) > 0 {
aCtx.ProjectRef, err = qp.findProject(projectId)
if err != nil {
return nil, err
}
}
return aCtx, nil
}
// Takes a request for a task's file to be copied from
// one s3 location to another. Ensures that if the destination
// file path already exists, no file copy is performed.
func S3CopyHandler(w http.ResponseWriter, r *http.Request) {
task := plugin.GetTask(r)
if task == nil {
http.Error(w, "task not found", http.StatusNotFound)
return
}
s3CopyReq := &S3CopyRequest{}
err := util.ReadJSONInto(r.Body, s3CopyReq)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "error reading push request: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Get the version for this task, so we can check if it has
// any already-done pushes
v, err := version.FindOne(version.ById(task.Version))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error querying task %v with version id %v: %v",
task.Id, task.Version, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Check for an already-pushed file with this same file path,
// but from a conflicting or newer commit sequence num
if v == nil {
evergreen.Logger.Logf(slogger.ERROR, "no version found for build %v", task.BuildId)
http.Error(w, "version not found", http.StatusNotFound)
return
}
copyFromLocation := strings.Join([]string{s3CopyReq.S3SourceBucket, s3CopyReq.S3SourcePath}, "/")
copyToLocation := strings.Join([]string{s3CopyReq.S3DestinationBucket, s3CopyReq.S3DestinationPath}, "/")
newestPushLog, err := model.FindPushLogAfter(copyToLocation, v.RevisionOrderNumber)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error querying for push log at %v: %v", copyToLocation, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if newestPushLog != nil {
evergreen.Logger.Logf(slogger.ERROR, "conflict with existing pushed file: "+
"%v", copyToLocation)
http.Error(w, "conflicting push target for this file already exists.", http.StatusConflict)
return
}
// It's now safe to put the file in its permanent location.
newPushLog := model.NewPushLog(v, task, copyToLocation)
err = newPushLog.Insert()
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "failed to create new push log: %v %v", newPushLog, err)
http.Error(w, fmt.Sprintf("failed to create push log: %v", err), http.StatusInternalServerError)
return
}
// Now copy the file into the permanent location
auth := &aws.Auth{
AccessKey: s3CopyReq.AwsKey,
SecretKey: s3CopyReq.AwsSecret,
}
evergreen.Logger.Logf(slogger.INFO, "performing S3 copy: '%v' => '%v'",
copyFromLocation, copyToLocation)
_, err = util.RetryArithmeticBackoff(func() error {
err := thirdparty.S3CopyFile(auth,
s3CopyReq.S3SourceBucket,
s3CopyReq.S3SourcePath,
s3CopyReq.S3DestinationBucket,
s3CopyReq.S3DestinationPath,
string(s3.PublicRead),
)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "S3 copy failed for task %v, "+
"retrying: %v", task.Id, err)
return util.RetriableError{err}
} else {
err := newPushLog.UpdateStatus(model.PushLogSuccess)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed"+
" for task %v: %v", task.Id, err)
}
return err
}
}, s3CopyRetryNumRetries, s3CopyRetrySleepTimeSec*time.Second)
if err != nil {
message := fmt.Sprintf("S3 copy failed for task %v: %v", task.Id, err)
evergreen.Logger.Logf(slogger.ERROR, message)
err = newPushLog.UpdateStatus(model.PushLogFailed)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed: %v", err)
}
http.Error(w, message, http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, "S3 copy Successful")
}
// getTasksForLatestVersion sends back the TaskJSON data associated with the latest version.
func getTasksForLatestVersion(w http.ResponseWriter, r *http.Request) {
project := mux.Vars(r)["project_id"]
name := mux.Vars(r)["name"]
skip, err := util.GetIntValue(r, "skip", 0)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if skip < 0 {
http.Error(w, "negative skip", http.StatusBadRequest)
return
}
pipeline := []bson.M{
// match on name and project
{"$match": bson.M{
NameKey: name,
ProjectIdKey: project,
}},
// sort on the revision number
{"$sort": bson.M{
RevisionOrderNumberKey: -1,
}},
// group by version id
{"$group": bson.M{
"_id": bson.M{
"r": "$" + RevisionOrderNumberKey,
"vid": "$" + VersionIdKey,
},
"t": bson.M{
"$push": bson.M{
"d": "$" + DataKey,
"t_id": "$" + TaskIdKey,
"tn": "$" + TaskNameKey,
"var": "$" + VariantKey,
},
},
}},
// sort on the _id
{"$sort": bson.M{
"_id.r": -1,
}},
{"$skip": skip},
{"$limit": 2},
}
tasksForVersions := []TasksForVersion{}
err = db.Aggregate(collection, pipeline, &tasksForVersions)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(tasksForVersions) == 0 {
http.Error(w, "no tasks found", http.StatusNotFound)
return
}
// we default have another revision
lastRevision := false
currentVersion := tasksForVersions[0]
// if there is only one version, then we are at the last revision.
if len(tasksForVersions) < 2 {
lastRevision = true
}
// get the version commit info
v, err := version.FindOne(version.ById(currentVersion.Id.VersionId))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if v == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
commitInfo := CommitInfo{
Author: v.Author,
Message: v.Message,
CreateTime: v.CreateTime,
Revision: v.Revision,
VersionId: v.Id,
}
data := VersionData{currentVersion.Tasks, commitInfo, lastRevision}
plugin.WriteJSON(w, http.StatusOK, data)
}
func (uis *UIServer) variantHistory(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
variant := mux.Vars(r)["variant"]
beforeCommitId := r.FormValue("before")
isJson := (r.FormValue("format") == "json")
var beforeCommit *version.Version
var err error
beforeCommit = nil
if beforeCommitId != "" {
beforeCommit, err = version.FindOne(version.ById(beforeCommitId))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if beforeCommit == nil {
evergreen.Logger.Logf(slogger.WARN, "'before' was specified but query returned nil")
}
}
project, err := model.FindProject("", projCtx.ProjectRef)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
bv := project.FindBuildVariant(variant)
if bv == nil {
http.Error(w, "variant not found", http.StatusNotFound)
return
}
iter := model.NewBuildVariantHistoryIterator(variant, bv.Name, projCtx.Project.Identifier)
tasks, versions, err := iter.GetItems(beforeCommit, 50)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
var suites []string
for _, task := range bv.Tasks {
suites = append(suites, task.Name)
}
sort.Strings(suites)
data := struct {
Variant string
Tasks []bson.M
TaskNames []string
Versions []version.Version
Project string
}{variant, tasks, suites, versions, projCtx.Project.Identifier}
if isJson {
uis.WriteJSON(w, http.StatusOK, data)
return
}
uis.WriteHTML(w, http.StatusOK, struct {
ProjectData projectContext
User *user.DBUser
Flashes []interface{}
Data interface{}
}{projCtx, GetUser(r), []interface{}{}, data}, "base",
"build_variant_history.html", "base_angular.html", "menu.html")
}
func TestTaskHistory(t *testing.T) {
Convey("With a task history iterator", t, func() {
buildVariants := []string{"bv_0", "bv_1", "bv_2"}
projectName := "project"
taskHistoryIterator := NewTaskHistoryIterator(evergreen.CompileStage,
buildVariants, projectName)
Convey("when finding task history items", func() {
testutil.HandleTestingErr(db.ClearCollections(version.Collection, TasksCollection),
t, "Error clearing test collections")
for i := 10; i < 20; i++ {
projectToUse := projectName
if i == 14 {
projectToUse = "otherBranch"
}
vid := fmt.Sprintf("v%v", i)
ver := &version.Version{
Id: vid,
RevisionOrderNumber: i,
Revision: vid,
Requester: evergreen.RepotrackerVersionRequester,
Project: projectToUse,
}
testutil.HandleTestingErr(ver.Insert(), t,
"Error inserting version")
for j := 0; j < 3; j++ {
task := &Task{
Id: fmt.Sprintf("t%v_%v", i, j),
BuildVariant: fmt.Sprintf("bv_%v", j),
DisplayName: evergreen.CompileStage,
RevisionOrderNumber: i,
Revision: vid,
Requester: evergreen.RepotrackerVersionRequester,
Project: projectToUse,
}
testutil.HandleTestingErr(task.Insert(), t,
"Error inserting task")
}
}
Convey("the specified number of task history items should be"+
" fetched, starting at the specified version", func() {
taskHistoryChunk, err := taskHistoryIterator.GetChunk(nil, 5, 0, false)
versions := taskHistoryChunk.Versions
tasks := taskHistoryChunk.Tasks
So(err, ShouldBeNil)
So(taskHistoryChunk.Exhausted.Before, ShouldBeFalse)
So(taskHistoryChunk.Exhausted.After, ShouldBeTrue)
So(len(versions), ShouldEqual, 5)
So(len(tasks), ShouldEqual, len(versions))
So(versions[0].Id, ShouldEqual, tasks[0]["_id"])
So(versions[len(versions)-1].Id, ShouldEqual, "v15")
So(tasks[len(tasks)-1]["_id"], ShouldEqual,
versions[len(versions)-1].Id)
})
Convey("tasks from a different project should be filtered"+
" out", func() {
vBefore, err := version.FindOne(version.ById("v15"))
So(err, ShouldBeNil)
taskHistoryChunk, err := taskHistoryIterator.GetChunk(vBefore, 5, 0, false)
versions := taskHistoryChunk.Versions
tasks := taskHistoryChunk.Tasks
So(err, ShouldBeNil)
So(taskHistoryChunk.Exhausted.Before, ShouldBeTrue)
So(taskHistoryChunk.Exhausted.After, ShouldBeFalse)
// Should skip 14 because its in another project
So(versions[0].Id, ShouldEqual, "v13")
So(versions[0].Id, ShouldEqual, tasks[0]["_id"])
So(len(tasks), ShouldEqual, 4)
So(len(tasks), ShouldEqual, len(versions))
So(tasks[len(tasks)-1]["_id"], ShouldEqual,
versions[len(versions)-1].Id)
})
})
})
}
func getVersionHistory(versionId string, N int) ([]version.Version, error) {
v, err := version.FindOne(version.ById(versionId))
if err != nil {
return nil, err
} else if v == nil {
return nil, fmt.Errorf("Version '%v' not found", versionId)
}
// Versions in the same push event, assuming that no two push events happen at the exact same time
// Never want more than 2N+1 versions, so make sure we add a limit
siblingVersions, err := version.Find(db.Query(
bson.M{
"order": v.RevisionOrderNumber,
"r": evergreen.RepotrackerVersionRequester,
"branch": v.Project,
}).WithoutFields(version.ConfigKey).Sort([]string{"order"}).Limit(2*N + 1))
if err != nil {
return nil, err
}
versionIndex := -1
for i := 0; i < len(siblingVersions); i++ {
if siblingVersions[i].Id == v.Id {
versionIndex = i
}
}
numSiblings := len(siblingVersions) - 1
versions := siblingVersions
if versionIndex < N {
// There are less than N later versions from the same push event
// N subsequent versions plus the specified one
subsequentVersions, err := version.Find(
//TODO encapsulate this query in version pkg
db.Query(bson.M{
"order": bson.M{"$gt": v.RevisionOrderNumber},
"r": evergreen.RepotrackerVersionRequester,
"branch": v.Project,
}).WithoutFields(version.ConfigKey).Sort([]string{"order"}).Limit(N - versionIndex))
if err != nil {
return nil, err
}
// Reverse the second array so we have the versions ordered "newest one first"
for i := 0; i < len(subsequentVersions)/2; i++ {
subsequentVersions[i], subsequentVersions[len(subsequentVersions)-1-i] = subsequentVersions[len(subsequentVersions)-1-i], subsequentVersions[i]
}
versions = append(subsequentVersions, versions...)
}
if numSiblings-versionIndex < N {
previousVersions, err := version.Find(db.Query(bson.M{
"order": bson.M{"$lt": v.RevisionOrderNumber},
"r": evergreen.RepotrackerVersionRequester,
"branch": v.Project,
}).WithoutFields(version.ConfigKey).Sort([]string{"-order"}).Limit(N))
if err != nil {
return nil, err
}
versions = append(versions, previousVersions...)
}
return versions, nil
}
func (uis *UIServer) modifyVersion(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
user := MustHaveUser(r)
if projCtx.Project == nil || projCtx.Version == nil {
http.Error(w, "not found", http.StatusNotFound)
return
}
jsonMap := struct {
Action string `json:"action"`
Active bool `json:"active"`
Abort bool `json:"abort"`
Priority int64 `json:"priority"`
TaskIds []string `json:"task_ids"`
}{}
err := util.ReadJSONInto(r.Body, &jsonMap)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// determine what action needs to be taken
switch jsonMap.Action {
case "restart":
if err = model.RestartVersion(projCtx.Version.Id, jsonMap.TaskIds, jsonMap.Abort, user.Id); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
case "set_active":
if jsonMap.Abort {
if err = model.AbortVersion(projCtx.Version.Id); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
if err = model.SetVersionActivation(projCtx.Version.Id, jsonMap.Active, user.Id); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
case "set_priority":
if jsonMap.Priority > evergreen.MaxTaskPriority {
if !uis.isSuperUser(user) {
http.Error(w, fmt.Sprintf("Insufficient access to set priority %v, can only set priority less than or equal to %v", jsonMap.Priority, evergreen.MaxTaskPriority),
http.StatusBadRequest)
return
}
}
if err = model.SetVersionPriority(projCtx.Version.Id, jsonMap.Priority); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
default:
uis.WriteJSON(w, http.StatusBadRequest, fmt.Sprintf("Unrecognized action: %v", jsonMap.Action))
return
}
// After the version has been modified, re-load it from DB and send back the up-to-date view
// to the client.
projCtx.Version, err = version.FindOne(version.ById(projCtx.Version.Id))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
versionAsUI := uiVersion{
Version: *projCtx.Version,
RepoOwner: projCtx.ProjectRef.Owner,
Repo: projCtx.ProjectRef.Repo,
}
dbBuilds, err := build.Find(build.ByIds(projCtx.Version.BuildIds))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
uiBuilds := make([]uiBuild, 0, len(projCtx.Version.BuildIds))
for _, build := range dbBuilds {
buildAsUI := uiBuild{Build: build}
uiTasks := make([]uiTask, 0, len(build.Tasks))
for _, t := range build.Tasks {
uiTasks = append(uiTasks,
uiTask{
Task: task.Task{Id: t.Id, Activated: t.Activated,
StartTime: t.StartTime, TimeTaken: t.TimeTaken, Status: t.Status,
Details: t.StatusDetails, DisplayName: t.DisplayName},
})
if t.Activated {
versionAsUI.ActiveTasks++
}
}
buildAsUI.Tasks = uiTasks
uiBuilds = append(uiBuilds, buildAsUI)
}
versionAsUI.Builds = uiBuilds
uis.WriteJSON(w, http.StatusOK, versionAsUI)
}
// Constructs all versions stored from recent repository revisions
// The additional complexity is due to support for project modifications on patch builds.
// We need to parse the remote config as it existed when each revision was created.
// The return value is the most recent version created as a result of storing the revisions.
// This function is idempotent with regard to storing the same version multiple times.
func (repoTracker *RepoTracker) StoreRevisions(revisions []model.Revision) (newestVersion *version.Version, err error) {
defer func() {
if newestVersion != nil {
// Fetch the updated version doc, so that we include buildvariants in the result
newestVersion, err = version.FindOne(version.ById(newestVersion.Id))
}
}()
ref := repoTracker.ProjectRef
for i := len(revisions) - 1; i >= 0; i-- {
revision := revisions[i].Revision
evergreen.Logger.Logf(slogger.INFO, "Processing revision %v in project %v", revision, ref.Identifier)
// We check if the version exists here so we can avoid fetching the github config unnecessarily
existingVersion, err := version.FindOne(version.ByProjectIdAndRevision(ref.Identifier, revisions[i].Revision))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR,
"Error looking up version at %v for project %v: %v", ref.Identifier, revision, err)
}
if existingVersion != nil {
evergreen.Logger.Logf(slogger.INFO,
"Skipping creation of version for project %v, revision %v since"+
" we already have a record for it", ref.Identifier, revision)
// We bind newestVersion here since we still need to return the most recent
// version, even if it already exists
newestVersion = existingVersion
continue
}
// Create the stub of the version (not stored in DB yet)
v, err := NewVersionFromRevision(ref, revisions[i])
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error creating version for project %v: %v", ref.Identifier, err)
}
err = sanityCheckOrderNum(v.RevisionOrderNumber, ref.Identifier)
if err != nil { // something seriously wrong (bad data in db?) so fail now
panic(err)
}
project, err := repoTracker.GetProjectConfig(revision)
if err != nil {
projectError, isProjectError := err.(projectConfigError)
if isProjectError {
// Store just the stub version with the project errors
v.Errors = projectError.errors
if err := v.Insert(); err != nil {
evergreen.Logger.Logf(slogger.ERROR,
"Failed storing stub version for project %v: %v", ref.Identifier, err)
return nil, err
}
newestVersion = v
continue
} else {
// Fatal error - don't store the stub
evergreen.Logger.Logf(slogger.INFO,
"Failed to get config for project %v at revision %v: %v", ref.Identifier, revision, err)
return nil, err
}
}
// We have a config, so turn it into a usable yaml string to store with the version doc
projectYamlBytes, err := yaml.Marshal(project)
if err != nil {
return nil, fmt.Errorf("Error marshalling config: %v", err)
}
v.Config = string(projectYamlBytes)
// We rebind newestVersion each iteration, so the last binding will be the newest version
err = createVersionItems(v, ref, project)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error creating version items for %v in project %v: %v",
v.Id, ref.Identifier, err)
return nil, err
}
newestVersion = v
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Unable to store revision %v for project %v: %v:",
revision, ref.Identifier, err)
return nil, err
}
}
return
}
