// Helper to make the appropriate query to the versions collection for what
// we will need. "before" indicates whether to fetch versions before or
// after the passed-in task.
func makeVersionsQuery(anchorOrderNum int, projectId string, versionsToFetch int, before bool) ([]version.Version, error) {
// decide how the versions we want relative to the task's revision order number
ronQuery := bson.M{"$gt": anchorOrderNum}
if before {
ronQuery = bson.M{"$lt": anchorOrderNum}
}
// switch how to sort the versions
sortVersions := []string{version.RevisionOrderNumberKey}
if before {
sortVersions = []string{"-" + version.RevisionOrderNumberKey}
}
// fetch the versions
return version.Find(
db.Query(bson.M{
version.IdentifierKey: projectId,
version.RevisionOrderNumberKey: ronQuery,
}).WithFields(
version.RevisionOrderNumberKey,
version.RevisionKey,
version.MessageKey,
version.CreateTimeKey,
).Sort(sortVersions).Limit(versionsToFetch))
}
func (iter *taskHistoryIterator) findAllVersions(v *version.Version, numRevisions int, before, include bool) ([]version.Version, bool, error) {
versionQuery := bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.IdentifierKey: iter.ProjectName,
}
// If including the specified version in the result, then should
// get an additional revision
if include {
numRevisions++
}
// Determine the comparator to use based on whether the revisions
// come before/after the specified version
compare, order := "$gt", version.RevisionOrderNumberKey
if before {
compare, order = "$lt", fmt.Sprintf("-%v", version.RevisionOrderNumberKey)
if include {
compare = "$lte"
}
} else if include {
compare = "$gte"
}
if v != nil {
versionQuery[version.RevisionOrderNumberKey] = bson.M{compare: v.RevisionOrderNumber}
}
// Get the next numRevisions, plus an additional one to check if have
// reached the beginning/end of history
versions, err := version.Find(
db.Query(versionQuery).WithFields(
version.IdKey,
version.RevisionOrderNumberKey,
version.RevisionKey,
version.MessageKey,
version.CreateTimeKey,
).Sort([]string{order}).Limit(numRevisions + 1))
// Check if there were fewer results returned by the query than what
// the limit was set as
exhausted := len(versions) <= numRevisions
if !exhausted {
// Exclude the last version because we actually only wanted
// `numRevisions` number of commits
versions = versions[:len(versions)-1]
}
// The iterator can only be exhausted if an actual version was specified
exhausted = exhausted || (v == nil && numRevisions == 0)
if !before {
// Reverse the order so that the most recent version is first
for i, j := 0, len(versions)-1; i < j; i, j = i+1, j-1 {
versions[i], versions[j] = versions[j], versions[i]
}
}
return versions, exhausted, err
}
func getTimelineData(projectName, requester string, versionsToSkip, versionsPerPage int) (*timelineData, error) {
data := &timelineData{}
// get the total number of versions in the database (used for pagination)
totalVersions, err := version.Count(version.ByProjectId(projectName))
if err != nil {
return nil, err
}
data.TotalVersions = totalVersions
q := version.ByMostRecentForRequester(projectName, requester).WithoutFields(version.ConfigKey).
Skip(versionsToSkip * versionsPerPage).Limit(versionsPerPage)
// get the most recent versions, to display in their entirety on the page
versionsFromDB, err := version.Find(q)
if err != nil {
return nil, err
}
// create the necessary uiVersion struct for each version
uiVersions := make([]uiVersion, len(versionsFromDB))
for versionIdx, version := range versionsFromDB {
versionAsUI := uiVersion{Version: version}
uiVersions[versionIdx] = versionAsUI
buildIds := version.BuildIds
dbBuilds, err := build.Find(build.ByIds(buildIds))
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Ids: %v", buildIds)
}
buildsMap := make(map[string]build.Build)
for _, dbBuild := range dbBuilds {
buildsMap[dbBuild.Id] = dbBuild
}
uiBuilds := make([]uiBuild, len(dbBuilds))
for buildIdx, buildId := range buildIds {
build := buildsMap[buildId]
buildAsUI := uiBuild{Build: build}
uiBuilds[buildIdx] = buildAsUI
}
versionAsUI.Builds = uiBuilds
uiVersions[versionIdx] = versionAsUI
}
data.Versions = uiVersions
return data, nil
}
// Helper function to fetch a group of versions and their associated builds.
// Returns the versions themselves, as well as a map of version id -> the
// builds that are a part of the version (unsorted).
func fetchVersionsAndAssociatedBuilds(project *model.Project, skip int, numVersions int) ([]version.Version, map[string][]build.Build, error) {
// fetch the versions from the db
versionsFromDB, err := version.Find(version.ByProjectId(project.Identifier).
WithFields(
version.RevisionKey,
version.ErrorsKey,
version.WarningsKey,
version.IgnoredKey,
version.MessageKey,
version.AuthorKey,
version.RevisionOrderNumberKey,
version.CreateTimeKey,
).Sort([]string{"-" + version.RevisionOrderNumberKey}).Skip(skip).Limit(numVersions))
if err != nil {
return nil, nil, fmt.Errorf("error fetching versions from database: %v", err)
}
// create a slice of the version ids (used to fetch the builds)
versionIds := make([]string, 0, len(versionsFromDB))
for _, v := range versionsFromDB {
versionIds = append(versionIds, v.Id)
}
// fetch all of the builds (with only relevant fields)
buildsFromDb, err := build.Find(
build.ByVersions(versionIds).
WithFields(build.BuildVariantKey, build.TasksKey, build.VersionKey))
if err != nil {
return nil, nil, fmt.Errorf("error fetching builds from database: %v", err)
}
// group the builds by version
buildsByVersion := map[string][]build.Build{}
for _, build := range buildsFromDb {
buildsByVersion[build.Version] = append(buildsByVersion[build.Version], build)
}
return versionsFromDB, buildsByVersion, nil
}
//versionFind redirects to the correct version page based on the gitHash and versionId given.
//It finds the version associated with the versionId and gitHash and redirects to /version/{version_id}.
func (uis *UIServer) versionFind(w http.ResponseWriter, r *http.Request) {
id := mux.Vars(r)["project_id"]
revision := mux.Vars(r)["revision"]
if len(revision) < 5 {
http.Error(w, "revision not long enough: must be at least 5 characters", http.StatusBadRequest)
return
}
foundVersions, err := version.Find(version.ByProjectIdAndRevisionPrefix(id, revision).Limit(2))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if len(foundVersions) == 0 {
uis.WriteJSON(w, http.StatusNotFound, fmt.Sprintf("Version Not Found: %v - %v", id, revision))
return
}
if len(foundVersions) > 1 {
uis.WriteJSON(w, http.StatusBadRequest, fmt.Sprintf("Multiple versions found: %v - %v", id, revision))
return
}
http.Redirect(w, r, fmt.Sprintf("/version/%v", foundVersions[0].Id), http.StatusFound)
}
func getVersionHistory(versionId string, N int) ([]version.Version, error) {
v, err := version.FindOne(version.ById(versionId))
if err != nil {
return nil, err
} else if v == nil {
return nil, fmt.Errorf("Version '%v' not found", versionId)
}
// Versions in the same push event, assuming that no two push events happen at the exact same time
// Never want more than 2N+1 versions, so make sure we add a limit
siblingVersions, err := version.Find(db.Query(
bson.M{
"order": v.RevisionOrderNumber,
"r": evergreen.RepotrackerVersionRequester,
"branch": v.Project,
}).WithoutFields(version.ConfigKey).Sort([]string{"order"}).Limit(2*N + 1))
if err != nil {
return nil, err
}
versionIndex := -1
for i := 0; i < len(siblingVersions); i++ {
if siblingVersions[i].Id == v.Id {
versionIndex = i
}
}
numSiblings := len(siblingVersions) - 1
versions := siblingVersions
if versionIndex < N {
// There are less than N later versions from the same push event
// N subsequent versions plus the specified one
subsequentVersions, err := version.Find(
//TODO encapsulate this query in version pkg
db.Query(bson.M{
"order": bson.M{"$gt": v.RevisionOrderNumber},
"r": evergreen.RepotrackerVersionRequester,
"branch": v.Project,
}).WithoutFields(version.ConfigKey).Sort([]string{"order"}).Limit(N - versionIndex))
if err != nil {
return nil, err
}
// Reverse the second array so we have the versions ordered "newest one first"
for i := 0; i < len(subsequentVersions)/2; i++ {
subsequentVersions[i], subsequentVersions[len(subsequentVersions)-1-i] = subsequentVersions[len(subsequentVersions)-1-i], subsequentVersions[i]
}
versions = append(subsequentVersions, versions...)
}
if numSiblings-versionIndex < N {
previousVersions, err := version.Find(db.Query(bson.M{
"order": bson.M{"$lt": v.RevisionOrderNumber},
"r": evergreen.RepotrackerVersionRequester,
"branch": v.Project,
}).WithoutFields(version.ConfigKey).Sort([]string{"-order"}).Limit(N))
if err != nil {
return nil, err
}
versions = append(versions, previousVersions...)
}
return versions, nil
}
// taskTimingJSON sends over the task data for a certain task of a certain build variant
func (uis *UIServer) taskTimingJSON(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
beforeTaskId := r.FormValue("before")
limit, err := getIntValue(r, "limit", 50)
if err != nil {
uis.LoggedError(w, r, http.StatusBadRequest, err)
return
}
buildVariant := mux.Vars(r)["build_variant"]
taskName := mux.Vars(r)["task_name"]
request := mux.Vars(r)["request"]
if projCtx.Project == nil {
uis.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("not found"))
return
}
bv := projCtx.Project.FindBuildVariant(buildVariant)
if bv == nil {
uis.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("build variant %v not found", buildVariant))
return
}
var versionIds []string
data := UIStats{}
// if its all tasks find the build
if taskName == "" || taskName == "All Tasks" {
// TODO: switch this to be a query on the builds TaskCache
builds, err := build.Find(build.ByProjectAndVariant(projCtx.Project.Identifier, buildVariant, request).
WithFields(build.IdKey, build.CreateTimeKey, build.VersionKey,
build.TimeTakenKey, build.TasksKey, build.FinishTimeKey, build.StartTimeKey).
Sort([]string{"-" + build.RevisionOrderNumberKey}).
Limit(limit))
if err != nil {
uis.LoggedError(w, r, http.StatusBadRequest, err)
return
}
versionIds = make([]string, 0, len(builds))
uiBuilds := []*UIBuild{}
// get the versions for every single task that was returned
for _, build := range builds {
// create a UITask
b := &UIBuild{
Id: build.Id,
CreateTime: build.CreateTime,
StartTime: build.StartTime,
FinishTime: build.FinishTime,
Version: build.Version,
Status: build.Status,
TimeTaken: int64(build.TimeTaken),
Tasks: build.Tasks,
}
uiBuilds = append(uiBuilds, b)
versionIds = append(versionIds, b.Version)
}
data.Builds = uiBuilds
} else {
foundTask := false
for _, task := range bv.Tasks {
if task.Name == taskName {
foundTask = true
break
}
}
if !foundTask {
uis.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("no task named '%v'", taskName))
return
}
tasks, err := model.FindCompletedTasksByVariantAndName(projCtx.Project.Identifier,
buildVariant, taskName, request, limit, beforeTaskId)
if err != nil {
uis.LoggedError(w, r, http.StatusNotFound, err)
return
}
uiTasks := []*UITask{}
versionIds = make([]string, 0, len(tasks))
// get the versions for every single task that was returned
for _, task := range tasks {
// create a UITask
t := &UITask{
Id: task.Id,
CreateTime: task.CreateTime,
DispatchTime: task.DispatchTime,
PushTime: task.PushTime,
ScheduledTime: task.ScheduledTime,
StartTime: task.StartTime,
FinishTime: task.FinishTime,
Version: task.Version,
Status: task.Status,
Host: task.HostId,
Distro: task.DistroId,
}
uiTasks = append(uiTasks, t)
versionIds = append(versionIds, task.Version)
}
data.Tasks = uiTasks
}
// Populate the versions field if with commits, otherwise patches field
if request == evergreen.RepotrackerVersionRequester {
versions, err := version.Find(version.ByIds(versionIds).
WithFields(version.IdKey, version.CreateTimeKey, version.MessageKey,
version.AuthorKey, version.RevisionKey, version.RevisionOrderNumberKey).
Sort([]string{"-" + version.RevisionOrderNumberKey}))
if err != nil {
uis.LoggedError(w, r, http.StatusBadRequest, err)
return
}
data.Versions = versions
} else {
// patches
patches, err := patch.Find(patch.ByVersions(versionIds).
WithFields(patch.IdKey, patch.CreateTimeKey, patch.DescriptionKey, patch.AuthorKey))
if err != nil {
uis.LoggedError(w, r, http.StatusBadRequest, err)
return
}
data.Patches = patches
}
uis.WriteJSON(w, http.StatusOK, data)
}
// Returns versions and tasks grouped by gitspec, newest first (sorted by order number desc)
func (self *buildVariantHistoryIterator) GetItems(beforeCommit *version.Version, numRevisions int) ([]bson.M, []version.Version, error) {
session, dbobj, err := db.GetGlobalSessionFactory().GetSession()
defer session.Close()
if err != nil {
return nil, nil, err
}
var versionQuery db.Q
if beforeCommit != nil {
versionQuery = db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.RevisionOrderNumberKey: bson.M{"$lt": beforeCommit.RevisionOrderNumber},
version.IdentifierKey: self.ProjectName,
version.BuildVariantsKey: bson.M{
"$elemMatch": bson.M{
version.BuildStatusVariantKey: self.BuildVariantInVersion,
},
},
})
} else {
versionQuery = db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.IdentifierKey: self.ProjectName,
version.BuildVariantsKey: bson.M{
"$elemMatch": bson.M{
version.BuildStatusVariantKey: self.BuildVariantInVersion,
},
},
})
}
versionQuery = versionQuery.WithFields(
version.IdKey,
version.RevisionOrderNumberKey,
version.RevisionKey,
version.MessageKey,
version.CreateTimeKey,
).Sort([]string{"-" + version.RevisionOrderNumberKey}).Limit(numRevisions)
//Get the next numCommits
versions, err := version.Find(versionQuery)
if err != nil {
return nil, nil, err
}
if len(versions) == 0 {
return nil, []version.Version{}, nil
}
//versionEndBoundary is the *earliest* version which should be included in results
versionEndBoundary := versions[len(versions)-1]
matchFilter := bson.M{
TaskRequesterKey: evergreen.RepotrackerVersionRequester,
TaskBuildVariantKey: self.BuildVariantInTask,
TaskProjectKey: self.ProjectName,
}
if beforeCommit != nil {
matchFilter[TaskRevisionOrderNumberKey] = bson.M{
"$gte": versionEndBoundary.RevisionOrderNumber,
"$lt": beforeCommit.RevisionOrderNumber,
}
} else {
matchFilter[TaskRevisionOrderNumberKey] = bson.M{
"$gte": versionEndBoundary.RevisionOrderNumber,
}
}
pipeline := dbobj.C(TasksCollection).Pipe(
[]bson.M{
{"$match": matchFilter},
bson.M{"$sort": bson.D{{TaskRevisionOrderNumberKey, 1}}},
bson.M{
"$group": bson.M{
"_id": "$" + TaskRevisionKey,
"order": bson.M{"$first": "$" + TaskRevisionOrderNumberKey},
"tasks": bson.M{
"$push": bson.M{
"_id": "$" + TaskIdKey,
"status": "$" + TaskStatusKey,
"task_end_details": "$" + TaskDetailsKey,
"activated": "$" + TaskActivatedKey,
"time_taken": "$" + TaskTimeTakenKey,
"display_name": "$" + TaskDisplayNameKey,
},
},
},
},
bson.M{"$sort": bson.M{TaskRevisionOrderNumberKey: -1, TaskDisplayNameKey: 1}},
},
)
var output []bson.M
err = pipeline.All(&output)
if err != nil {
return nil, nil, err
}
return output, versions, nil
}
// Returns a JSON response of an array with the NumRecentVersions
// most recent versions (sorted on commit order number descending).
func (restapi restAPI) getRecentVersions(w http.ResponseWriter, r *http.Request) {
projectId := mux.Vars(r)["project_id"]
versions, err := version.Find(version.ByMostRecentForRequester(projectId, evergreen.RepotrackerVersionRequester).Limit(10))
if err != nil {
msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
// Create a slice of version ids to find all relevant builds
versionIds := make([]string, 0, len(versions))
// Cache the order of versions in a map for lookup by their id
versionIdx := make(map[string]int, len(versions))
for i, version := range versions {
versionIds = append(versionIds, version.Id)
versionIdx[version.Id] = i
}
// Find all builds corresponding the set of version ids
builds, err := build.Find(
build.ByVersions(versionIds).
WithFields(build.BuildVariantKey, build.DisplayNameKey, build.TasksKey, build.VersionKey))
if err != nil {
msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
result := recentVersionsContent{
Project: projectId,
Versions: make([]versionLessInfo, 0, len(versions)),
}
for _, version := range versions {
versionInfo := versionLessInfo{
Id: version.Id,
Author: version.Author,
Revision: version.Revision,
Message: version.Message,
Builds: make(versionByBuild),
}
result.Versions = append(result.Versions, versionInfo)
}
for _, build := range builds {
buildInfo := versionBuildInfo{
Id: build.Id,
Name: build.DisplayName,
Tasks: make(versionByBuildByTask, len(build.Tasks)),
}
for _, task := range build.Tasks {
buildInfo.Tasks[task.DisplayName] = versionStatus{
Id: task.Id,
Status: task.Status,
TimeTaken: task.TimeTaken,
}
}
versionInfo := result.Versions[versionIdx[build.Version]]
versionInfo.Builds[build.BuildVariant] = buildInfo
}
restapi.WriteJSON(w, http.StatusOK, result)
return
}
func (uis *UIServer) grid(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
if projCtx.Project == nil {
http.Error(w, "Project not found", http.StatusNotFound)
}
// If no version was specified in the URL, grab the latest version on the project
if projCtx.Version == nil {
v, err := version.Find(version.ByMostRecentForRequester(projCtx.Project.Identifier, evergreen.RepotrackerVersionRequester).Limit(1))
if err != nil {
http.Error(w, fmt.Sprintf("Error finding version: %v", err), http.StatusInternalServerError)
return
}
if len(v) > 0 {
projCtx.Version = &v[0]
}
}
var versions map[string]version.Version
var cells grid.Grid
var failures grid.Failures
var depth int
var err error
d := mux.Vars(r)["depth"]
if d == "" {
depth = defaultGridDepth
} else {
depth, err = strconv.Atoi(d)
if err != nil {
http.Error(w, fmt.Sprintf("Error converting depth: %v", err), http.StatusBadRequest)
return
}
if depth < 0 {
http.Error(w, fmt.Sprintf("Depth must be non-negative, got %v", depth), http.StatusBadRequest)
return
}
}
if projCtx.Version != nil {
recentVersions, err := version.Find(version.
ByProjectIdAndOrder(projCtx.Version.Project, projCtx.Version.RevisionOrderNumber).
WithFields(version.IdKey, version.RevisionKey, version.RevisionOrderNumberKey, version.MessageKey).
Sort([]string{"-" + version.RevisionOrderNumberKey}).
Limit(depth + 1))
if err != nil {
http.Error(w, fmt.Sprintf("Error fetching versions: %v", err), http.StatusInternalServerError)
return
}
versions = make(map[string]version.Version, len(recentVersions))
for _, v := range recentVersions {
versions[v.Revision] = v
}
cells, err = grid.FetchCells(*projCtx.Version, depth)
if err != nil {
http.Error(w, fmt.Sprintf("Error fetching builds: %v", err), http.StatusInternalServerError)
return
}
failures, err = grid.FetchFailures(*projCtx.Version, depth)
if err != nil {
http.Error(w, fmt.Sprintf("Error fetching builds: %v", err), http.StatusInternalServerError)
return
}
} else {
versions = make(map[string]version.Version)
cells = make(grid.Grid, 0)
failures = make(grid.Failures, 0)
}
uis.WriteHTML(w, http.StatusOK, struct {
ProjectData projectContext
Versions map[string]version.Version
GridCells grid.Grid
Failures grid.Failures
User *user.DBUser
}{projCtx, versions, cells, failures, GetUser(r)}, "base", "grid.html", "base_angular.html", "menu.html")
}
func (uis *UIServer) patchTimelineJson(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
pageNum, err := strconv.Atoi(r.FormValue("page"))
if err != nil {
pageNum = 0
}
skip := pageNum * DefaultLimit
user := mux.Vars(r)["user_id"]
var patches []patch.Patch
if len(user) > 0 {
patches, err = patch.Find(patch.ByUser(user).
Project(patch.ExcludePatchDiff).
Sort([]string{"-" + patch.CreateTimeKey}).
Skip(skip).Limit(DefaultLimit))
} else {
patches, err = patch.Find(patch.ByProject(projCtx.Project.Identifier).
Sort([]string{"-" + patch.CreateTimeKey}).
Project(patch.ExcludePatchDiff).
Skip(skip).Limit(DefaultLimit))
}
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error fetching patches for %v: %v", projCtx.Project.Identifier, err))
return
}
versionIds := make([]string, 0, len(patches))
uiPatches := make([]uiPatch, 0, len(patches))
for _, patch := range patches {
if patch.Version != "" {
versionIds = append(versionIds, patch.Version)
}
baseVersion, err := version.FindOne(version.ByProjectIdAndRevision(patch.Project, patch.Githash))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
var baseVersionId string
if baseVersion != nil {
baseVersionId = baseVersion.Id
}
patch.Patches = nil
uiPatches = append(uiPatches, uiPatch{Patch: patch, BaseVersionId: baseVersionId})
}
versions, err := version.Find(version.ByIds(versionIds).WithoutFields(version.ConfigKey))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error fetching versions for patches: %v", err))
return
}
versionsMap := map[string]*uiVersion{}
for _, version := range versions {
versionUI, err := PopulateUIVersion(&version)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
versionsMap[version.Id] = versionUI
}
data := struct {
VersionsMap map[string]*uiVersion
UIPatches []uiPatch
PageNum int
}{versionsMap, uiPatches, pageNum}
uis.WriteJSON(w, http.StatusOK, data)
}