func getTimelineData(projectName, requester string, versionsToSkip, versionsPerPage int) (*timelineData, error) {
data := &timelineData{}
// get the total number of versions in the database (used for pagination)
totalVersions, err := version.Count(version.ByProjectId(projectName))
if err != nil {
return nil, err
}
data.TotalVersions = totalVersions
q := version.ByMostRecentForRequester(projectName, requester).WithoutFields(version.ConfigKey).
Skip(versionsToSkip * versionsPerPage).Limit(versionsPerPage)
// get the most recent versions, to display in their entirety on the page
versionsFromDB, err := version.Find(q)
if err != nil {
return nil, err
}
// create the necessary uiVersion struct for each version
uiVersions := make([]uiVersion, len(versionsFromDB))
for versionIdx, version := range versionsFromDB {
versionAsUI := uiVersion{Version: version}
uiVersions[versionIdx] = versionAsUI
buildIds := version.BuildIds
dbBuilds, err := build.Find(build.ByIds(buildIds))
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Ids: %v", buildIds)
}
buildsMap := make(map[string]build.Build)
for _, dbBuild := range dbBuilds {
buildsMap[dbBuild.Id] = dbBuild
}
uiBuilds := make([]uiBuild, len(dbBuilds))
for buildIdx, buildId := range buildIds {
build := buildsMap[buildId]
buildAsUI := uiBuild{Build: build}
uiBuilds[buildIdx] = buildAsUI
}
versionAsUI.Builds = uiBuilds
uiVersions[versionIdx] = versionAsUI
}
data.Versions = uiVersions
return data, nil
}
// Verifies that the given revision order number is higher than the latest number stored for the project.
func sanityCheckOrderNum(revOrderNum int, projectId string) error {
latest, err := version.FindOne(version.ByMostRecentForRequester(projectId, evergreen.RepotrackerVersionRequester))
if err != nil {
return fmt.Errorf("Error getting latest version: %v", err.Error())
}
// When there are no versions in the db yet, sanity check is moot
if latest != nil {
if revOrderNum <= latest.RevisionOrderNumber {
return fmt.Errorf("Commit order number isn't greater than last stored version's: %v <= %v",
revOrderNum, latest.RevisionOrderNumber)
}
}
return nil
}
func TestStoreRepositoryRevisions(t *testing.T) {
dropTestDB(t)
testutil.ConfigureIntegrationTest(t, testConfig, "TestStoreRepositoryRevisions")
Convey("When storing revisions gotten from a repository...", t, func() {
err := testutil.CreateTestLocalConfig(testConfig, "mci-test", "")
So(err, ShouldBeNil)
repoTracker := RepoTracker{testConfig, projectRef, NewGithubRepositoryPoller(projectRef,
testConfig.Credentials["github"])}
// insert distros used in testing.
d := distro.Distro{Id: "test-distro-one"}
So(d.Insert(), ShouldBeNil)
d.Id = "test-distro-two"
So(d.Insert(), ShouldBeNil)
Convey("On storing a single repo revision, we expect a version to be created"+
" in the database for this project, which should be retrieved when we search"+
" for this project's most recent version", func() {
createTime := time.Now()
revisionOne := *createTestRevision("firstRevision", createTime)
revisions := []model.Revision{revisionOne}
resultVersion, err := repoTracker.StoreRevisions(revisions)
testutil.HandleTestingErr(err, t, "Error storing repository revisions %v")
newestVersion, err := version.FindOne(version.ByMostRecentForRequester(projectRef.String(), evergreen.RepotrackerVersionRequester))
testutil.HandleTestingErr(err, t, "Error retreiving newest version %v")
So(resultVersion, ShouldResemble, newestVersion)
})
Convey("On storing several repo revisions, we expect a version to be created "+
"for each revision", func() {
createTime := time.Now()
laterCreateTime := createTime.Add(time.Duration(4 * time.Hour))
revisionOne := *createTestRevision("one", laterCreateTime)
revisionTwo := *createTestRevision("two", createTime)
revisions := []model.Revision{revisionOne, revisionTwo}
_, err := repoTracker.StoreRevisions(revisions)
testutil.HandleTestingErr(err, t, "Error storing repository revisions %v")
versionOne, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionOne.Revision))
testutil.HandleTestingErr(err, t, "Error retrieving first stored version %v")
versionTwo, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionTwo.Revision))
testutil.HandleTestingErr(err, t, "Error retreiving second stored version %v")
So(versionOne.Revision, ShouldEqual, revisionOne.Revision)
So(versionTwo.Revision, ShouldEqual, revisionTwo.Revision)
})
Reset(func() {
dropTestDB(t)
})
})
Convey("When storing versions from repositories with remote configuration files...", t, func() {
project := createTestProject(nil, nil)
revisions := []model.Revision{
*createTestRevision("foo", time.Now().Add(1*time.Minute)),
}
poller := NewMockRepoPoller(project, revisions)
repoTracker := RepoTracker{
testConfig,
&model.ProjectRef{
Identifier: "testproject",
BatchTime: 10,
},
poller,
}
// insert distros used in testing.
d := distro.Distro{Id: "test-distro-one"}
So(d.Insert(), ShouldBeNil)
d.Id = "test-distro-two"
So(d.Insert(), ShouldBeNil)
Convey("We should not fetch configs for versions we already have stored.",
func() {
So(poller.ConfigGets, ShouldBeZeroValue)
// Store revisions the first time
_, err := repoTracker.StoreRevisions(revisions)
So(err, ShouldBeNil)
// We should have fetched the config once for each revision
So(poller.ConfigGets, ShouldEqual, len(revisions))
// Store them again
_, err = repoTracker.StoreRevisions(revisions)
So(err, ShouldBeNil)
// We shouldn't have fetched the config any additional times
// since we have already stored these versions
So(poller.ConfigGets, ShouldEqual, len(revisions))
},
)
Convey("We should handle invalid configuration files gracefully by storing a stub version",
func() {
errStrs := []string{"Someone dun' goof'd"}
poller.setNextError(projectConfigError{errStrs})
stubVersion, err := repoTracker.StoreRevisions(revisions)
// We want this error to get swallowed so a config error
// doesn't stop additional versions from getting created
So(err, ShouldBeNil)
So(stubVersion.Errors, ShouldResemble, errStrs)
},
)
Convey("If there is an error other than a config error while fetching a config, we should fail hard",
func() {
unexpectedError := errors.New("Something terrible has happened!!")
poller.setNextError(unexpectedError)
v, err := repoTracker.StoreRevisions(revisions)
So(v, ShouldBeNil)
So(err, ShouldEqual, unexpectedError)
},
)
Reset(func() {
dropTestDB(t)
})
})
}
// The FetchRevisions method is used by a RepoTracker to run the pipeline for
// tracking repositories. It performs everything from polling the repository to
// persisting any changes retrieved from the repository reference.
func (repoTracker *RepoTracker) FetchRevisions(numNewRepoRevisionsToFetch int) (
err error) {
settings := repoTracker.Settings
projectRef := repoTracker.ProjectRef
projectIdentifier := projectRef.String()
if !projectRef.Enabled {
evergreen.Logger.Logf(slogger.INFO, "Skipping disabled project “%v”", projectRef)
return nil
}
repository, err := model.FindRepository(projectIdentifier)
if err != nil {
return fmt.Errorf("error finding repository '%v': %v",
projectIdentifier, err)
}
var revisions []model.Revision
var lastRevision string
if repository != nil {
lastRevision = repository.LastRevision
}
if lastRevision == "" {
// if this is the first time we're running the tracker for this project,
// fetch the most recent `numNewRepoRevisionsToFetch` revisions
evergreen.Logger.Logf(slogger.INFO, "No last recorded repository revision "+
"for “%v”. Proceeding to fetch most recent %v revisions",
projectRef, numNewRepoRevisionsToFetch)
revisions, err = repoTracker.GetRecentRevisions(numNewRepoRevisionsToFetch)
} else {
evergreen.Logger.Logf(slogger.INFO, "Last recorded repository revision for "+
"“%v” is “%v”", projectRef, lastRevision)
revisions, err = repoTracker.GetRevisionsSince(lastRevision,
settings.RepoTracker.MaxRepoRevisionsToSearch)
}
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error fetching revisions for "+
"repository “%v”: %v", projectRef, err)
repoTracker.sendFailureNotification(lastRevision, err)
return nil
}
var lastVersion *version.Version
if len(revisions) > 0 {
lastVersion, err = repoTracker.StoreRevisions(revisions)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error storing revisions for "+
"repository %v: %v", projectRef, err)
return err
}
err = model.UpdateLastRevision(lastVersion.Identifier, lastVersion.Revision)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error updating last revision for "+
"repository %v: %v", projectRef, err)
return err
}
} else {
lastVersion, err = version.FindOne(version.ByMostRecentForRequester(projectIdentifier, evergreen.RepotrackerVersionRequester))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error getting most recent version for "+
"repository %v: %v", projectRef, err)
return err
}
}
if lastVersion == nil {
evergreen.Logger.Logf(slogger.WARN, "no version to activate for repository %v", projectIdentifier)
return nil
}
err = repoTracker.activateElapsedBuilds(lastVersion)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error activating variants: %v", err)
return err
}
return nil
}
// Returns a JSON response of an array with the NumRecentVersions
// most recent versions (sorted on commit order number descending).
func (restapi restAPI) getRecentVersions(w http.ResponseWriter, r *http.Request) {
projectId := mux.Vars(r)["project_id"]
versions, err := version.Find(version.ByMostRecentForRequester(projectId, evergreen.RepotrackerVersionRequester).Limit(10))
if err != nil {
msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
// Create a slice of version ids to find all relevant builds
versionIds := make([]string, 0, len(versions))
// Cache the order of versions in a map for lookup by their id
versionIdx := make(map[string]int, len(versions))
for i, version := range versions {
versionIds = append(versionIds, version.Id)
versionIdx[version.Id] = i
}
// Find all builds corresponding the set of version ids
builds, err := build.Find(
build.ByVersions(versionIds).
WithFields(build.BuildVariantKey, build.DisplayNameKey, build.TasksKey, build.VersionKey))
if err != nil {
msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
result := recentVersionsContent{
Project: projectId,
Versions: make([]versionLessInfo, 0, len(versions)),
}
for _, version := range versions {
versionInfo := versionLessInfo{
Id: version.Id,
Author: version.Author,
Revision: version.Revision,
Message: version.Message,
Builds: make(versionByBuild),
}
result.Versions = append(result.Versions, versionInfo)
}
for _, build := range builds {
buildInfo := versionBuildInfo{
Id: build.Id,
Name: build.DisplayName,
Tasks: make(versionByBuildByTask, len(build.Tasks)),
}
for _, task := range build.Tasks {
buildInfo.Tasks[task.DisplayName] = versionStatus{
Id: task.Id,
Status: task.Status,
TimeTaken: task.TimeTaken,
}
}
versionInfo := result.Versions[versionIdx[build.Version]]
versionInfo.Builds[build.BuildVariant] = buildInfo
}
restapi.WriteJSON(w, http.StatusOK, result)
return
}
func (uis *UIServer) grid(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
if projCtx.Project == nil {
http.Error(w, "Project not found", http.StatusNotFound)
}
// If no version was specified in the URL, grab the latest version on the project
if projCtx.Version == nil {
v, err := version.Find(version.ByMostRecentForRequester(projCtx.Project.Identifier, evergreen.RepotrackerVersionRequester).Limit(1))
if err != nil {
http.Error(w, fmt.Sprintf("Error finding version: %v", err), http.StatusInternalServerError)
return
}
if len(v) > 0 {
projCtx.Version = &v[0]
}
}
var versions map[string]version.Version
var cells grid.Grid
var failures grid.Failures
var depth int
var err error
d := mux.Vars(r)["depth"]
if d == "" {
depth = defaultGridDepth
} else {
depth, err = strconv.Atoi(d)
if err != nil {
http.Error(w, fmt.Sprintf("Error converting depth: %v", err), http.StatusBadRequest)
return
}
if depth < 0 {
http.Error(w, fmt.Sprintf("Depth must be non-negative, got %v", depth), http.StatusBadRequest)
return
}
}
if projCtx.Version != nil {
recentVersions, err := version.Find(version.
ByProjectIdAndOrder(projCtx.Version.Project, projCtx.Version.RevisionOrderNumber).
WithFields(version.IdKey, version.RevisionKey, version.RevisionOrderNumberKey, version.MessageKey).
Sort([]string{"-" + version.RevisionOrderNumberKey}).
Limit(depth + 1))
if err != nil {
http.Error(w, fmt.Sprintf("Error fetching versions: %v", err), http.StatusInternalServerError)
return
}
versions = make(map[string]version.Version, len(recentVersions))
for _, v := range recentVersions {
versions[v.Revision] = v
}
cells, err = grid.FetchCells(*projCtx.Version, depth)
if err != nil {
http.Error(w, fmt.Sprintf("Error fetching builds: %v", err), http.StatusInternalServerError)
return
}
failures, err = grid.FetchFailures(*projCtx.Version, depth)
if err != nil {
http.Error(w, fmt.Sprintf("Error fetching builds: %v", err), http.StatusInternalServerError)
return
}
} else {
versions = make(map[string]version.Version)
cells = make(grid.Grid, 0)
failures = make(grid.Failures, 0)
}
uis.WriteHTML(w, http.StatusOK, struct {
ProjectData projectContext
Versions map[string]version.Version
GridCells grid.Grid
Failures grid.Failures
User *user.DBUser
}{projCtx, versions, cells, failures, GetUser(r)}, "base", "grid.html", "base_angular.html", "menu.html")
}
// Given a project name and a list of build variants, return the latest version
// on which all the given build variants succeeded. Gives up after 100 versions.
func FindLastPassingVersionForBuildVariants(project *Project, buildVariantNames []string) (*version.Version, error) {
if len(buildVariantNames) == 0 {
return nil, fmt.Errorf("No build variants specified!")
}
// Get latest commit order number for this project
latestVersion, err := version.FindOne(db.Query(
version.ByMostRecentForRequester(project.Identifier, evergreen.RepotrackerVersionRequester).
WithFields(version.RevisionOrderNumberKey)))
if err != nil {
return nil, fmt.Errorf("Error getting latest version: %v", err)
}
if latestVersion == nil {
return nil, nil
}
mostRecentRevisionOrderNumber := latestVersion.RevisionOrderNumber
// Earliest commit order number to consider
leastRecentRevisionOrderNumber := mostRecentRevisionOrderNumber - StaleVersionCutoff
if leastRecentRevisionOrderNumber < 0 {
leastRecentRevisionOrderNumber = 0
}
pipeline := []bson.M{
// Limit ourselves to builds for non-stale versions and the given project
// and build variants
{
"$match": bson.M{
build.ProjectKey: project.Identifier,
build.RevisionOrderNumberKey: bson.M{"$gte": leastRecentRevisionOrderNumber},
build.BuildVariantKey: bson.M{"$in": buildVariantNames},
build.StatusKey: evergreen.BuildSucceeded,
},
},
// Sum up the number of builds that succeeded for each commit order number
{
"$group": bson.M{
"_id": fmt.Sprintf("$%v", build.RevisionOrderNumberKey),
"numSucceeded": bson.M{
"$sum": 1,
},
},
},
// Find builds that succeeded on all of the requested build variants
{
"$match": bson.M{"numSucceeded": len(buildVariantNames)},
},
// Order by commit order number, descending
{
"$sort": bson.M{"_id": -1},
},
// Get the highest commit order number where builds succeeded on all the
// requested build variants
{
"$limit": 1,
},
}
var result []bson.M
err = db.Aggregate(build.Collection, pipeline, &result)
if err != nil {
return nil, fmt.Errorf("Aggregation failed: %v", err)
}
if len(result) == 0 {
return nil, nil
}
// Get the version corresponding to the resulting commit order number
v, err := version.FindOne(
db.Query(bson.M{
version.RequesterKey: evergreen.RepotrackerVersionRequester,
version.IdentifierKey: project.Identifier,
version.RevisionOrderNumberKey: result[0]["_id"],
}))
if err != nil {
return nil, err
}
if v == nil {
return nil, fmt.Errorf("Couldn't find version with id `%v` after "+
"successful aggregation.", result[0]["_id"])
}
return v, nil
}