Beispiel #1
0
func getTimelineData(projectName, requester string, versionsToSkip, versionsPerPage int) (*timelineData, error) {
	data := &timelineData{}

	// get the total number of versions in the database (used for pagination)
	totalVersions, err := version.Count(version.ByProjectId(projectName))
	if err != nil {
		return nil, err
	}
	data.TotalVersions = totalVersions

	q := version.ByMostRecentForRequester(projectName, requester).WithoutFields(version.ConfigKey).
		Skip(versionsToSkip * versionsPerPage).Limit(versionsPerPage)

	// get the most recent versions, to display in their entirety on the page
	versionsFromDB, err := version.Find(q)
	if err != nil {
		return nil, err
	}

	// create the necessary uiVersion struct for each version
	uiVersions := make([]uiVersion, len(versionsFromDB))
	for versionIdx, version := range versionsFromDB {
		versionAsUI := uiVersion{Version: version}
		uiVersions[versionIdx] = versionAsUI

		buildIds := version.BuildIds
		dbBuilds, err := build.Find(build.ByIds(buildIds))
		if err != nil {
			evergreen.Logger.Errorf(slogger.ERROR, "Ids: %v", buildIds)
		}

		buildsMap := make(map[string]build.Build)
		for _, dbBuild := range dbBuilds {
			buildsMap[dbBuild.Id] = dbBuild
		}

		uiBuilds := make([]uiBuild, len(dbBuilds))
		for buildIdx, buildId := range buildIds {
			build := buildsMap[buildId]
			buildAsUI := uiBuild{Build: build}
			uiBuilds[buildIdx] = buildAsUI
		}
		versionAsUI.Builds = uiBuilds
		uiVersions[versionIdx] = versionAsUI
	}

	data.Versions = uiVersions
	return data, nil
}
Beispiel #2
0
// Verifies that the given revision order number is higher than the latest number stored for the project.
func sanityCheckOrderNum(revOrderNum int, projectId string) error {
	latest, err := version.FindOne(version.ByMostRecentForRequester(projectId, evergreen.RepotrackerVersionRequester))
	if err != nil {
		return fmt.Errorf("Error getting latest version: %v", err.Error())
	}

	// When there are no versions in the db yet, sanity check is moot
	if latest != nil {
		if revOrderNum <= latest.RevisionOrderNumber {
			return fmt.Errorf("Commit order number isn't greater than last stored version's: %v <= %v",
				revOrderNum, latest.RevisionOrderNumber)
		}
	}
	return nil
}
func TestStoreRepositoryRevisions(t *testing.T) {
	dropTestDB(t)
	testutil.ConfigureIntegrationTest(t, testConfig, "TestStoreRepositoryRevisions")
	Convey("When storing revisions gotten from a repository...", t, func() {
		err := testutil.CreateTestLocalConfig(testConfig, "mci-test", "")
		So(err, ShouldBeNil)
		repoTracker := RepoTracker{testConfig, projectRef, NewGithubRepositoryPoller(projectRef,
			testConfig.Credentials["github"])}

		// insert distros used in testing.
		d := distro.Distro{Id: "test-distro-one"}
		So(d.Insert(), ShouldBeNil)
		d.Id = "test-distro-two"
		So(d.Insert(), ShouldBeNil)

		Convey("On storing a single repo revision, we expect a version to be created"+
			" in the database for this project, which should be retrieved when we search"+
			" for this project's most recent version", func() {
			createTime := time.Now()
			revisionOne := *createTestRevision("firstRevision", createTime)
			revisions := []model.Revision{revisionOne}

			resultVersion, err := repoTracker.StoreRevisions(revisions)
			testutil.HandleTestingErr(err, t, "Error storing repository revisions %v")

			newestVersion, err := version.FindOne(version.ByMostRecentForRequester(projectRef.String(), evergreen.RepotrackerVersionRequester))
			testutil.HandleTestingErr(err, t, "Error retreiving newest version %v")

			So(resultVersion, ShouldResemble, newestVersion)
		})

		Convey("On storing several repo revisions, we expect a version to be created "+
			"for each revision", func() {
			createTime := time.Now()
			laterCreateTime := createTime.Add(time.Duration(4 * time.Hour))

			revisionOne := *createTestRevision("one", laterCreateTime)
			revisionTwo := *createTestRevision("two", createTime)

			revisions := []model.Revision{revisionOne, revisionTwo}

			_, err := repoTracker.StoreRevisions(revisions)
			testutil.HandleTestingErr(err, t, "Error storing repository revisions %v")

			versionOne, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionOne.Revision))
			testutil.HandleTestingErr(err, t, "Error retrieving first stored version %v")
			versionTwo, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionTwo.Revision))
			testutil.HandleTestingErr(err, t, "Error retreiving second stored version %v")

			So(versionOne.Revision, ShouldEqual, revisionOne.Revision)
			So(versionTwo.Revision, ShouldEqual, revisionTwo.Revision)
		})

		Reset(func() {
			dropTestDB(t)
		})
	})

	Convey("When storing versions from repositories with remote configuration files...", t, func() {

		project := createTestProject(nil, nil)

		revisions := []model.Revision{
			*createTestRevision("foo", time.Now().Add(1*time.Minute)),
		}

		poller := NewMockRepoPoller(project, revisions)

		repoTracker := RepoTracker{
			testConfig,
			&model.ProjectRef{
				Identifier: "testproject",
				BatchTime:  10,
			},
			poller,
		}

		// insert distros used in testing.
		d := distro.Distro{Id: "test-distro-one"}
		So(d.Insert(), ShouldBeNil)
		d.Id = "test-distro-two"
		So(d.Insert(), ShouldBeNil)

		Convey("We should not fetch configs for versions we already have stored.",
			func() {
				So(poller.ConfigGets, ShouldBeZeroValue)
				// Store revisions the first time
				_, err := repoTracker.StoreRevisions(revisions)
				So(err, ShouldBeNil)
				// We should have fetched the config once for each revision
				So(poller.ConfigGets, ShouldEqual, len(revisions))

				// Store them again
				_, err = repoTracker.StoreRevisions(revisions)
				So(err, ShouldBeNil)
				// We shouldn't have fetched the config any additional times
				// since we have already stored these versions
				So(poller.ConfigGets, ShouldEqual, len(revisions))
			},
		)

		Convey("We should handle invalid configuration files gracefully by storing a stub version",
			func() {
				errStrs := []string{"Someone dun' goof'd"}
				poller.setNextError(projectConfigError{errStrs})
				stubVersion, err := repoTracker.StoreRevisions(revisions)
				// We want this error to get swallowed so a config error
				// doesn't stop additional versions from getting created
				So(err, ShouldBeNil)
				So(stubVersion.Errors, ShouldResemble, errStrs)
			},
		)

		Convey("If there is an error other than a config error while fetching a config, we should fail hard",
			func() {
				unexpectedError := errors.New("Something terrible has happened!!")
				poller.setNextError(unexpectedError)
				v, err := repoTracker.StoreRevisions(revisions)
				So(v, ShouldBeNil)
				So(err, ShouldEqual, unexpectedError)
			},
		)

		Reset(func() {
			dropTestDB(t)
		})

	})
}
Beispiel #4
0
// The FetchRevisions method is used by a RepoTracker to run the pipeline for
// tracking repositories. It performs everything from polling the repository to
// persisting any changes retrieved from the repository reference.
func (repoTracker *RepoTracker) FetchRevisions(numNewRepoRevisionsToFetch int) (
	err error) {
	settings := repoTracker.Settings
	projectRef := repoTracker.ProjectRef
	projectIdentifier := projectRef.String()

	if !projectRef.Enabled {
		evergreen.Logger.Logf(slogger.INFO, "Skipping disabled project “%v”", projectRef)
		return nil
	}

	repository, err := model.FindRepository(projectIdentifier)
	if err != nil {
		return fmt.Errorf("error finding repository '%v': %v",
			projectIdentifier, err)
	}

	var revisions []model.Revision
	var lastRevision string

	if repository != nil {
		lastRevision = repository.LastRevision
	}

	if lastRevision == "" {
		// if this is the first time we're running the tracker for this project,
		// fetch the most recent `numNewRepoRevisionsToFetch` revisions
		evergreen.Logger.Logf(slogger.INFO, "No last recorded repository revision "+
			"for “%v”. Proceeding to fetch most recent %v revisions",
			projectRef, numNewRepoRevisionsToFetch)
		revisions, err = repoTracker.GetRecentRevisions(numNewRepoRevisionsToFetch)
	} else {
		evergreen.Logger.Logf(slogger.INFO, "Last recorded repository revision for "+
			"“%v” is “%v”", projectRef, lastRevision)
		revisions, err = repoTracker.GetRevisionsSince(lastRevision,
			settings.RepoTracker.MaxRepoRevisionsToSearch)
	}

	if err != nil {
		evergreen.Logger.Logf(slogger.ERROR, "error fetching revisions for "+
			"repository “%v”: %v", projectRef, err)
		repoTracker.sendFailureNotification(lastRevision, err)
		return nil
	}

	var lastVersion *version.Version

	if len(revisions) > 0 {
		lastVersion, err = repoTracker.StoreRevisions(revisions)
		if err != nil {
			evergreen.Logger.Logf(slogger.ERROR, "error storing revisions for "+
				"repository %v: %v", projectRef, err)
			return err
		}
		err = model.UpdateLastRevision(lastVersion.Identifier, lastVersion.Revision)
		if err != nil {
			evergreen.Logger.Logf(slogger.ERROR, "error updating last revision for "+
				"repository %v: %v", projectRef, err)
			return err
		}
	} else {
		lastVersion, err = version.FindOne(version.ByMostRecentForRequester(projectIdentifier, evergreen.RepotrackerVersionRequester))
		if err != nil {
			evergreen.Logger.Logf(slogger.ERROR, "error getting most recent version for "+
				"repository %v: %v", projectRef, err)
			return err
		}
	}

	if lastVersion == nil {
		evergreen.Logger.Logf(slogger.WARN, "no version to activate for repository %v", projectIdentifier)
		return nil
	}

	err = repoTracker.activateElapsedBuilds(lastVersion)

	if err != nil {
		evergreen.Logger.Logf(slogger.ERROR, "error activating variants: %v", err)
		return err
	}

	return nil
}
Beispiel #5
0
// Returns a JSON response of an array with the NumRecentVersions
// most recent versions (sorted on commit order number descending).
func (restapi restAPI) getRecentVersions(w http.ResponseWriter, r *http.Request) {
	projectId := mux.Vars(r)["project_id"]

	versions, err := version.Find(version.ByMostRecentForRequester(projectId, evergreen.RepotrackerVersionRequester).Limit(10))
	if err != nil {
		msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId)
		evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
		restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
		return
	}

	// Create a slice of version ids to find all relevant builds
	versionIds := make([]string, 0, len(versions))

	// Cache the order of versions in a map for lookup by their id
	versionIdx := make(map[string]int, len(versions))

	for i, version := range versions {
		versionIds = append(versionIds, version.Id)
		versionIdx[version.Id] = i
	}

	// Find all builds corresponding the set of version ids
	builds, err := build.Find(
		build.ByVersions(versionIds).
			WithFields(build.BuildVariantKey, build.DisplayNameKey, build.TasksKey, build.VersionKey))
	if err != nil {
		msg := fmt.Sprintf("Error finding recent versions of project '%v'", projectId)
		evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
		restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
		return
	}

	result := recentVersionsContent{
		Project:  projectId,
		Versions: make([]versionLessInfo, 0, len(versions)),
	}

	for _, version := range versions {
		versionInfo := versionLessInfo{
			Id:       version.Id,
			Author:   version.Author,
			Revision: version.Revision,
			Message:  version.Message,
			Builds:   make(versionByBuild),
		}

		result.Versions = append(result.Versions, versionInfo)
	}

	for _, build := range builds {
		buildInfo := versionBuildInfo{
			Id:    build.Id,
			Name:  build.DisplayName,
			Tasks: make(versionByBuildByTask, len(build.Tasks)),
		}

		for _, task := range build.Tasks {
			buildInfo.Tasks[task.DisplayName] = versionStatus{
				Id:        task.Id,
				Status:    task.Status,
				TimeTaken: task.TimeTaken,
			}
		}

		versionInfo := result.Versions[versionIdx[build.Version]]
		versionInfo.Builds[build.BuildVariant] = buildInfo
	}

	restapi.WriteJSON(w, http.StatusOK, result)
	return
}
Beispiel #6
0
func (uis *UIServer) grid(w http.ResponseWriter, r *http.Request) {
	projCtx := MustHaveProjectContext(r)
	if projCtx.Project == nil {
		http.Error(w, "Project not found", http.StatusNotFound)
	}

	// If no version was specified in the URL, grab the latest version on the project
	if projCtx.Version == nil {
		v, err := version.Find(version.ByMostRecentForRequester(projCtx.Project.Identifier, evergreen.RepotrackerVersionRequester).Limit(1))
		if err != nil {
			http.Error(w, fmt.Sprintf("Error finding version: %v", err), http.StatusInternalServerError)
			return
		}
		if len(v) > 0 {
			projCtx.Version = &v[0]
		}
	}

	var versions map[string]version.Version
	var cells grid.Grid
	var failures grid.Failures
	var depth int
	var err error

	d := mux.Vars(r)["depth"]
	if d == "" {
		depth = defaultGridDepth
	} else {
		depth, err = strconv.Atoi(d)
		if err != nil {
			http.Error(w, fmt.Sprintf("Error converting depth: %v", err), http.StatusBadRequest)
			return
		}
		if depth < 0 {
			http.Error(w, fmt.Sprintf("Depth must be non-negative, got %v", depth), http.StatusBadRequest)
			return
		}
	}

	if projCtx.Version != nil {
		recentVersions, err := version.Find(version.
			ByProjectIdAndOrder(projCtx.Version.Project, projCtx.Version.RevisionOrderNumber).
			WithFields(version.IdKey, version.RevisionKey, version.RevisionOrderNumberKey, version.MessageKey).
			Sort([]string{"-" + version.RevisionOrderNumberKey}).
			Limit(depth + 1))
		if err != nil {
			http.Error(w, fmt.Sprintf("Error fetching versions: %v", err), http.StatusInternalServerError)
			return
		}

		versions = make(map[string]version.Version, len(recentVersions))
		for _, v := range recentVersions {
			versions[v.Revision] = v
		}

		cells, err = grid.FetchCells(*projCtx.Version, depth)
		if err != nil {
			http.Error(w, fmt.Sprintf("Error fetching builds: %v", err), http.StatusInternalServerError)
			return
		}

		failures, err = grid.FetchFailures(*projCtx.Version, depth)
		if err != nil {
			http.Error(w, fmt.Sprintf("Error fetching builds: %v", err), http.StatusInternalServerError)
			return
		}
	} else {
		versions = make(map[string]version.Version)
		cells = make(grid.Grid, 0)
		failures = make(grid.Failures, 0)
	}
	uis.WriteHTML(w, http.StatusOK, struct {
		ProjectData projectContext
		Versions    map[string]version.Version
		GridCells   grid.Grid
		Failures    grid.Failures
		User        *user.DBUser
	}{projCtx, versions, cells, failures, GetUser(r)}, "base", "grid.html", "base_angular.html", "menu.html")
}
Beispiel #7
0
// Given a project name and a list of build variants, return the latest version
// on which all the given build variants succeeded. Gives up after 100 versions.
func FindLastPassingVersionForBuildVariants(project *Project, buildVariantNames []string) (*version.Version, error) {
	if len(buildVariantNames) == 0 {
		return nil, fmt.Errorf("No build variants specified!")
	}

	// Get latest commit order number for this project
	latestVersion, err := version.FindOne(db.Query(
		version.ByMostRecentForRequester(project.Identifier, evergreen.RepotrackerVersionRequester).
			WithFields(version.RevisionOrderNumberKey)))
	if err != nil {
		return nil, fmt.Errorf("Error getting latest version: %v", err)
	}
	if latestVersion == nil {
		return nil, nil
	}

	mostRecentRevisionOrderNumber := latestVersion.RevisionOrderNumber

	// Earliest commit order number to consider
	leastRecentRevisionOrderNumber := mostRecentRevisionOrderNumber - StaleVersionCutoff
	if leastRecentRevisionOrderNumber < 0 {
		leastRecentRevisionOrderNumber = 0
	}

	pipeline := []bson.M{
		// Limit ourselves to builds for non-stale versions and the given project
		// and build variants
		{
			"$match": bson.M{
				build.ProjectKey:             project.Identifier,
				build.RevisionOrderNumberKey: bson.M{"$gte": leastRecentRevisionOrderNumber},
				build.BuildVariantKey:        bson.M{"$in": buildVariantNames},
				build.StatusKey:              evergreen.BuildSucceeded,
			},
		},
		// Sum up the number of builds that succeeded for each commit order number
		{
			"$group": bson.M{
				"_id": fmt.Sprintf("$%v", build.RevisionOrderNumberKey),
				"numSucceeded": bson.M{
					"$sum": 1,
				},
			},
		},
		// Find builds that succeeded on all of the requested build variants
		{
			"$match": bson.M{"numSucceeded": len(buildVariantNames)},
		},
		// Order by commit order number, descending
		{
			"$sort": bson.M{"_id": -1},
		},
		// Get the highest commit order number where builds succeeded on all the
		// requested build variants
		{
			"$limit": 1,
		},
	}

	var result []bson.M
	err = db.Aggregate(build.Collection, pipeline, &result)

	if err != nil {
		return nil, fmt.Errorf("Aggregation failed: %v", err)
	}

	if len(result) == 0 {
		return nil, nil
	}

	// Get the version corresponding to the resulting commit order number
	v, err := version.FindOne(
		db.Query(bson.M{
			version.RequesterKey:           evergreen.RepotrackerVersionRequester,
			version.IdentifierKey:          project.Identifier,
			version.RevisionOrderNumberKey: result[0]["_id"],
		}))
	if err != nil {
		return nil, err
	}
	if v == nil {
		return nil, fmt.Errorf("Couldn't find version with id `%v` after "+
			"successful aggregation.", result[0]["_id"])
	}
	return v, nil
}