func (m *ManifestPlugin) GetManifest(w http.ResponseWriter, r *http.Request) { project := mux.Vars(r)["project_id"] revision := mux.Vars(r)["revision"] version, err := version.FindOne(version.ByProjectIdAndRevision(project, revision)) if err != nil { http.Error(w, fmt.Sprintf("error getting version for project %v with revision %v: %v", project, revision, err), http.StatusBadRequest) return } if version == nil { http.Error(w, fmt.Sprintf("version not found for project %v, with revision %v", project, revision), http.StatusNotFound) return } foundManifest, err := manifest.FindOne(manifest.ById(version.Id)) if err != nil { http.Error(w, fmt.Sprintf("error getting manifest with version id %v: %v", version.Id, err), http.StatusBadRequest) return } if foundManifest == nil { http.Error(w, fmt.Sprintf("manifest not found for version %v", version.Id), http.StatusNotFound) return } plugin.WriteJSON(w, http.StatusOK, foundManifest) return }
// Returns a JSON response with the marshalled output of the version // specified by its revision and project name in the request. func (restapi restAPI) getVersionInfoViaRevision(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) projectId := vars["project_id"] revision := vars["revision"] srcVersion, err := version.FindOne(version.ByProjectIdAndRevision(projectId, revision)) if err != nil || srcVersion == nil { msg := fmt.Sprintf("Error finding revision '%v' for project '%v'", revision, projectId) statusCode := http.StatusNotFound if err != nil { evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err) statusCode = http.StatusInternalServerError } restapi.WriteJSON(w, statusCode, responseError{Message: msg}) return } destVersion := &restVersion{} copyVersion(srcVersion, destVersion) for _, buildStatus := range srcVersion.BuildVariants { destVersion.BuildVariants = append(destVersion.BuildVariants, buildStatus.BuildVariant) evergreen.Logger.Logf(slogger.ERROR, "adding BuildVariant %v", buildStatus.BuildVariant) } restapi.WriteJSON(w, http.StatusOK, destVersion) return }
func FindProject(revision string, projectRef *ProjectRef) (*Project, error) { if projectRef == nil { return nil, fmt.Errorf("projectRef given is nil") } if projectRef.Identifier == "" { return nil, fmt.Errorf("Invalid project with blank identifier") } project := &Project{} project.Identifier = projectRef.Identifier // when the revision is empty we find the last known good configuration from the versions // If the last known good configuration does not exist, // load the configuration from the local config in the project ref. if revision == "" { lastGoodVersion, err := version.FindOne(version.ByLastKnownGoodConfig(projectRef.Identifier)) if err != nil { return nil, fmt.Errorf("Error finding recent valid version for %v: %v", projectRef.Identifier, err) } if lastGoodVersion != nil { // for new repositories, we don't want to error out when we don't have // any versions stored in the database so we default to the skeletal // information we already have from the project file on disk err = LoadProjectInto([]byte(lastGoodVersion.Config), projectRef.Identifier, project) if err != nil { return nil, fmt.Errorf("Error loading project from "+ "last good version for project, %v: %v", lastGoodVersion.Identifier, err) } } else { // Check to see if there is a local configuration in the project ref if projectRef.LocalConfig != "" { err = LoadProjectInto([]byte(projectRef.LocalConfig), projectRef.Identifier, project) if err != nil { return nil, fmt.Errorf("Error loading local config for project ref, %v : %v", projectRef.Identifier, err) } } } } if revision != "" { // we immediately return an error if the repotracker version isn't found // for the given project at the given revision version, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revision)) if err != nil { return nil, fmt.Errorf("error fetching version for project %v revision %v: %v", projectRef.Identifier, revision, err) } if version == nil { // fall back to the skeletal project return project, nil } project = &Project{} if err = LoadProjectInto([]byte(version.Config), projectRef.Identifier, project); err != nil { return nil, fmt.Errorf("Error loading project from version: %v", err) } } return project, nil }
func TestStoreRepositoryRevisions(t *testing.T) { dropTestDB(t) testutil.ConfigureIntegrationTest(t, testConfig, "TestStoreRepositoryRevisions") Convey("When storing revisions gotten from a repository...", t, func() { err := testutil.CreateTestLocalConfig(testConfig, "mci-test", "") So(err, ShouldBeNil) repoTracker := RepoTracker{testConfig, projectRef, NewGithubRepositoryPoller(projectRef, testConfig.Credentials["github"])} // insert distros used in testing. d := distro.Distro{Id: "test-distro-one"} So(d.Insert(), ShouldBeNil) d.Id = "test-distro-two" So(d.Insert(), ShouldBeNil) Convey("On storing a single repo revision, we expect a version to be created"+ " in the database for this project, which should be retrieved when we search"+ " for this project's most recent version", func() { createTime := time.Now() revisionOne := *createTestRevision("firstRevision", createTime) revisions := []model.Revision{revisionOne} resultVersion, err := repoTracker.StoreRevisions(revisions) testutil.HandleTestingErr(err, t, "Error storing repository revisions %v") newestVersion, err := version.FindOne(version.ByMostRecentForRequester(projectRef.String(), evergreen.RepotrackerVersionRequester)) testutil.HandleTestingErr(err, t, "Error retreiving newest version %v") So(resultVersion, ShouldResemble, newestVersion) }) Convey("On storing several repo revisions, we expect a version to be created "+ "for each revision", func() { createTime := time.Now() laterCreateTime := createTime.Add(time.Duration(4 * time.Hour)) revisionOne := *createTestRevision("one", laterCreateTime) revisionTwo := *createTestRevision("two", createTime) revisions := []model.Revision{revisionOne, revisionTwo} _, err := repoTracker.StoreRevisions(revisions) testutil.HandleTestingErr(err, t, "Error storing repository revisions %v") versionOne, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionOne.Revision)) testutil.HandleTestingErr(err, t, "Error retrieving first stored version %v") versionTwo, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionTwo.Revision)) testutil.HandleTestingErr(err, t, "Error retreiving second stored version %v") So(versionOne.Revision, ShouldEqual, revisionOne.Revision) So(versionTwo.Revision, ShouldEqual, revisionTwo.Revision) }) Reset(func() { dropTestDB(t) }) }) Convey("When storing versions from repositories with remote configuration files...", t, func() { project := createTestProject(nil, nil) revisions := []model.Revision{ *createTestRevision("foo", time.Now().Add(1*time.Minute)), } poller := NewMockRepoPoller(project, revisions) repoTracker := RepoTracker{ testConfig, &model.ProjectRef{ Identifier: "testproject", BatchTime: 10, }, poller, } // insert distros used in testing. d := distro.Distro{Id: "test-distro-one"} So(d.Insert(), ShouldBeNil) d.Id = "test-distro-two" So(d.Insert(), ShouldBeNil) Convey("We should not fetch configs for versions we already have stored.", func() { So(poller.ConfigGets, ShouldBeZeroValue) // Store revisions the first time _, err := repoTracker.StoreRevisions(revisions) So(err, ShouldBeNil) // We should have fetched the config once for each revision So(poller.ConfigGets, ShouldEqual, len(revisions)) // Store them again _, err = repoTracker.StoreRevisions(revisions) So(err, ShouldBeNil) // We shouldn't have fetched the config any additional times // since we have already stored these versions So(poller.ConfigGets, ShouldEqual, len(revisions)) }, ) Convey("We should handle invalid configuration files gracefully by storing a stub version", func() { errStrs := []string{"Someone dun' goof'd"} poller.setNextError(projectConfigError{errStrs}) stubVersion, err := repoTracker.StoreRevisions(revisions) // We want this error to get swallowed so a config error // doesn't stop additional versions from getting created So(err, ShouldBeNil) So(stubVersion.Errors, ShouldResemble, errStrs) }, ) Convey("If there is an error other than a config error while fetching a config, we should fail hard", func() { unexpectedError := errors.New("Something terrible has happened!!") poller.setNextError(unexpectedError) v, err := repoTracker.StoreRevisions(revisions) So(v, ShouldBeNil) So(err, ShouldEqual, unexpectedError) }, ) Reset(func() { dropTestDB(t) }) }) }
// Serves the task history page itself. func (uis *UIServer) taskHistoryPage(w http.ResponseWriter, r *http.Request) { projCtx := MustHaveProjectContext(r) if projCtx.Project == nil { http.Error(w, "not found", http.StatusNotFound) return } taskName := mux.Vars(r)["task_name"] var chunk model.TaskHistoryChunk var v *version.Version var before bool var err error if strBefore := r.FormValue("before"); strBefore != "" { if before, err = strconv.ParseBool(strBefore); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } } buildVariants := projCtx.Project.GetVariantsWithTask(taskName) if revision := r.FormValue("revision"); revision != "" { v, err = version.FindOne(version.ByProjectIdAndRevision(projCtx.Project.Identifier, revision)) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } } taskHistoryIterator := model.NewTaskHistoryIterator(taskName, buildVariants, projCtx.Project.Identifier) if r.FormValue("format") == "" { if v != nil { chunk, err = taskHistoryIterator.GetChunk(v, InitRevisionsBefore, InitRevisionsAfter, true) } else { // Load the most recent MaxNumRevisions if a particular // version was unspecified chunk, err = taskHistoryIterator.GetChunk(v, MaxNumRevisions, NoRevisions, false) } } else if before { chunk, err = taskHistoryIterator.GetChunk(v, MaxNumRevisions, NoRevisions, false) } else { chunk, err = taskHistoryIterator.GetChunk(v, NoRevisions, MaxNumRevisions, false) } if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } data := taskHistoryPageData{ TaskName: taskName, Tasks: chunk.Tasks, Variants: buildVariants, FailedTests: chunk.FailedTests, Versions: chunk.Versions, ExhaustedBefore: chunk.Exhausted.Before, ExhaustedAfter: chunk.Exhausted.After, SelectedRevision: r.FormValue("revision"), } switch r.FormValue("format") { case "json": uis.WriteJSON(w, http.StatusOK, data) return default: uis.WriteHTML(w, http.StatusOK, struct { ProjectData projectContext User *user.DBUser Flashes []interface{} Data taskHistoryPageData }{projCtx, GetUser(r), []interface{}{}, data}, "base", "task_history.html", "base_angular.html", "menu.html") } }
// Constructs all versions stored from recent repository revisions // The additional complexity is due to support for project modifications on patch builds. // We need to parse the remote config as it existed when each revision was created. // The return value is the most recent version created as a result of storing the revisions. // This function is idempotent with regard to storing the same version multiple times. func (repoTracker *RepoTracker) StoreRevisions(revisions []model.Revision) (newestVersion *version.Version, err error) { defer func() { if newestVersion != nil { // Fetch the updated version doc, so that we include buildvariants in the result newestVersion, err = version.FindOne(version.ById(newestVersion.Id)) } }() ref := repoTracker.ProjectRef for i := len(revisions) - 1; i >= 0; i-- { revision := revisions[i].Revision evergreen.Logger.Logf(slogger.INFO, "Processing revision %v in project %v", revision, ref.Identifier) // We check if the version exists here so we can avoid fetching the github config unnecessarily existingVersion, err := version.FindOne(version.ByProjectIdAndRevision(ref.Identifier, revisions[i].Revision)) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error looking up version at %v for project %v: %v", ref.Identifier, revision, err) } if existingVersion != nil { evergreen.Logger.Logf(slogger.INFO, "Skipping creation of version for project %v, revision %v since"+ " we already have a record for it", ref.Identifier, revision) // We bind newestVersion here since we still need to return the most recent // version, even if it already exists newestVersion = existingVersion continue } // Create the stub of the version (not stored in DB yet) v, err := NewVersionFromRevision(ref, revisions[i]) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error creating version for project %v: %v", ref.Identifier, err) } err = sanityCheckOrderNum(v.RevisionOrderNumber, ref.Identifier) if err != nil { // something seriously wrong (bad data in db?) so fail now panic(err) } project, err := repoTracker.GetProjectConfig(revision) if err != nil { projectError, isProjectError := err.(projectConfigError) if isProjectError { // Store just the stub version with the project errors v.Errors = projectError.errors if err := v.Insert(); err != nil { evergreen.Logger.Logf(slogger.ERROR, "Failed storing stub version for project %v: %v", ref.Identifier, err) return nil, err } newestVersion = v continue } else { // Fatal error - don't store the stub evergreen.Logger.Logf(slogger.INFO, "Failed to get config for project %v at revision %v: %v", ref.Identifier, revision, err) return nil, err } } // We have a config, so turn it into a usable yaml string to store with the version doc projectYamlBytes, err := yaml.Marshal(project) if err != nil { return nil, fmt.Errorf("Error marshalling config: %v", err) } v.Config = string(projectYamlBytes) // We rebind newestVersion each iteration, so the last binding will be the newest version err = createVersionItems(v, ref, project) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error creating version items for %v in project %v: %v", v.Id, ref.Identifier, err) return nil, err } newestVersion = v if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Unable to store revision %v for project %v: %v:", revision, ref.Identifier, err) return nil, err } } return }
func (uis *UIServer) patchTimelineJson(w http.ResponseWriter, r *http.Request) { projCtx := MustHaveProjectContext(r) pageNum, err := strconv.Atoi(r.FormValue("page")) if err != nil { pageNum = 0 } skip := pageNum * DefaultLimit user := mux.Vars(r)["user_id"] var patches []patch.Patch if len(user) > 0 { patches, err = patch.Find(patch.ByUser(user). Project(patch.ExcludePatchDiff). Sort([]string{"-" + patch.CreateTimeKey}). Skip(skip).Limit(DefaultLimit)) } else { patches, err = patch.Find(patch.ByProject(projCtx.Project.Identifier). Sort([]string{"-" + patch.CreateTimeKey}). Project(patch.ExcludePatchDiff). Skip(skip).Limit(DefaultLimit)) } if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error fetching patches for %v: %v", projCtx.Project.Identifier, err)) return } versionIds := make([]string, 0, len(patches)) uiPatches := make([]uiPatch, 0, len(patches)) for _, patch := range patches { if patch.Version != "" { versionIds = append(versionIds, patch.Version) } baseVersion, err := version.FindOne(version.ByProjectIdAndRevision(patch.Project, patch.Githash)) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } var baseVersionId string if baseVersion != nil { baseVersionId = baseVersion.Id } patch.Patches = nil uiPatches = append(uiPatches, uiPatch{Patch: patch, BaseVersionId: baseVersionId}) } versions, err := version.Find(version.ByIds(versionIds).WithoutFields(version.ConfigKey)) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error fetching versions for patches: %v", err)) return } versionsMap := map[string]*uiVersion{} for _, version := range versions { versionUI, err := PopulateUIVersion(&version) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } versionsMap[version.Id] = versionUI } data := struct { VersionsMap map[string]*uiVersion UIPatches []uiPatch PageNum int }{versionsMap, uiPatches, pageNum} uis.WriteJSON(w, http.StatusOK, data) }