func (m *ManifestPlugin) GetManifest(w http.ResponseWriter, r *http.Request) {
project := mux.Vars(r)["project_id"]
revision := mux.Vars(r)["revision"]
version, err := version.FindOne(version.ByProjectIdAndRevision(project, revision))
if err != nil {
http.Error(w, fmt.Sprintf("error getting version for project %v with revision %v: %v",
project, revision, err), http.StatusBadRequest)
return
}
if version == nil {
http.Error(w, fmt.Sprintf("version not found for project %v, with revision %v", project, revision),
http.StatusNotFound)
return
}
foundManifest, err := manifest.FindOne(manifest.ById(version.Id))
if err != nil {
http.Error(w, fmt.Sprintf("error getting manifest with version id %v: %v",
version.Id, err), http.StatusBadRequest)
return
}
if foundManifest == nil {
http.Error(w, fmt.Sprintf("manifest not found for version %v", version.Id), http.StatusNotFound)
return
}
plugin.WriteJSON(w, http.StatusOK, foundManifest)
return
}
// Returns a JSON response with the marshalled output of the version
// specified by its revision and project name in the request.
func (restapi restAPI) getVersionInfoViaRevision(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
projectId := vars["project_id"]
revision := vars["revision"]
srcVersion, err := version.FindOne(version.ByProjectIdAndRevision(projectId, revision))
if err != nil || srcVersion == nil {
msg := fmt.Sprintf("Error finding revision '%v' for project '%v'", revision, projectId)
statusCode := http.StatusNotFound
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
statusCode = http.StatusInternalServerError
}
restapi.WriteJSON(w, statusCode, responseError{Message: msg})
return
}
destVersion := &restVersion{}
copyVersion(srcVersion, destVersion)
for _, buildStatus := range srcVersion.BuildVariants {
destVersion.BuildVariants = append(destVersion.BuildVariants, buildStatus.BuildVariant)
evergreen.Logger.Logf(slogger.ERROR, "adding BuildVariant %v", buildStatus.BuildVariant)
}
restapi.WriteJSON(w, http.StatusOK, destVersion)
return
}
func FindProject(revision string, projectRef *ProjectRef) (*Project, error) {
if projectRef == nil {
return nil, fmt.Errorf("projectRef given is nil")
}
if projectRef.Identifier == "" {
return nil, fmt.Errorf("Invalid project with blank identifier")
}
project := &Project{}
project.Identifier = projectRef.Identifier
// when the revision is empty we find the last known good configuration from the versions
// If the last known good configuration does not exist,
// load the configuration from the local config in the project ref.
if revision == "" {
lastGoodVersion, err := version.FindOne(version.ByLastKnownGoodConfig(projectRef.Identifier))
if err != nil {
return nil, fmt.Errorf("Error finding recent valid version for %v: %v", projectRef.Identifier, err)
}
if lastGoodVersion != nil {
// for new repositories, we don't want to error out when we don't have
// any versions stored in the database so we default to the skeletal
// information we already have from the project file on disk
err = LoadProjectInto([]byte(lastGoodVersion.Config), projectRef.Identifier, project)
if err != nil {
return nil, fmt.Errorf("Error loading project from "+
"last good version for project, %v: %v", lastGoodVersion.Identifier, err)
}
} else {
// Check to see if there is a local configuration in the project ref
if projectRef.LocalConfig != "" {
err = LoadProjectInto([]byte(projectRef.LocalConfig), projectRef.Identifier, project)
if err != nil {
return nil, fmt.Errorf("Error loading local config for project ref, %v : %v", projectRef.Identifier, err)
}
}
}
}
if revision != "" {
// we immediately return an error if the repotracker version isn't found
// for the given project at the given revision
version, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revision))
if err != nil {
return nil, fmt.Errorf("error fetching version for project %v revision %v: %v", projectRef.Identifier, revision, err)
}
if version == nil {
// fall back to the skeletal project
return project, nil
}
project = &Project{}
if err = LoadProjectInto([]byte(version.Config), projectRef.Identifier, project); err != nil {
return nil, fmt.Errorf("Error loading project from version: %v", err)
}
}
return project, nil
}
func TestStoreRepositoryRevisions(t *testing.T) {
dropTestDB(t)
testutil.ConfigureIntegrationTest(t, testConfig, "TestStoreRepositoryRevisions")
Convey("When storing revisions gotten from a repository...", t, func() {
err := testutil.CreateTestLocalConfig(testConfig, "mci-test", "")
So(err, ShouldBeNil)
repoTracker := RepoTracker{testConfig, projectRef, NewGithubRepositoryPoller(projectRef,
testConfig.Credentials["github"])}
// insert distros used in testing.
d := distro.Distro{Id: "test-distro-one"}
So(d.Insert(), ShouldBeNil)
d.Id = "test-distro-two"
So(d.Insert(), ShouldBeNil)
Convey("On storing a single repo revision, we expect a version to be created"+
" in the database for this project, which should be retrieved when we search"+
" for this project's most recent version", func() {
createTime := time.Now()
revisionOne := *createTestRevision("firstRevision", createTime)
revisions := []model.Revision{revisionOne}
resultVersion, err := repoTracker.StoreRevisions(revisions)
testutil.HandleTestingErr(err, t, "Error storing repository revisions %v")
newestVersion, err := version.FindOne(version.ByMostRecentForRequester(projectRef.String(), evergreen.RepotrackerVersionRequester))
testutil.HandleTestingErr(err, t, "Error retreiving newest version %v")
So(resultVersion, ShouldResemble, newestVersion)
})
Convey("On storing several repo revisions, we expect a version to be created "+
"for each revision", func() {
createTime := time.Now()
laterCreateTime := createTime.Add(time.Duration(4 * time.Hour))
revisionOne := *createTestRevision("one", laterCreateTime)
revisionTwo := *createTestRevision("two", createTime)
revisions := []model.Revision{revisionOne, revisionTwo}
_, err := repoTracker.StoreRevisions(revisions)
testutil.HandleTestingErr(err, t, "Error storing repository revisions %v")
versionOne, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionOne.Revision))
testutil.HandleTestingErr(err, t, "Error retrieving first stored version %v")
versionTwo, err := version.FindOne(version.ByProjectIdAndRevision(projectRef.Identifier, revisionTwo.Revision))
testutil.HandleTestingErr(err, t, "Error retreiving second stored version %v")
So(versionOne.Revision, ShouldEqual, revisionOne.Revision)
So(versionTwo.Revision, ShouldEqual, revisionTwo.Revision)
})
Reset(func() {
dropTestDB(t)
})
})
Convey("When storing versions from repositories with remote configuration files...", t, func() {
project := createTestProject(nil, nil)
revisions := []model.Revision{
*createTestRevision("foo", time.Now().Add(1*time.Minute)),
}
poller := NewMockRepoPoller(project, revisions)
repoTracker := RepoTracker{
testConfig,
&model.ProjectRef{
Identifier: "testproject",
BatchTime: 10,
},
poller,
}
// insert distros used in testing.
d := distro.Distro{Id: "test-distro-one"}
So(d.Insert(), ShouldBeNil)
d.Id = "test-distro-two"
So(d.Insert(), ShouldBeNil)
Convey("We should not fetch configs for versions we already have stored.",
func() {
So(poller.ConfigGets, ShouldBeZeroValue)
// Store revisions the first time
_, err := repoTracker.StoreRevisions(revisions)
So(err, ShouldBeNil)
// We should have fetched the config once for each revision
So(poller.ConfigGets, ShouldEqual, len(revisions))
// Store them again
_, err = repoTracker.StoreRevisions(revisions)
So(err, ShouldBeNil)
// We shouldn't have fetched the config any additional times
// since we have already stored these versions
So(poller.ConfigGets, ShouldEqual, len(revisions))
},
)
Convey("We should handle invalid configuration files gracefully by storing a stub version",
func() {
errStrs := []string{"Someone dun' goof'd"}
poller.setNextError(projectConfigError{errStrs})
stubVersion, err := repoTracker.StoreRevisions(revisions)
// We want this error to get swallowed so a config error
// doesn't stop additional versions from getting created
So(err, ShouldBeNil)
So(stubVersion.Errors, ShouldResemble, errStrs)
},
)
Convey("If there is an error other than a config error while fetching a config, we should fail hard",
func() {
unexpectedError := errors.New("Something terrible has happened!!")
poller.setNextError(unexpectedError)
v, err := repoTracker.StoreRevisions(revisions)
So(v, ShouldBeNil)
So(err, ShouldEqual, unexpectedError)
},
)
Reset(func() {
dropTestDB(t)
})
})
}
// Serves the task history page itself.
func (uis *UIServer) taskHistoryPage(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
if projCtx.Project == nil {
http.Error(w, "not found", http.StatusNotFound)
return
}
taskName := mux.Vars(r)["task_name"]
var chunk model.TaskHistoryChunk
var v *version.Version
var before bool
var err error
if strBefore := r.FormValue("before"); strBefore != "" {
if before, err = strconv.ParseBool(strBefore); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
buildVariants := projCtx.Project.GetVariantsWithTask(taskName)
if revision := r.FormValue("revision"); revision != "" {
v, err = version.FindOne(version.ByProjectIdAndRevision(projCtx.Project.Identifier, revision))
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
taskHistoryIterator := model.NewTaskHistoryIterator(taskName, buildVariants, projCtx.Project.Identifier)
if r.FormValue("format") == "" {
if v != nil {
chunk, err = taskHistoryIterator.GetChunk(v, InitRevisionsBefore, InitRevisionsAfter, true)
} else {
// Load the most recent MaxNumRevisions if a particular
// version was unspecified
chunk, err = taskHistoryIterator.GetChunk(v, MaxNumRevisions, NoRevisions, false)
}
} else if before {
chunk, err = taskHistoryIterator.GetChunk(v, MaxNumRevisions, NoRevisions, false)
} else {
chunk, err = taskHistoryIterator.GetChunk(v, NoRevisions, MaxNumRevisions, false)
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
data := taskHistoryPageData{
TaskName: taskName,
Tasks: chunk.Tasks,
Variants: buildVariants,
FailedTests: chunk.FailedTests,
Versions: chunk.Versions,
ExhaustedBefore: chunk.Exhausted.Before,
ExhaustedAfter: chunk.Exhausted.After,
SelectedRevision: r.FormValue("revision"),
}
switch r.FormValue("format") {
case "json":
uis.WriteJSON(w, http.StatusOK, data)
return
default:
uis.WriteHTML(w, http.StatusOK, struct {
ProjectData projectContext
User *user.DBUser
Flashes []interface{}
Data taskHistoryPageData
}{projCtx, GetUser(r), []interface{}{}, data}, "base",
"task_history.html", "base_angular.html", "menu.html")
}
}
// Constructs all versions stored from recent repository revisions
// The additional complexity is due to support for project modifications on patch builds.
// We need to parse the remote config as it existed when each revision was created.
// The return value is the most recent version created as a result of storing the revisions.
// This function is idempotent with regard to storing the same version multiple times.
func (repoTracker *RepoTracker) StoreRevisions(revisions []model.Revision) (newestVersion *version.Version, err error) {
defer func() {
if newestVersion != nil {
// Fetch the updated version doc, so that we include buildvariants in the result
newestVersion, err = version.FindOne(version.ById(newestVersion.Id))
}
}()
ref := repoTracker.ProjectRef
for i := len(revisions) - 1; i >= 0; i-- {
revision := revisions[i].Revision
evergreen.Logger.Logf(slogger.INFO, "Processing revision %v in project %v", revision, ref.Identifier)
// We check if the version exists here so we can avoid fetching the github config unnecessarily
existingVersion, err := version.FindOne(version.ByProjectIdAndRevision(ref.Identifier, revisions[i].Revision))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR,
"Error looking up version at %v for project %v: %v", ref.Identifier, revision, err)
}
if existingVersion != nil {
evergreen.Logger.Logf(slogger.INFO,
"Skipping creation of version for project %v, revision %v since"+
" we already have a record for it", ref.Identifier, revision)
// We bind newestVersion here since we still need to return the most recent
// version, even if it already exists
newestVersion = existingVersion
continue
}
// Create the stub of the version (not stored in DB yet)
v, err := NewVersionFromRevision(ref, revisions[i])
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error creating version for project %v: %v", ref.Identifier, err)
}
err = sanityCheckOrderNum(v.RevisionOrderNumber, ref.Identifier)
if err != nil { // something seriously wrong (bad data in db?) so fail now
panic(err)
}
project, err := repoTracker.GetProjectConfig(revision)
if err != nil {
projectError, isProjectError := err.(projectConfigError)
if isProjectError {
// Store just the stub version with the project errors
v.Errors = projectError.errors
if err := v.Insert(); err != nil {
evergreen.Logger.Logf(slogger.ERROR,
"Failed storing stub version for project %v: %v", ref.Identifier, err)
return nil, err
}
newestVersion = v
continue
} else {
// Fatal error - don't store the stub
evergreen.Logger.Logf(slogger.INFO,
"Failed to get config for project %v at revision %v: %v", ref.Identifier, revision, err)
return nil, err
}
}
// We have a config, so turn it into a usable yaml string to store with the version doc
projectYamlBytes, err := yaml.Marshal(project)
if err != nil {
return nil, fmt.Errorf("Error marshalling config: %v", err)
}
v.Config = string(projectYamlBytes)
// We rebind newestVersion each iteration, so the last binding will be the newest version
err = createVersionItems(v, ref, project)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error creating version items for %v in project %v: %v",
v.Id, ref.Identifier, err)
return nil, err
}
newestVersion = v
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Unable to store revision %v for project %v: %v:",
revision, ref.Identifier, err)
return nil, err
}
}
return
}
func (uis *UIServer) patchTimelineJson(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
pageNum, err := strconv.Atoi(r.FormValue("page"))
if err != nil {
pageNum = 0
}
skip := pageNum * DefaultLimit
user := mux.Vars(r)["user_id"]
var patches []patch.Patch
if len(user) > 0 {
patches, err = patch.Find(patch.ByUser(user).
Project(patch.ExcludePatchDiff).
Sort([]string{"-" + patch.CreateTimeKey}).
Skip(skip).Limit(DefaultLimit))
} else {
patches, err = patch.Find(patch.ByProject(projCtx.Project.Identifier).
Sort([]string{"-" + patch.CreateTimeKey}).
Project(patch.ExcludePatchDiff).
Skip(skip).Limit(DefaultLimit))
}
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error fetching patches for %v: %v", projCtx.Project.Identifier, err))
return
}
versionIds := make([]string, 0, len(patches))
uiPatches := make([]uiPatch, 0, len(patches))
for _, patch := range patches {
if patch.Version != "" {
versionIds = append(versionIds, patch.Version)
}
baseVersion, err := version.FindOne(version.ByProjectIdAndRevision(patch.Project, patch.Githash))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
var baseVersionId string
if baseVersion != nil {
baseVersionId = baseVersion.Id
}
patch.Patches = nil
uiPatches = append(uiPatches, uiPatch{Patch: patch, BaseVersionId: baseVersionId})
}
versions, err := version.Find(version.ByIds(versionIds).WithoutFields(version.ConfigKey))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error fetching versions for patches: %v", err))
return
}
versionsMap := map[string]*uiVersion{}
for _, version := range versions {
versionUI, err := PopulateUIVersion(&version)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
versionsMap[version.Id] = versionUI
}
data := struct {
VersionsMap map[string]*uiVersion
UIPatches []uiPatch
PageNum int
}{versionsMap, uiPatches, pageNum}
uis.WriteJSON(w, http.StatusOK, data)
}