func (uis *UIServer) projectPage(w http.ResponseWriter, r *http.Request) { _ = MustHaveProjectContext(r) _ = MustHaveUser(r) vars := mux.Vars(r) id := vars["project_id"] projRef, err := model.FindOneProjectRef(id) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } projVars, err := model.FindOneProjectVars(id) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } data := struct { ProjectRef *model.ProjectRef ProjectVars *model.ProjectVars }{projRef, projVars} // the project context has all projects so make the ui list using all projects uis.WriteJSON(w, http.StatusOK, data) }
// Creates a project ref local config that can be used for testing, with the string identifier given // and the local config from a path func CreateTestLocalConfig(testSettings *evergreen.Settings, projectName, projectPath string) error { if projectPath == "" { config, err := evergreen.FindConfig(testSettings.ConfigDir) if err != nil { return err } projectPath = filepath.Join(config, "project", fmt.Sprintf("%v.yml", projectName)) } projectRef, err := model.FindOneProjectRef(projectName) if err != nil { return err } if projectRef == nil { projectRef = &model.ProjectRef{} } data, err := ioutil.ReadFile(projectPath) if err != nil { return err } err = yaml.Unmarshal(data, projectRef) if err != nil { return err } projectRef.LocalConfig = string(data) return projectRef.Upsert() }
func TestCheckProjectSyntax(t *testing.T) { Convey("When validating a project's syntax", t, func() { Convey("if the project passes all of the validation funcs, no errors"+ " should be returned", func() { distros := []distro.Distro{ {Id: "test-distro-one"}, {Id: "test-distro-two"}, } err := testutil.CreateTestLocalConfig(projectValidatorConf, "project_test", "") So(err, ShouldBeNil) projectRef, err := model.FindOneProjectRef("project_test") So(err, ShouldBeNil) for _, d := range distros { So(d.Insert(), ShouldBeNil) } project, err := model.FindProject("", projectRef) So(err, ShouldBeNil) So(CheckProjectSyntax(project), ShouldResemble, []ValidationError{}) }) Reset(func() { db.Clear(distro.Collection) }) }) }
// reachedFailureLimit returns true if task for the previous failure transition alert // happened too long ago, as determined by some magic math. func reachedFailureLimit(taskId string) (bool, error) { t, err := task.FindOne(task.ById(taskId)) if err != nil { return false, err } if t == nil { return false, fmt.Errorf("task %v not found", taskId) } pr, err := model.FindOneProjectRef(t.Project) if err != nil { return false, err } if pr == nil { return false, fmt.Errorf("project ref %v not found", t.Project) } p, err := model.FindProject(t.Revision, pr) if err != nil { return false, err } if p == nil { return false, fmt.Errorf("project %v not found for revision %v", t.Project, t.Revision) } v := p.FindBuildVariant(t.BuildVariant) if v == nil { return false, fmt.Errorf("build variant %v does not exist in project", t.BuildVariant) } batchTime := pr.GetBatchTime(v) reached := time.Since(t.FinishTime) > (time.Duration(batchTime) * time.Minute * failureLimitMultiplier) return reached, nil }
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub // and applying the patch to the latest remote configuration. The error returned can be a validation error. func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) { if p.Version != "" { return nil, fmt.Errorf("Patch %v already finalized", p.Version) } projectRef, err := model.FindOneProjectRef(p.Project) if err != nil { return nil, err } // try to get the remote project file data at the requested revision var projectFileBytes []byte projectFileURL := thirdparty.GetGithubFileURL( projectRef.Owner, projectRef.Repo, projectRef.RemotePath, p.Githash, ) githubFile, err := thirdparty.GetGithubFile(settings.Credentials["github"], projectFileURL) if err != nil { // if the project file doesn't exist, but our patch includes a project file, // we try to apply the diff and proceed. if !(p.ConfigChanged(projectRef.RemotePath) && thirdparty.IsFileNotFound(err)) { // return an error if the github error is network/auth-related or we aren't patching the config return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err) } } else { // we successfully got the project file in base64, so we decode it projectFileBytes, err = base64.StdEncoding.DecodeString(githubFile.Content) if err != nil { return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err) } } project := &model.Project{} if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil { return nil, err } // apply remote configuration patch if needed if p.ConfigChanged(projectRef.RemotePath) { project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes)) if err != nil { return nil, fmt.Errorf("Could not patch remote configuration file: %v", err) } // overwrite project fields with the project ref to disallow tracking a // different project or doing other crazy things via config patches errs := CheckProjectSyntax(project) if len(errs) != 0 { var message string for _, err := range errs { message += fmt.Sprintf("\n\t=> %v", err) } return nil, fmt.Errorf(message) } } return project, nil }
func TestProjectRef(t *testing.T) { Convey("When inserting a project ref", t, func() { err := modelutil.CreateTestLocalConfig(patchTestConfig, "mci-test", "") So(err, ShouldBeNil) projectRef, err := model.FindOneProjectRef("mci-test") So(err, ShouldBeNil) So(projectRef, ShouldNotBeNil) So(projectRef.Identifier, ShouldEqual, "mci-test") }) }
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub // and applying the patch to the latest remote configuration. The error returned can be a validation error. func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) { if p.Version != "" { return nil, fmt.Errorf("Patch %v already finalized", p.Version) } projectRef, err := model.FindOneProjectRef(p.Project) if err != nil { return nil, err } // get the remote file at the requested revision projectFileURL := thirdparty.GetGithubFileURL( projectRef.Owner, projectRef.Repo, projectRef.RemotePath, p.Githash, ) githubFile, err := thirdparty.GetGithubFile( settings.Credentials["github"], projectFileURL, ) if err != nil { return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err) } projectFileBytes, err := base64.StdEncoding.DecodeString(githubFile.Content) if err != nil { return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err) } project := &model.Project{} if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil { return nil, err } // apply remote configuration patch if needed if p.ConfigChanged(projectRef.RemotePath) { project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes)) if err != nil { return nil, fmt.Errorf("Could not patch remote configuration file: %v", err) } // overwrite project fields with the project ref to disallow tracking a // different project or doing other crazy things via config patches errs := CheckProjectSyntax(project) if len(errs) != 0 { var message string for _, err := range errs { message += fmt.Sprintf("\n\t=> %v", err) } return nil, fmt.Errorf(message) } } return project, nil }
// gets the project ref project name corresponding to this identifier func getProjectRef(identifier string) (projectRef *model.ProjectRef, err error) { if cachedProjectRef[identifier] == nil { projectRef, err = model.FindOneProjectRef(identifier) if err != nil { return } cachedProjectRecords[identifier] = projectRef return projectRef, nil } return cachedProjectRef[identifier], nil }
// fetchProjectRef returns a project ref given the project identifier func (as *APIServer) fetchProjectRef(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["identifier"] projectRef, err := model.FindOneProjectRef(id) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } if projectRef == nil { http.Error(w, fmt.Sprintf("no project found named '%v'", id), http.StatusNotFound) return } as.WriteJSON(w, http.StatusOK, projectRef) }
func (as *APIServer) listVariants(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["projectId"] projectRef, err := model.FindOneProjectRef(id) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } project, err := model.FindProject("", projectRef) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } as.WriteJSON(w, http.StatusOK, project.BuildVariants) }
// findProject is a wrapper around FindProjectRef that caches results by their ID to prevent // redundantly querying for the same projectref over and over again. func (qp *QueueProcessor) findProject(projectId string) (*model.ProjectRef, error) { if qp.projectsCache == nil { // lazily initialize the cache qp.projectsCache = map[string]*model.ProjectRef{} } if project, ok := qp.projectsCache[projectId]; ok { return project, nil } project, err := model.FindOneProjectRef(projectId) if err != nil { return nil, err } if project == nil { return nil, nil } qp.projectsCache[projectId] = project return project, nil }
func (uis *UIServer) addProject(w http.ResponseWriter, r *http.Request) { dbUser := MustHaveUser(r) _ = MustHaveProjectContext(r) vars := mux.Vars(r) id := vars["project_id"] projectRef, err := model.FindOneProjectRef(id) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } if projectRef != nil { http.Error(w, "Project already exists", http.StatusInternalServerError) return } newProject := model.ProjectRef{ Identifier: id, Enabled: true, Tracked: true, RepoKind: "github", } err = newProject.Insert() if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } allProjects, err := uis.filterAuthorizedProjects(dbUser) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } data := struct { Available bool ProjectId string AllProjects []model.ProjectRef }{true, id, allProjects} uis.WriteJSON(w, http.StatusOK, data) }
func (as *APIServer) GetProjectRef(w http.ResponseWriter, r *http.Request) { task := MustHaveTask(r) p, err := model.FindOneProjectRef(task.Project) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } if p == nil { http.Error(w, "project ref not found", http.StatusNotFound) return } as.WriteJSON(w, http.StatusOK, p) }
func (restapi restAPI) getTaskHistory(w http.ResponseWriter, r *http.Request) { taskName := mux.Vars(r)["task_name"] projectName := r.FormValue("project") projectRef, err := model.FindOneProjectRef(projectName) if err != nil || projectRef == nil { msg := fmt.Sprintf("Error finding project '%v'", projectName) statusCode := http.StatusNotFound if err != nil { evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err) statusCode = http.StatusInternalServerError } restapi.WriteJSON(w, statusCode, responseError{Message: msg}) return } project, err := model.FindProject("", projectRef) if err != nil { msg := fmt.Sprintf("Error finding project '%v'", projectName) evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err) restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg}) return } buildVariants := project.GetVariantsWithTask(taskName) iter := model.NewTaskHistoryIterator(taskName, buildVariants, project.Identifier) chunk, err := iter.GetChunk(nil, MaxNumRevisions, NoRevisions, false) if err != nil { msg := fmt.Sprintf("Error finding history for task '%v'", taskName) evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err) restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg}) return } restapi.WriteJSON(w, http.StatusOK, chunk) return }
// setRevision sets the latest revision in the Repository // database to the revision sent from the projects page. func (uis *UIServer) setRevision(w http.ResponseWriter, r *http.Request) { MustHaveUser(r) vars := mux.Vars(r) id := vars["project_id"] data, err := ioutil.ReadAll(r.Body) if err != nil { uis.LoggedError(w, r, http.StatusNotFound, err) return } revision := string(data) if revision == "" { uis.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("revision sent was empty")) return } // update the latest revision to be the revision id err = model.UpdateLastRevision(id, revision) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } // update the projectRef too projectRef, err := model.FindOneProjectRef(id) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } projectRef.RepotrackerError.Exists = false projectRef.RepotrackerError.InvalidRevision = "" projectRef.RepotrackerError.MergeBaseRevision = "" err = projectRef.Upsert() if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } uis.WriteJSON(w, http.StatusOK, nil) }
func (as *APIServer) listTasks(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) id := vars["projectId"] projectRef, err := model.FindOneProjectRef(id) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } project, err := model.FindProject("", projectRef) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // zero out the depends on and commands fields because they are // unnecessary and may not get marshaled properly for i := range project.Tasks { project.Tasks[i].DependsOn = []model.TaskDependency{} project.Tasks[i].Commands = []model.PluginCommandConf{} } as.WriteJSON(w, http.StatusOK, project.Tasks) }
func (uis *UIServer) modifyProject(w http.ResponseWriter, r *http.Request) { _ = MustHaveUser(r) vars := mux.Vars(r) id := vars["project_id"] projectRef, err := model.FindOneProjectRef(id) if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } if projectRef == nil { http.Error(w, "Project not found", http.StatusNotFound) return } responseRef := struct { Identifier string `json:"id"` DisplayName string `json:"display_name"` RemotePath string `json:"remote_path"` BatchTime int `json:"batch_time"` DeactivatePrevious bool `json:"deactivate_previous"` Branch string `json:"branch_name"` ProjVarsMap map[string]string `json:"project_vars"` Enabled bool `json:"enabled"` Private bool `json:"private"` Owner string `json:"owner_name"` Repo string `json:"repo_name"` AlertConfig map[string][]struct { Provider string `json:"provider"` Settings map[string]interface{} `json:"settings"` } `json:"alert_config"` }{} err = util.ReadJSONInto(r.Body, &responseRef) if err != nil { http.Error(w, fmt.Sprintf("Error parsing request body %v", err), http.StatusInternalServerError) return } projectRef.DisplayName = responseRef.DisplayName projectRef.RemotePath = responseRef.RemotePath projectRef.BatchTime = responseRef.BatchTime projectRef.Branch = responseRef.Branch projectRef.Enabled = responseRef.Enabled projectRef.Private = responseRef.Private projectRef.Owner = responseRef.Owner projectRef.DeactivatePrevious = responseRef.DeactivatePrevious projectRef.Repo = responseRef.Repo projectRef.Identifier = id projectRef.Alerts = map[string][]model.AlertConfig{} for triggerId, alerts := range responseRef.AlertConfig { //TODO validate the triggerID, provider, and settings. for _, alert := range alerts { projectRef.Alerts[triggerId] = append(projectRef.Alerts[triggerId], model.AlertConfig{ Provider: alert.Provider, Settings: bson.M(alert.Settings), }) } } err = projectRef.Upsert() if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } //modify project vars if necessary projectVars := model.ProjectVars{id, responseRef.ProjVarsMap} _, err = projectVars.Upsert() if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } allProjects, err := model.FindAllProjectRefs() if err != nil { uis.LoggedError(w, r, http.StatusInternalServerError, err) return } data := struct { AllProjects []model.ProjectRef }{allProjects} uis.WriteJSON(w, http.StatusOK, data) }
// ManifestLoadHandler attempts to get the manifest, if it exists it updates the expansions and returns // If it does not exist it performs GitHub API calls for each of the project's modules and gets // the head revision of the branch and inserts it into the manifest collection. // If there is a duplicate key error, then do a find on the manifest again. func (mp *ManifestPlugin) ManifestLoadHandler(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) projectRef, err := model.FindOneProjectRef(task.Project) if err != nil { http.Error(w, fmt.Sprintf("projectRef not found for project %v: %v", task.Project, err), http.StatusNotFound) return } project, err := model.FindProject("", projectRef) if err != nil { http.Error(w, fmt.Sprintf("project not found for ProjectRef %v: %v", projectRef.Identifier, err), http.StatusNotFound) return } if project == nil { http.Error(w, fmt.Sprintf("empty project not found for ProjectRef %v: %v", projectRef.Identifier, err), http.StatusNotFound) return } // try to get the manifest currentManifest, err := manifest.FindOne(manifest.ById(task.Version)) if err != nil { http.Error(w, fmt.Sprintf("error retrieving manifest with version id %v: %v", task.Version, err), http.StatusNotFound) return } if currentManifest != nil { plugin.WriteJSON(w, http.StatusOK, currentManifest) return } if task.Version == "" { http.Error(w, fmt.Sprintf("versionId is empty"), http.StatusNotFound) return } // attempt to insert a manifest after making GitHub API calls newManifest := &manifest.Manifest{ Id: task.Version, Revision: task.Revision, ProjectName: task.Project, Branch: project.Branch, } // populate modules modules := make(map[string]*manifest.Module) for _, module := range project.Modules { owner, repo := module.GetRepoOwnerAndName() gitBranch, err := thirdparty.GetBranchEvent(mp.OAuthCredentials, owner, repo, module.Branch) if err != nil { http.Error(w, fmt.Sprintf("error retrieving getting git branch for module %v: %v", module.Name, err), http.StatusNotFound) return } modules[module.Name] = &manifest.Module{ Branch: module.Branch, Revision: gitBranch.Commit.SHA, Repo: repo, Owner: owner, URL: gitBranch.Commit.Url, } } newManifest.Modules = modules duplicate, err := newManifest.TryInsert() if err != nil { http.Error(w, fmt.Sprintf("error inserting manifest for %v: %v", newManifest.ProjectName, err), http.StatusNotFound) return } // if it is a duplicate, load the manifest again` if duplicate { // try to get the manifest m, err := manifest.FindOne(manifest.ById(task.Version)) if err != nil { http.Error(w, fmt.Sprintf("error getting latest manifest for %v: %v", newManifest.ProjectName, err), http.StatusNotFound) return } if m != nil { plugin.WriteJSON(w, http.StatusOK, m) return } } // no duplicate key error, use the manifest just created. plugin.WriteJSON(w, http.StatusOK, newManifest) return }
// LoadProjectContext builds a projectContext from vars in the request's URL. // This is done by reading in specific variables and inferring other required // context variables when necessary (e.g. loading a project based on the task). func (uis *UIServer) LoadProjectContext(rw http.ResponseWriter, r *http.Request) (projectContext, error) { user := GetUser(r) vars := mux.Vars(r) proj := &projectContext{} taskId := vars["task_id"] buildId := vars["build_id"] versionId := vars["version_id"] patchId := vars["patch_id"] err := proj.populateProjectRefs(user != nil) if err != nil { return *proj, err } projectId, err := proj.populateTaskBuildVersion(taskId, buildId, versionId) if err != nil { return *proj, err } err = proj.populatePatch(patchId) if err != nil { return *proj, err } if proj.Patch != nil && len(projectId) == 0 { projectId = proj.Patch.Project } // Try to infer project ID - if we don't already have it from the task/build/version, try to // get it from the URL if len(projectId) == 0 { projectId = vars["project_id"] } usingDefault := false // Still don't have a project ID to use, check if the user's cookie contains one if len(projectId) == 0 { cookie, err := r.Cookie(ProjectCookieName) if err == nil { projectId = cookie.Value } usingDefault = true } // Still no project ID found anywhere, just use the default one according to config. if len(projectId) == 0 { projectId = uis.Settings.Ui.DefaultProject usingDefault = true } // Try to load project for the ID we found, and set cookie with it for subsequent requests if len(projectId) > 0 { // Also lookup the ProjectRef itself and add it to context. proj.ProjectRef, err = model.FindOneProjectRef(projectId) if err != nil { return *proj, err } // If we used the default project or a cookie to choose the project, // but that project doesn't exist, choose the first one in the list. if usingDefault && proj.ProjectRef == nil { if len(proj.AllProjects) > 0 { proj.ProjectRef = &proj.AllProjects[0] } } if proj.ProjectRef != nil { proj.Project, err = model.FindProject("", proj.ProjectRef) if err != nil { return *proj, err } if proj.Project != nil { // A project was found, update the project cookie for subsequent request. http.SetCookie(rw, &http.Cookie{ Name: ProjectCookieName, Value: projectId, Path: "", Expires: time.Now().Add(7 * 24 * time.Hour), }) } } } proj.AuthRedirect = uis.UserManager.IsRedirect() return *proj, nil }
func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) { p, err := getPatchFromRequest(r) if err != nil { as.WriteJSON(w, http.StatusBadRequest, err.Error()) return } moduleName := r.FormValue("module") patchContent := r.FormValue("patch") githash := r.FormValue("githash") projectRef, err := model.FindOneProjectRef(p.Project) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting project ref with id %v: %v", p.Project, err)) return } project, err := model.FindProject("", projectRef) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting patch: %v", err)) return } if project == nil { as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("can't find project: %v", p.Project)) return } module, err := project.GetModuleByName(moduleName) if err != nil || module == nil { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("No such module")) return } gitOutput, err := thirdparty.GitApplyNumstat(patchContent) if err != nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Invalid patch: %v", err)) return } if gitOutput == nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Empty diff")) return } summaries, err := thirdparty.ParseGitSummary(gitOutput) if err != nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Can't validate patch: %v", err)) return } repoOwner, repo := module.GetRepoOwnerAndName() commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials[projectRef.RepoKind], repoOwner, repo, githash) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } if commitInfo == nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("commit hash doesn't seem to exist")) return } modulePatch := patch.ModulePatch{ ModuleName: moduleName, Githash: githash, PatchSet: patch.PatchSet{ Patch: patchContent, Summary: summaries, // thirdparty.GetPatchSummary(apiRequest.PatchContent), }, } if err = p.UpdateModulePatch(modulePatch); err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } as.WriteJSON(w, http.StatusOK, "Patch module updated") return }
func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) { user := MustHaveUser(r) apiRequest := PatchAPIRequest{ ProjectFileName: r.FormValue("project"), ModuleName: r.FormValue("module"), Githash: r.FormValue("githash"), PatchContent: r.FormValue("patch"), BuildVariants: strings.Split(r.FormValue("buildvariants"), ","), } description := r.FormValue("desc") projId := r.FormValue("project") projectRef, err := model.FindOneProjectRef(projId) if err != nil { message := fmt.Errorf("Error locating project ref '%v': %v", projId, err) as.LoggedError(w, r, http.StatusInternalServerError, message) return } project, err := model.FindProject("", projectRef) if err != nil { message := fmt.Errorf("Error locating project '%v' from '%v': %v", projId, as.Settings.ConfigDir, err) as.LoggedError(w, r, http.StatusInternalServerError, message) return } if project == nil { as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("project %v not found", projId)) return } patchMetadata, err := apiRequest.Validate(as.Settings.Credentials[projectRef.RepoKind]) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid patch: %v", err)) return } if patchMetadata == nil { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("patch metadata is empty")) return } if apiRequest.ModuleName != "" { as.WriteJSON(w, http.StatusBadRequest, "module not allowed when creating new patches (must be added in a subsequent request)") return } patchProjectRef, err := model.FindOneProjectRef(patchMetadata.Project.Identifier) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid projectRef: %v", err)) return } if patchProjectRef == nil { as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("Empty patch project Ref")) return } commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials[projectRef.RepoKind], patchProjectRef.Owner, patchProjectRef.Repo, apiRequest.Githash) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } if commitInfo == nil { as.WriteJSON(w, http.StatusBadRequest, "That commit doesn't seem to exist.") return } createTime := time.Now() patchDoc := &patch.Patch{ Id: bson.NewObjectId(), Description: description, Author: user.Id, Project: apiRequest.ProjectFileName, Githash: apiRequest.Githash, CreateTime: createTime, Status: evergreen.PatchCreated, BuildVariants: apiRequest.BuildVariants, Tasks: nil, Patches: []patch.ModulePatch{ patch.ModulePatch{ ModuleName: "", Githash: apiRequest.Githash, PatchSet: patch.PatchSet{ Patch: apiRequest.PatchContent, Summary: patchMetadata.Summaries, }, }, }, } // set the patch number based on patch author patchDoc.PatchNumber, err = user.IncPatchNumber() if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error computing patch num %v", err)) return } if err = patchDoc.Insert(); err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error inserting patch: %v", err)) return } if strings.ToLower(r.FormValue("finalize")) == "true" { if _, err = validator.ValidateAndFinalize(patchDoc, &as.Settings); err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } } as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc}) }
func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) { user := MustHaveUser(r) var apiRequest PatchAPIRequest var projId, description string var finalize bool if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" { patchContent := r.FormValue("patch") if patchContent == "" { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty")) return } apiRequest = PatchAPIRequest{ ProjectFileName: r.FormValue("project"), ModuleName: r.FormValue("module"), Githash: r.FormValue("githash"), PatchContent: r.FormValue("patch"), BuildVariants: strings.Split(r.FormValue("buildvariants"), ","), } projId = r.FormValue("project") description = r.FormValue("desc") finalize = strings.ToLower(r.FormValue("finalize")) == "true" } else { data := struct { Description string `json:"desc"` Project string `json:"project"` Patch string `json:"patch"` Githash string `json:"githash"` Variants string `json:"buildvariants"` Tasks []string `json:"tasks"` Finalize bool `json:"finalize"` }{} if err := util.ReadJSONInto(r.Body, &data); err != nil { as.LoggedError(w, r, http.StatusBadRequest, err) return } if len(data.Patch) > patch.SizeLimit { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Patch is too large.")) } if len(data.Patch) == 0 { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty")) return } description = data.Description projId = data.Project finalize = data.Finalize apiRequest = PatchAPIRequest{ ProjectFileName: data.Project, ModuleName: r.FormValue("module"), Githash: data.Githash, PatchContent: data.Patch, BuildVariants: strings.Split(data.Variants, ","), Tasks: data.Tasks, } } projectRef, err := model.FindOneProjectRef(projId) if err != nil { message := fmt.Errorf("Error locating project ref '%v': %v", projId, err) as.LoggedError(w, r, http.StatusInternalServerError, message) return } project, err := model.FindProject("", projectRef) if err != nil { message := fmt.Errorf("Error locating project '%v' from '%v': %v", projId, as.Settings.ConfigDir, err) as.LoggedError(w, r, http.StatusInternalServerError, message) return } if project == nil { as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("project %v not found", projId)) return } patchMetadata, err := apiRequest.Validate(as.Settings.Credentials["github"]) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid patch: %v", err)) return } if patchMetadata == nil { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("patch metadata is empty")) return } if apiRequest.ModuleName != "" { as.WriteJSON(w, http.StatusBadRequest, "module not allowed when creating new patches (must be added in a subsequent request)") return } patchProjectRef, err := model.FindOneProjectRef(patchMetadata.Project.Identifier) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid projectRef: %v", err)) return } if patchProjectRef == nil { as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("Empty patch project Ref")) return } commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"], patchProjectRef.Owner, patchProjectRef.Repo, apiRequest.Githash) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } if commitInfo == nil { as.WriteJSON(w, http.StatusBadRequest, "That commit doesn't seem to exist.") return } createTime := time.Now() // create a new object ID to use as reference for the patch data patchFileId := bson.NewObjectId().Hex() patchDoc := &patch.Patch{ Id: bson.NewObjectId(), Description: description, Author: user.Id, Project: apiRequest.ProjectFileName, Githash: apiRequest.Githash, CreateTime: createTime, Status: evergreen.PatchCreated, BuildVariants: apiRequest.BuildVariants, Tasks: apiRequest.Tasks, Patches: []patch.ModulePatch{ patch.ModulePatch{ ModuleName: "", Githash: apiRequest.Githash, PatchSet: patch.PatchSet{ PatchFileId: patchFileId, Summary: patchMetadata.Summaries, }, }, }, } // write the patch content into a GridFS file under a new ObjectId. err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(apiRequest.PatchContent)) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err)) return } // Get and validate patched config and add it to the patch document patchedProject, err := validator.GetPatchedProject(patchDoc, &as.Settings) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("invalid patched config: %v", err)) return } projectYamlBytes, err := yaml.Marshal(patchedProject) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error marshalling patched config: %v", err)) return } // set the patch number based on patch author patchDoc.PatchNumber, err = user.IncPatchNumber() if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error computing patch num %v", err)) return } patchDoc.PatchedConfig = string(projectYamlBytes) patchDoc.ClearPatchData() //expand tasks and build variants and include dependencies buildVariants := patchDoc.BuildVariants if len(patchDoc.BuildVariants) == 1 && patchDoc.BuildVariants[0] == "all" { buildVariants = make([]string, 0) for _, buildVariant := range patchedProject.BuildVariants { if buildVariant.Disabled { continue } buildVariants = append(buildVariants, buildVariant.Name) } } tasks := patchDoc.Tasks if len(patchDoc.Tasks) == 1 && patchDoc.Tasks[0] == "all" { tasks = make([]string, 0) for _, t := range patchedProject.Tasks { tasks = append(tasks, t.Name) } } // update variant and tasks to include dependencies patchDoc.BuildVariants, patchDoc.Tasks = model.IncludePatchDependencies( project, buildVariants, tasks) if err = patchDoc.Insert(); err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error inserting patch: %v", err)) return } if finalize { if _, err = model.FinalizePatch(patchDoc, &as.Settings); err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } } as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc}) }
// CreatePatch checks an API request to see if it is safe and sane. // Returns the relevant patch metadata, the patch document, and any errors that occur. func (pr *PatchAPIRequest) CreatePatch(finalize bool, oauthToken string, dbUser *user.DBUser, settings *evergreen.Settings) (*model.Project, *patch.Patch, error) { var repoOwner, repo string var module *model.Module projectRef, err := model.FindOneProjectRef(pr.ProjectId) if err != nil { return nil, nil, fmt.Errorf("Could not find project ref %v : %v", pr.ProjectId, err) } repoOwner = projectRef.Owner repo = projectRef.Repo if len(pr.Githash) != 40 { return nil, nil, fmt.Errorf("invalid githash") } gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash) if err != nil { return nil, nil, fmt.Errorf("could not find base revision %v for project %v: %v", pr.Githash, projectRef.Identifier, err) } if gitCommit == nil { return nil, nil, fmt.Errorf("commit hash %v doesn't seem to exist", pr.Githash) } summaries, err := getSummaries(pr.PatchContent) if err != nil { return nil, nil, err } isEmpty := (pr.PatchContent == "") if finalize && (len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "") { return nil, nil, fmt.Errorf("no buildvariants specified") } createTime := time.Now() // create a new object ID to use as reference for the patch data patchFileId := bson.NewObjectId().Hex() patchDoc := &patch.Patch{ Id: bson.NewObjectId(), Description: pr.Description, Author: dbUser.Id, Project: pr.ProjectId, Githash: pr.Githash, CreateTime: createTime, Status: evergreen.PatchCreated, BuildVariants: pr.BuildVariants, Tasks: pr.Tasks, IsEmpty: isEmpty, Patches: []patch.ModulePatch{ { ModuleName: "", Githash: pr.Githash, PatchSet: patch.PatchSet{ Patch: pr.PatchContent, PatchFileId: patchFileId, Summary: summaries, }, }, }, } // Get and validate patched config and add it to the patch document project, err := validator.GetPatchedProject(patchDoc, settings) if err != nil { return nil, nil, fmt.Errorf("invalid patched config: %v", err) } if pr.ModuleName != "" { // is there a module? validate it. module, err = project.GetModuleByName(pr.ModuleName) if err != nil { return nil, nil, fmt.Errorf("could not find module %v: %v", pr.ModuleName, err) } if module == nil { return nil, nil, fmt.Errorf("no module named %v", pr.ModuleName) } } // verify that all variants exists for _, buildVariant := range pr.BuildVariants { if buildVariant == "all" || buildVariant == "" { continue } bv := project.FindBuildVariant(buildVariant) if bv == nil { return nil, nil, fmt.Errorf("No such buildvariant: %v", buildVariant) } } // write the patch content into a GridFS file under a new ObjectId after validating. err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(pr.PatchContent)) if err != nil { return nil, nil, fmt.Errorf("failed to write patch file to db: %v", err) } // add the project config projectYamlBytes, err := yaml.Marshal(project) if err != nil { return nil, nil, fmt.Errorf("error marshalling patched config: %v", err) } // set the patch number based on patch author patchDoc.PatchNumber, err = dbUser.IncPatchNumber() if err != nil { return nil, nil, fmt.Errorf("error computing patch num %v", err) } patchDoc.PatchedConfig = string(projectYamlBytes) patchDoc.ClearPatchData() return project, patchDoc, nil }
func (as *APIServer) EndTask(w http.ResponseWriter, r *http.Request) { finishTime := time.Now() taskEndResponse := &apimodels.TaskEndResponse{} task := MustHaveTask(r) details := &apimodels.TaskEndDetail{} if err := util.ReadJSONInto(r.Body, details); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } // Check that finishing status is a valid constant if details.Status != evergreen.TaskSucceeded && details.Status != evergreen.TaskFailed && details.Status != evergreen.TaskUndispatched { msg := fmt.Errorf("Invalid end status '%v' for task %v", details.Status, task.Id) as.LoggedError(w, r, http.StatusBadRequest, msg) return } projectRef, err := model.FindOneProjectRef(task.Project) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) } if projectRef == nil { as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("empty projectRef for task")) return } project, err := model.FindProject("", projectRef) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } if !getGlobalLock(r.RemoteAddr, task.Id) { as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout) return } defer releaseGlobalLock(r.RemoteAddr, task.Id) // mark task as finished err = task.MarkEnd(APIServerLockTitle, finishTime, details, project, projectRef.DeactivatePrevious) if err != nil { message := fmt.Errorf("Error calling mark finish on task %v : %v", task.Id, err) as.LoggedError(w, r, http.StatusInternalServerError, message) return } if task.Requester != evergreen.PatchVersionRequester { alerts.RunTaskFailureTriggers(task) } else { //TODO(EVG-223) process patch-specific triggers } // if task was aborted, reset to inactive if details.Status == evergreen.TaskUndispatched { if err = model.SetTaskActivated(task.Id, "", false); err != nil { message := fmt.Sprintf("Error deactivating task after abort: %v", err) evergreen.Logger.Logf(slogger.ERROR, message) taskEndResponse.Message = message as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse) return } as.taskFinished(w, task, finishTime) return } // update the bookkeeping entry for the task err = bookkeeping.UpdateExpectedDuration(task, task.TimeTaken) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "Error updating expected duration: %v", err) } // log the task as finished evergreen.Logger.Logf(slogger.INFO, "Successfully marked task %v as finished", task.Id) // construct and return the appropriate response for the agent as.taskFinished(w, task, finishTime) }
// Validate checks an API request to see if it is safe and sane. // Returns the relevant patch metadata and any errors that occur. func (pr *PatchAPIRequest) Validate(oauthToken string) (*PatchMetadata, error) { var repoOwner, repo string var module *model.Module projectRef, err := model.FindOneProjectRef(pr.ProjectFileName) if err != nil { return nil, fmt.Errorf("Could not find project ref %v : %v", pr.ProjectFileName, err) } repoOwner = projectRef.Owner repo = projectRef.Repo // validate the project file project, err := model.FindProject("", projectRef) if err != nil { return nil, fmt.Errorf("Could not find project file %v: %v", pr.ProjectFileName, err) } if project == nil { return nil, fmt.Errorf("No such project file named %v", pr.ProjectFileName) } if pr.ModuleName != "" { // is there a module? validate it. module, err = project.GetModuleByName(pr.ModuleName) if err != nil { return nil, fmt.Errorf("could not find module %v: %v", pr.ModuleName, err) } if module == nil { return nil, fmt.Errorf("no module named %v", pr.ModuleName) } repoOwner, repo = module.GetRepoOwnerAndName() } if len(pr.Githash) != 40 { return nil, fmt.Errorf("invalid githash") } gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash) if err != nil { return nil, fmt.Errorf("could not find base revision %v for project %v: %v", pr.Githash, projectRef.Identifier, err) } if gitCommit == nil { return nil, fmt.Errorf("commit hash %v doesn't seem to exist", pr.Githash) } gitOutput, err := thirdparty.GitApplyNumstat(pr.PatchContent) if err != nil { return nil, fmt.Errorf("couldn't validate patch: %v", err) } if gitOutput == nil { return nil, fmt.Errorf("couldn't validate patch: git apply --numstat returned empty") } summaries, err := thirdparty.ParseGitSummary(gitOutput) if err != nil { return nil, fmt.Errorf("couldn't validate patch: %v", err) } if len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "" { return nil, fmt.Errorf("no buildvariants specified") } // verify that this build variant exists for _, buildVariant := range pr.BuildVariants { if buildVariant == "all" { continue } bv := project.FindBuildVariant(buildVariant) if bv == nil { return nil, fmt.Errorf("No such buildvariant: %v", buildVariant) } } return &PatchMetadata{pr.Githash, project, module, pr.BuildVariants, summaries}, nil }
func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) { p, err := getPatchFromRequest(r) if err != nil { as.WriteJSON(w, http.StatusBadRequest, err.Error()) return } var moduleName, patchContent, githash string if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" { moduleName, patchContent, githash = r.FormValue("module"), r.FormValue("patch"), r.FormValue("githash") } else { data := struct { Module string `json:"module"` Patch string `json:"patch"` Githash string `json:"githash"` }{} if err := util.ReadJSONInto(r.Body, &data); err != nil { as.LoggedError(w, r, http.StatusBadRequest, err) return } moduleName, patchContent, githash = data.Module, data.Patch, data.Githash } if len(patchContent) == 0 { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty")) return } projectRef, err := model.FindOneProjectRef(p.Project) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting project ref with id %v: %v", p.Project, err)) return } project, err := model.FindProject("", projectRef) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting patch: %v", err)) return } if project == nil { as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("can't find project: %v", p.Project)) return } module, err := project.GetModuleByName(moduleName) if err != nil || module == nil { as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("No such module", moduleName)) return } gitOutput, err := thirdparty.GitApplyNumstat(patchContent) if err != nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Invalid patch: %v", err)) return } if gitOutput == nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Empty diff")) return } summaries, err := thirdparty.ParseGitSummary(gitOutput) if err != nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Can't validate patch: %v", err)) return } repoOwner, repo := module.GetRepoOwnerAndName() commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"], repoOwner, repo, githash) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } if commitInfo == nil { as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("commit hash doesn't seem to exist")) return } // write the patch content into a GridFS file under a new ObjectId. patchFileId := bson.NewObjectId().Hex() err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(patchContent)) if err != nil { as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err)) return } modulePatch := patch.ModulePatch{ ModuleName: moduleName, Githash: githash, PatchSet: patch.PatchSet{ PatchFileId: patchFileId, Summary: summaries, }, } if err = p.UpdateModulePatch(modulePatch); err != nil { as.LoggedError(w, r, http.StatusInternalServerError, err) return } as.WriteJSON(w, http.StatusOK, "Patch module updated") return }