func (uis *UIServer) projectPage(w http.ResponseWriter, r *http.Request) {
_ = MustHaveProjectContext(r)
_ = MustHaveUser(r)
vars := mux.Vars(r)
id := vars["project_id"]
projRef, err := model.FindOneProjectRef(id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
projVars, err := model.FindOneProjectVars(id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
data := struct {
ProjectRef *model.ProjectRef
ProjectVars *model.ProjectVars
}{projRef, projVars}
// the project context has all projects so make the ui list using all projects
uis.WriteJSON(w, http.StatusOK, data)
}
// Creates a project ref local config that can be used for testing, with the string identifier given
// and the local config from a path
func CreateTestLocalConfig(testSettings *evergreen.Settings, projectName, projectPath string) error {
if projectPath == "" {
config, err := evergreen.FindConfig(testSettings.ConfigDir)
if err != nil {
return err
}
projectPath = filepath.Join(config, "project", fmt.Sprintf("%v.yml", projectName))
}
projectRef, err := model.FindOneProjectRef(projectName)
if err != nil {
return err
}
if projectRef == nil {
projectRef = &model.ProjectRef{}
}
data, err := ioutil.ReadFile(projectPath)
if err != nil {
return err
}
err = yaml.Unmarshal(data, projectRef)
if err != nil {
return err
}
projectRef.LocalConfig = string(data)
return projectRef.Upsert()
}
func TestCheckProjectSyntax(t *testing.T) {
Convey("When validating a project's syntax", t, func() {
Convey("if the project passes all of the validation funcs, no errors"+
" should be returned", func() {
distros := []distro.Distro{
{Id: "test-distro-one"},
{Id: "test-distro-two"},
}
err := testutil.CreateTestLocalConfig(projectValidatorConf, "project_test", "")
So(err, ShouldBeNil)
projectRef, err := model.FindOneProjectRef("project_test")
So(err, ShouldBeNil)
for _, d := range distros {
So(d.Insert(), ShouldBeNil)
}
project, err := model.FindProject("", projectRef)
So(err, ShouldBeNil)
So(CheckProjectSyntax(project), ShouldResemble, []ValidationError{})
})
Reset(func() {
db.Clear(distro.Collection)
})
})
}
// reachedFailureLimit returns true if task for the previous failure transition alert
// happened too long ago, as determined by some magic math.
func reachedFailureLimit(taskId string) (bool, error) {
t, err := task.FindOne(task.ById(taskId))
if err != nil {
return false, err
}
if t == nil {
return false, fmt.Errorf("task %v not found", taskId)
}
pr, err := model.FindOneProjectRef(t.Project)
if err != nil {
return false, err
}
if pr == nil {
return false, fmt.Errorf("project ref %v not found", t.Project)
}
p, err := model.FindProject(t.Revision, pr)
if err != nil {
return false, err
}
if p == nil {
return false, fmt.Errorf("project %v not found for revision %v", t.Project, t.Revision)
}
v := p.FindBuildVariant(t.BuildVariant)
if v == nil {
return false, fmt.Errorf("build variant %v does not exist in project", t.BuildVariant)
}
batchTime := pr.GetBatchTime(v)
reached := time.Since(t.FinishTime) > (time.Duration(batchTime) * time.Minute * failureLimitMultiplier)
return reached, nil
}
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
if p.Version != "" {
return nil, fmt.Errorf("Patch %v already finalized", p.Version)
}
projectRef, err := model.FindOneProjectRef(p.Project)
if err != nil {
return nil, err
}
// try to get the remote project file data at the requested revision
var projectFileBytes []byte
projectFileURL := thirdparty.GetGithubFileURL(
projectRef.Owner,
projectRef.Repo,
projectRef.RemotePath,
p.Githash,
)
githubFile, err := thirdparty.GetGithubFile(settings.Credentials["github"], projectFileURL)
if err != nil {
// if the project file doesn't exist, but our patch includes a project file,
// we try to apply the diff and proceed.
if !(p.ConfigChanged(projectRef.RemotePath) && thirdparty.IsFileNotFound(err)) {
// return an error if the github error is network/auth-related or we aren't patching the config
return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
}
} else {
// we successfully got the project file in base64, so we decode it
projectFileBytes, err = base64.StdEncoding.DecodeString(githubFile.Content)
if err != nil {
return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
}
}
project := &model.Project{}
if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
return nil, err
}
// apply remote configuration patch if needed
if p.ConfigChanged(projectRef.RemotePath) {
project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
if err != nil {
return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
}
// overwrite project fields with the project ref to disallow tracking a
// different project or doing other crazy things via config patches
errs := CheckProjectSyntax(project)
if len(errs) != 0 {
var message string
for _, err := range errs {
message += fmt.Sprintf("\n\t=> %v", err)
}
return nil, fmt.Errorf(message)
}
}
return project, nil
}
func TestProjectRef(t *testing.T) {
Convey("When inserting a project ref", t, func() {
err := modelutil.CreateTestLocalConfig(patchTestConfig, "mci-test", "")
So(err, ShouldBeNil)
projectRef, err := model.FindOneProjectRef("mci-test")
So(err, ShouldBeNil)
So(projectRef, ShouldNotBeNil)
So(projectRef.Identifier, ShouldEqual, "mci-test")
})
}
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
if p.Version != "" {
return nil, fmt.Errorf("Patch %v already finalized", p.Version)
}
projectRef, err := model.FindOneProjectRef(p.Project)
if err != nil {
return nil, err
}
// get the remote file at the requested revision
projectFileURL := thirdparty.GetGithubFileURL(
projectRef.Owner,
projectRef.Repo,
projectRef.RemotePath,
p.Githash,
)
githubFile, err := thirdparty.GetGithubFile(
settings.Credentials["github"],
projectFileURL,
)
if err != nil {
return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
}
projectFileBytes, err := base64.StdEncoding.DecodeString(githubFile.Content)
if err != nil {
return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
}
project := &model.Project{}
if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
return nil, err
}
// apply remote configuration patch if needed
if p.ConfigChanged(projectRef.RemotePath) {
project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
if err != nil {
return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
}
// overwrite project fields with the project ref to disallow tracking a
// different project or doing other crazy things via config patches
errs := CheckProjectSyntax(project)
if len(errs) != 0 {
var message string
for _, err := range errs {
message += fmt.Sprintf("\n\t=> %v", err)
}
return nil, fmt.Errorf(message)
}
}
return project, nil
}
// gets the project ref project name corresponding to this identifier
func getProjectRef(identifier string) (projectRef *model.ProjectRef,
err error) {
if cachedProjectRef[identifier] == nil {
projectRef, err = model.FindOneProjectRef(identifier)
if err != nil {
return
}
cachedProjectRecords[identifier] = projectRef
return projectRef, nil
}
return cachedProjectRef[identifier], nil
}
// fetchProjectRef returns a project ref given the project identifier
func (as *APIServer) fetchProjectRef(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["identifier"]
projectRef, err := model.FindOneProjectRef(id)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if projectRef == nil {
http.Error(w, fmt.Sprintf("no project found named '%v'", id), http.StatusNotFound)
return
}
as.WriteJSON(w, http.StatusOK, projectRef)
}
func (as *APIServer) listVariants(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["projectId"]
projectRef, err := model.FindOneProjectRef(id)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
as.WriteJSON(w, http.StatusOK, project.BuildVariants)
}
// findProject is a wrapper around FindProjectRef that caches results by their ID to prevent
// redundantly querying for the same projectref over and over again.
func (qp *QueueProcessor) findProject(projectId string) (*model.ProjectRef, error) {
if qp.projectsCache == nil { // lazily initialize the cache
qp.projectsCache = map[string]*model.ProjectRef{}
}
if project, ok := qp.projectsCache[projectId]; ok {
return project, nil
}
project, err := model.FindOneProjectRef(projectId)
if err != nil {
return nil, err
}
if project == nil {
return nil, nil
}
qp.projectsCache[projectId] = project
return project, nil
}
func (uis *UIServer) addProject(w http.ResponseWriter, r *http.Request) {
dbUser := MustHaveUser(r)
_ = MustHaveProjectContext(r)
vars := mux.Vars(r)
id := vars["project_id"]
projectRef, err := model.FindOneProjectRef(id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if projectRef != nil {
http.Error(w, "Project already exists", http.StatusInternalServerError)
return
}
newProject := model.ProjectRef{
Identifier: id,
Enabled: true,
Tracked: true,
RepoKind: "github",
}
err = newProject.Insert()
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
allProjects, err := uis.filterAuthorizedProjects(dbUser)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
data := struct {
Available bool
ProjectId string
AllProjects []model.ProjectRef
}{true, id, allProjects}
uis.WriteJSON(w, http.StatusOK, data)
}
func (as *APIServer) GetProjectRef(w http.ResponseWriter, r *http.Request) {
task := MustHaveTask(r)
p, err := model.FindOneProjectRef(task.Project)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if p == nil {
http.Error(w, "project ref not found", http.StatusNotFound)
return
}
as.WriteJSON(w, http.StatusOK, p)
}
func (restapi restAPI) getTaskHistory(w http.ResponseWriter, r *http.Request) {
taskName := mux.Vars(r)["task_name"]
projectName := r.FormValue("project")
projectRef, err := model.FindOneProjectRef(projectName)
if err != nil || projectRef == nil {
msg := fmt.Sprintf("Error finding project '%v'", projectName)
statusCode := http.StatusNotFound
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
statusCode = http.StatusInternalServerError
}
restapi.WriteJSON(w, statusCode, responseError{Message: msg})
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
msg := fmt.Sprintf("Error finding project '%v'", projectName)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
buildVariants := project.GetVariantsWithTask(taskName)
iter := model.NewTaskHistoryIterator(taskName, buildVariants, project.Identifier)
chunk, err := iter.GetChunk(nil, MaxNumRevisions, NoRevisions, false)
if err != nil {
msg := fmt.Sprintf("Error finding history for task '%v'", taskName)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
restapi.WriteJSON(w, http.StatusOK, chunk)
return
}
// setRevision sets the latest revision in the Repository
// database to the revision sent from the projects page.
func (uis *UIServer) setRevision(w http.ResponseWriter, r *http.Request) {
MustHaveUser(r)
vars := mux.Vars(r)
id := vars["project_id"]
data, err := ioutil.ReadAll(r.Body)
if err != nil {
uis.LoggedError(w, r, http.StatusNotFound, err)
return
}
revision := string(data)
if revision == "" {
uis.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("revision sent was empty"))
return
}
// update the latest revision to be the revision id
err = model.UpdateLastRevision(id, revision)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
// update the projectRef too
projectRef, err := model.FindOneProjectRef(id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
projectRef.RepotrackerError.Exists = false
projectRef.RepotrackerError.InvalidRevision = ""
projectRef.RepotrackerError.MergeBaseRevision = ""
err = projectRef.Upsert()
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
uis.WriteJSON(w, http.StatusOK, nil)
}
func (as *APIServer) listTasks(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
id := vars["projectId"]
projectRef, err := model.FindOneProjectRef(id)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// zero out the depends on and commands fields because they are
// unnecessary and may not get marshaled properly
for i := range project.Tasks {
project.Tasks[i].DependsOn = []model.TaskDependency{}
project.Tasks[i].Commands = []model.PluginCommandConf{}
}
as.WriteJSON(w, http.StatusOK, project.Tasks)
}
func (uis *UIServer) modifyProject(w http.ResponseWriter, r *http.Request) {
_ = MustHaveUser(r)
vars := mux.Vars(r)
id := vars["project_id"]
projectRef, err := model.FindOneProjectRef(id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if projectRef == nil {
http.Error(w, "Project not found", http.StatusNotFound)
return
}
responseRef := struct {
Identifier string `json:"id"`
DisplayName string `json:"display_name"`
RemotePath string `json:"remote_path"`
BatchTime int `json:"batch_time"`
DeactivatePrevious bool `json:"deactivate_previous"`
Branch string `json:"branch_name"`
ProjVarsMap map[string]string `json:"project_vars"`
Enabled bool `json:"enabled"`
Private bool `json:"private"`
Owner string `json:"owner_name"`
Repo string `json:"repo_name"`
AlertConfig map[string][]struct {
Provider string `json:"provider"`
Settings map[string]interface{} `json:"settings"`
} `json:"alert_config"`
}{}
err = util.ReadJSONInto(r.Body, &responseRef)
if err != nil {
http.Error(w, fmt.Sprintf("Error parsing request body %v", err), http.StatusInternalServerError)
return
}
projectRef.DisplayName = responseRef.DisplayName
projectRef.RemotePath = responseRef.RemotePath
projectRef.BatchTime = responseRef.BatchTime
projectRef.Branch = responseRef.Branch
projectRef.Enabled = responseRef.Enabled
projectRef.Private = responseRef.Private
projectRef.Owner = responseRef.Owner
projectRef.DeactivatePrevious = responseRef.DeactivatePrevious
projectRef.Repo = responseRef.Repo
projectRef.Identifier = id
projectRef.Alerts = map[string][]model.AlertConfig{}
for triggerId, alerts := range responseRef.AlertConfig {
//TODO validate the triggerID, provider, and settings.
for _, alert := range alerts {
projectRef.Alerts[triggerId] = append(projectRef.Alerts[triggerId], model.AlertConfig{
Provider: alert.Provider,
Settings: bson.M(alert.Settings),
})
}
}
err = projectRef.Upsert()
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
//modify project vars if necessary
projectVars := model.ProjectVars{id, responseRef.ProjVarsMap}
_, err = projectVars.Upsert()
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
allProjects, err := model.FindAllProjectRefs()
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
data := struct {
AllProjects []model.ProjectRef
}{allProjects}
uis.WriteJSON(w, http.StatusOK, data)
}
// ManifestLoadHandler attempts to get the manifest, if it exists it updates the expansions and returns
// If it does not exist it performs GitHub API calls for each of the project's modules and gets
// the head revision of the branch and inserts it into the manifest collection.
// If there is a duplicate key error, then do a find on the manifest again.
func (mp *ManifestPlugin) ManifestLoadHandler(w http.ResponseWriter, r *http.Request) {
task := plugin.GetTask(r)
projectRef, err := model.FindOneProjectRef(task.Project)
if err != nil {
http.Error(w, fmt.Sprintf("projectRef not found for project %v: %v", task.Project, err), http.StatusNotFound)
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
http.Error(w, fmt.Sprintf("project not found for ProjectRef %v: %v", projectRef.Identifier, err),
http.StatusNotFound)
return
}
if project == nil {
http.Error(w, fmt.Sprintf("empty project not found for ProjectRef %v: %v", projectRef.Identifier, err),
http.StatusNotFound)
return
}
// try to get the manifest
currentManifest, err := manifest.FindOne(manifest.ById(task.Version))
if err != nil {
http.Error(w, fmt.Sprintf("error retrieving manifest with version id %v: %v", task.Version, err),
http.StatusNotFound)
return
}
if currentManifest != nil {
plugin.WriteJSON(w, http.StatusOK, currentManifest)
return
}
if task.Version == "" {
http.Error(w, fmt.Sprintf("versionId is empty"), http.StatusNotFound)
return
}
// attempt to insert a manifest after making GitHub API calls
newManifest := &manifest.Manifest{
Id: task.Version,
Revision: task.Revision,
ProjectName: task.Project,
Branch: project.Branch,
}
// populate modules
modules := make(map[string]*manifest.Module)
for _, module := range project.Modules {
owner, repo := module.GetRepoOwnerAndName()
gitBranch, err := thirdparty.GetBranchEvent(mp.OAuthCredentials, owner, repo, module.Branch)
if err != nil {
http.Error(w, fmt.Sprintf("error retrieving getting git branch for module %v: %v", module.Name, err),
http.StatusNotFound)
return
}
modules[module.Name] = &manifest.Module{
Branch: module.Branch,
Revision: gitBranch.Commit.SHA,
Repo: repo,
Owner: owner,
URL: gitBranch.Commit.Url,
}
}
newManifest.Modules = modules
duplicate, err := newManifest.TryInsert()
if err != nil {
http.Error(w, fmt.Sprintf("error inserting manifest for %v: %v", newManifest.ProjectName, err),
http.StatusNotFound)
return
}
// if it is a duplicate, load the manifest again`
if duplicate {
// try to get the manifest
m, err := manifest.FindOne(manifest.ById(task.Version))
if err != nil {
http.Error(w, fmt.Sprintf("error getting latest manifest for %v: %v", newManifest.ProjectName, err),
http.StatusNotFound)
return
}
if m != nil {
plugin.WriteJSON(w, http.StatusOK, m)
return
}
}
// no duplicate key error, use the manifest just created.
plugin.WriteJSON(w, http.StatusOK, newManifest)
return
}
// LoadProjectContext builds a projectContext from vars in the request's URL.
// This is done by reading in specific variables and inferring other required
// context variables when necessary (e.g. loading a project based on the task).
func (uis *UIServer) LoadProjectContext(rw http.ResponseWriter, r *http.Request) (projectContext, error) {
user := GetUser(r)
vars := mux.Vars(r)
proj := &projectContext{}
taskId := vars["task_id"]
buildId := vars["build_id"]
versionId := vars["version_id"]
patchId := vars["patch_id"]
err := proj.populateProjectRefs(user != nil)
if err != nil {
return *proj, err
}
projectId, err := proj.populateTaskBuildVersion(taskId, buildId, versionId)
if err != nil {
return *proj, err
}
err = proj.populatePatch(patchId)
if err != nil {
return *proj, err
}
if proj.Patch != nil && len(projectId) == 0 {
projectId = proj.Patch.Project
}
// Try to infer project ID - if we don't already have it from the task/build/version, try to
// get it from the URL
if len(projectId) == 0 {
projectId = vars["project_id"]
}
usingDefault := false
// Still don't have a project ID to use, check if the user's cookie contains one
if len(projectId) == 0 {
cookie, err := r.Cookie(ProjectCookieName)
if err == nil {
projectId = cookie.Value
}
usingDefault = true
}
// Still no project ID found anywhere, just use the default one according to config.
if len(projectId) == 0 {
projectId = uis.Settings.Ui.DefaultProject
usingDefault = true
}
// Try to load project for the ID we found, and set cookie with it for subsequent requests
if len(projectId) > 0 {
// Also lookup the ProjectRef itself and add it to context.
proj.ProjectRef, err = model.FindOneProjectRef(projectId)
if err != nil {
return *proj, err
}
// If we used the default project or a cookie to choose the project,
// but that project doesn't exist, choose the first one in the list.
if usingDefault && proj.ProjectRef == nil {
if len(proj.AllProjects) > 0 {
proj.ProjectRef = &proj.AllProjects[0]
}
}
if proj.ProjectRef != nil {
proj.Project, err = model.FindProject("", proj.ProjectRef)
if err != nil {
return *proj, err
}
if proj.Project != nil {
// A project was found, update the project cookie for subsequent request.
http.SetCookie(rw, &http.Cookie{
Name: ProjectCookieName,
Value: projectId,
Path: "",
Expires: time.Now().Add(7 * 24 * time.Hour),
})
}
}
}
proj.AuthRedirect = uis.UserManager.IsRedirect()
return *proj, nil
}
func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) {
p, err := getPatchFromRequest(r)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, err.Error())
return
}
moduleName := r.FormValue("module")
patchContent := r.FormValue("patch")
githash := r.FormValue("githash")
projectRef, err := model.FindOneProjectRef(p.Project)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting project ref with id %v: %v", p.Project, err))
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting patch: %v", err))
return
}
if project == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("can't find project: %v", p.Project))
return
}
module, err := project.GetModuleByName(moduleName)
if err != nil || module == nil {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("No such module"))
return
}
gitOutput, err := thirdparty.GitApplyNumstat(patchContent)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Invalid patch: %v", err))
return
}
if gitOutput == nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Empty diff"))
return
}
summaries, err := thirdparty.ParseGitSummary(gitOutput)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Can't validate patch: %v", err))
return
}
repoOwner, repo := module.GetRepoOwnerAndName()
commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials[projectRef.RepoKind], repoOwner, repo, githash)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if commitInfo == nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("commit hash doesn't seem to exist"))
return
}
modulePatch := patch.ModulePatch{
ModuleName: moduleName,
Githash: githash,
PatchSet: patch.PatchSet{
Patch: patchContent,
Summary: summaries, // thirdparty.GetPatchSummary(apiRequest.PatchContent),
},
}
if err = p.UpdateModulePatch(modulePatch); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
as.WriteJSON(w, http.StatusOK, "Patch module updated")
return
}
func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) {
user := MustHaveUser(r)
apiRequest := PatchAPIRequest{
ProjectFileName: r.FormValue("project"),
ModuleName: r.FormValue("module"),
Githash: r.FormValue("githash"),
PatchContent: r.FormValue("patch"),
BuildVariants: strings.Split(r.FormValue("buildvariants"), ","),
}
description := r.FormValue("desc")
projId := r.FormValue("project")
projectRef, err := model.FindOneProjectRef(projId)
if err != nil {
message := fmt.Errorf("Error locating project ref '%v': %v",
projId, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
message := fmt.Errorf("Error locating project '%v' from '%v': %v",
projId, as.Settings.ConfigDir, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
if project == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("project %v not found", projId))
return
}
patchMetadata, err := apiRequest.Validate(as.Settings.Credentials[projectRef.RepoKind])
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid patch: %v", err))
return
}
if patchMetadata == nil {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("patch metadata is empty"))
return
}
if apiRequest.ModuleName != "" {
as.WriteJSON(w, http.StatusBadRequest,
"module not allowed when creating new patches (must be added in a subsequent request)")
return
}
patchProjectRef, err := model.FindOneProjectRef(patchMetadata.Project.Identifier)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid projectRef: %v", err))
return
}
if patchProjectRef == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("Empty patch project Ref"))
return
}
commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials[projectRef.RepoKind],
patchProjectRef.Owner,
patchProjectRef.Repo,
apiRequest.Githash)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if commitInfo == nil {
as.WriteJSON(w, http.StatusBadRequest, "That commit doesn't seem to exist.")
return
}
createTime := time.Now()
patchDoc := &patch.Patch{
Id: bson.NewObjectId(),
Description: description,
Author: user.Id,
Project: apiRequest.ProjectFileName,
Githash: apiRequest.Githash,
CreateTime: createTime,
Status: evergreen.PatchCreated,
BuildVariants: apiRequest.BuildVariants,
Tasks: nil,
Patches: []patch.ModulePatch{
patch.ModulePatch{
ModuleName: "",
Githash: apiRequest.Githash,
PatchSet: patch.PatchSet{
Patch: apiRequest.PatchContent,
Summary: patchMetadata.Summaries,
},
},
},
}
// set the patch number based on patch author
patchDoc.PatchNumber, err = user.IncPatchNumber()
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error computing patch num %v", err))
return
}
if err = patchDoc.Insert(); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error inserting patch: %v", err))
return
}
if strings.ToLower(r.FormValue("finalize")) == "true" {
if _, err = validator.ValidateAndFinalize(patchDoc, &as.Settings); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
}
as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc})
}
func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) {
user := MustHaveUser(r)
var apiRequest PatchAPIRequest
var projId, description string
var finalize bool
if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
patchContent := r.FormValue("patch")
if patchContent == "" {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
return
}
apiRequest = PatchAPIRequest{
ProjectFileName: r.FormValue("project"),
ModuleName: r.FormValue("module"),
Githash: r.FormValue("githash"),
PatchContent: r.FormValue("patch"),
BuildVariants: strings.Split(r.FormValue("buildvariants"), ","),
}
projId = r.FormValue("project")
description = r.FormValue("desc")
finalize = strings.ToLower(r.FormValue("finalize")) == "true"
} else {
data := struct {
Description string `json:"desc"`
Project string `json:"project"`
Patch string `json:"patch"`
Githash string `json:"githash"`
Variants string `json:"buildvariants"`
Tasks []string `json:"tasks"`
Finalize bool `json:"finalize"`
}{}
if err := util.ReadJSONInto(r.Body, &data); err != nil {
as.LoggedError(w, r, http.StatusBadRequest, err)
return
}
if len(data.Patch) > patch.SizeLimit {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Patch is too large."))
}
if len(data.Patch) == 0 {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
return
}
description = data.Description
projId = data.Project
finalize = data.Finalize
apiRequest = PatchAPIRequest{
ProjectFileName: data.Project,
ModuleName: r.FormValue("module"),
Githash: data.Githash,
PatchContent: data.Patch,
BuildVariants: strings.Split(data.Variants, ","),
Tasks: data.Tasks,
}
}
projectRef, err := model.FindOneProjectRef(projId)
if err != nil {
message := fmt.Errorf("Error locating project ref '%v': %v",
projId, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
message := fmt.Errorf("Error locating project '%v' from '%v': %v",
projId, as.Settings.ConfigDir, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
if project == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("project %v not found", projId))
return
}
patchMetadata, err := apiRequest.Validate(as.Settings.Credentials["github"])
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid patch: %v", err))
return
}
if patchMetadata == nil {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("patch metadata is empty"))
return
}
if apiRequest.ModuleName != "" {
as.WriteJSON(w, http.StatusBadRequest,
"module not allowed when creating new patches (must be added in a subsequent request)")
return
}
patchProjectRef, err := model.FindOneProjectRef(patchMetadata.Project.Identifier)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid projectRef: %v", err))
return
}
if patchProjectRef == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("Empty patch project Ref"))
return
}
commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"],
patchProjectRef.Owner,
patchProjectRef.Repo,
apiRequest.Githash)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if commitInfo == nil {
as.WriteJSON(w, http.StatusBadRequest, "That commit doesn't seem to exist.")
return
}
createTime := time.Now()
// create a new object ID to use as reference for the patch data
patchFileId := bson.NewObjectId().Hex()
patchDoc := &patch.Patch{
Id: bson.NewObjectId(),
Description: description,
Author: user.Id,
Project: apiRequest.ProjectFileName,
Githash: apiRequest.Githash,
CreateTime: createTime,
Status: evergreen.PatchCreated,
BuildVariants: apiRequest.BuildVariants,
Tasks: apiRequest.Tasks,
Patches: []patch.ModulePatch{
patch.ModulePatch{
ModuleName: "",
Githash: apiRequest.Githash,
PatchSet: patch.PatchSet{
PatchFileId: patchFileId,
Summary: patchMetadata.Summaries,
},
},
},
}
// write the patch content into a GridFS file under a new ObjectId.
err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(apiRequest.PatchContent))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err))
return
}
// Get and validate patched config and add it to the patch document
patchedProject, err := validator.GetPatchedProject(patchDoc, &as.Settings)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("invalid patched config: %v", err))
return
}
projectYamlBytes, err := yaml.Marshal(patchedProject)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error marshalling patched config: %v", err))
return
}
// set the patch number based on patch author
patchDoc.PatchNumber, err = user.IncPatchNumber()
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error computing patch num %v", err))
return
}
patchDoc.PatchedConfig = string(projectYamlBytes)
patchDoc.ClearPatchData()
//expand tasks and build variants and include dependencies
buildVariants := patchDoc.BuildVariants
if len(patchDoc.BuildVariants) == 1 && patchDoc.BuildVariants[0] == "all" {
buildVariants = make([]string, 0)
for _, buildVariant := range patchedProject.BuildVariants {
if buildVariant.Disabled {
continue
}
buildVariants = append(buildVariants, buildVariant.Name)
}
}
tasks := patchDoc.Tasks
if len(patchDoc.Tasks) == 1 && patchDoc.Tasks[0] == "all" {
tasks = make([]string, 0)
for _, t := range patchedProject.Tasks {
tasks = append(tasks, t.Name)
}
}
// update variant and tasks to include dependencies
patchDoc.BuildVariants, patchDoc.Tasks = model.IncludePatchDependencies(
project, buildVariants, tasks)
if err = patchDoc.Insert(); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error inserting patch: %v", err))
return
}
if finalize {
if _, err = model.FinalizePatch(patchDoc, &as.Settings); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
}
as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc})
}
// CreatePatch checks an API request to see if it is safe and sane.
// Returns the relevant patch metadata, the patch document, and any errors that occur.
func (pr *PatchAPIRequest) CreatePatch(finalize bool, oauthToken string,
dbUser *user.DBUser, settings *evergreen.Settings) (*model.Project, *patch.Patch, error) {
var repoOwner, repo string
var module *model.Module
projectRef, err := model.FindOneProjectRef(pr.ProjectId)
if err != nil {
return nil, nil, fmt.Errorf("Could not find project ref %v : %v", pr.ProjectId, err)
}
repoOwner = projectRef.Owner
repo = projectRef.Repo
if len(pr.Githash) != 40 {
return nil, nil, fmt.Errorf("invalid githash")
}
gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash)
if err != nil {
return nil, nil, fmt.Errorf("could not find base revision %v for project %v: %v",
pr.Githash, projectRef.Identifier, err)
}
if gitCommit == nil {
return nil, nil, fmt.Errorf("commit hash %v doesn't seem to exist", pr.Githash)
}
summaries, err := getSummaries(pr.PatchContent)
if err != nil {
return nil, nil, err
}
isEmpty := (pr.PatchContent == "")
if finalize && (len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "") {
return nil, nil, fmt.Errorf("no buildvariants specified")
}
createTime := time.Now()
// create a new object ID to use as reference for the patch data
patchFileId := bson.NewObjectId().Hex()
patchDoc := &patch.Patch{
Id: bson.NewObjectId(),
Description: pr.Description,
Author: dbUser.Id,
Project: pr.ProjectId,
Githash: pr.Githash,
CreateTime: createTime,
Status: evergreen.PatchCreated,
BuildVariants: pr.BuildVariants,
Tasks: pr.Tasks,
IsEmpty: isEmpty,
Patches: []patch.ModulePatch{
{
ModuleName: "",
Githash: pr.Githash,
PatchSet: patch.PatchSet{
Patch: pr.PatchContent,
PatchFileId: patchFileId,
Summary: summaries,
},
},
},
}
// Get and validate patched config and add it to the patch document
project, err := validator.GetPatchedProject(patchDoc, settings)
if err != nil {
return nil, nil, fmt.Errorf("invalid patched config: %v", err)
}
if pr.ModuleName != "" {
// is there a module? validate it.
module, err = project.GetModuleByName(pr.ModuleName)
if err != nil {
return nil, nil, fmt.Errorf("could not find module %v: %v", pr.ModuleName, err)
}
if module == nil {
return nil, nil, fmt.Errorf("no module named %v", pr.ModuleName)
}
}
// verify that all variants exists
for _, buildVariant := range pr.BuildVariants {
if buildVariant == "all" || buildVariant == "" {
continue
}
bv := project.FindBuildVariant(buildVariant)
if bv == nil {
return nil, nil, fmt.Errorf("No such buildvariant: %v", buildVariant)
}
}
// write the patch content into a GridFS file under a new ObjectId after validating.
err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(pr.PatchContent))
if err != nil {
return nil, nil, fmt.Errorf("failed to write patch file to db: %v", err)
}
// add the project config
projectYamlBytes, err := yaml.Marshal(project)
if err != nil {
return nil, nil, fmt.Errorf("error marshalling patched config: %v", err)
}
// set the patch number based on patch author
patchDoc.PatchNumber, err = dbUser.IncPatchNumber()
if err != nil {
return nil, nil, fmt.Errorf("error computing patch num %v", err)
}
patchDoc.PatchedConfig = string(projectYamlBytes)
patchDoc.ClearPatchData()
return project, patchDoc, nil
}
func (as *APIServer) EndTask(w http.ResponseWriter, r *http.Request) {
finishTime := time.Now()
taskEndResponse := &apimodels.TaskEndResponse{}
task := MustHaveTask(r)
details := &apimodels.TaskEndDetail{}
if err := util.ReadJSONInto(r.Body, details); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Check that finishing status is a valid constant
if details.Status != evergreen.TaskSucceeded &&
details.Status != evergreen.TaskFailed &&
details.Status != evergreen.TaskUndispatched {
msg := fmt.Errorf("Invalid end status '%v' for task %v", details.Status, task.Id)
as.LoggedError(w, r, http.StatusBadRequest, msg)
return
}
projectRef, err := model.FindOneProjectRef(task.Project)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
}
if projectRef == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("empty projectRef for task"))
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if !getGlobalLock(r.RemoteAddr, task.Id) {
as.LoggedError(w, r, http.StatusInternalServerError, ErrLockTimeout)
return
}
defer releaseGlobalLock(r.RemoteAddr, task.Id)
// mark task as finished
err = task.MarkEnd(APIServerLockTitle, finishTime, details, project, projectRef.DeactivatePrevious)
if err != nil {
message := fmt.Errorf("Error calling mark finish on task %v : %v", task.Id, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
if task.Requester != evergreen.PatchVersionRequester {
alerts.RunTaskFailureTriggers(task)
} else {
//TODO(EVG-223) process patch-specific triggers
}
// if task was aborted, reset to inactive
if details.Status == evergreen.TaskUndispatched {
if err = model.SetTaskActivated(task.Id, "", false); err != nil {
message := fmt.Sprintf("Error deactivating task after abort: %v", err)
evergreen.Logger.Logf(slogger.ERROR, message)
taskEndResponse.Message = message
as.WriteJSON(w, http.StatusInternalServerError, taskEndResponse)
return
}
as.taskFinished(w, task, finishTime)
return
}
// update the bookkeeping entry for the task
err = bookkeeping.UpdateExpectedDuration(task, task.TimeTaken)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "Error updating expected duration: %v",
err)
}
// log the task as finished
evergreen.Logger.Logf(slogger.INFO, "Successfully marked task %v as finished", task.Id)
// construct and return the appropriate response for the agent
as.taskFinished(w, task, finishTime)
}
// Validate checks an API request to see if it is safe and sane.
// Returns the relevant patch metadata and any errors that occur.
func (pr *PatchAPIRequest) Validate(oauthToken string) (*PatchMetadata, error) {
var repoOwner, repo string
var module *model.Module
projectRef, err := model.FindOneProjectRef(pr.ProjectFileName)
if err != nil {
return nil, fmt.Errorf("Could not find project ref %v : %v", pr.ProjectFileName, err)
}
repoOwner = projectRef.Owner
repo = projectRef.Repo
// validate the project file
project, err := model.FindProject("", projectRef)
if err != nil {
return nil, fmt.Errorf("Could not find project file %v: %v",
pr.ProjectFileName, err)
}
if project == nil {
return nil, fmt.Errorf("No such project file named %v", pr.ProjectFileName)
}
if pr.ModuleName != "" {
// is there a module? validate it.
module, err = project.GetModuleByName(pr.ModuleName)
if err != nil {
return nil, fmt.Errorf("could not find module %v: %v", pr.ModuleName, err)
}
if module == nil {
return nil, fmt.Errorf("no module named %v", pr.ModuleName)
}
repoOwner, repo = module.GetRepoOwnerAndName()
}
if len(pr.Githash) != 40 {
return nil, fmt.Errorf("invalid githash")
}
gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash)
if err != nil {
return nil, fmt.Errorf("could not find base revision %v for project %v: %v",
pr.Githash, projectRef.Identifier, err)
}
if gitCommit == nil {
return nil, fmt.Errorf("commit hash %v doesn't seem to exist", pr.Githash)
}
gitOutput, err := thirdparty.GitApplyNumstat(pr.PatchContent)
if err != nil {
return nil, fmt.Errorf("couldn't validate patch: %v", err)
}
if gitOutput == nil {
return nil, fmt.Errorf("couldn't validate patch: git apply --numstat returned empty")
}
summaries, err := thirdparty.ParseGitSummary(gitOutput)
if err != nil {
return nil, fmt.Errorf("couldn't validate patch: %v", err)
}
if len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "" {
return nil, fmt.Errorf("no buildvariants specified")
}
// verify that this build variant exists
for _, buildVariant := range pr.BuildVariants {
if buildVariant == "all" {
continue
}
bv := project.FindBuildVariant(buildVariant)
if bv == nil {
return nil, fmt.Errorf("No such buildvariant: %v", buildVariant)
}
}
return &PatchMetadata{pr.Githash, project, module, pr.BuildVariants, summaries}, nil
}
func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) {
p, err := getPatchFromRequest(r)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, err.Error())
return
}
var moduleName, patchContent, githash string
if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
moduleName, patchContent, githash = r.FormValue("module"), r.FormValue("patch"), r.FormValue("githash")
} else {
data := struct {
Module string `json:"module"`
Patch string `json:"patch"`
Githash string `json:"githash"`
}{}
if err := util.ReadJSONInto(r.Body, &data); err != nil {
as.LoggedError(w, r, http.StatusBadRequest, err)
return
}
moduleName, patchContent, githash = data.Module, data.Patch, data.Githash
}
if len(patchContent) == 0 {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
return
}
projectRef, err := model.FindOneProjectRef(p.Project)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting project ref with id %v: %v", p.Project, err))
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting patch: %v", err))
return
}
if project == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("can't find project: %v", p.Project))
return
}
module, err := project.GetModuleByName(moduleName)
if err != nil || module == nil {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("No such module", moduleName))
return
}
gitOutput, err := thirdparty.GitApplyNumstat(patchContent)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Invalid patch: %v", err))
return
}
if gitOutput == nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Empty diff"))
return
}
summaries, err := thirdparty.ParseGitSummary(gitOutput)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Can't validate patch: %v", err))
return
}
repoOwner, repo := module.GetRepoOwnerAndName()
commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"], repoOwner, repo, githash)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if commitInfo == nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("commit hash doesn't seem to exist"))
return
}
// write the patch content into a GridFS file under a new ObjectId.
patchFileId := bson.NewObjectId().Hex()
err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(patchContent))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err))
return
}
modulePatch := patch.ModulePatch{
ModuleName: moduleName,
Githash: githash,
PatchSet: patch.PatchSet{
PatchFileId: patchFileId,
Summary: summaries,
},
}
if err = p.UpdateModulePatch(modulePatch); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
as.WriteJSON(w, http.StatusOK, "Patch module updated")
return
}