// insertTask creates a TaskJSON document with the data sent in the request body. func insertTask(w http.ResponseWriter, r *http.Request) { t := plugin.GetTask(r) if t == nil { http.Error(w, "task not found", http.StatusNotFound) return } name := mux.Vars(r)["name"] rawData := map[string]interface{}{} err := util.ReadJSONInto(r.Body, &rawData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } jsonBlob := TaskJSON{ TaskId: t.Id, TaskName: t.DisplayName, Name: name, BuildId: t.BuildId, Variant: t.BuildVariant, ProjectId: t.Project, VersionId: t.Version, CreateTime: t.CreateTime, Revision: t.Revision, RevisionOrderNumber: t.RevisionOrderNumber, Data: rawData, IsPatch: t.Requester == evergreen.PatchVersionRequester, } _, err = db.Upsert(collection, bson.M{TaskIdKey: t.Id, NameKey: name}, jsonBlob) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, "ok") return }
// getTask finds a json document by using thex task that is in the plugin. func getTaskByName(w http.ResponseWriter, r *http.Request) { t := plugin.GetTask(r) if t == nil { http.Error(w, "task not found", http.StatusNotFound) return } name := mux.Vars(r)["name"] taskName := mux.Vars(r)["task_name"] var jsonForTask TaskJSON err := db.FindOneQ(collection, db.Query(bson.M{VersionIdKey: t.Version, BuildIdKey: t.BuildId, NameKey: name, TaskNameKey: taskName}), &jsonForTask) if err != nil { if err == mgo.ErrNotFound { plugin.WriteJSON(w, http.StatusNotFound, nil) return } http.Error(w, err.Error(), http.StatusInternalServerError) return } if len(r.FormValue("full")) != 0 { // if specified, include the json data's container as well plugin.WriteJSON(w, http.StatusOK, jsonForTask) return } plugin.WriteJSON(w, http.StatusOK, jsonForTask.Data) }
//XXX remove this once the transition is complete... //AttachResultsHandler is an API hook for receiving and updating test results func AttachResultsHandler(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) if task == nil { message := "Cannot find task for attach results request" evergreen.Logger.Errorf(slogger.ERROR, message) http.Error(w, message, http.StatusBadRequest) return } results := &model.TestResults{} err := util.ReadJSONInto(r.Body, results) if err != nil { message := fmt.Sprintf("error reading test results: %v", err) evergreen.Logger.Errorf(slogger.ERROR, message) http.Error(w, message, http.StatusBadRequest) return } // set test result of task if err := task.SetResults(results.Results); err != nil { message := fmt.Sprintf("Error calling set results on task %v: %v", task.Id, err) evergreen.Logger.Errorf(slogger.ERROR, message) http.Error(w, message, http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, "Test results successfully attached") }
// AttachFilesHandler updates file mappings for a task or build func AttachFilesHandler(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) evergreen.Logger.Logf(slogger.INFO, "Attaching files to task %v", task.Id) fileEntry := &artifact.Entry{} fileEntry.TaskId = task.Id fileEntry.TaskDisplayName = task.DisplayName fileEntry.BuildId = task.BuildId err := util.ReadJSONInto(r.Body, &fileEntry.Files) if err != nil { message := fmt.Sprintf("error reading file definitions: %v", err) evergreen.Logger.Errorf(slogger.ERROR, message) http.Error(w, message, http.StatusBadRequest) return } if err := fileEntry.Upsert(); err != nil { message := fmt.Sprintf("Error updating artifact file info: %v", err) evergreen.Logger.Errorf(slogger.ERROR, message) http.Error(w, message, http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, fmt.Sprintf("Artifact files for task %v successfully attached", task.Id)) }
// getTaskHistory finds previous tasks by task name. func apiGetTaskHistory(w http.ResponseWriter, r *http.Request) { t := plugin.GetTask(r) if t == nil { http.Error(w, "task not found", http.StatusNotFound) return } getTaskHistory(t, w, r) }
// servePatch is the API hook for returning patch data as json func servePatch(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) patch, err := task.FetchPatch() if err != nil { msg := fmt.Sprintf("error fetching patch for task %v from db: %v", task.Id, err) evergreen.Logger.Logf(slogger.ERROR, msg) http.Error(w, msg, http.StatusInternalServerError) return } if patch == nil { msg := fmt.Sprintf("no patch found for task %v", task.Id) evergreen.Logger.Errorf(slogger.ERROR, msg) http.Error(w, msg, http.StatusNotFound) return } plugin.WriteJSON(w, http.StatusOK, patch) }
func MockPluginEcho(w http.ResponseWriter, request *http.Request) { arg1 := mux.Vars(request)["param1"] arg2, err := strconv.Atoi(mux.Vars(request)["param2"]) if err != nil { http.Error(w, "bad val for param2", http.StatusBadRequest) return } newTask := plugin.GetTask(request) if newTask != nil { //task should have been populated for us, by the API server plugin.WriteJSON(w, http.StatusOK, map[string]string{ "echo": fmt.Sprintf("%v/%v/%v", arg1, arg2, newTask.Id), }) return } http.Error(w, "couldn't get task from context", http.StatusInternalServerError) }
// getTaskForVariant finds a task by name and variant and finds // the document in the json collection associated with that task's id. func getTaskForVariant(w http.ResponseWriter, r *http.Request) { t := plugin.GetTask(r) if t == nil { http.Error(w, "task not found", http.StatusNotFound) return } name := mux.Vars(r)["name"] taskName := mux.Vars(r)["task_name"] variantId := mux.Vars(r)["variant"] // Find the task for the other variant, if it exists ts, err := task.Find(db.Query(bson.M{task.VersionKey: t.Version, task.BuildVariantKey: variantId, task.DisplayNameKey: taskName}).Limit(1)) if err != nil { if err == mgo.ErrNotFound { plugin.WriteJSON(w, http.StatusNotFound, nil) return } http.Error(w, err.Error(), http.StatusInternalServerError) return } if len(ts) == 0 { plugin.WriteJSON(w, http.StatusNotFound, nil) return } otherVariantTask := ts[0] var jsonForTask TaskJSON err = db.FindOneQ(collection, db.Query(bson.M{TaskIdKey: otherVariantTask.Id, NameKey: name}), &jsonForTask) if err != nil { if err == mgo.ErrNotFound { plugin.WriteJSON(w, http.StatusNotFound, nil) return } http.Error(w, err.Error(), http.StatusInternalServerError) return } if len(r.FormValue("full")) != 0 { // if specified, include the json data's container as well plugin.WriteJSON(w, http.StatusOK, jsonForTask) return } plugin.WriteJSON(w, http.StatusOK, jsonForTask.Data) }
// getTaskByTag returns a TaskJSON with a specific task name and tag func getTaskByTag(w http.ResponseWriter, r *http.Request) { t := plugin.GetTask(r) if t == nil { http.Error(w, "task not found", http.StatusNotFound) return } tagged := []TaskJSON{} jsonQuery := db.Query(bson.M{ ProjectIdKey: t.Project, VariantKey: t.BuildVariant, TaskNameKey: mux.Vars(r)["task_name"], TagKey: bson.M{"$exists": true, "$ne": ""}, NameKey: mux.Vars(r)["name"]}) err := db.FindAllQ(collection, jsonQuery, &tagged) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, tagged) }
// FetchVarsHandler is an API hook for returning the project variables // associated with a task's project. func FetchVarsHandler(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) if task == nil { http.Error(w, "task not found", http.StatusNotFound) return } projectVars, err := model.FindOneProjectVars(task.Project) if err != nil { message := fmt.Sprintf("Failed to fetch vars for task %v: %v", task.Id, err) evergreen.Logger.Logf(slogger.ERROR, message) http.Error(w, message, http.StatusInternalServerError) return } if projectVars == nil { plugin.WriteJSON(w, http.StatusOK, ExpansionVars{}) return } plugin.WriteJSON(w, http.StatusOK, projectVars.Vars) return }
// Takes a request for a task's file to be copied from // one s3 location to another. Ensures that if the destination // file path already exists, no file copy is performed. func S3CopyHandler(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) if task == nil { http.Error(w, "task not found", http.StatusNotFound) return } s3CopyReq := &S3CopyRequest{} err := util.ReadJSONInto(r.Body, s3CopyReq) if err != nil { evergreen.Logger.Errorf(slogger.ERROR, "error reading push request: %v", err) http.Error(w, err.Error(), http.StatusBadRequest) return } // Get the version for this task, so we can check if it has // any already-done pushes v, err := version.FindOne(version.ById(task.Version)) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error querying task %v with version id %v: %v", task.Id, task.Version, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } // Check for an already-pushed file with this same file path, // but from a conflicting or newer commit sequence num if v == nil { evergreen.Logger.Logf(slogger.ERROR, "no version found for build %v", task.BuildId) http.Error(w, "version not found", http.StatusNotFound) return } copyFromLocation := strings.Join([]string{s3CopyReq.S3SourceBucket, s3CopyReq.S3SourcePath}, "/") copyToLocation := strings.Join([]string{s3CopyReq.S3DestinationBucket, s3CopyReq.S3DestinationPath}, "/") newestPushLog, err := model.FindPushLogAfter(copyToLocation, v.RevisionOrderNumber) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error querying for push log at %v: %v", copyToLocation, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } if newestPushLog != nil { evergreen.Logger.Logf(slogger.ERROR, "conflict with existing pushed file: "+ "%v", copyToLocation) http.Error(w, "conflicting push target for this file already exists.", http.StatusConflict) return } // It's now safe to put the file in its permanent location. newPushLog := model.NewPushLog(v, task, copyToLocation) err = newPushLog.Insert() if err != nil { evergreen.Logger.Logf(slogger.ERROR, "failed to create new push log: %v %v", newPushLog, err) http.Error(w, fmt.Sprintf("failed to create push log: %v", err), http.StatusInternalServerError) return } // Now copy the file into the permanent location auth := &aws.Auth{ AccessKey: s3CopyReq.AwsKey, SecretKey: s3CopyReq.AwsSecret, } evergreen.Logger.Logf(slogger.INFO, "performing S3 copy: '%v' => '%v'", copyFromLocation, copyToLocation) _, err = util.RetryArithmeticBackoff(func() error { err := thirdparty.S3CopyFile(auth, s3CopyReq.S3SourceBucket, s3CopyReq.S3SourcePath, s3CopyReq.S3DestinationBucket, s3CopyReq.S3DestinationPath, string(s3.PublicRead), ) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "S3 copy failed for task %v, "+ "retrying: %v", task.Id, err) return util.RetriableError{err} } else { err := newPushLog.UpdateStatus(model.PushLogSuccess) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed"+ " for task %v: %v", task.Id, err) } return err } }, s3CopyRetryNumRetries, s3CopyRetrySleepTimeSec*time.Second) if err != nil { message := fmt.Sprintf("S3 copy failed for task %v: %v", task.Id, err) evergreen.Logger.Logf(slogger.ERROR, message) err = newPushLog.UpdateStatus(model.PushLogFailed) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed: %v", err) } http.Error(w, message, http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, "S3 copy Successful") }
// ManifestLoadHandler attempts to get the manifest, if it exists it updates the expansions and returns // If it does not exist it performs GitHub API calls for each of the project's modules and gets // the head revision of the branch and inserts it into the manifest collection. // If there is a duplicate key error, then do a find on the manifest again. func (mp *ManifestPlugin) ManifestLoadHandler(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) projectRef, err := model.FindOneProjectRef(task.Project) if err != nil { http.Error(w, fmt.Sprintf("projectRef not found for project %v: %v", task.Project, err), http.StatusNotFound) return } project, err := model.FindProject("", projectRef) if err != nil { http.Error(w, fmt.Sprintf("project not found for ProjectRef %v: %v", projectRef.Identifier, err), http.StatusNotFound) return } if project == nil { http.Error(w, fmt.Sprintf("empty project not found for ProjectRef %v: %v", projectRef.Identifier, err), http.StatusNotFound) return } // try to get the manifest currentManifest, err := manifest.FindOne(manifest.ById(task.Version)) if err != nil { http.Error(w, fmt.Sprintf("error retrieving manifest with version id %v: %v", task.Version, err), http.StatusNotFound) return } if currentManifest != nil { plugin.WriteJSON(w, http.StatusOK, currentManifest) return } if task.Version == "" { http.Error(w, fmt.Sprintf("versionId is empty"), http.StatusNotFound) return } // attempt to insert a manifest after making GitHub API calls newManifest := &manifest.Manifest{ Id: task.Version, Revision: task.Revision, ProjectName: task.Project, Branch: project.Branch, } // populate modules modules := make(map[string]*manifest.Module) for _, module := range project.Modules { owner, repo := module.GetRepoOwnerAndName() gitBranch, err := thirdparty.GetBranchEvent(mp.OAuthCredentials, owner, repo, module.Branch) if err != nil { http.Error(w, fmt.Sprintf("error retrieving getting git branch for module %v: %v", module.Name, err), http.StatusNotFound) return } modules[module.Name] = &manifest.Module{ Branch: module.Branch, Revision: gitBranch.Commit.SHA, Repo: repo, Owner: owner, URL: gitBranch.Commit.Url, } } newManifest.Modules = modules duplicate, err := newManifest.TryInsert() if err != nil { http.Error(w, fmt.Sprintf("error inserting manifest for %v: %v", newManifest.ProjectName, err), http.StatusNotFound) return } // if it is a duplicate, load the manifest again` if duplicate { // try to get the manifest m, err := manifest.FindOne(manifest.ById(task.Version)) if err != nil { http.Error(w, fmt.Sprintf("error getting latest manifest for %v: %v", newManifest.ProjectName, err), http.StatusNotFound) return } if m != nil { plugin.WriteJSON(w, http.StatusOK, m) return } } // no duplicate key error, use the manifest just created. plugin.WriteJSON(w, http.StatusOK, newManifest) return }
// GetRoutes returns an API route for serving patch data. func (jsp *JSONPlugin) GetAPIHandler() http.Handler { r := mux.NewRouter() r.HandleFunc("/tags/{task_name}/{name}", func(w http.ResponseWriter, r *http.Request) { t := plugin.GetTask(r) if t == nil { http.Error(w, "task not found", http.StatusNotFound) return } tagged := []TaskJSON{} jsonQuery := db.Query(bson.M{ "project_id": t.Project, "variant": t.BuildVariant, "task_name": mux.Vars(r)["task_name"], "tag": bson.M{"$exists": true, "$ne": ""}, "name": mux.Vars(r)["name"]}) err := db.FindAllQ(collection, jsonQuery, &tagged) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, tagged) }) r.HandleFunc("/history/{task_name}/{name}", func(w http.ResponseWriter, r *http.Request) { t := plugin.GetTask(r) if t == nil { http.Error(w, "task not found", http.StatusNotFound) return } before := []TaskJSON{} jsonQuery := db.Query(bson.M{ "project_id": t.Project, "variant": t.BuildVariant, "order": bson.M{"$lte": t.RevisionOrderNumber}, "task_name": mux.Vars(r)["task_name"], "is_patch": false, "name": mux.Vars(r)["name"]}).Sort([]string{"-order"}).Limit(100) err := db.FindAllQ(collection, jsonQuery, &before) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } //reverse order of "before" because we had to sort it backwards to apply the limit correctly: for i, j := 0, len(before)-1; i < j; i, j = i+1, j-1 { before[i], before[j] = before[j], before[i] } after := []TaskJSON{} jsonAfterQuery := db.Query(bson.M{ "project_id": t.Project, "variant": t.BuildVariant, "order": bson.M{"$gt": t.RevisionOrderNumber}, "task_name": mux.Vars(r)["task_name"], "is_patch": false, "name": mux.Vars(r)["name"]}).Sort([]string{"order"}).Limit(100) err = db.FindAllQ(collection, jsonAfterQuery, &after) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } //concatenate before + after before = append(before, after...) plugin.WriteJSON(w, http.StatusOK, before) }) r.HandleFunc("/data/{name}", func(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) if task == nil { http.Error(w, "task not found", http.StatusNotFound) return } name := mux.Vars(r)["name"] rawData := map[string]interface{}{} err := util.ReadJSONInto(r.Body, &rawData) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } jsonBlob := TaskJSON{ TaskId: task.Id, TaskName: task.DisplayName, Name: name, BuildId: task.BuildId, Variant: task.BuildVariant, ProjectId: task.Project, VersionId: task.Version, Revision: task.Revision, RevisionOrderNumber: task.RevisionOrderNumber, Data: rawData, IsPatch: task.Requester == evergreen.PatchVersionRequester, } _, err = db.Upsert(collection, bson.M{"task_id": task.Id, "name": name}, jsonBlob) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, "ok") return }) r.HandleFunc("/data/{task_name}/{name}", func(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) if task == nil { http.Error(w, "task not found", http.StatusNotFound) return } name := mux.Vars(r)["name"] taskName := mux.Vars(r)["task_name"] var jsonForTask TaskJSON err := db.FindOneQ(collection, db.Query(bson.M{"version_id": task.Version, "build_id": task.BuildId, "name": name, "task_name": taskName}), &jsonForTask) if err != nil { if err == mgo.ErrNotFound { plugin.WriteJSON(w, http.StatusNotFound, nil) return } http.Error(w, err.Error(), http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, jsonForTask.Data) }) r.HandleFunc("/data/{task_name}/{name}/{variant}", func(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) if task == nil { http.Error(w, "task not found", http.StatusNotFound) return } name := mux.Vars(r)["name"] taskName := mux.Vars(r)["task_name"] variantId := mux.Vars(r)["variant"] // Find the task for the other variant, if it exists ts, err := model.FindTasks(db.Query(bson.M{model.TaskVersionKey: task.Version, model.TaskBuildVariantKey: variantId, model.TaskDisplayNameKey: taskName}).Limit(1)) if err != nil { if err == mgo.ErrNotFound { plugin.WriteJSON(w, http.StatusNotFound, nil) return } http.Error(w, err.Error(), http.StatusInternalServerError) return } if len(ts) == 0 { plugin.WriteJSON(w, http.StatusNotFound, nil) return } otherVariantTask := ts[0] var jsonForTask TaskJSON err = db.FindOneQ(collection, db.Query(bson.M{"task_id": otherVariantTask.Id, "name": name}), &jsonForTask) if err != nil { if err == mgo.ErrNotFound { plugin.WriteJSON(w, http.StatusNotFound, nil) return } http.Error(w, err.Error(), http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, jsonForTask.Data) }) return r }