// TaskPostResults posts a set of test results for the communicator's task. func (t *TaskJSONCommunicator) TaskPostResults(results *model.TestResults) error { retriableSendFile := util.RetriableFunc( func() error { resp, err := t.tryPostJSON("results", results) if resp != nil { defer resp.Body.Close() } if err != nil { err := fmt.Errorf("error posting results: %v", err) return util.RetriableError{err} } body, _ := ioutil.ReadAll(resp.Body) bodyErr := fmt.Errorf("error posting results (%v): %v", resp.StatusCode, string(body)) switch resp.StatusCode { case http.StatusOK: return nil case http.StatusBadRequest: return bodyErr default: return util.RetriableError{bodyErr} } }, ) retryFail, err := util.RetryArithmeticBackoff(retriableSendFile, 10, 1) if retryFail { return fmt.Errorf("attaching test results failed after %v tries: %v", 10, err) } return nil }
// PostTaskFiles is used by the PluginCommunicator interface for attaching task files. func (t *TaskJSONCommunicator) PostTaskFiles(task_files []*artifact.File) error { retriableSendFile := util.RetriableFunc( func() error { resp, err := t.TryPostJSON("files", task_files) if resp != nil { defer resp.Body.Close() } if err != nil { err := fmt.Errorf("error posting results: %v", err) return util.RetriableError{err} } body, readAllErr := ioutil.ReadAll(resp.Body) bodyErr := fmt.Errorf("error posting results (%v): %v", resp.StatusCode, string(body)) if readAllErr != nil { return bodyErr } switch resp.StatusCode { case http.StatusOK: return nil case http.StatusBadRequest: return bodyErr default: return util.RetriableError{bodyErr} } }, ) retryFail, err := util.RetryArithmeticBackoff(retriableSendFile, 10, 1) if retryFail { return fmt.Errorf("attaching task files failed after %v tries: %v", 10, err) } return nil }
// Wrapper around the Put() function to retry it func (s3pc *S3PutCommand) PutWithRetry(log plugin.Logger, com plugin.PluginCommunicator) error { retriablePut := util.RetriableFunc( func() error { err := s3pc.Put() if err != nil { if err == errSkippedFile { return err } log.LogExecution(slogger.ERROR, "Error putting to s3 bucket: %v", err) return util.RetriableError{err} } return nil }, ) retryFail, err := util.RetryArithmeticBackoff(retriablePut, maxS3PutAttempts, s3PutSleep) if err == errSkippedFile { log.LogExecution(slogger.INFO, "S3 put skipped optional missing file.") return nil } if retryFail { log.LogExecution(slogger.ERROR, "S3 put failed with error: %v", err) return err } return s3pc.AttachTaskFiles(log, com) }
// Helper function to send notifications func TrySendNotification(recipients []string, subject, body string, mailer Mailer) (err error) { // evergreen.Logger.Logf(slogger.DEBUG, "address: %v subject: %v body: %v", recipients, subject, body) // return nil _, err = util.RetryArithmeticBackoff(func() error { err = mailer.SendMail(recipients, subject, body) if err != nil { evergreen.Logger.Errorf(slogger.ERROR, "Error sending notification: %v", err) return util.RetriableError{err} } return nil }, NumSmtpRetries, SmtpSleepTime) return err }
// TaskPostTestLog posts a test log for a communicator's task. func (t *TaskJSONCommunicator) TaskPostTestLog(log *model.TestLog) (string, error) { var logId string retriableSendFile := util.RetriableFunc( func() error { resp, err := t.TryPostJSON("test_logs", log) if err != nil { err := fmt.Errorf("error posting logs: %v", err) if resp != nil { resp.Body.Close() } return util.RetriableError{err} } if resp.StatusCode == http.StatusOK { logReply := struct { Id string `json:"_id"` }{} err = util.ReadJSONInto(resp.Body, &logReply) if err != nil { return err } logId = logReply.Id return nil } bodyErr, err := ioutil.ReadAll(resp.Body) if err != nil { return util.RetriableError{err} } if resp.StatusCode == http.StatusBadRequest { return fmt.Errorf("bad request posting logs: %v", string(bodyErr)) } return util.RetriableError{fmt.Errorf("failed posting logs: %v", string(bodyErr))} }, ) retryFail, err := util.RetryArithmeticBackoff(retriableSendFile, 10, 1) if retryFail { return "", fmt.Errorf("attaching test logs failed after %v tries: %v", 10, err) } return logId, nil }
// getPatchContents() dereferences any patch files that are stored externally, fetching them from // the API server, and setting them into the patch object. func (gapc GitApplyPatchCommand) getPatchContents(conf *model.TaskConfig, com plugin.PluginCommunicator, log plugin.Logger, p *patch.Patch) error { for i, patchPart := range p.Patches { // If the patch isn't stored externally, no need to do anything. if patchPart.PatchSet.PatchFileId == "" { continue } // otherwise, fetch the contents and load it into the patch object log.LogExecution(slogger.INFO, "Fetching patch contents for %v", patchPart.PatchSet.PatchFileId) var result []byte retriableGet := util.RetriableFunc( func() error { resp, err := com.TaskGetJSON(fmt.Sprintf("%s/%s", GitPatchFilePath, patchPart.PatchSet.PatchFileId)) if resp != nil { defer resp.Body.Close() } if err != nil { //Some generic error trying to connect - try again log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err) return util.RetriableError{err} } if resp != nil && resp.StatusCode != http.StatusOK { log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode) resp.Body.Close() return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)} } result, err = ioutil.ReadAll(resp.Body) if err != nil { return err } return nil }) _, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second) if err != nil { return err } p.Patches[i].PatchSet.Patch = string(result) } return nil }
// Wrapper around the Put() function to retry it func (self *S3PutCommand) PutWithRetry(pluginLogger plugin.Logger, pluginComm plugin.PluginCommunicator) error { retriablePut := util.RetriableFunc( func() error { err := self.Put() if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error putting to s3"+ " bucket: %v", err) return util.RetriableError{err} } return nil }, ) retryFail, err := util.RetryArithmeticBackoff(retriablePut, maxS3PutAttempts, s3PutSleep) if retryFail { pluginLogger.LogExecution(slogger.ERROR, "S3 put failed with error: %v", err) return err } return self.AttachTaskFiles(pluginLogger, pluginComm) }
// Wrapper around the Get() function to retry it func (self *S3GetCommand) GetWithRetry(pluginLogger plugin.Logger) error { retriableGet := util.RetriableFunc( func() error { pluginLogger.LogTask(slogger.INFO, "Fetching %v from"+ " s3 bucket %v", self.RemoteFile, self.Bucket) err := self.Get() if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error getting from"+ " s3 bucket: %v", err) return util.RetriableError{err} } return nil }, ) retryFail, err := util.RetryArithmeticBackoff(retriableGet, MaxS3GetAttempts, S3GetSleep) if retryFail { pluginLogger.LogExecution(slogger.ERROR, "S3 get failed with error: %v", err) return err } return nil }
// Load performs a GET on /manifest/load func (mfc *ManifestLoadCommand) Load(log plugin.Logger, pluginCom plugin.PluginCommunicator, conf *model.TaskConfig) error { var loadedManifest *manifest.Manifest var err error retriableGet := util.RetriableFunc( func() error { resp, err := pluginCom.TaskGetJSON(ManifestLoadAPIEndpoint) if resp != nil { defer resp.Body.Close() } if err != nil { //Some generic error trying to connect - try again log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err) return util.RetriableError{err} } if resp != nil && resp.StatusCode != http.StatusOK { log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode) return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)} } err = util.ReadJSONInto(resp.Body, &loadedManifest) if err != nil { return err } return nil }) _, err = util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second) if err != nil { return err } if loadedManifest == nil { return fmt.Errorf("Manifest is empty") } mfc.updateExpansions(loadedManifest, conf) return nil }
// Takes a request for a task's file to be copied from // one s3 location to another. Ensures that if the destination // file path already exists, no file copy is performed. func S3CopyHandler(w http.ResponseWriter, r *http.Request) { task := plugin.GetTask(r) if task == nil { http.Error(w, "task not found", http.StatusNotFound) return } s3CopyReq := &S3CopyRequest{} err := util.ReadJSONInto(r.Body, s3CopyReq) if err != nil { evergreen.Logger.Errorf(slogger.ERROR, "error reading push request: %v", err) http.Error(w, err.Error(), http.StatusBadRequest) return } // Get the version for this task, so we can check if it has // any already-done pushes v, err := version.FindOne(version.ById(task.Version)) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error querying task %v with version id %v: %v", task.Id, task.Version, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } // Check for an already-pushed file with this same file path, // but from a conflicting or newer commit sequence num if v == nil { evergreen.Logger.Logf(slogger.ERROR, "no version found for build %v", task.BuildId) http.Error(w, "version not found", http.StatusNotFound) return } copyFromLocation := strings.Join([]string{s3CopyReq.S3SourceBucket, s3CopyReq.S3SourcePath}, "/") copyToLocation := strings.Join([]string{s3CopyReq.S3DestinationBucket, s3CopyReq.S3DestinationPath}, "/") newestPushLog, err := model.FindPushLogAfter(copyToLocation, v.RevisionOrderNumber) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "error querying for push log at %v: %v", copyToLocation, err) http.Error(w, err.Error(), http.StatusInternalServerError) return } if newestPushLog != nil { evergreen.Logger.Logf(slogger.ERROR, "conflict with existing pushed file: "+ "%v", copyToLocation) http.Error(w, "conflicting push target for this file already exists.", http.StatusConflict) return } // It's now safe to put the file in its permanent location. newPushLog := model.NewPushLog(v, task, copyToLocation) err = newPushLog.Insert() if err != nil { evergreen.Logger.Logf(slogger.ERROR, "failed to create new push log: %v %v", newPushLog, err) http.Error(w, fmt.Sprintf("failed to create push log: %v", err), http.StatusInternalServerError) return } // Now copy the file into the permanent location auth := &aws.Auth{ AccessKey: s3CopyReq.AwsKey, SecretKey: s3CopyReq.AwsSecret, } evergreen.Logger.Logf(slogger.INFO, "performing S3 copy: '%v' => '%v'", copyFromLocation, copyToLocation) _, err = util.RetryArithmeticBackoff(func() error { err := thirdparty.S3CopyFile(auth, s3CopyReq.S3SourceBucket, s3CopyReq.S3SourcePath, s3CopyReq.S3DestinationBucket, s3CopyReq.S3DestinationPath, string(s3.PublicRead), ) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "S3 copy failed for task %v, "+ "retrying: %v", task.Id, err) return util.RetriableError{err} } else { err := newPushLog.UpdateStatus(model.PushLogSuccess) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed"+ " for task %v: %v", task.Id, err) } return err } }, s3CopyRetryNumRetries, s3CopyRetrySleepTimeSec*time.Second) if err != nil { message := fmt.Sprintf("S3 copy failed for task %v: %v", task.Id, err) evergreen.Logger.Logf(slogger.ERROR, message) err = newPushLog.UpdateStatus(model.PushLogFailed) if err != nil { evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed: %v", err) } http.Error(w, message, http.StatusInternalServerError) return } plugin.WriteJSON(w, http.StatusOK, "S3 copy Successful") }
// GetPatch tries to get the patch data from the server in json format, // and unmarhals it into a patch struct. The GET request is attempted // multiple times upon failure. func (gapc GitApplyPatchCommand) GetPatch(conf *model.TaskConfig, pluginCom plugin.PluginCommunicator, pluginLogger plugin.Logger) (*patch.Patch, error) { patch := &patch.Patch{} retriableGet := util.RetriableFunc( func() error { resp, err := pluginCom.TaskGetJSON(GitPatchPath) if resp != nil { defer resp.Body.Close() } if err != nil { //Some generic error trying to connect - try again pluginLogger.LogExecution(slogger.WARN, "Error connecting to API server: %v", err) return util.RetriableError{err} } if resp != nil && resp.StatusCode == http.StatusNotFound { //nothing broke, but no patch was found for task Id - no retry body, err := ioutil.ReadAll(resp.Body) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading response body") } msg := fmt.Sprintf("no patch found for task: %v", string(body)) pluginLogger.LogExecution(slogger.WARN, msg) return fmt.Errorf(msg) } if resp != nil && resp.StatusCode == http.StatusInternalServerError { //something went wrong in api server body, err := ioutil.ReadAll(resp.Body) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading response body") } msg := fmt.Sprintf("error fetching patch from server: %v", string(body)) pluginLogger.LogExecution(slogger.WARN, msg) return util.RetriableError{ fmt.Errorf(msg), } } if resp != nil && resp.StatusCode == http.StatusConflict { //wrong secret body, err := ioutil.ReadAll(resp.Body) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading response body") } msg := fmt.Sprintf("secret conflict: %v", string(body)) pluginLogger.LogExecution(slogger.ERROR, msg) return fmt.Errorf(msg) } if resp == nil { pluginLogger.LogExecution(slogger.WARN, "Empty response from API server") return util.RetriableError{fmt.Errorf("empty response")} } else { err = util.ReadJSONInto(resp.Body, patch) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading json into patch struct: %v", err) return util.RetriableError{err} } return nil } }, ) retryFail, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second) if retryFail { return nil, fmt.Errorf("getting patch failed after %v tries: %v", 10, err) } if err != nil { return nil, fmt.Errorf("getting patch failed: %v", err) } return patch, nil }