// TaskPostResults posts a set of test results for the communicator's task.
func (t *TaskJSONCommunicator) TaskPostResults(results *model.TestResults) error {
retriableSendFile := util.RetriableFunc(
func() error {
resp, err := t.tryPostJSON("results", results)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
err := fmt.Errorf("error posting results: %v", err)
return util.RetriableError{err}
}
body, _ := ioutil.ReadAll(resp.Body)
bodyErr := fmt.Errorf("error posting results (%v): %v",
resp.StatusCode, string(body))
switch resp.StatusCode {
case http.StatusOK:
return nil
case http.StatusBadRequest:
return bodyErr
default:
return util.RetriableError{bodyErr}
}
},
)
retryFail, err := util.RetryArithmeticBackoff(retriableSendFile, 10, 1)
if retryFail {
return fmt.Errorf("attaching test results failed after %v tries: %v", 10, err)
}
return nil
}
// PostTaskFiles is used by the PluginCommunicator interface for attaching task files.
func (t *TaskJSONCommunicator) PostTaskFiles(task_files []*artifact.File) error {
retriableSendFile := util.RetriableFunc(
func() error {
resp, err := t.TryPostJSON("files", task_files)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
err := fmt.Errorf("error posting results: %v", err)
return util.RetriableError{err}
}
body, readAllErr := ioutil.ReadAll(resp.Body)
bodyErr := fmt.Errorf("error posting results (%v): %v",
resp.StatusCode, string(body))
if readAllErr != nil {
return bodyErr
}
switch resp.StatusCode {
case http.StatusOK:
return nil
case http.StatusBadRequest:
return bodyErr
default:
return util.RetriableError{bodyErr}
}
},
)
retryFail, err := util.RetryArithmeticBackoff(retriableSendFile, 10, 1)
if retryFail {
return fmt.Errorf("attaching task files failed after %v tries: %v", 10, err)
}
return nil
}
// Wrapper around the Put() function to retry it
func (s3pc *S3PutCommand) PutWithRetry(log plugin.Logger, com plugin.PluginCommunicator) error {
retriablePut := util.RetriableFunc(
func() error {
err := s3pc.Put()
if err != nil {
if err == errSkippedFile {
return err
}
log.LogExecution(slogger.ERROR, "Error putting to s3 bucket: %v", err)
return util.RetriableError{err}
}
return nil
},
)
retryFail, err := util.RetryArithmeticBackoff(retriablePut, maxS3PutAttempts, s3PutSleep)
if err == errSkippedFile {
log.LogExecution(slogger.INFO, "S3 put skipped optional missing file.")
return nil
}
if retryFail {
log.LogExecution(slogger.ERROR, "S3 put failed with error: %v", err)
return err
}
return s3pc.AttachTaskFiles(log, com)
}
// Helper function to send notifications
func TrySendNotification(recipients []string, subject, body string, mailer Mailer) (err error) {
// evergreen.Logger.Logf(slogger.DEBUG, "address: %v subject: %v body: %v", recipients, subject, body)
// return nil
_, err = util.RetryArithmeticBackoff(func() error {
err = mailer.SendMail(recipients, subject, body)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "Error sending notification: %v", err)
return util.RetriableError{err}
}
return nil
}, NumSmtpRetries, SmtpSleepTime)
return err
}
// TaskPostTestLog posts a test log for a communicator's task.
func (t *TaskJSONCommunicator) TaskPostTestLog(log *model.TestLog) (string, error) {
var logId string
retriableSendFile := util.RetriableFunc(
func() error {
resp, err := t.TryPostJSON("test_logs", log)
if err != nil {
err := fmt.Errorf("error posting logs: %v", err)
if resp != nil {
resp.Body.Close()
}
return util.RetriableError{err}
}
if resp.StatusCode == http.StatusOK {
logReply := struct {
Id string `json:"_id"`
}{}
err = util.ReadJSONInto(resp.Body, &logReply)
if err != nil {
return err
}
logId = logReply.Id
return nil
}
bodyErr, err := ioutil.ReadAll(resp.Body)
if err != nil {
return util.RetriableError{err}
}
if resp.StatusCode == http.StatusBadRequest {
return fmt.Errorf("bad request posting logs: %v", string(bodyErr))
}
return util.RetriableError{fmt.Errorf("failed posting logs: %v", string(bodyErr))}
},
)
retryFail, err := util.RetryArithmeticBackoff(retriableSendFile, 10, 1)
if retryFail {
return "", fmt.Errorf("attaching test logs failed after %v tries: %v", 10, err)
}
return logId, nil
}
// getPatchContents() dereferences any patch files that are stored externally, fetching them from
// the API server, and setting them into the patch object.
func (gapc GitApplyPatchCommand) getPatchContents(conf *model.TaskConfig, com plugin.PluginCommunicator, log plugin.Logger, p *patch.Patch) error {
for i, patchPart := range p.Patches {
// If the patch isn't stored externally, no need to do anything.
if patchPart.PatchSet.PatchFileId == "" {
continue
}
// otherwise, fetch the contents and load it into the patch object
log.LogExecution(slogger.INFO, "Fetching patch contents for %v", patchPart.PatchSet.PatchFileId)
var result []byte
retriableGet := util.RetriableFunc(
func() error {
resp, err := com.TaskGetJSON(fmt.Sprintf("%s/%s", GitPatchFilePath, patchPart.PatchSet.PatchFileId))
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
//Some generic error trying to connect - try again
log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
return util.RetriableError{err}
}
if resp != nil && resp.StatusCode != http.StatusOK {
log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode)
resp.Body.Close()
return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)}
}
result, err = ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return nil
})
_, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second)
if err != nil {
return err
}
p.Patches[i].PatchSet.Patch = string(result)
}
return nil
}
// Wrapper around the Put() function to retry it
func (self *S3PutCommand) PutWithRetry(pluginLogger plugin.Logger,
pluginComm plugin.PluginCommunicator) error {
retriablePut := util.RetriableFunc(
func() error {
err := self.Put()
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error putting to s3"+
" bucket: %v", err)
return util.RetriableError{err}
}
return nil
},
)
retryFail, err := util.RetryArithmeticBackoff(retriablePut,
maxS3PutAttempts, s3PutSleep)
if retryFail {
pluginLogger.LogExecution(slogger.ERROR, "S3 put failed with error: %v",
err)
return err
}
return self.AttachTaskFiles(pluginLogger, pluginComm)
}
// Wrapper around the Get() function to retry it
func (self *S3GetCommand) GetWithRetry(pluginLogger plugin.Logger) error {
retriableGet := util.RetriableFunc(
func() error {
pluginLogger.LogTask(slogger.INFO, "Fetching %v from"+
" s3 bucket %v", self.RemoteFile, self.Bucket)
err := self.Get()
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error getting from"+
" s3 bucket: %v", err)
return util.RetriableError{err}
}
return nil
},
)
retryFail, err := util.RetryArithmeticBackoff(retriableGet,
MaxS3GetAttempts, S3GetSleep)
if retryFail {
pluginLogger.LogExecution(slogger.ERROR, "S3 get failed with error: %v",
err)
return err
}
return nil
}
// Load performs a GET on /manifest/load
func (mfc *ManifestLoadCommand) Load(log plugin.Logger, pluginCom plugin.PluginCommunicator, conf *model.TaskConfig) error {
var loadedManifest *manifest.Manifest
var err error
retriableGet := util.RetriableFunc(
func() error {
resp, err := pluginCom.TaskGetJSON(ManifestLoadAPIEndpoint)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
//Some generic error trying to connect - try again
log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
return util.RetriableError{err}
}
if resp != nil && resp.StatusCode != http.StatusOK {
log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode)
return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)}
}
err = util.ReadJSONInto(resp.Body, &loadedManifest)
if err != nil {
return err
}
return nil
})
_, err = util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second)
if err != nil {
return err
}
if loadedManifest == nil {
return fmt.Errorf("Manifest is empty")
}
mfc.updateExpansions(loadedManifest, conf)
return nil
}
// Takes a request for a task's file to be copied from
// one s3 location to another. Ensures that if the destination
// file path already exists, no file copy is performed.
func S3CopyHandler(w http.ResponseWriter, r *http.Request) {
task := plugin.GetTask(r)
if task == nil {
http.Error(w, "task not found", http.StatusNotFound)
return
}
s3CopyReq := &S3CopyRequest{}
err := util.ReadJSONInto(r.Body, s3CopyReq)
if err != nil {
evergreen.Logger.Errorf(slogger.ERROR, "error reading push request: %v", err)
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Get the version for this task, so we can check if it has
// any already-done pushes
v, err := version.FindOne(version.ById(task.Version))
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error querying task %v with version id %v: %v",
task.Id, task.Version, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Check for an already-pushed file with this same file path,
// but from a conflicting or newer commit sequence num
if v == nil {
evergreen.Logger.Logf(slogger.ERROR, "no version found for build %v", task.BuildId)
http.Error(w, "version not found", http.StatusNotFound)
return
}
copyFromLocation := strings.Join([]string{s3CopyReq.S3SourceBucket, s3CopyReq.S3SourcePath}, "/")
copyToLocation := strings.Join([]string{s3CopyReq.S3DestinationBucket, s3CopyReq.S3DestinationPath}, "/")
newestPushLog, err := model.FindPushLogAfter(copyToLocation, v.RevisionOrderNumber)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "error querying for push log at %v: %v", copyToLocation, err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if newestPushLog != nil {
evergreen.Logger.Logf(slogger.ERROR, "conflict with existing pushed file: "+
"%v", copyToLocation)
http.Error(w, "conflicting push target for this file already exists.", http.StatusConflict)
return
}
// It's now safe to put the file in its permanent location.
newPushLog := model.NewPushLog(v, task, copyToLocation)
err = newPushLog.Insert()
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "failed to create new push log: %v %v", newPushLog, err)
http.Error(w, fmt.Sprintf("failed to create push log: %v", err), http.StatusInternalServerError)
return
}
// Now copy the file into the permanent location
auth := &aws.Auth{
AccessKey: s3CopyReq.AwsKey,
SecretKey: s3CopyReq.AwsSecret,
}
evergreen.Logger.Logf(slogger.INFO, "performing S3 copy: '%v' => '%v'",
copyFromLocation, copyToLocation)
_, err = util.RetryArithmeticBackoff(func() error {
err := thirdparty.S3CopyFile(auth,
s3CopyReq.S3SourceBucket,
s3CopyReq.S3SourcePath,
s3CopyReq.S3DestinationBucket,
s3CopyReq.S3DestinationPath,
string(s3.PublicRead),
)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "S3 copy failed for task %v, "+
"retrying: %v", task.Id, err)
return util.RetriableError{err}
} else {
err := newPushLog.UpdateStatus(model.PushLogSuccess)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed"+
" for task %v: %v", task.Id, err)
}
return err
}
}, s3CopyRetryNumRetries, s3CopyRetrySleepTimeSec*time.Second)
if err != nil {
message := fmt.Sprintf("S3 copy failed for task %v: %v", task.Id, err)
evergreen.Logger.Logf(slogger.ERROR, message)
err = newPushLog.UpdateStatus(model.PushLogFailed)
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "updating pushlog status failed: %v", err)
}
http.Error(w, message, http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, "S3 copy Successful")
}
// GetPatch tries to get the patch data from the server in json format,
// and unmarhals it into a patch struct. The GET request is attempted
// multiple times upon failure.
func (gapc GitApplyPatchCommand) GetPatch(conf *model.TaskConfig,
pluginCom plugin.PluginCommunicator, pluginLogger plugin.Logger) (*patch.Patch, error) {
patch := &patch.Patch{}
retriableGet := util.RetriableFunc(
func() error {
resp, err := pluginCom.TaskGetJSON(GitPatchPath)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
//Some generic error trying to connect - try again
pluginLogger.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
return util.RetriableError{err}
}
if resp != nil && resp.StatusCode == http.StatusNotFound {
//nothing broke, but no patch was found for task Id - no retry
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error reading response body")
}
msg := fmt.Sprintf("no patch found for task: %v", string(body))
pluginLogger.LogExecution(slogger.WARN, msg)
return fmt.Errorf(msg)
}
if resp != nil && resp.StatusCode == http.StatusInternalServerError {
//something went wrong in api server
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error reading response body")
}
msg := fmt.Sprintf("error fetching patch from server: %v", string(body))
pluginLogger.LogExecution(slogger.WARN, msg)
return util.RetriableError{
fmt.Errorf(msg),
}
}
if resp != nil && resp.StatusCode == http.StatusConflict {
//wrong secret
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error reading response body")
}
msg := fmt.Sprintf("secret conflict: %v", string(body))
pluginLogger.LogExecution(slogger.ERROR, msg)
return fmt.Errorf(msg)
}
if resp == nil {
pluginLogger.LogExecution(slogger.WARN, "Empty response from API server")
return util.RetriableError{fmt.Errorf("empty response")}
} else {
err = util.ReadJSONInto(resp.Body, patch)
if err != nil {
pluginLogger.LogExecution(slogger.ERROR,
"Error reading json into patch struct: %v", err)
return util.RetriableError{err}
}
return nil
}
},
)
retryFail, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second)
if retryFail {
return nil, fmt.Errorf("getting patch failed after %v tries: %v", 10, err)
}
if err != nil {
return nil, fmt.Errorf("getting patch failed: %v", err)
}
return patch, nil
}
