// AttachTaskFiles is responsible for sending the // specified file to the API Server func (c *S3CopyCommand) AttachTaskFiles(pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator, request S3CopyRequest) error { remotePath := filepath.ToSlash(request.S3DestinationPath) fileLink := s3baseURL + request.S3DestinationBucket + "/" + remotePath displayName := request.S3DisplayName if displayName == "" { displayName = filepath.Base(request.S3SourcePath) } pluginLogger.LogExecution(slogger.INFO, "attaching file with name %v", displayName) file := artifact.File{ Name: displayName, Link: fileLink, } files := []*artifact.File{&file} err := pluginCom.PostTaskFiles(files) if err != nil { return fmt.Errorf("Attach files failed: %v", err) } pluginLogger.LogExecution(slogger.INFO, "API attach files call succeeded") return nil }
func (mc *MockCommand) Execute(logger plugin.Logger, pluginCom plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error { resp, err := pluginCom.TaskGetJSON(fmt.Sprintf("blah/%s/%d", mc.Param1, mc.Param2)) if resp != nil { defer resp.Body.Close() } if resp == nil { return fmt.Errorf("Received nil HTTP response from api server") } jsonReply := map[string]string{} err = util.ReadJSONInto(resp.Body, &jsonReply) if err != nil { return err } if resp.StatusCode != http.StatusOK { return fmt.Errorf("Got bad status code from API response: %v, body: %v", resp.StatusCode, jsonReply) } expectedEchoReply := fmt.Sprintf("%v/%v/%v", mc.Param1, mc.Param2, conf.Task.Id) if jsonReply["echo"] != expectedEchoReply { return fmt.Errorf("Wrong echo reply! Wanted %v, got %v", expectedEchoReply, jsonReply["echo"]) } return nil }
// Execute fetches the expansions from the API server func (incCmd *IncCommand) Execute(pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error { err := plugin.ExpandValues(incCmd, conf.Expansions) if err != nil { return err } keyVal := &KeyVal{} resp, err := pluginCom.TaskPostJSON(IncRoute, incCmd.Key) if err != nil { return err } if resp == nil { return fmt.Errorf("received nil response from inc API call") } else { defer resp.Body.Close() } if resp.StatusCode != http.StatusOK { return fmt.Errorf("unexpected status code: %v", resp.StatusCode) } err = util.ReadJSONInto(resp.Body, keyVal) if err != nil { return fmt.Errorf("Failed to read JSON reply: %v", err) } conf.Expansions.Put(incCmd.Destination, fmt.Sprintf("%d", keyVal.Value)) return nil }
func (jsc *JSONSendCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error { if jsc.File == "" { return fmt.Errorf("'file' param must not be blank") } if jsc.DataName == "" { return fmt.Errorf("'name' param must not be blank") } errChan := make(chan error) go func() { // attempt to open the file fileLoc := filepath.Join(conf.WorkDir, jsc.File) jsonFile, err := os.Open(fileLoc) if err != nil { errChan <- fmt.Errorf("Couldn't open json file: '%v'", err) return } jsonData := map[string]interface{}{} err = util.ReadJSONInto(jsonFile, &jsonData) if err != nil { errChan <- fmt.Errorf("File contained invalid json: %v", err) return } retriablePost := util.RetriableFunc( func() error { log.LogTask(slogger.INFO, "Posting JSON") resp, err := com.TaskPostJSON(fmt.Sprintf("data/%v", jsc.DataName), jsonData) if resp != nil { defer resp.Body.Close() } if err != nil { return util.RetriableError{err} } if resp.StatusCode != http.StatusOK { return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)} } return nil }, ) _, err = util.Retry(retriablePost, 10, 3*time.Second) errChan <- err }() select { case err := <-errChan: if err != nil { log.LogTask(slogger.ERROR, "Sending json data failed: %v", err) } return err case <-stop: log.LogExecution(slogger.INFO, "Received abort signal, stopping.") return nil } }
func (jgc *JSONHistoryCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error { err := plugin.ExpandValues(jgc, conf.Expansions) if err != nil { return err } if jgc.File == "" { return fmt.Errorf("'file' param must not be blank") } if jgc.DataName == "" { return fmt.Errorf("'name' param must not be blank") } if jgc.TaskName == "" { return fmt.Errorf("'task' param must not be blank") } if jgc.File != "" && !filepath.IsAbs(jgc.File) { jgc.File = filepath.Join(conf.WorkDir, jgc.File) } endpoint := fmt.Sprintf("history/%s/%s", jgc.TaskName, jgc.DataName) if jgc.Tags { endpoint = fmt.Sprintf("tags/%s/%s", jgc.TaskName, jgc.DataName) } retriableGet := util.RetriableFunc( func() error { resp, err := com.TaskGetJSON(endpoint) if resp != nil { defer resp.Body.Close() } if err != nil { //Some generic error trying to connect - try again log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err) return util.RetriableError{err} } if resp.StatusCode == http.StatusOK { jsonBytes, err := ioutil.ReadAll(resp.Body) if err != nil { return err } return ioutil.WriteFile(jgc.File, jsonBytes, 0755) } if resp.StatusCode != http.StatusOK { if resp.StatusCode == http.StatusNotFound { return fmt.Errorf("No JSON data found") } return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)} } return nil }, ) _, err = util.Retry(retriableGet, 10, 3*time.Second) return err }
// SendJSONLogs is responsible for sending the specified logs // to the API Server. If successful, it returns a log ID that can be used // to refer to the log object in test results. func SendJSONLogs(taskConfig *model.TaskConfig, pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator, logs *model.TestLog) (string, error) { pluginLogger.LogExecution(slogger.INFO, "Attaching test logs for %v", logs.Name) logId, err := pluginCom.TaskPostTestLog(logs) if err != nil { return "", err } pluginLogger.LogTask(slogger.INFO, "Attach test logs succeeded") return logId, nil }
// SendJSONResults is responsible for sending the // specified file to the API Server func SendJSONResults(taskConfig *model.TaskConfig, pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator, results *model.TestResults) error { pluginLogger.LogExecution(slogger.INFO, "Attaching test results") err := pluginCom.TaskPostResults(results) if err != nil { return err } pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded") return nil }
// S3Copy is responsible for carrying out the core of the S3CopyPlugin's // function - it makes an API calls to copy a given staged file to it's final // production destination func (scc *S3CopyCommand) S3Copy(taskConfig *model.TaskConfig, pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator) error { for _, s3CopyFile := range scc.S3CopyFiles { if len(s3CopyFile.BuildVariants) > 0 && !util.SliceContains( s3CopyFile.BuildVariants, taskConfig.BuildVariant.Name) { continue } pluginLogger.LogExecution(slogger.INFO, "Making API push copy call to "+ "transfer %v/%v => %v/%v", s3CopyFile.Source.Bucket, s3CopyFile.Source.Path, s3CopyFile.Destination.Bucket, s3CopyFile.Destination.Path) s3CopyReq := S3CopyRequest{ AwsKey: scc.AwsKey, AwsSecret: scc.AwsSecret, S3SourceBucket: s3CopyFile.Source.Bucket, S3SourcePath: s3CopyFile.Source.Path, S3DestinationBucket: s3CopyFile.Destination.Bucket, S3DestinationPath: s3CopyFile.Destination.Path, S3DisplayName: s3CopyFile.DisplayName, } resp, err := pluginCom.TaskPostJSON(s3CopyAPIEndpoint, s3CopyReq) if resp != nil { defer resp.Body.Close() } if resp != nil && resp.StatusCode != http.StatusOK { body, _ := ioutil.ReadAll(resp.Body) return fmt.Errorf("S3 push copy failed (%v): %v", resp.StatusCode, string(body)) } if err != nil { body, _ := ioutil.ReadAll(resp.Body) return fmt.Errorf("S3 push copy failed (%v): %v", resp.StatusCode, string(body)) } pluginLogger.LogExecution(slogger.INFO, "API push copy call succeeded") err = scc.AttachTaskFiles(pluginLogger, pluginCom, s3CopyReq) if err != nil { body, readAllErr := ioutil.ReadAll(resp.Body) if readAllErr != nil { return fmt.Errorf("Error: %v", err) } return fmt.Errorf("Error: %v, (%v): %v", resp.StatusCode, err, string(body)) } } return nil }
// getPatchContents() dereferences any patch files that are stored externally, fetching them from // the API server, and setting them into the patch object. func (gapc GitApplyPatchCommand) getPatchContents(conf *model.TaskConfig, com plugin.PluginCommunicator, log plugin.Logger, p *patch.Patch) error { for i, patchPart := range p.Patches { // If the patch isn't stored externally, no need to do anything. if patchPart.PatchSet.PatchFileId == "" { continue } // otherwise, fetch the contents and load it into the patch object log.LogExecution(slogger.INFO, "Fetching patch contents for %v", patchPart.PatchSet.PatchFileId) var result []byte retriableGet := util.RetriableFunc( func() error { resp, err := com.TaskGetJSON(fmt.Sprintf("%s/%s", GitPatchFilePath, patchPart.PatchSet.PatchFileId)) if resp != nil { defer resp.Body.Close() } if err != nil { //Some generic error trying to connect - try again log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err) return util.RetriableError{err} } if resp != nil && resp.StatusCode != http.StatusOK { log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode) resp.Body.Close() return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)} } result, err = ioutil.ReadAll(resp.Body) if err != nil { return err } return nil }) _, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second) if err != nil { return err } p.Patches[i].PatchSet.Patch = string(result) } return nil }
// SendJSONResults is responsible for sending the // specified file to the API Server func SendJSONResults(taskConfig *model.TaskConfig, pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator, results *task.TestResults) error { for i, res := range results.Results { if res.LogRaw != "" { pluginLogger.LogExecution(slogger.INFO, "Attaching raw test logs") testLogs := &model.TestLog{ Name: res.TestFile, Task: taskConfig.Task.Id, TaskExecution: taskConfig.Task.Execution, Lines: []string{res.LogRaw}, } id, err := pluginCom.TaskPostTestLog(testLogs) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error posting raw logs from results: %v", err) } else { results.Results[i].LogId = id } // clear the logs from the TestResult struct after it has been saved in the test logs. Since they are // being saved in the test_logs collection, we can clear them to prevent them from being saved in the task // collection. results.Results[i].LogRaw = "" } } pluginLogger.LogExecution(slogger.INFO, "Attaching test results") err := pluginCom.TaskPostResults(results) if err != nil { return err } pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded") return nil }
// AttachTaskFiles is responsible for sending the // specified file to the API Server func (s3pc *S3PutCommand) AttachTaskFiles(log plugin.Logger, com plugin.PluginCommunicator) error { remoteFile := filepath.ToSlash(s3pc.RemoteFile) fileLink := s3baseURL + s3pc.Bucket + "/" + remoteFile displayName := s3pc.DisplayName if displayName == "" { displayName = filepath.Base(s3pc.LocalFile) } file := &artifact.File{ Name: displayName, Link: fileLink, Visibility: s3pc.Visibility, } err := com.PostTaskFiles([]*artifact.File{file}) if err != nil { return fmt.Errorf("Attach files failed: %v", err) } log.LogExecution(slogger.INFO, "API attach files call succeeded") return nil }
// Load performs a GET on /manifest/load func (mfc *ManifestLoadCommand) Load(log plugin.Logger, pluginCom plugin.PluginCommunicator, conf *model.TaskConfig) error { var loadedManifest *manifest.Manifest var err error retriableGet := util.RetriableFunc( func() error { resp, err := pluginCom.TaskGetJSON(ManifestLoadAPIEndpoint) if resp != nil { defer resp.Body.Close() } if err != nil { //Some generic error trying to connect - try again log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err) return util.RetriableError{err} } if resp != nil && resp.StatusCode != http.StatusOK { log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode) return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)} } err = util.ReadJSONInto(resp.Body, &loadedManifest) if err != nil { return err } return nil }) _, err = util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second) if err != nil { return err } if loadedManifest == nil { return fmt.Errorf("Manifest is empty") } mfc.updateExpansions(loadedManifest, conf) return nil }
// GetPatch tries to get the patch data from the server in json format, // and unmarhals it into a patch struct. The GET request is attempted // multiple times upon failure. func (gapc GitApplyPatchCommand) GetPatch(conf *model.TaskConfig, pluginCom plugin.PluginCommunicator, pluginLogger plugin.Logger) (*patch.Patch, error) { patch := &patch.Patch{} retriableGet := util.RetriableFunc( func() error { resp, err := pluginCom.TaskGetJSON(GitPatchPath) if resp != nil { defer resp.Body.Close() } if err != nil { //Some generic error trying to connect - try again pluginLogger.LogExecution(slogger.WARN, "Error connecting to API server: %v", err) return util.RetriableError{err} } if resp != nil && resp.StatusCode == http.StatusNotFound { //nothing broke, but no patch was found for task Id - no retry body, err := ioutil.ReadAll(resp.Body) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading response body") } msg := fmt.Sprintf("no patch found for task: %v", string(body)) pluginLogger.LogExecution(slogger.WARN, msg) return fmt.Errorf(msg) } if resp != nil && resp.StatusCode == http.StatusInternalServerError { //something went wrong in api server body, err := ioutil.ReadAll(resp.Body) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading response body") } msg := fmt.Sprintf("error fetching patch from server: %v", string(body)) pluginLogger.LogExecution(slogger.WARN, msg) return util.RetriableError{ fmt.Errorf(msg), } } if resp != nil && resp.StatusCode == http.StatusConflict { //wrong secret body, err := ioutil.ReadAll(resp.Body) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading response body") } msg := fmt.Sprintf("secret conflict: %v", string(body)) pluginLogger.LogExecution(slogger.ERROR, msg) return fmt.Errorf(msg) } if resp == nil { pluginLogger.LogExecution(slogger.WARN, "Empty response from API server") return util.RetriableError{fmt.Errorf("empty response")} } else { err = util.ReadJSONInto(resp.Body, patch) if err != nil { pluginLogger.LogExecution(slogger.ERROR, "Error reading json into patch struct: %v", err) return util.RetriableError{err} } return nil } }, ) retryFail, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second) if retryFail { return nil, fmt.Errorf("getting patch failed after %v tries: %v", 10, err) } if err != nil { return nil, fmt.Errorf("getting patch failed: %v", err) } return patch, nil }
// Execute parses the specified output files and sends the test results found in them // back to the server. func (pfCmd *ParseFilesCommand) Execute(pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig, stop chan bool) error { if err := plugin.ExpandValues(pfCmd, taskConfig.Expansions); err != nil { msg := fmt.Sprintf("error expanding params: %v", err) pluginLogger.LogTask(slogger.ERROR, "Error parsing gotest files: %v", msg) return fmt.Errorf(msg) } // make sure the file patterns are relative to the task's working directory for idx, file := range pfCmd.Files { pfCmd.Files[idx] = filepath.Join(taskConfig.WorkDir, file) } // will be all files containing test results outputFiles, err := pfCmd.AllOutputFiles() if err != nil { return fmt.Errorf("error obtaining names of output files: %v", err) } // make sure we're parsing something if len(outputFiles) == 0 { return fmt.Errorf("no files found to be parsed") } // parse all of the files logs, results, err := ParseTestOutputFiles(outputFiles, stop, pluginLogger, taskConfig) if err != nil { return fmt.Errorf("error parsing output results: %v", err) } // ship all of the test logs off to the server pluginLogger.LogTask(slogger.INFO, "Sending test logs to server...") allResults := []TestResult{} for idx, log := range logs { logId := "" if logId, err = pluginCom.TaskPostTestLog(&log); err != nil { // continue on error to let the other logs be posted pluginLogger.LogTask(slogger.ERROR, "Error posting log: %v", err) } // add all of the test results that correspond to that log to the // full list of results for _, result := range results[idx] { result.LogId = logId allResults = append(allResults, result) } } pluginLogger.LogTask(slogger.INFO, "Finished posting logs to server") // convert everything resultsAsModel := ToModelTestResults(taskConfig.Task, allResults) // ship the parsed results off to the server pluginLogger.LogTask(slogger.INFO, "Sending parsed results to server...") if err := pluginCom.TaskPostResults(&resultsAsModel); err != nil { return fmt.Errorf("error posting parsed results to server: %v", err) } pluginLogger.LogTask(slogger.INFO, "Successfully sent parsed results to server") return nil }
func (self *RunTestCommand) Execute(pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig, stop chan bool) error { if err := plugin.ExpandValues(self, taskConfig.Expansions); err != nil { msg := fmt.Sprintf("error expanding params: %v", err) pluginLogger.LogTask(slogger.ERROR, "Error updating test configs: %v", msg) return fmt.Errorf(msg) } // define proper working directory if self.WorkDir != "" { self.WorkDir = filepath.Join(taskConfig.WorkDir, self.WorkDir) } else { self.WorkDir = taskConfig.WorkDir } pluginLogger.LogTask(slogger.INFO, "Running tests with working dir '%v'", self.WorkDir) if os.Getenv("GOPATH") == "" { pluginLogger.LogTask(slogger.WARN, "No GOPATH; setting GOPATH to working dir") err := os.Setenv("GOPATH", self.WorkDir) if err != nil { return err } } var results []TestResult allPassed := true // run all tests, concat results. Hold onto failures until the end for idx, test := range self.Tests { // kill the execution if API server requests select { case <-stop: return fmt.Errorf("command was stopped") default: // no stop signal } // update test directory test.Dir = filepath.Join(self.WorkDir, test.Dir) suiteName := getSuiteNameFromDir(idx, test.Dir) parser := &VanillaParser{Suite: suiteName} pluginLogger.LogTask( slogger.INFO, "Running go test with '%v' in '%v'", test.Args, test.Dir) if len(test.EnvironmentVariables) > 0 { pluginLogger.LogTask( slogger.INFO, "Adding environment variables to gotest: %#v", test.EnvironmentVariables) } passed, err := RunAndParseTests(test, parser, pluginLogger, stop) logLines := parser.Logs() for _, log := range logLines { pluginLogger.LogTask(slogger.INFO, ">>> %v", log) } pluginLogger.LogTask(slogger.INFO, "Sending logs to API server (%v lines)", len(logLines)) testLog := &model.TestLog{ Name: suiteName, Task: taskConfig.Task.Id, TaskExecution: taskConfig.Task.Execution, Lines: logLines, } logId, err := pluginCom.TaskPostTestLog(testLog) if err != nil { pluginLogger.LogTask(slogger.ERROR, "error posting test log: %v", err) } if passed != true { allPassed = false pluginLogger.LogTask(slogger.WARN, "Test suite failed, continuing...") } if err != nil { pluginLogger.LogTask(slogger.ERROR, "Error running and parsing test '%v': %v", test.Dir, err) continue } // get the results of this individual test, and set the log id // appropriately testResults := parser.Results() for _, result := range testResults { result.LogId = logId results = append(results, result) } } pluginLogger.LogTask(slogger.INFO, "Sending go test results to server") modelResults := ToModelTestResults(taskConfig.Task, results) err := pluginCom.TaskPostResults(&modelResults) if err != nil { return fmt.Errorf("error posting results: %v", err) } if allPassed { return nil } else { return fmt.Errorf("test failures") } }
// SendJSONResults is responsible for sending the // specified file to the API Server func (self *AttachTaskFilesCommand) SendTaskFiles(taskConfig *model.TaskConfig, pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator) error { // log each file attachment for name, link := range self.Files { pluginLogger.LogTask(slogger.INFO, "Attaching file: %v -> %v", name, link) } retriableSendFile := util.RetriableFunc( func() error { resp, err := pluginCom.TaskPostJSON( AttachTaskFilesAPIEndpoint, self.Files.Array(), ) if resp != nil { defer resp.Body.Close() } if resp != nil && resp.StatusCode == http.StatusConflict { body, _ := ioutil.ReadAll(resp.Body) msg := fmt.Sprintf( "secret conflict while posting task files: %v", string(body)) pluginLogger.LogTask(slogger.ERROR, msg) return fmt.Errorf(msg) } if resp != nil && resp.StatusCode == http.StatusBadRequest { body, _ := ioutil.ReadAll(resp.Body) msg := fmt.Sprintf( "error posting task files (%v): %v", resp.StatusCode, string(body)) pluginLogger.LogTask(slogger.ERROR, msg) return fmt.Errorf(msg) } if resp != nil && resp.StatusCode != http.StatusOK { body, _ := ioutil.ReadAll(resp.Body) msg := fmt.Sprintf("error posting task files (%v): %v", resp.StatusCode, string(body)) pluginLogger.LogExecution(slogger.WARN, msg) return util.RetriableError{err} } if err != nil { msg := fmt.Sprintf("error posting files: %v", err) pluginLogger.LogExecution(slogger.WARN, msg) return util.RetriableError{fmt.Errorf(msg)} } return nil }, ) retryFail, err := util.Retry(retriableSendFile, AttachResultsPostRetries, AttachResultsRetrySleepSec) if retryFail { return fmt.Errorf("Attach files failed after %v tries: %v", AttachResultsPostRetries, err) } if err != nil { return fmt.Errorf("Attach files failed: %v", err) } pluginLogger.LogExecution(slogger.INFO, "API attach files call succeeded") return nil }