// ParseTestOutputFiles parses all of the files that are passed in, and returns the
// test logs and test results found within.
func ParseTestOutputFiles(outputFiles []string, stop chan bool,
pluginLogger plugin.Logger, taskConfig *model.TaskConfig) ([]model.TestLog,
[][]TestResult, error) {
var results [][]TestResult
var logs []model.TestLog
// now, open all the files, and parse the test results
for _, outputFile := range outputFiles {
// kill the execution if API server requests
select {
case <-stop:
return nil, nil, fmt.Errorf("command was stopped")
default:
// no stop signal
}
// assume that the name of the file, stripping off the ".suite" extension if present,
// is the name of the suite being tested
_, suiteName := filepath.Split(outputFile)
suiteName = strings.TrimSuffix(suiteName, ".suite")
// open the file
fileReader, err := os.Open(outputFile)
if err != nil {
// don't bomb out on a single bad file
pluginLogger.LogTask(slogger.ERROR, "Unable to open file '%v' for parsing: %v",
outputFile, err)
continue
}
defer fileReader.Close()
// parse the output logs
parser := &VanillaParser{Suite: suiteName}
if err := parser.Parse(fileReader); err != nil {
// continue on error
pluginLogger.LogTask(slogger.ERROR, "Error parsing file '%v': %v",
outputFile, err)
continue
}
// build up the test logs
logLines := parser.Logs()
testLog := model.TestLog{
Name: suiteName,
Task: taskConfig.Task.Id,
TaskExecution: taskConfig.Task.Execution,
Lines: logLines,
}
// save the results
results = append(results, parser.Results())
logs = append(logs, testLog)
}
return logs, results, nil
}
func (jsc *JSONSendCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {
if jsc.File == "" {
return fmt.Errorf("'file' param must not be blank")
}
if jsc.DataName == "" {
return fmt.Errorf("'name' param must not be blank")
}
errChan := make(chan error)
go func() {
// attempt to open the file
fileLoc := filepath.Join(conf.WorkDir, jsc.File)
jsonFile, err := os.Open(fileLoc)
if err != nil {
errChan <- fmt.Errorf("Couldn't open json file: '%v'", err)
return
}
jsonData := map[string]interface{}{}
err = util.ReadJSONInto(jsonFile, &jsonData)
if err != nil {
errChan <- fmt.Errorf("File contained invalid json: %v", err)
return
}
retriablePost := util.RetriableFunc(
func() error {
log.LogTask(slogger.INFO, "Posting JSON")
resp, err := com.TaskPostJSON(fmt.Sprintf("data/%v", jsc.DataName), jsonData)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
return util.RetriableError{err}
}
if resp.StatusCode != http.StatusOK {
return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)}
}
return nil
},
)
_, err = util.Retry(retriablePost, 10, 3*time.Second)
errChan <- err
}()
select {
case err := <-errChan:
if err != nil {
log.LogTask(slogger.ERROR, "Sending json data failed: %v", err)
}
return err
case <-stop:
log.LogExecution(slogger.INFO, "Received abort signal, stopping.")
return nil
}
}
// Execute fetches the expansions from the API server
func (self *FetchVarsCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator,
conf *model.TaskConfig,
stop chan bool) error {
pluginLogger.LogTask(slogger.ERROR, "Expansions.fetch deprecated")
return nil
}
// SendJSONLogs is responsible for sending the specified logs
// to the API Server. If successful, it returns a log ID that can be used
// to refer to the log object in test results.
func SendJSONLogs(taskConfig *model.TaskConfig, pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, logs *model.TestLog) (string, error) {
pluginLogger.LogExecution(slogger.INFO, "Attaching test logs for %v", logs.Name)
logId, err := pluginCom.TaskPostTestLog(logs)
if err != nil {
return "", err
}
pluginLogger.LogTask(slogger.INFO, "Attach test logs succeeded")
return logId, nil
}
func (self *AttachXUnitResultsCommand) parseAndUploadResults(
taskConfig *model.TaskConfig, pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator) error {
tests := []model.TestResult{}
logs := []*model.TestLog{}
logIdxToTestIdx := []int{}
reportFilePaths, err := getFilePaths(taskConfig.WorkDir, self.File)
if err != nil {
return err
}
for _, reportFileLoc := range reportFilePaths {
file, err := os.Open(reportFileLoc)
if err != nil {
return fmt.Errorf("couldn't open xunit file: '%v'", err)
}
testSuites, err := xunit.ParseXMLResults(file)
if err != nil {
return fmt.Errorf("error parsing xunit file: '%v'", err)
}
err = file.Close()
if err != nil {
return fmt.Errorf("error closing xunit file: '%v'", err)
}
// go through all the tests
for _, suite := range testSuites {
for _, tc := range suite.TestCases {
// logs are only created when a test case does not succeed
test, log := tc.ToModelTestResultAndLog(taskConfig.Task)
if log != nil {
logs = append(logs, log)
logIdxToTestIdx = append(logIdxToTestIdx, len(tests))
}
tests = append(tests, test)
}
}
}
for i, log := range logs {
logId, err := SendJSONLogs(taskConfig, pluginLogger, pluginCom, log)
if err != nil {
pluginLogger.LogTask(slogger.WARN, "Error uploading logs for %v", log.Name)
continue
}
tests[logIdxToTestIdx[i]].LogId = logId
tests[logIdxToTestIdx[i]].LineNum = 1
}
return SendJSONResults(taskConfig, pluginLogger, pluginCom, &model.TestResults{tests})
}
// SendJSONResults is responsible for sending the
// specified file to the API Server
func SendJSONResults(taskConfig *model.TaskConfig,
pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator,
results *model.TestResults) error {
pluginLogger.LogExecution(slogger.INFO, "Attaching test results")
err := pluginCom.TaskPostResults(results)
if err != nil {
return err
}
pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded")
return nil
}
// Execute carries out the AttachResultsCommand command - this is required
// to satisfy the 'Command' interface
func (self *AttachTaskFilesCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator,
taskConfig *model.TaskConfig,
stop chan bool) error {
if err := self.expandAttachTaskFilesCommand(taskConfig); err != nil {
msg := fmt.Sprintf("error expanding params: %v", err)
pluginLogger.LogTask(slogger.ERROR, "Error updating task files: %v", msg)
return fmt.Errorf(msg)
}
pluginLogger.LogTask(slogger.INFO, "Sending task file links to server")
return self.SendTaskFiles(taskConfig, pluginLogger, pluginCom)
}
// Implementation of Execute. Expands the parameters, and then fetches the
// resource from s3.
func (self *S3GetCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, conf *model.TaskConfig,
stop chan bool) error {
// expand necessary params
if err := self.expandParams(conf); err != nil {
return err
}
// validate the params
if err := self.validateParams(); err != nil {
return fmt.Errorf("expanded params are not valid: %v", err)
}
if !self.shouldRunForVariant(conf.BuildVariant.Name) {
pluginLogger.LogTask(slogger.INFO, "Skipping S3 get of remote file %v for variant %v",
self.RemoteFile,
conf.BuildVariant.Name)
return nil
}
// if the local file or extract_to is a relative path, join it to the
// working dir
if self.LocalFile != "" && !filepath.IsAbs(self.LocalFile) {
self.LocalFile = filepath.Join(conf.WorkDir, self.LocalFile)
}
if self.ExtractTo != "" && !filepath.IsAbs(self.ExtractTo) {
self.ExtractTo = filepath.Join(conf.WorkDir, self.ExtractTo)
}
errChan := make(chan error)
go func() {
errChan <- self.GetWithRetry(pluginLogger)
}()
select {
case err := <-errChan:
return err
case <-stop:
pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
" execution of S3 Get Command")
return nil
}
}
// Execute updates the expansions. Fulfills Command interface.
func (self *UpdateCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {
err := self.ExecuteUpdates(conf)
if err != nil {
return err
}
if self.YamlFile != "" {
pluginLogger.LogTask(slogger.INFO, "Updating expansions with keys from file: %v", self.YamlFile)
filename := filepath.Join(conf.WorkDir, self.YamlFile)
err := conf.Expansions.UpdateFromYaml(filename)
if err != nil {
return err
}
}
return nil
}
// Implementation of Execute. Expands the parameters, and then puts the
// resource to s3.
func (s3pc *S3PutCommand) Execute(log plugin.Logger,
com plugin.PluginCommunicator, conf *model.TaskConfig,
stop chan bool) error {
// expand necessary params
if err := s3pc.expandParams(conf); err != nil {
return err
}
// validate the params
if err := s3pc.validateParams(); err != nil {
return fmt.Errorf("expanded params are not valid: %v", err)
}
if !s3pc.shouldRunForVariant(conf.BuildVariant.Name) {
log.LogTask(slogger.INFO, "Skipping S3 put of local file %v for variant %v",
s3pc.LocalFile,
conf.BuildVariant.Name)
return nil
}
// if the local file is a relative path, join it to the work dir
if !filepath.IsAbs(s3pc.LocalFile) {
s3pc.LocalFile = filepath.Join(conf.WorkDir, s3pc.LocalFile)
}
log.LogTask(slogger.INFO, "Putting %v into path %v in s3 bucket %v",
s3pc.LocalFile, s3pc.RemoteFile, s3pc.Bucket)
errChan := make(chan error)
go func() {
errChan <- s3pc.PutWithRetry(log, com)
}()
select {
case err := <-errChan:
return err
case <-stop:
log.LogExecution(slogger.INFO, "Received signal to terminate execution of S3 Put Command")
return nil
}
}
// SendJSONResults is responsible for sending the
// specified file to the API Server
func SendJSONResults(taskConfig *model.TaskConfig,
pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator,
results *task.TestResults) error {
for i, res := range results.Results {
if res.LogRaw != "" {
pluginLogger.LogExecution(slogger.INFO, "Attaching raw test logs")
testLogs := &model.TestLog{
Name: res.TestFile,
Task: taskConfig.Task.Id,
TaskExecution: taskConfig.Task.Execution,
Lines: []string{res.LogRaw},
}
id, err := pluginCom.TaskPostTestLog(testLogs)
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error posting raw logs from results: %v", err)
} else {
results.Results[i].LogId = id
}
// clear the logs from the TestResult struct after it has been saved in the test logs. Since they are
// being saved in the test_logs collection, we can clear them to prevent them from being saved in the task
// collection.
results.Results[i].LogRaw = ""
}
}
pluginLogger.LogExecution(slogger.INFO, "Attaching test results")
err := pluginCom.TaskPostResults(results)
if err != nil {
return err
}
pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded")
return nil
}
// Wrapper around the Get() function to retry it
func (self *S3GetCommand) GetWithRetry(pluginLogger plugin.Logger) error {
retriableGet := util.RetriableFunc(
func() error {
pluginLogger.LogTask(slogger.INFO, "Fetching %v from"+
" s3 bucket %v", self.RemoteFile, self.Bucket)
err := self.Get()
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error getting from"+
" s3 bucket: %v", err)
return util.RetriableError{err}
}
return nil
},
)
retryFail, err := util.RetryArithmeticBackoff(retriableGet,
MaxS3GetAttempts, S3GetSleep)
if retryFail {
pluginLogger.LogExecution(slogger.ERROR, "S3 get failed with error: %v",
err)
return err
}
return nil
}
func cleanup(key string, log plugin.Logger) error {
pids, err := listProc()
if err != nil {
return err
}
pidMarker := fmt.Sprintf("EVR_AGENT_PID=%v", os.Getpid())
taskMarker := fmt.Sprintf("EVR_TASK_ID=%v", key)
for _, pid := range pids {
env, err := getEnv(pid)
if err != nil {
continue
}
if envHasMarkers(env, pidMarker, taskMarker) {
p := os.Process{}
p.Pid = pid
if err := p.Kill(); err != nil {
log.LogSystem(slogger.INFO, "Cleanup killing %v failed: %v", pid, err)
} else {
log.LogTask(slogger.INFO, "Cleanup killed process %v", pid)
}
}
}
return nil
}
// SendJSONResults is responsible for sending the
// specified file to the API Server
func (self *AttachTaskFilesCommand) SendTaskFiles(taskConfig *model.TaskConfig,
pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator) error {
// log each file attachment
for name, link := range self.Files {
pluginLogger.LogTask(slogger.INFO,
"Attaching file: %v -> %v", name, link)
}
retriableSendFile := util.RetriableFunc(
func() error {
resp, err := pluginCom.TaskPostJSON(
AttachTaskFilesAPIEndpoint,
self.Files.Array(),
)
if resp != nil {
defer resp.Body.Close()
}
if resp != nil && resp.StatusCode == http.StatusConflict {
body, _ := ioutil.ReadAll(resp.Body)
msg := fmt.Sprintf(
"secret conflict while posting task files: %v", string(body))
pluginLogger.LogTask(slogger.ERROR, msg)
return fmt.Errorf(msg)
}
if resp != nil && resp.StatusCode == http.StatusBadRequest {
body, _ := ioutil.ReadAll(resp.Body)
msg := fmt.Sprintf(
"error posting task files (%v): %v", resp.StatusCode, string(body))
pluginLogger.LogTask(slogger.ERROR, msg)
return fmt.Errorf(msg)
}
if resp != nil && resp.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(resp.Body)
msg := fmt.Sprintf("error posting task files (%v): %v",
resp.StatusCode, string(body))
pluginLogger.LogExecution(slogger.WARN, msg)
return util.RetriableError{err}
}
if err != nil {
msg := fmt.Sprintf("error posting files: %v", err)
pluginLogger.LogExecution(slogger.WARN, msg)
return util.RetriableError{fmt.Errorf(msg)}
}
return nil
},
)
retryFail, err := util.Retry(retriableSendFile, AttachResultsPostRetries,
AttachResultsRetrySleepSec)
if retryFail {
return fmt.Errorf("Attach files failed after %v tries: %v",
AttachResultsPostRetries, err)
}
if err != nil {
return fmt.Errorf("Attach files failed: %v", err)
}
pluginLogger.LogExecution(slogger.INFO, "API attach files call succeeded")
return nil
}
// Execute parses the specified output files and sends the test results found in them
// back to the server.
func (pfCmd *ParseFilesCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
stop chan bool) error {
if err := plugin.ExpandValues(pfCmd, taskConfig.Expansions); err != nil {
msg := fmt.Sprintf("error expanding params: %v", err)
pluginLogger.LogTask(slogger.ERROR, "Error parsing gotest files: %v", msg)
return fmt.Errorf(msg)
}
// make sure the file patterns are relative to the task's working directory
for idx, file := range pfCmd.Files {
pfCmd.Files[idx] = filepath.Join(taskConfig.WorkDir, file)
}
// will be all files containing test results
outputFiles, err := pfCmd.AllOutputFiles()
if err != nil {
return fmt.Errorf("error obtaining names of output files: %v", err)
}
// make sure we're parsing something
if len(outputFiles) == 0 {
return fmt.Errorf("no files found to be parsed")
}
// parse all of the files
logs, results, err := ParseTestOutputFiles(outputFiles, stop, pluginLogger, taskConfig)
if err != nil {
return fmt.Errorf("error parsing output results: %v", err)
}
// ship all of the test logs off to the server
pluginLogger.LogTask(slogger.INFO, "Sending test logs to server...")
allResults := []TestResult{}
for idx, log := range logs {
logId := ""
if logId, err = pluginCom.TaskPostTestLog(&log); err != nil {
// continue on error to let the other logs be posted
pluginLogger.LogTask(slogger.ERROR, "Error posting log: %v", err)
}
// add all of the test results that correspond to that log to the
// full list of results
for _, result := range results[idx] {
result.LogId = logId
allResults = append(allResults, result)
}
}
pluginLogger.LogTask(slogger.INFO, "Finished posting logs to server")
// convert everything
resultsAsModel := ToModelTestResults(taskConfig.Task, allResults)
// ship the parsed results off to the server
pluginLogger.LogTask(slogger.INFO, "Sending parsed results to server...")
if err := pluginCom.TaskPostResults(&resultsAsModel); err != nil {
return fmt.Errorf("error posting parsed results to server: %v", err)
}
pluginLogger.LogTask(slogger.INFO, "Successfully sent parsed results to server")
return nil
}
func (self *RunTestCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
stop chan bool) error {
if err := plugin.ExpandValues(self, taskConfig.Expansions); err != nil {
msg := fmt.Sprintf("error expanding params: %v", err)
pluginLogger.LogTask(slogger.ERROR, "Error updating test configs: %v",
msg)
return fmt.Errorf(msg)
}
// define proper working directory
if self.WorkDir != "" {
self.WorkDir = filepath.Join(taskConfig.WorkDir, self.WorkDir)
} else {
self.WorkDir = taskConfig.WorkDir
}
pluginLogger.LogTask(slogger.INFO,
"Running tests with working dir '%v'", self.WorkDir)
if os.Getenv("GOPATH") == "" {
pluginLogger.LogTask(slogger.WARN, "No GOPATH; setting GOPATH to working dir")
err := os.Setenv("GOPATH", self.WorkDir)
if err != nil {
return err
}
}
var results []TestResult
allPassed := true
// run all tests, concat results. Hold onto failures until the end
for idx, test := range self.Tests {
// kill the execution if API server requests
select {
case <-stop:
return fmt.Errorf("command was stopped")
default:
// no stop signal
}
// update test directory
test.Dir = filepath.Join(self.WorkDir, test.Dir)
suiteName := getSuiteNameFromDir(idx, test.Dir)
parser := &VanillaParser{Suite: suiteName}
pluginLogger.LogTask(
slogger.INFO, "Running go test with '%v' in '%v'", test.Args, test.Dir)
if len(test.EnvironmentVariables) > 0 {
pluginLogger.LogTask(
slogger.INFO, "Adding environment variables to gotest: %#v",
test.EnvironmentVariables)
}
passed, err := RunAndParseTests(test, parser, pluginLogger, stop)
logLines := parser.Logs()
for _, log := range logLines {
pluginLogger.LogTask(slogger.INFO, ">>> %v", log)
}
pluginLogger.LogTask(slogger.INFO,
"Sending logs to API server (%v lines)", len(logLines))
testLog := &model.TestLog{
Name: suiteName,
Task: taskConfig.Task.Id,
TaskExecution: taskConfig.Task.Execution,
Lines: logLines,
}
logId, err := pluginCom.TaskPostTestLog(testLog)
if err != nil {
pluginLogger.LogTask(slogger.ERROR, "error posting test log: %v", err)
}
if passed != true {
allPassed = false
pluginLogger.LogTask(slogger.WARN, "Test suite failed, continuing...")
}
if err != nil {
pluginLogger.LogTask(slogger.ERROR,
"Error running and parsing test '%v': %v", test.Dir, err)
continue
}
// get the results of this individual test, and set the log id
// appropriately
testResults := parser.Results()
for _, result := range testResults {
result.LogId = logId
results = append(results, result)
}
}
pluginLogger.LogTask(slogger.INFO, "Sending go test results to server")
modelResults := ToModelTestResults(taskConfig.Task, results)
err := pluginCom.TaskPostResults(&modelResults)
if err != nil {
return fmt.Errorf("error posting results: %v", err)
}
if allPassed {
return nil
} else {
return fmt.Errorf("test failures")
}
}
func RunAndParseTests(conf TestConfig, parser Parser,
pluginLogger plugin.Logger, stop <-chan bool) (bool, error) {
originalDir, err := os.Getwd()
if err != nil {
return false, fmt.Errorf("could not get working directory: %v", err)
}
// cd to given folder, but reset at the end
if err = os.Chdir(conf.Dir); err != nil {
return false, fmt.Errorf(
"could not change working directory to %v from %v: %v",
conf.Dir, originalDir, err)
}
defer os.Chdir(originalDir)
cmd := exec.Command("sh", "-c", "go test -v "+conf.Args)
if len(conf.EnvironmentVariables) > 0 {
cmd = exec.Command("sh", "-c", strings.Join(conf.EnvironmentVariables, " ")+
" go test -v "+conf.Args)
}
// display stderr but continue in the unlikely case we can't get it
stderr, err := cmd.StderrPipe()
if err == nil {
go io.Copy(pluginLogger.GetTaskLogWriter(slogger.ERROR), stderr)
} else {
pluginLogger.LogTask(
slogger.WARN, "couldn't get stderr from command: %v", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return false, fmt.Errorf("could not read stdout: %v", err)
}
if err := cmd.Start(); err != nil {
return false, fmt.Errorf(
"could not start test 'go test -v %v': %v", conf.Args, err)
}
if err := parser.Parse(stdout); err != nil {
// log and keep going if we have trouble parsing; corrupt output
// does not mean that the tests are failing--let the exit code
// determine that!
pluginLogger.LogTask(slogger.ERROR, "error parsing test output: %v", err)
}
// wait in a goroutine, so we can
// kill running tests if they time out
waitErrChan := make(chan error)
go func() {
waitErrChan <- cmd.Wait()
}()
// wait for execution completion or a stop signal
select {
case <-stop:
cmd.Process.Kill()
return false, fmt.Errorf("command was stopped")
case err = <-waitErrChan:
// if an error is returned, the test either failed or failed to
// be copied properly. We must distinguish these below
if err != nil {
// if we finished but failed, return failure with no error
if cmd.ProcessState.Exited() && !cmd.ProcessState.Success() {
return false, nil
} else {
return false, fmt.Errorf("error ending test: %v", err)
}
}
return true, nil
}
}
