Ejemplo n.º 1
0
// ParseTestOutputFiles parses all of the files that are passed in, and returns the
// test logs and test results found within.
func ParseTestOutputFiles(outputFiles []string, stop chan bool,
	pluginLogger plugin.Logger, taskConfig *model.TaskConfig) ([]model.TestLog,
	[][]TestResult, error) {

	var results [][]TestResult
	var logs []model.TestLog

	// now, open all the files, and parse the test results
	for _, outputFile := range outputFiles {

		// kill the execution if API server requests
		select {
		case <-stop:
			return nil, nil, fmt.Errorf("command was stopped")
		default:
			// no stop signal
		}

		// assume that the name of the file, stripping off the ".suite" extension if present,
		// is the name of the suite being tested
		_, suiteName := filepath.Split(outputFile)
		suiteName = strings.TrimSuffix(suiteName, ".suite")

		// open the file
		fileReader, err := os.Open(outputFile)
		if err != nil {
			// don't bomb out on a single bad file
			pluginLogger.LogTask(slogger.ERROR, "Unable to open file '%v' for parsing: %v",
				outputFile, err)
			continue
		}
		defer fileReader.Close()

		// parse the output logs
		parser := &VanillaParser{Suite: suiteName}
		if err := parser.Parse(fileReader); err != nil {
			// continue on error
			pluginLogger.LogTask(slogger.ERROR, "Error parsing file '%v': %v",
				outputFile, err)
			continue
		}

		// build up the test logs
		logLines := parser.Logs()
		testLog := model.TestLog{
			Name:          suiteName,
			Task:          taskConfig.Task.Id,
			TaskExecution: taskConfig.Task.Execution,
			Lines:         logLines,
		}

		// save the results
		results = append(results, parser.Results())
		logs = append(logs, testLog)

	}

	return logs, results, nil

}
Ejemplo n.º 2
0
Archivo: json.go Proyecto: sr527/json
func (jsc *JSONSendCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {
	if jsc.File == "" {
		return fmt.Errorf("'file' param must not be blank")
	}
	if jsc.DataName == "" {
		return fmt.Errorf("'name' param must not be blank")
	}

	errChan := make(chan error)
	go func() {
		// attempt to open the file
		fileLoc := filepath.Join(conf.WorkDir, jsc.File)
		jsonFile, err := os.Open(fileLoc)
		if err != nil {
			errChan <- fmt.Errorf("Couldn't open json file: '%v'", err)
			return
		}

		jsonData := map[string]interface{}{}
		err = util.ReadJSONInto(jsonFile, &jsonData)
		if err != nil {
			errChan <- fmt.Errorf("File contained invalid json: %v", err)
			return
		}

		retriablePost := util.RetriableFunc(
			func() error {
				log.LogTask(slogger.INFO, "Posting JSON")
				resp, err := com.TaskPostJSON(fmt.Sprintf("data/%v", jsc.DataName), jsonData)
				if resp != nil {
					defer resp.Body.Close()
				}
				if err != nil {
					return util.RetriableError{err}
				}
				if resp.StatusCode != http.StatusOK {
					return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)}
				}
				return nil
			},
		)

		_, err = util.Retry(retriablePost, 10, 3*time.Second)
		errChan <- err
	}()

	select {
	case err := <-errChan:
		if err != nil {
			log.LogTask(slogger.ERROR, "Sending json data failed: %v", err)
		}
		return err
	case <-stop:
		log.LogExecution(slogger.INFO, "Received abort signal, stopping.")
		return nil
	}
}
Ejemplo n.º 3
0
// Execute fetches the expansions from the API server
func (self *FetchVarsCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	pluginLogger.LogTask(slogger.ERROR, "Expansions.fetch deprecated")
	return nil

}
Ejemplo n.º 4
0
// SendJSONLogs is responsible for sending the specified logs
// to the API Server. If successful, it returns a log ID that can be used
// to refer to the log object in test results.
func SendJSONLogs(taskConfig *model.TaskConfig, pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, logs *model.TestLog) (string, error) {
	pluginLogger.LogExecution(slogger.INFO, "Attaching test logs for %v", logs.Name)
	logId, err := pluginCom.TaskPostTestLog(logs)
	if err != nil {
		return "", err
	}
	pluginLogger.LogTask(slogger.INFO, "Attach test logs succeeded")
	return logId, nil
}
Ejemplo n.º 5
0
func (self *AttachXUnitResultsCommand) parseAndUploadResults(
	taskConfig *model.TaskConfig, pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator) error {
	tests := []model.TestResult{}
	logs := []*model.TestLog{}
	logIdxToTestIdx := []int{}

	reportFilePaths, err := getFilePaths(taskConfig.WorkDir, self.File)
	if err != nil {
		return err
	}

	for _, reportFileLoc := range reportFilePaths {
		file, err := os.Open(reportFileLoc)
		if err != nil {
			return fmt.Errorf("couldn't open xunit file: '%v'", err)
		}

		testSuites, err := xunit.ParseXMLResults(file)
		if err != nil {
			return fmt.Errorf("error parsing xunit file: '%v'", err)
		}

		err = file.Close()
		if err != nil {
			return fmt.Errorf("error closing xunit file: '%v'", err)
		}

		// go through all the tests
		for _, suite := range testSuites {
			for _, tc := range suite.TestCases {
				// logs are only created when a test case does not succeed
				test, log := tc.ToModelTestResultAndLog(taskConfig.Task)
				if log != nil {
					logs = append(logs, log)
					logIdxToTestIdx = append(logIdxToTestIdx, len(tests))
				}
				tests = append(tests, test)
			}
		}
	}

	for i, log := range logs {
		logId, err := SendJSONLogs(taskConfig, pluginLogger, pluginCom, log)
		if err != nil {
			pluginLogger.LogTask(slogger.WARN, "Error uploading logs for %v", log.Name)
			continue
		}
		tests[logIdxToTestIdx[i]].LogId = logId
		tests[logIdxToTestIdx[i]].LineNum = 1
	}

	return SendJSONResults(taskConfig, pluginLogger, pluginCom, &model.TestResults{tests})
}
Ejemplo n.º 6
0
// SendJSONResults is responsible for sending the
// specified file to the API Server
func SendJSONResults(taskConfig *model.TaskConfig,
	pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator,
	results *model.TestResults) error {

	pluginLogger.LogExecution(slogger.INFO, "Attaching test results")
	err := pluginCom.TaskPostResults(results)
	if err != nil {
		return err
	}

	pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded")
	return nil
}
Ejemplo n.º 7
0
// Execute carries out the AttachResultsCommand command - this is required
// to satisfy the 'Command' interface
func (self *AttachTaskFilesCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	taskConfig *model.TaskConfig,
	stop chan bool) error {

	if err := self.expandAttachTaskFilesCommand(taskConfig); err != nil {
		msg := fmt.Sprintf("error expanding params: %v", err)
		pluginLogger.LogTask(slogger.ERROR, "Error updating task files: %v", msg)
		return fmt.Errorf(msg)
	}

	pluginLogger.LogTask(slogger.INFO, "Sending task file links to server")
	return self.SendTaskFiles(taskConfig, pluginLogger, pluginCom)
}
Ejemplo n.º 8
0
// Implementation of Execute.  Expands the parameters, and then fetches the
// resource from s3.
func (self *S3GetCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, conf *model.TaskConfig,
	stop chan bool) error {

	// expand necessary params
	if err := self.expandParams(conf); err != nil {
		return err
	}

	// validate the params
	if err := self.validateParams(); err != nil {
		return fmt.Errorf("expanded params are not valid: %v", err)
	}

	if !self.shouldRunForVariant(conf.BuildVariant.Name) {
		pluginLogger.LogTask(slogger.INFO, "Skipping S3 get of remote file %v for variant %v",
			self.RemoteFile,
			conf.BuildVariant.Name)
		return nil
	}

	// if the local file or extract_to is a relative path, join it to the
	// working dir
	if self.LocalFile != "" && !filepath.IsAbs(self.LocalFile) {
		self.LocalFile = filepath.Join(conf.WorkDir, self.LocalFile)
	}
	if self.ExtractTo != "" && !filepath.IsAbs(self.ExtractTo) {
		self.ExtractTo = filepath.Join(conf.WorkDir, self.ExtractTo)
	}

	errChan := make(chan error)
	go func() {
		errChan <- self.GetWithRetry(pluginLogger)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of S3 Get Command")
		return nil
	}

}
Ejemplo n.º 9
0
// Execute updates the expansions. Fulfills Command interface.
func (self *UpdateCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {

	err := self.ExecuteUpdates(conf)
	if err != nil {
		return err
	}

	if self.YamlFile != "" {
		pluginLogger.LogTask(slogger.INFO, "Updating expansions with keys from file: %v", self.YamlFile)
		filename := filepath.Join(conf.WorkDir, self.YamlFile)
		err := conf.Expansions.UpdateFromYaml(filename)
		if err != nil {
			return err
		}
	}
	return nil

}
Ejemplo n.º 10
0
// Implementation of Execute.  Expands the parameters, and then puts the
// resource to s3.
func (s3pc *S3PutCommand) Execute(log plugin.Logger,
	com plugin.PluginCommunicator, conf *model.TaskConfig,
	stop chan bool) error {

	// expand necessary params
	if err := s3pc.expandParams(conf); err != nil {
		return err
	}

	// validate the params
	if err := s3pc.validateParams(); err != nil {
		return fmt.Errorf("expanded params are not valid: %v", err)
	}

	if !s3pc.shouldRunForVariant(conf.BuildVariant.Name) {
		log.LogTask(slogger.INFO, "Skipping S3 put of local file %v for variant %v",
			s3pc.LocalFile,
			conf.BuildVariant.Name)
		return nil
	}

	// if the local file is a relative path, join it to the work dir
	if !filepath.IsAbs(s3pc.LocalFile) {
		s3pc.LocalFile = filepath.Join(conf.WorkDir, s3pc.LocalFile)
	}

	log.LogTask(slogger.INFO, "Putting %v into path %v in s3 bucket %v",
		s3pc.LocalFile, s3pc.RemoteFile, s3pc.Bucket)

	errChan := make(chan error)
	go func() {
		errChan <- s3pc.PutWithRetry(log, com)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		log.LogExecution(slogger.INFO, "Received signal to terminate execution of S3 Put Command")
		return nil
	}

}
Ejemplo n.º 11
0
// SendJSONResults is responsible for sending the
// specified file to the API Server
func SendJSONResults(taskConfig *model.TaskConfig,
	pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator,
	results *task.TestResults) error {
	for i, res := range results.Results {

		if res.LogRaw != "" {
			pluginLogger.LogExecution(slogger.INFO, "Attaching raw test logs")
			testLogs := &model.TestLog{
				Name:          res.TestFile,
				Task:          taskConfig.Task.Id,
				TaskExecution: taskConfig.Task.Execution,
				Lines:         []string{res.LogRaw},
			}

			id, err := pluginCom.TaskPostTestLog(testLogs)
			if err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Error posting raw logs from results: %v", err)
			} else {
				results.Results[i].LogId = id
			}

			// clear the logs from the TestResult struct after it has been saved in the test logs. Since they are
			// being saved in the test_logs collection, we can clear them to prevent them from being saved in the task
			// collection.
			results.Results[i].LogRaw = ""

		}
	}

	pluginLogger.LogExecution(slogger.INFO, "Attaching test results")
	err := pluginCom.TaskPostResults(results)
	if err != nil {
		return err
	}

	pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded")
	return nil
}
Ejemplo n.º 12
0
// Wrapper around the Get() function to retry it
func (self *S3GetCommand) GetWithRetry(pluginLogger plugin.Logger) error {
	retriableGet := util.RetriableFunc(
		func() error {
			pluginLogger.LogTask(slogger.INFO, "Fetching %v from"+
				" s3 bucket %v", self.RemoteFile, self.Bucket)
			err := self.Get()
			if err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Error getting from"+
					" s3 bucket: %v", err)
				return util.RetriableError{err}
			}
			return nil
		},
	)

	retryFail, err := util.RetryArithmeticBackoff(retriableGet,
		MaxS3GetAttempts, S3GetSleep)
	if retryFail {
		pluginLogger.LogExecution(slogger.ERROR, "S3 get failed with error: %v",
			err)
		return err
	}
	return nil
}
Ejemplo n.º 13
0
func cleanup(key string, log plugin.Logger) error {
	pids, err := listProc()
	if err != nil {
		return err
	}
	pidMarker := fmt.Sprintf("EVR_AGENT_PID=%v", os.Getpid())
	taskMarker := fmt.Sprintf("EVR_TASK_ID=%v", key)
	for _, pid := range pids {
		env, err := getEnv(pid)
		if err != nil {
			continue
		}
		if envHasMarkers(env, pidMarker, taskMarker) {
			p := os.Process{}
			p.Pid = pid
			if err := p.Kill(); err != nil {
				log.LogSystem(slogger.INFO, "Cleanup killing %v failed: %v", pid, err)
			} else {
				log.LogTask(slogger.INFO, "Cleanup killed process %v", pid)
			}
		}
	}
	return nil
}
Ejemplo n.º 14
0
// SendJSONResults is responsible for sending the
// specified file to the API Server
func (self *AttachTaskFilesCommand) SendTaskFiles(taskConfig *model.TaskConfig,
	pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator) error {

	// log each file attachment
	for name, link := range self.Files {
		pluginLogger.LogTask(slogger.INFO,
			"Attaching file: %v -> %v", name, link)
	}

	retriableSendFile := util.RetriableFunc(
		func() error {
			resp, err := pluginCom.TaskPostJSON(
				AttachTaskFilesAPIEndpoint,
				self.Files.Array(),
			)
			if resp != nil {
				defer resp.Body.Close()
			}
			if resp != nil && resp.StatusCode == http.StatusConflict {
				body, _ := ioutil.ReadAll(resp.Body)
				msg := fmt.Sprintf(
					"secret conflict while posting task files: %v", string(body))
				pluginLogger.LogTask(slogger.ERROR, msg)
				return fmt.Errorf(msg)
			}
			if resp != nil && resp.StatusCode == http.StatusBadRequest {
				body, _ := ioutil.ReadAll(resp.Body)
				msg := fmt.Sprintf(
					"error posting task files (%v): %v", resp.StatusCode, string(body))
				pluginLogger.LogTask(slogger.ERROR, msg)
				return fmt.Errorf(msg)
			}
			if resp != nil && resp.StatusCode != http.StatusOK {
				body, _ := ioutil.ReadAll(resp.Body)
				msg := fmt.Sprintf("error posting task files (%v): %v",
					resp.StatusCode, string(body))
				pluginLogger.LogExecution(slogger.WARN, msg)
				return util.RetriableError{err}
			}
			if err != nil {
				msg := fmt.Sprintf("error posting files: %v", err)
				pluginLogger.LogExecution(slogger.WARN, msg)
				return util.RetriableError{fmt.Errorf(msg)}
			}
			return nil
		},
	)

	retryFail, err := util.Retry(retriableSendFile, AttachResultsPostRetries,
		AttachResultsRetrySleepSec)

	if retryFail {
		return fmt.Errorf("Attach files failed after %v tries: %v",
			AttachResultsPostRetries, err)
	}
	if err != nil {
		return fmt.Errorf("Attach files failed: %v", err)
	}

	pluginLogger.LogExecution(slogger.INFO, "API attach files call succeeded")
	return nil
}
Ejemplo n.º 15
0
// Execute parses the specified output files and sends the test results found in them
// back to the server.
func (pfCmd *ParseFilesCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(pfCmd, taskConfig.Expansions); err != nil {
		msg := fmt.Sprintf("error expanding params: %v", err)
		pluginLogger.LogTask(slogger.ERROR, "Error parsing gotest files: %v", msg)
		return fmt.Errorf(msg)
	}

	// make sure the file patterns are relative to the task's working directory
	for idx, file := range pfCmd.Files {
		pfCmd.Files[idx] = filepath.Join(taskConfig.WorkDir, file)
	}

	// will be all files containing test results
	outputFiles, err := pfCmd.AllOutputFiles()
	if err != nil {
		return fmt.Errorf("error obtaining names of output files: %v", err)
	}

	// make sure we're parsing something
	if len(outputFiles) == 0 {
		return fmt.Errorf("no files found to be parsed")
	}

	// parse all of the files
	logs, results, err := ParseTestOutputFiles(outputFiles, stop, pluginLogger, taskConfig)
	if err != nil {
		return fmt.Errorf("error parsing output results: %v", err)
	}

	// ship all of the test logs off to the server
	pluginLogger.LogTask(slogger.INFO, "Sending test logs to server...")
	allResults := []TestResult{}
	for idx, log := range logs {

		logId := ""
		if logId, err = pluginCom.TaskPostTestLog(&log); err != nil {
			// continue on error to let the other logs be posted
			pluginLogger.LogTask(slogger.ERROR, "Error posting log: %v", err)
		}

		// add all of the test results that correspond to that log to the
		// full list of results
		for _, result := range results[idx] {
			result.LogId = logId
			allResults = append(allResults, result)
		}

	}
	pluginLogger.LogTask(slogger.INFO, "Finished posting logs to server")

	// convert everything
	resultsAsModel := ToModelTestResults(taskConfig.Task, allResults)

	// ship the parsed results off to the server
	pluginLogger.LogTask(slogger.INFO, "Sending parsed results to server...")
	if err := pluginCom.TaskPostResults(&resultsAsModel); err != nil {
		return fmt.Errorf("error posting parsed results to server: %v", err)
	}
	pluginLogger.LogTask(slogger.INFO, "Successfully sent parsed results to server")

	return nil

}
Ejemplo n.º 16
0
func (self *RunTestCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(self, taskConfig.Expansions); err != nil {
		msg := fmt.Sprintf("error expanding params: %v", err)
		pluginLogger.LogTask(slogger.ERROR, "Error updating test configs: %v",
			msg)
		return fmt.Errorf(msg)
	}

	// define proper working directory
	if self.WorkDir != "" {
		self.WorkDir = filepath.Join(taskConfig.WorkDir, self.WorkDir)
	} else {
		self.WorkDir = taskConfig.WorkDir
	}
	pluginLogger.LogTask(slogger.INFO,
		"Running tests with working dir '%v'", self.WorkDir)

	if os.Getenv("GOPATH") == "" {
		pluginLogger.LogTask(slogger.WARN, "No GOPATH; setting GOPATH to working dir")
		err := os.Setenv("GOPATH", self.WorkDir)
		if err != nil {
			return err
		}
	}

	var results []TestResult
	allPassed := true

	// run all tests, concat results. Hold onto failures until the end
	for idx, test := range self.Tests {
		// kill the execution if API server requests
		select {
		case <-stop:
			return fmt.Errorf("command was stopped")
		default:
			// no stop signal
		}

		// update test directory
		test.Dir = filepath.Join(self.WorkDir, test.Dir)
		suiteName := getSuiteNameFromDir(idx, test.Dir)

		parser := &VanillaParser{Suite: suiteName}
		pluginLogger.LogTask(
			slogger.INFO, "Running go test with '%v' in '%v'", test.Args, test.Dir)
		if len(test.EnvironmentVariables) > 0 {
			pluginLogger.LogTask(
				slogger.INFO, "Adding environment variables to gotest: %#v",
				test.EnvironmentVariables)
		}
		passed, err := RunAndParseTests(test, parser, pluginLogger, stop)

		logLines := parser.Logs()
		for _, log := range logLines {
			pluginLogger.LogTask(slogger.INFO, ">>> %v", log)
		}

		pluginLogger.LogTask(slogger.INFO,
			"Sending logs to API server (%v lines)", len(logLines))
		testLog := &model.TestLog{
			Name:          suiteName,
			Task:          taskConfig.Task.Id,
			TaskExecution: taskConfig.Task.Execution,
			Lines:         logLines,
		}

		logId, err := pluginCom.TaskPostTestLog(testLog)
		if err != nil {
			pluginLogger.LogTask(slogger.ERROR, "error posting test log: %v", err)
		}

		if passed != true {
			allPassed = false
			pluginLogger.LogTask(slogger.WARN, "Test suite failed, continuing...")
		}
		if err != nil {
			pluginLogger.LogTask(slogger.ERROR,
				"Error running and parsing test '%v': %v", test.Dir, err)
			continue
		}

		// get the results of this individual test, and set the log id
		// appropriately
		testResults := parser.Results()
		for _, result := range testResults {
			result.LogId = logId
			results = append(results, result)
		}
	}

	pluginLogger.LogTask(slogger.INFO, "Sending go test results to server")
	modelResults := ToModelTestResults(taskConfig.Task, results)
	err := pluginCom.TaskPostResults(&modelResults)
	if err != nil {
		return fmt.Errorf("error posting results: %v", err)
	}

	if allPassed {
		return nil
	} else {
		return fmt.Errorf("test failures")
	}
}
Ejemplo n.º 17
0
func RunAndParseTests(conf TestConfig, parser Parser,
	pluginLogger plugin.Logger, stop <-chan bool) (bool, error) {

	originalDir, err := os.Getwd()
	if err != nil {
		return false, fmt.Errorf("could not get working directory: %v", err)
	}

	// cd to given folder, but reset at the end
	if err = os.Chdir(conf.Dir); err != nil {
		return false, fmt.Errorf(
			"could not change working directory to %v from %v: %v",
			conf.Dir, originalDir, err)
	}
	defer os.Chdir(originalDir)

	cmd := exec.Command("sh", "-c", "go test -v "+conf.Args)
	if len(conf.EnvironmentVariables) > 0 {
		cmd = exec.Command("sh", "-c", strings.Join(conf.EnvironmentVariables, " ")+
			" go test -v "+conf.Args)
	}

	// display stderr but continue in the unlikely case we can't get it
	stderr, err := cmd.StderrPipe()
	if err == nil {
		go io.Copy(pluginLogger.GetTaskLogWriter(slogger.ERROR), stderr)
	} else {
		pluginLogger.LogTask(
			slogger.WARN, "couldn't get stderr from command: %v", err)
	}

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return false, fmt.Errorf("could not read stdout: %v", err)
	}
	if err := cmd.Start(); err != nil {
		return false, fmt.Errorf(
			"could not start test 'go test -v %v': %v", conf.Args, err)
	}
	if err := parser.Parse(stdout); err != nil {
		// log and keep going if we have trouble parsing; corrupt output
		// does not mean that the tests are failing--let the exit code
		// determine that!
		pluginLogger.LogTask(slogger.ERROR, "error parsing test output: %v", err)
	}

	// wait in a goroutine, so we can
	// kill running tests if they time out
	waitErrChan := make(chan error)
	go func() {
		waitErrChan <- cmd.Wait()
	}()

	// wait for execution completion or a stop signal
	select {
	case <-stop:
		cmd.Process.Kill()
		return false, fmt.Errorf("command was stopped")
	case err = <-waitErrChan:
		// if an error is returned, the test either failed or failed to
		// be copied properly. We must distinguish these below
		if err != nil {
			// if we finished but failed, return failure with no error
			if cmd.ProcessState.Exited() && !cmd.ProcessState.Success() {
				return false, nil
			} else {
				return false, fmt.Errorf("error ending test: %v", err)
			}
		}
		return true, nil
	}
}