// Execute carries out the S3CopyCommand command - this is required
// to satisfy the 'Command' interface
func (scc *S3CopyCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator,
taskConfig *model.TaskConfig,
stop chan bool) error {
// expand the S3 copy parameters before running the task
if err := plugin.ExpandValues(scc, taskConfig.Expansions); err != nil {
return err
}
// validate the S3 copy parameters before running the task
if err := scc.validateS3CopyParams(); err != nil {
return err
}
errChan := make(chan error)
go func() {
errChan <- scc.S3Copy(taskConfig, pluginLogger, pluginCom)
}()
select {
case err := <-errChan:
return err
case <-stop:
pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
" execution of S3 copy command")
return nil
}
}
// Execute fetches the expansions from the API server
func (incCmd *IncCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, conf *model.TaskConfig,
stop chan bool) error {
err := plugin.ExpandValues(incCmd, conf.Expansions)
if err != nil {
return err
}
keyVal := &KeyVal{}
resp, err := pluginCom.TaskPostJSON(IncRoute, incCmd.Key)
if err != nil {
return err
}
if resp == nil {
return fmt.Errorf("received nil response from inc API call")
} else {
defer resp.Body.Close()
}
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: %v", resp.StatusCode)
}
err = util.ReadJSONInto(resp.Body, keyVal)
if err != nil {
return fmt.Errorf("Failed to read JSON reply: %v", err)
}
conf.Expansions.Put(incCmd.Destination, fmt.Sprintf("%d", keyVal.Value))
return nil
}
func (jgc *JSONHistoryCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {
err := plugin.ExpandValues(jgc, conf.Expansions)
if err != nil {
return err
}
if jgc.File == "" {
return fmt.Errorf("'file' param must not be blank")
}
if jgc.DataName == "" {
return fmt.Errorf("'name' param must not be blank")
}
if jgc.TaskName == "" {
return fmt.Errorf("'task' param must not be blank")
}
if jgc.File != "" && !filepath.IsAbs(jgc.File) {
jgc.File = filepath.Join(conf.WorkDir, jgc.File)
}
endpoint := fmt.Sprintf("history/%s/%s", jgc.TaskName, jgc.DataName)
if jgc.Tags {
endpoint = fmt.Sprintf("tags/%s/%s", jgc.TaskName, jgc.DataName)
}
retriableGet := util.RetriableFunc(
func() error {
resp, err := com.TaskGetJSON(endpoint)
if resp != nil {
defer resp.Body.Close()
}
if err != nil {
//Some generic error trying to connect - try again
log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
return util.RetriableError{err}
}
if resp.StatusCode == http.StatusOK {
jsonBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return ioutil.WriteFile(jgc.File, jsonBytes, 0755)
}
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
return fmt.Errorf("No JSON data found")
}
return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)}
}
return nil
},
)
_, err = util.Retry(retriableGet, 10, 3*time.Second)
return err
}
// Execute builds the archive.
func (self *TarGzPackCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator,
conf *model.TaskConfig,
stop chan bool) error {
if err := plugin.ExpandValues(self, conf.Expansions); err != nil {
return fmt.Errorf("error expanding params: %v", err)
}
// if the source dir is a relative path, join it to the working dir
if !filepath.IsAbs(self.SourceDir) {
self.SourceDir = filepath.Join(conf.WorkDir, self.SourceDir)
}
// if the target is a relative path, join it to the working dir
if !filepath.IsAbs(self.Target) {
self.Target = filepath.Join(conf.WorkDir, self.Target)
}
errChan := make(chan error)
filesArchived := -1
go func() {
var err error
filesArchived, err = self.BuildArchive(conf.WorkDir, pluginLogger)
errChan <- err
}()
select {
case err := <-errChan:
if err != nil {
return err
}
if filesArchived == 0 {
deleteErr := os.Remove(self.Target)
if deleteErr != nil {
pluginLogger.LogExecution(slogger.INFO, "Error deleting empty archive: %v", deleteErr)
}
}
return nil
case <-stop:
pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
" execution of targz pack command")
return nil
}
}
// Implementation of Execute, to unpack the archive.
func (self *TarGzUnpackCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator,
conf *model.TaskConfig,
stop chan bool) error {
if err := plugin.ExpandValues(self, conf.Expansions); err != nil {
return fmt.Errorf("error expanding params: %v", err)
}
errChan := make(chan error)
go func() {
errChan <- self.UnpackArchive()
}()
select {
case err := <-errChan:
return err
case <-stop:
pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
" execution of targz unpack command")
return nil
}
}
// Apply the expansions from the relevant task config to all appropriate
// fields of the S3GetCommand.
func (self *S3GetCommand) expandParams(conf *model.TaskConfig) error {
return plugin.ExpandValues(self, conf.Expansions)
}
// Execute parses the specified output files and sends the test results found in them
// back to the server.
func (pfCmd *ParseFilesCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
stop chan bool) error {
if err := plugin.ExpandValues(pfCmd, taskConfig.Expansions); err != nil {
msg := fmt.Sprintf("error expanding params: %v", err)
pluginLogger.LogTask(slogger.ERROR, "Error parsing gotest files: %v", msg)
return fmt.Errorf(msg)
}
// make sure the file patterns are relative to the task's working directory
for idx, file := range pfCmd.Files {
pfCmd.Files[idx] = filepath.Join(taskConfig.WorkDir, file)
}
// will be all files containing test results
outputFiles, err := pfCmd.AllOutputFiles()
if err != nil {
return fmt.Errorf("error obtaining names of output files: %v", err)
}
// make sure we're parsing something
if len(outputFiles) == 0 {
return fmt.Errorf("no files found to be parsed")
}
// parse all of the files
logs, results, err := ParseTestOutputFiles(outputFiles, stop, pluginLogger, taskConfig)
if err != nil {
return fmt.Errorf("error parsing output results: %v", err)
}
// ship all of the test logs off to the server
pluginLogger.LogTask(slogger.INFO, "Sending test logs to server...")
allResults := []TestResult{}
for idx, log := range logs {
logId := ""
if logId, err = pluginCom.TaskPostTestLog(&log); err != nil {
// continue on error to let the other logs be posted
pluginLogger.LogTask(slogger.ERROR, "Error posting log: %v", err)
}
// add all of the test results that correspond to that log to the
// full list of results
for _, result := range results[idx] {
result.LogId = logId
allResults = append(allResults, result)
}
}
pluginLogger.LogTask(slogger.INFO, "Finished posting logs to server")
// convert everything
resultsAsModel := ToModelTestResults(taskConfig.Task, allResults)
// ship the parsed results off to the server
pluginLogger.LogTask(slogger.INFO, "Sending parsed results to server...")
if err := pluginCom.TaskPostResults(&resultsAsModel); err != nil {
return fmt.Errorf("error posting parsed results to server: %v", err)
}
pluginLogger.LogTask(slogger.INFO, "Successfully sent parsed results to server")
return nil
}
func (self *RunTestCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
stop chan bool) error {
if err := plugin.ExpandValues(self, taskConfig.Expansions); err != nil {
msg := fmt.Sprintf("error expanding params: %v", err)
pluginLogger.LogTask(slogger.ERROR, "Error updating test configs: %v",
msg)
return fmt.Errorf(msg)
}
// define proper working directory
if self.WorkDir != "" {
self.WorkDir = filepath.Join(taskConfig.WorkDir, self.WorkDir)
} else {
self.WorkDir = taskConfig.WorkDir
}
pluginLogger.LogTask(slogger.INFO,
"Running tests with working dir '%v'", self.WorkDir)
if os.Getenv("GOPATH") == "" {
pluginLogger.LogTask(slogger.WARN, "No GOPATH; setting GOPATH to working dir")
err := os.Setenv("GOPATH", self.WorkDir)
if err != nil {
return err
}
}
var results []TestResult
allPassed := true
// run all tests, concat results. Hold onto failures until the end
for idx, test := range self.Tests {
// kill the execution if API server requests
select {
case <-stop:
return fmt.Errorf("command was stopped")
default:
// no stop signal
}
// update test directory
test.Dir = filepath.Join(self.WorkDir, test.Dir)
suiteName := getSuiteNameFromDir(idx, test.Dir)
parser := &VanillaParser{Suite: suiteName}
pluginLogger.LogTask(
slogger.INFO, "Running go test with '%v' in '%v'", test.Args, test.Dir)
if len(test.EnvironmentVariables) > 0 {
pluginLogger.LogTask(
slogger.INFO, "Adding environment variables to gotest: %#v",
test.EnvironmentVariables)
}
passed, err := RunAndParseTests(test, parser, pluginLogger, stop)
logLines := parser.Logs()
for _, log := range logLines {
pluginLogger.LogTask(slogger.INFO, ">>> %v", log)
}
pluginLogger.LogTask(slogger.INFO,
"Sending logs to API server (%v lines)", len(logLines))
testLog := &model.TestLog{
Name: suiteName,
Task: taskConfig.Task.Id,
TaskExecution: taskConfig.Task.Execution,
Lines: logLines,
}
logId, err := pluginCom.TaskPostTestLog(testLog)
if err != nil {
pluginLogger.LogTask(slogger.ERROR, "error posting test log: %v", err)
}
if passed != true {
allPassed = false
pluginLogger.LogTask(slogger.WARN, "Test suite failed, continuing...")
}
if err != nil {
pluginLogger.LogTask(slogger.ERROR,
"Error running and parsing test '%v': %v", test.Dir, err)
continue
}
// get the results of this individual test, and set the log id
// appropriately
testResults := parser.Results()
for _, result := range testResults {
result.LogId = logId
results = append(results, result)
}
}
pluginLogger.LogTask(slogger.INFO, "Sending go test results to server")
modelResults := ToModelTestResults(taskConfig.Task, results)
err := pluginCom.TaskPostResults(&modelResults)
if err != nil {
return fmt.Errorf("error posting results: %v", err)
}
if allPassed {
return nil
} else {
return fmt.Errorf("test failures")
}
}
// Execute gets the source code required by the project
func (ggpc *GitGetProjectCommand) Execute(pluginLogger plugin.Logger,
pluginCom plugin.PluginCommunicator,
conf *model.TaskConfig,
stop chan bool) error {
// expand the github parameters before running the task
if err := plugin.ExpandValues(&ggpc.Revisions, conf.Expansions); err != nil {
return err
}
location, err := conf.ProjectRef.Location()
if err != nil {
return err
}
gitCommands := []string{
fmt.Sprintf("set -o errexit"),
fmt.Sprintf("set -o verbose"),
fmt.Sprintf("rm -rf %v", ggpc.Directory),
fmt.Sprintf("git clone %v '%v'", location, ggpc.Directory),
fmt.Sprintf("cd %v; git checkout %v", ggpc.Directory, conf.Task.Revision),
}
cmdsJoined := strings.Join(gitCommands, "\n")
fetchSourceCmd := &command.LocalCommand{
CmdString: cmdsJoined,
WorkingDirectory: conf.WorkDir,
Stdout: pluginLogger.GetTaskLogWriter(slogger.INFO),
Stderr: pluginLogger.GetTaskLogWriter(slogger.ERROR),
ScriptMode: true,
}
errChan := make(chan error)
go func() {
pluginLogger.LogExecution(slogger.INFO, "Fetching source from git...")
errChan <- fetchSourceCmd.Run()
pluginLogger.Flush()
}()
// wait until the command finishes or the stop channel is tripped
select {
case err := <-errChan:
if err != nil {
return err
}
case <-stop:
pluginLogger.LogExecution(slogger.INFO, "Got kill signal")
if fetchSourceCmd.Cmd != nil {
pluginLogger.LogExecution(slogger.INFO, "Stopping process: %v", fetchSourceCmd.Cmd.Process.Pid)
if err := fetchSourceCmd.Stop(); err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error occurred stopping process: %v", err)
}
}
return fmt.Errorf("Fetch command interrupted.")
}
// Fetch source for the modules
for _, moduleName := range conf.BuildVariant.Modules {
pluginLogger.LogExecution(slogger.INFO, "Fetching module: %v", moduleName)
module, err := conf.Project.GetModuleByName(moduleName)
if err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Couldn't get module %v: %v",
moduleName, err)
continue
}
if module == nil {
pluginLogger.LogExecution(slogger.ERROR, "No module found for %v",
moduleName)
continue
}
moduleBase := filepath.Join(module.Prefix, module.Name)
moduleDir := filepath.Join(conf.WorkDir, moduleBase, "/_")
err = os.MkdirAll(moduleDir, 0755)
if err != nil {
return err
}
// clear the destination
err = os.RemoveAll(moduleDir)
if err != nil {
return err
}
revision := ggpc.Revisions[moduleName]
// if there is no revision, then use the branch name
if revision == "" {
revision = module.Branch
}
moduleCmds := []string{
fmt.Sprintf("set -o errexit"),
fmt.Sprintf("set -o verbose"),
fmt.Sprintf("git clone %v '%v'", module.Repo, filepath.ToSlash(moduleBase)),
fmt.Sprintf("cd %v; git checkout '%v'", filepath.ToSlash(moduleBase), revision),
}
moduleFetchCmd := &command.LocalCommand{
CmdString: strings.Join(moduleCmds, "\n"),
WorkingDirectory: filepath.ToSlash(filepath.Join(conf.WorkDir, ggpc.Directory)),
Stdout: pluginLogger.GetTaskLogWriter(slogger.INFO),
Stderr: pluginLogger.GetTaskLogWriter(slogger.ERROR),
ScriptMode: true,
}
go func() {
errChan <- moduleFetchCmd.Run()
pluginLogger.Flush()
}()
// wait until the command finishes or the stop channel is tripped
select {
case err := <-errChan:
if err != nil {
return err
}
case <-stop:
pluginLogger.LogExecution(slogger.INFO, "Got kill signal")
if moduleFetchCmd.Cmd != nil {
pluginLogger.LogExecution(slogger.INFO, "Stopping process: %v", moduleFetchCmd.Cmd.Process.Pid)
if err := moduleFetchCmd.Stop(); err != nil {
pluginLogger.LogExecution(slogger.ERROR, "Error occurred stopping process: %v", err)
}
}
return fmt.Errorf("Fetch module command interrupted.")
}
}
return nil
}
// Apply the expansions from the relevant task config to all appropriate
// fields of the S3PutCommand.
func (s3pc *S3PutCommand) expandParams(conf *model.TaskConfig) error {
return plugin.ExpandValues(s3pc, conf.Expansions)
}
func (self *AttachTaskFilesCommand) expandAttachTaskFilesCommand(
taskConfig *model.TaskConfig) (err error) {
return plugin.ExpandValues(&self.Files, taskConfig.Expansions)
}