Beispiel #1
0
// Execute carries out the S3CopyCommand command - this is required
// to satisfy the 'Command' interface
func (scc *S3CopyCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	taskConfig *model.TaskConfig,
	stop chan bool) error {

	// expand the S3 copy parameters before running the task
	if err := plugin.ExpandValues(scc, taskConfig.Expansions); err != nil {
		return err
	}

	// validate the S3 copy parameters before running the task
	if err := scc.validateS3CopyParams(); err != nil {
		return err
	}

	errChan := make(chan error)
	go func() {
		errChan <- scc.S3Copy(taskConfig, pluginLogger, pluginCom)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of S3 copy command")
		return nil
	}
}
Beispiel #2
0
// Execute fetches the expansions from the API server
func (incCmd *IncCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, conf *model.TaskConfig,
	stop chan bool) error {

	err := plugin.ExpandValues(incCmd, conf.Expansions)
	if err != nil {
		return err
	}

	keyVal := &KeyVal{}
	resp, err := pluginCom.TaskPostJSON(IncRoute, incCmd.Key)
	if err != nil {
		return err
	}
	if resp == nil {
		return fmt.Errorf("received nil response from inc API call")
	} else {
		defer resp.Body.Close()
	}
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("unexpected status code: %v", resp.StatusCode)
	}

	err = util.ReadJSONInto(resp.Body, keyVal)
	if err != nil {
		return fmt.Errorf("Failed to read JSON reply: %v", err)
	}

	conf.Expansions.Put(incCmd.Destination, fmt.Sprintf("%d", keyVal.Value))
	return nil
}
Beispiel #3
0
func (jgc *JSONHistoryCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {
	err := plugin.ExpandValues(jgc, conf.Expansions)
	if err != nil {
		return err
	}

	if jgc.File == "" {
		return fmt.Errorf("'file' param must not be blank")
	}
	if jgc.DataName == "" {
		return fmt.Errorf("'name' param must not be blank")
	}
	if jgc.TaskName == "" {
		return fmt.Errorf("'task' param must not be blank")
	}

	if jgc.File != "" && !filepath.IsAbs(jgc.File) {
		jgc.File = filepath.Join(conf.WorkDir, jgc.File)
	}

	endpoint := fmt.Sprintf("history/%s/%s", jgc.TaskName, jgc.DataName)
	if jgc.Tags {
		endpoint = fmt.Sprintf("tags/%s/%s", jgc.TaskName, jgc.DataName)
	}

	retriableGet := util.RetriableFunc(
		func() error {
			resp, err := com.TaskGetJSON(endpoint)
			if resp != nil {
				defer resp.Body.Close()
			}
			if err != nil {
				//Some generic error trying to connect - try again
				log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
				return util.RetriableError{err}
			}

			if resp.StatusCode == http.StatusOK {
				jsonBytes, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					return err
				}
				return ioutil.WriteFile(jgc.File, jsonBytes, 0755)
			}
			if resp.StatusCode != http.StatusOK {
				if resp.StatusCode == http.StatusNotFound {
					return fmt.Errorf("No JSON data found")
				}
				return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)}
			}
			return nil
		},
	)
	_, err = util.Retry(retriableGet, 10, 3*time.Second)
	return err
}
Beispiel #4
0
// Execute builds the archive.
func (self *TarGzPackCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(self, conf.Expansions); err != nil {
		return fmt.Errorf("error expanding params: %v", err)
	}

	// if the source dir is a relative path, join it to the working dir
	if !filepath.IsAbs(self.SourceDir) {
		self.SourceDir = filepath.Join(conf.WorkDir, self.SourceDir)
	}

	// if the target is a relative path, join it to the working dir
	if !filepath.IsAbs(self.Target) {
		self.Target = filepath.Join(conf.WorkDir, self.Target)
	}

	errChan := make(chan error)
	filesArchived := -1
	go func() {
		var err error
		filesArchived, err = self.BuildArchive(conf.WorkDir, pluginLogger)
		errChan <- err
	}()

	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
		if filesArchived == 0 {
			deleteErr := os.Remove(self.Target)
			if deleteErr != nil {
				pluginLogger.LogExecution(slogger.INFO, "Error deleting empty archive: %v", deleteErr)
			}
		}
		return nil
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of targz pack command")
		return nil
	}

}
// Implementation of Execute, to unpack the archive.
func (self *TarGzUnpackCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(self, conf.Expansions); err != nil {
		return fmt.Errorf("error expanding params: %v", err)
	}

	errChan := make(chan error)
	go func() {
		errChan <- self.UnpackArchive()
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of targz unpack command")
		return nil
	}
}
Beispiel #6
0
// Apply the expansions from the relevant task config to all appropriate
// fields of the S3GetCommand.
func (self *S3GetCommand) expandParams(conf *model.TaskConfig) error {
	return plugin.ExpandValues(self, conf.Expansions)
}
// Execute parses the specified output files and sends the test results found in them
// back to the server.
func (pfCmd *ParseFilesCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(pfCmd, taskConfig.Expansions); err != nil {
		msg := fmt.Sprintf("error expanding params: %v", err)
		pluginLogger.LogTask(slogger.ERROR, "Error parsing gotest files: %v", msg)
		return fmt.Errorf(msg)
	}

	// make sure the file patterns are relative to the task's working directory
	for idx, file := range pfCmd.Files {
		pfCmd.Files[idx] = filepath.Join(taskConfig.WorkDir, file)
	}

	// will be all files containing test results
	outputFiles, err := pfCmd.AllOutputFiles()
	if err != nil {
		return fmt.Errorf("error obtaining names of output files: %v", err)
	}

	// make sure we're parsing something
	if len(outputFiles) == 0 {
		return fmt.Errorf("no files found to be parsed")
	}

	// parse all of the files
	logs, results, err := ParseTestOutputFiles(outputFiles, stop, pluginLogger, taskConfig)
	if err != nil {
		return fmt.Errorf("error parsing output results: %v", err)
	}

	// ship all of the test logs off to the server
	pluginLogger.LogTask(slogger.INFO, "Sending test logs to server...")
	allResults := []TestResult{}
	for idx, log := range logs {

		logId := ""
		if logId, err = pluginCom.TaskPostTestLog(&log); err != nil {
			// continue on error to let the other logs be posted
			pluginLogger.LogTask(slogger.ERROR, "Error posting log: %v", err)
		}

		// add all of the test results that correspond to that log to the
		// full list of results
		for _, result := range results[idx] {
			result.LogId = logId
			allResults = append(allResults, result)
		}

	}
	pluginLogger.LogTask(slogger.INFO, "Finished posting logs to server")

	// convert everything
	resultsAsModel := ToModelTestResults(taskConfig.Task, allResults)

	// ship the parsed results off to the server
	pluginLogger.LogTask(slogger.INFO, "Sending parsed results to server...")
	if err := pluginCom.TaskPostResults(&resultsAsModel); err != nil {
		return fmt.Errorf("error posting parsed results to server: %v", err)
	}
	pluginLogger.LogTask(slogger.INFO, "Successfully sent parsed results to server")

	return nil

}
Beispiel #8
0
func (self *RunTestCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, taskConfig *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(self, taskConfig.Expansions); err != nil {
		msg := fmt.Sprintf("error expanding params: %v", err)
		pluginLogger.LogTask(slogger.ERROR, "Error updating test configs: %v",
			msg)
		return fmt.Errorf(msg)
	}

	// define proper working directory
	if self.WorkDir != "" {
		self.WorkDir = filepath.Join(taskConfig.WorkDir, self.WorkDir)
	} else {
		self.WorkDir = taskConfig.WorkDir
	}
	pluginLogger.LogTask(slogger.INFO,
		"Running tests with working dir '%v'", self.WorkDir)

	if os.Getenv("GOPATH") == "" {
		pluginLogger.LogTask(slogger.WARN, "No GOPATH; setting GOPATH to working dir")
		err := os.Setenv("GOPATH", self.WorkDir)
		if err != nil {
			return err
		}
	}

	var results []TestResult
	allPassed := true

	// run all tests, concat results. Hold onto failures until the end
	for idx, test := range self.Tests {
		// kill the execution if API server requests
		select {
		case <-stop:
			return fmt.Errorf("command was stopped")
		default:
			// no stop signal
		}

		// update test directory
		test.Dir = filepath.Join(self.WorkDir, test.Dir)
		suiteName := getSuiteNameFromDir(idx, test.Dir)

		parser := &VanillaParser{Suite: suiteName}
		pluginLogger.LogTask(
			slogger.INFO, "Running go test with '%v' in '%v'", test.Args, test.Dir)
		if len(test.EnvironmentVariables) > 0 {
			pluginLogger.LogTask(
				slogger.INFO, "Adding environment variables to gotest: %#v",
				test.EnvironmentVariables)
		}
		passed, err := RunAndParseTests(test, parser, pluginLogger, stop)

		logLines := parser.Logs()
		for _, log := range logLines {
			pluginLogger.LogTask(slogger.INFO, ">>> %v", log)
		}

		pluginLogger.LogTask(slogger.INFO,
			"Sending logs to API server (%v lines)", len(logLines))
		testLog := &model.TestLog{
			Name:          suiteName,
			Task:          taskConfig.Task.Id,
			TaskExecution: taskConfig.Task.Execution,
			Lines:         logLines,
		}

		logId, err := pluginCom.TaskPostTestLog(testLog)
		if err != nil {
			pluginLogger.LogTask(slogger.ERROR, "error posting test log: %v", err)
		}

		if passed != true {
			allPassed = false
			pluginLogger.LogTask(slogger.WARN, "Test suite failed, continuing...")
		}
		if err != nil {
			pluginLogger.LogTask(slogger.ERROR,
				"Error running and parsing test '%v': %v", test.Dir, err)
			continue
		}

		// get the results of this individual test, and set the log id
		// appropriately
		testResults := parser.Results()
		for _, result := range testResults {
			result.LogId = logId
			results = append(results, result)
		}
	}

	pluginLogger.LogTask(slogger.INFO, "Sending go test results to server")
	modelResults := ToModelTestResults(taskConfig.Task, results)
	err := pluginCom.TaskPostResults(&modelResults)
	if err != nil {
		return fmt.Errorf("error posting results: %v", err)
	}

	if allPassed {
		return nil
	} else {
		return fmt.Errorf("test failures")
	}
}
// Execute gets the source code required by the project
func (ggpc *GitGetProjectCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	// expand the github parameters before running the task
	if err := plugin.ExpandValues(&ggpc.Revisions, conf.Expansions); err != nil {
		return err
	}

	location, err := conf.ProjectRef.Location()

	if err != nil {
		return err
	}

	gitCommands := []string{
		fmt.Sprintf("set -o errexit"),
		fmt.Sprintf("set -o verbose"),
		fmt.Sprintf("rm -rf %v", ggpc.Directory),
		fmt.Sprintf("git clone %v '%v'", location, ggpc.Directory),
		fmt.Sprintf("cd %v; git checkout %v", ggpc.Directory, conf.Task.Revision),
	}

	cmdsJoined := strings.Join(gitCommands, "\n")

	fetchSourceCmd := &command.LocalCommand{
		CmdString:        cmdsJoined,
		WorkingDirectory: conf.WorkDir,
		Stdout:           pluginLogger.GetTaskLogWriter(slogger.INFO),
		Stderr:           pluginLogger.GetTaskLogWriter(slogger.ERROR),
		ScriptMode:       true,
	}

	errChan := make(chan error)
	go func() {
		pluginLogger.LogExecution(slogger.INFO, "Fetching source from git...")
		errChan <- fetchSourceCmd.Run()
		pluginLogger.Flush()
	}()

	// wait until the command finishes or the stop channel is tripped
	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Got kill signal")
		if fetchSourceCmd.Cmd != nil {
			pluginLogger.LogExecution(slogger.INFO, "Stopping process: %v", fetchSourceCmd.Cmd.Process.Pid)
			if err := fetchSourceCmd.Stop(); err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Error occurred stopping process: %v", err)
			}
		}
		return fmt.Errorf("Fetch command interrupted.")
	}

	// Fetch source for the modules
	for _, moduleName := range conf.BuildVariant.Modules {
		pluginLogger.LogExecution(slogger.INFO, "Fetching module: %v", moduleName)
		module, err := conf.Project.GetModuleByName(moduleName)
		if err != nil {
			pluginLogger.LogExecution(slogger.ERROR, "Couldn't get module %v: %v",
				moduleName, err)
			continue
		}
		if module == nil {
			pluginLogger.LogExecution(slogger.ERROR, "No module found for %v",
				moduleName)
			continue
		}

		moduleBase := filepath.Join(module.Prefix, module.Name)
		moduleDir := filepath.Join(conf.WorkDir, moduleBase, "/_")

		err = os.MkdirAll(moduleDir, 0755)
		if err != nil {
			return err
		}
		// clear the destination
		err = os.RemoveAll(moduleDir)
		if err != nil {
			return err
		}

		revision := ggpc.Revisions[moduleName]

		// if there is no revision, then use the branch name
		if revision == "" {
			revision = module.Branch
		}

		moduleCmds := []string{
			fmt.Sprintf("set -o errexit"),
			fmt.Sprintf("set -o verbose"),
			fmt.Sprintf("git clone %v '%v'", module.Repo, filepath.ToSlash(moduleBase)),
			fmt.Sprintf("cd %v; git checkout '%v'", filepath.ToSlash(moduleBase), revision),
		}

		moduleFetchCmd := &command.LocalCommand{
			CmdString:        strings.Join(moduleCmds, "\n"),
			WorkingDirectory: filepath.ToSlash(filepath.Join(conf.WorkDir, ggpc.Directory)),
			Stdout:           pluginLogger.GetTaskLogWriter(slogger.INFO),
			Stderr:           pluginLogger.GetTaskLogWriter(slogger.ERROR),
			ScriptMode:       true,
		}

		go func() {
			errChan <- moduleFetchCmd.Run()
			pluginLogger.Flush()
		}()

		// wait until the command finishes or the stop channel is tripped
		select {
		case err := <-errChan:
			if err != nil {
				return err
			}
		case <-stop:
			pluginLogger.LogExecution(slogger.INFO, "Got kill signal")
			if moduleFetchCmd.Cmd != nil {
				pluginLogger.LogExecution(slogger.INFO, "Stopping process: %v", moduleFetchCmd.Cmd.Process.Pid)
				if err := moduleFetchCmd.Stop(); err != nil {
					pluginLogger.LogExecution(slogger.ERROR, "Error occurred stopping process: %v", err)
				}
			}
			return fmt.Errorf("Fetch module command interrupted.")
		}
	}

	return nil

}
Beispiel #10
0
// Apply the expansions from the relevant task config to all appropriate
// fields of the S3PutCommand.
func (s3pc *S3PutCommand) expandParams(conf *model.TaskConfig) error {
	return plugin.ExpandValues(s3pc, conf.Expansions)
}
Beispiel #11
0
func (self *AttachTaskFilesCommand) expandAttachTaskFilesCommand(
	taskConfig *model.TaskConfig) (err error) {
	return plugin.ExpandValues(&self.Files, taskConfig.Expansions)
}