Example #1
0
// Execute carries out the S3CopyCommand command - this is required
// to satisfy the 'Command' interface
func (scc *S3CopyCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	taskConfig *model.TaskConfig,
	stop chan bool) error {

	// expand the S3 copy parameters before running the task
	if err := plugin.ExpandValues(scc, taskConfig.Expansions); err != nil {
		return err
	}

	// validate the S3 copy parameters before running the task
	if err := scc.validateS3CopyParams(); err != nil {
		return err
	}

	errChan := make(chan error)
	go func() {
		errChan <- scc.S3Copy(taskConfig, pluginLogger, pluginCom)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of S3 copy command")
		return nil
	}
}
Example #2
0
// AttachTaskFiles is responsible for sending the
// specified file to the API Server
func (c *S3CopyCommand) AttachTaskFiles(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, request S3CopyRequest) error {

	remotePath := filepath.ToSlash(request.S3DestinationPath)
	fileLink := s3baseURL + request.S3DestinationBucket + "/" + remotePath

	displayName := request.S3DisplayName

	if displayName == "" {
		displayName = filepath.Base(request.S3SourcePath)
	}

	pluginLogger.LogExecution(slogger.INFO, "attaching file with name %v", displayName)
	file := artifact.File{
		Name: displayName,
		Link: fileLink,
	}

	files := []*artifact.File{&file}

	err := pluginCom.PostTaskFiles(files)
	if err != nil {
		return fmt.Errorf("Attach files failed: %v", err)
	}
	pluginLogger.LogExecution(slogger.INFO, "API attach files call succeeded")
	return nil
}
// Execute builds the archive.
func (self *TarGzPackCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	// if the source dir is a relative path, join it to the working dir
	if !filepath.IsAbs(self.SourceDir) {
		self.SourceDir = filepath.Join(conf.WorkDir, self.SourceDir)
	}

	// if the target is a relative path, join it to the working dir
	if !filepath.IsAbs(self.Target) {
		self.Target = filepath.Join(conf.WorkDir, self.Target)
	}

	errChan := make(chan error)
	go func() {
		errChan <- self.BuildArchive(conf.WorkDir, pluginLogger)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of targz pack command")
		return nil
	}

}
Example #4
0
File: json.go Project: sr527/json
func (jsc *JSONSendCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {
	if jsc.File == "" {
		return fmt.Errorf("'file' param must not be blank")
	}
	if jsc.DataName == "" {
		return fmt.Errorf("'name' param must not be blank")
	}

	errChan := make(chan error)
	go func() {
		// attempt to open the file
		fileLoc := filepath.Join(conf.WorkDir, jsc.File)
		jsonFile, err := os.Open(fileLoc)
		if err != nil {
			errChan <- fmt.Errorf("Couldn't open json file: '%v'", err)
			return
		}

		jsonData := map[string]interface{}{}
		err = util.ReadJSONInto(jsonFile, &jsonData)
		if err != nil {
			errChan <- fmt.Errorf("File contained invalid json: %v", err)
			return
		}

		retriablePost := util.RetriableFunc(
			func() error {
				log.LogTask(slogger.INFO, "Posting JSON")
				resp, err := com.TaskPostJSON(fmt.Sprintf("data/%v", jsc.DataName), jsonData)
				if resp != nil {
					defer resp.Body.Close()
				}
				if err != nil {
					return util.RetriableError{err}
				}
				if resp.StatusCode != http.StatusOK {
					return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)}
				}
				return nil
			},
		)

		_, err = util.Retry(retriablePost, 10, 3*time.Second)
		errChan <- err
	}()

	select {
	case err := <-errChan:
		if err != nil {
			log.LogTask(slogger.ERROR, "Sending json data failed: %v", err)
		}
		return err
	case <-stop:
		log.LogExecution(slogger.INFO, "Received abort signal, stopping.")
		return nil
	}
}
Example #5
0
File: json.go Project: sr527/json
func (jgc *JSONHistoryCommand) Execute(log plugin.Logger, com plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {
	err := plugin.ExpandValues(jgc, conf.Expansions)
	if err != nil {
		return err
	}

	if jgc.File == "" {
		return fmt.Errorf("'file' param must not be blank")
	}
	if jgc.DataName == "" {
		return fmt.Errorf("'name' param must not be blank")
	}
	if jgc.TaskName == "" {
		return fmt.Errorf("'task' param must not be blank")
	}

	if jgc.File != "" && !filepath.IsAbs(jgc.File) {
		jgc.File = filepath.Join(conf.WorkDir, jgc.File)
	}

	endpoint := fmt.Sprintf("history/%s/%s", jgc.TaskName, jgc.DataName)
	if jgc.Tags {
		endpoint = fmt.Sprintf("tags/%s/%s", jgc.TaskName, jgc.DataName)
	}

	retriableGet := util.RetriableFunc(
		func() error {
			resp, err := com.TaskGetJSON(endpoint)
			if resp != nil {
				defer resp.Body.Close()
			}
			if err != nil {
				//Some generic error trying to connect - try again
				log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
				return util.RetriableError{err}
			}

			if resp.StatusCode == http.StatusOK {
				jsonBytes, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					return err
				}
				return ioutil.WriteFile(jgc.File, jsonBytes, 0755)
			}
			if resp.StatusCode != http.StatusOK {
				if resp.StatusCode == http.StatusNotFound {
					return fmt.Errorf("No JSON data found")
				}
				return util.RetriableError{fmt.Errorf("unexpected status code %v", resp.StatusCode)}
			}
			return nil
		},
	)
	_, err = util.Retry(retriableGet, 10, 3*time.Second)
	return err
}
Example #6
0
// SendJSONLogs is responsible for sending the specified logs
// to the API Server. If successful, it returns a log ID that can be used
// to refer to the log object in test results.
func SendJSONLogs(taskConfig *model.TaskConfig, pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, logs *model.TestLog) (string, error) {
	pluginLogger.LogExecution(slogger.INFO, "Attaching test logs for %v", logs.Name)
	logId, err := pluginCom.TaskPostTestLog(logs)
	if err != nil {
		return "", err
	}
	pluginLogger.LogTask(slogger.INFO, "Attach test logs succeeded")
	return logId, nil
}
Example #7
0
// SendJSONResults is responsible for sending the
// specified file to the API Server
func SendJSONResults(taskConfig *model.TaskConfig,
	pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator,
	results *model.TestResults) error {

	pluginLogger.LogExecution(slogger.INFO, "Attaching test results")
	err := pluginCom.TaskPostResults(results)
	if err != nil {
		return err
	}

	pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded")
	return nil
}
// Execute pulls the task's patch and then applies it
func (gapc *GitApplyPatchCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {

	//Apply patches only if necessary
	if conf.Task.Requester == evergreen.PatchVersionRequester {
		pluginLogger.LogExecution(slogger.INFO, "Fetching patch.")
		patch, err := gapc.GetPatch(conf, pluginCom, pluginLogger)
		if err != nil {
			pluginLogger.LogExecution(slogger.ERROR, "Failed to get patch: %v", err)
			return fmt.Errorf("Failed to get patch: %v", err)
		}

		err = gapc.getPatchContents(conf, pluginCom, pluginLogger, patch)
		if err != nil {
			pluginLogger.LogExecution(slogger.ERROR, "Failed to get patch contents: %v", err)
			return fmt.Errorf("Failed to get patch contents: %v", err)
		}

		err = gapc.applyPatch(conf, patch, pluginLogger)
		if err != nil {
			pluginLogger.LogExecution(slogger.INFO, "Failed to apply patch: %v", err)
			return fmt.Errorf("Failed to apply patch: %v", err)
		}
	}
	return nil
}
Example #9
0
// S3Copy is responsible for carrying out the core of the S3CopyPlugin's
// function - it makes an API calls to copy a given staged file to it's final
// production destination
func (scc *S3CopyCommand) S3Copy(taskConfig *model.TaskConfig,
	pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator) error {
	for _, s3CopyFile := range scc.S3CopyFiles {
		if len(s3CopyFile.BuildVariants) > 0 && !util.SliceContains(
			s3CopyFile.BuildVariants, taskConfig.BuildVariant.Name) {
			continue
		}

		pluginLogger.LogExecution(slogger.INFO, "Making API push copy call to "+
			"transfer %v/%v => %v/%v", s3CopyFile.Source.Bucket,
			s3CopyFile.Source.Path, s3CopyFile.Destination.Bucket,
			s3CopyFile.Destination.Path)

		s3CopyReq := S3CopyRequest{
			AwsKey:              scc.AwsKey,
			AwsSecret:           scc.AwsSecret,
			S3SourceBucket:      s3CopyFile.Source.Bucket,
			S3SourcePath:        s3CopyFile.Source.Path,
			S3DestinationBucket: s3CopyFile.Destination.Bucket,
			S3DestinationPath:   s3CopyFile.Destination.Path,
			S3DisplayName:       s3CopyFile.DisplayName,
		}
		resp, err := pluginCom.TaskPostJSON(s3CopyAPIEndpoint, s3CopyReq)
		if resp != nil {
			defer resp.Body.Close()
		}
		if resp != nil && resp.StatusCode != http.StatusOK {
			body, _ := ioutil.ReadAll(resp.Body)
			return fmt.Errorf("S3 push copy failed (%v): %v",
				resp.StatusCode, string(body))
		}
		if err != nil {
			body, _ := ioutil.ReadAll(resp.Body)
			return fmt.Errorf("S3 push copy failed (%v): %v",
				resp.StatusCode, string(body))
		}
		pluginLogger.LogExecution(slogger.INFO, "API push copy call succeeded")
		err = scc.AttachTaskFiles(pluginLogger, pluginCom, s3CopyReq)
		if err != nil {
			body, readAllErr := ioutil.ReadAll(resp.Body)
			if readAllErr != nil {
				return fmt.Errorf("Error: %v", err)
			}
			return fmt.Errorf("Error: %v, (%v): %v",
				resp.StatusCode, err, string(body))
		}
	}
	return nil
}
Example #10
0
// Execute builds the archive.
func (self *TarGzPackCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(self, conf.Expansions); err != nil {
		return fmt.Errorf("error expanding params: %v", err)
	}

	// if the source dir is a relative path, join it to the working dir
	if !filepath.IsAbs(self.SourceDir) {
		self.SourceDir = filepath.Join(conf.WorkDir, self.SourceDir)
	}

	// if the target is a relative path, join it to the working dir
	if !filepath.IsAbs(self.Target) {
		self.Target = filepath.Join(conf.WorkDir, self.Target)
	}

	errChan := make(chan error)
	filesArchived := -1
	go func() {
		var err error
		filesArchived, err = self.BuildArchive(conf.WorkDir, pluginLogger)
		errChan <- err
	}()

	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
		if filesArchived == 0 {
			deleteErr := os.Remove(self.Target)
			if deleteErr != nil {
				pluginLogger.LogExecution(slogger.INFO, "Error deleting empty archive: %v", deleteErr)
			}
		}
		return nil
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of targz pack command")
		return nil
	}

}
Example #11
0
// Implementation of Execute.  Expands the parameters, and then puts the
// resource to s3.
func (s3pc *S3PutCommand) Execute(log plugin.Logger,
	com plugin.PluginCommunicator, conf *model.TaskConfig,
	stop chan bool) error {

	// expand necessary params
	if err := s3pc.expandParams(conf); err != nil {
		return err
	}

	// validate the params
	if err := s3pc.validateParams(); err != nil {
		return fmt.Errorf("expanded params are not valid: %v", err)
	}

	if !s3pc.shouldRunForVariant(conf.BuildVariant.Name) {
		log.LogTask(slogger.INFO, "Skipping S3 put of local file %v for variant %v",
			s3pc.LocalFile,
			conf.BuildVariant.Name)
		return nil
	}

	if s3pc.isMulti() {
		log.LogTask(slogger.INFO, "Putting files matching filter %v into path %v in s3 bucket %v",
			s3pc.LocalFilesIncludeFilter, s3pc.RemoteFile, s3pc.Bucket)
	} else {
		if !filepath.IsAbs(s3pc.LocalFile) {
			s3pc.LocalFile = filepath.Join(conf.WorkDir, s3pc.LocalFile)
		}
		log.LogTask(slogger.INFO, "Putting %v into path %v in s3 bucket %v",
			s3pc.LocalFile, s3pc.RemoteFile, s3pc.Bucket)
	}

	errChan := make(chan error)
	go func() {
		errChan <- s3pc.PutWithRetry(log, com)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		log.LogExecution(slogger.INFO, "Received signal to terminate execution of S3 Put Command")
		return nil
	}

}
Example #12
0
// Implementation of Execute.  Expands the parameters, and then fetches the
// resource from s3.
func (self *S3GetCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, conf *model.TaskConfig,
	stop chan bool) error {

	// expand necessary params
	if err := self.expandParams(conf); err != nil {
		return err
	}

	// validate the params
	if err := self.validateParams(); err != nil {
		return fmt.Errorf("expanded params are not valid: %v", err)
	}

	if !self.shouldRunForVariant(conf.BuildVariant.Name) {
		pluginLogger.LogTask(slogger.INFO, "Skipping S3 get of remote file %v for variant %v",
			self.RemoteFile,
			conf.BuildVariant.Name)
		return nil
	}

	// if the local file or extract_to is a relative path, join it to the
	// working dir
	if self.LocalFile != "" && !filepath.IsAbs(self.LocalFile) {
		self.LocalFile = filepath.Join(conf.WorkDir, self.LocalFile)
	}
	if self.ExtractTo != "" && !filepath.IsAbs(self.ExtractTo) {
		self.ExtractTo = filepath.Join(conf.WorkDir, self.ExtractTo)
	}

	errChan := make(chan error)
	go func() {
		errChan <- self.GetWithRetry(pluginLogger)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of S3 Get Command")
		return nil
	}

}
Example #13
0
// Execute starts the shell with its given parameters.
func (cc *CleanupCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {
	defer func() {
		trackedTask = ""
	}()
	if trackedTask == "" && trackedTask != conf.Task.Id {
		pluginLogger.LogExecution(slogger.WARN, "Process tracking was not enabled for task, skipping cleanup.")
		return nil
	}

	pluginLogger.LogExecution(slogger.INFO, "Running process cleanup...")

	// Clean up all shell processes spawned during the execution of this task by this agent,
	// by calling the platform-specific "cleanup" function
	cleanup(conf.Task.Id, pluginLogger)
	return nil
}
Example #14
0
// Implementation of Execute.
func (mfc *ManifestLoadCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, conf *model.TaskConfig,
	stop chan bool) error {

	errChan := make(chan error)
	go func() {
		errChan <- mfc.Load(pluginLogger, pluginCom, conf)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of manifest load command")
		return nil
	}

}
// Implementation of Execute, to unpack the archive.
func (self *TarGzUnpackCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	errChan := make(chan error)
	go func() {
		errChan <- self.UnpackArchive()
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of targz unpack command")
		return nil
	}
}
Example #16
0
// Wrapper around the Put() function to retry it
func (s3pc *S3PutCommand) PutWithRetry(log plugin.Logger, com plugin.PluginCommunicator) error {
	retriablePut := util.RetriableFunc(
		func() error {
			err := s3pc.Put()
			if err != nil {
				if err == errSkippedFile {
					return err
				}
				log.LogExecution(slogger.ERROR, "Error putting to s3 bucket: %v", err)
				return util.RetriableError{err}
			}
			return nil
		},
	)

	retryFail, err := util.RetryArithmeticBackoff(retriablePut, maxS3PutAttempts, s3PutSleep)
	if err == errSkippedFile {
		log.LogExecution(slogger.INFO, "S3 put skipped optional missing file.")
		return nil
	}
	if retryFail {
		log.LogExecution(slogger.ERROR, "S3 put failed with error: %v", err)
		return err
	}
	return s3pc.AttachTaskFiles(log, com)
}
Example #17
0
// Execute carries out the AttachResultsCommand command - this is required
// to satisfy the 'Command' interface
func (self *AttachResultsCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	taskConfig *model.TaskConfig,
	stop chan bool) error {

	if err := self.expandAttachResultsParams(taskConfig); err != nil {
		return err
	}

	errChan := make(chan error)
	go func() {
		// attempt to open the file
		reportFileLoc := filepath.Join(taskConfig.WorkDir, self.FileLoc)
		reportFile, err := os.Open(reportFileLoc)
		if err != nil {
			errChan <- fmt.Errorf("Couldn't open report file: '%v'", err)
			return
		}
		results := &task.TestResults{}
		if err = util.ReadJSONInto(reportFile, results); err != nil {
			errChan <- fmt.Errorf("Couldn't read report file: '%v'", err)
			return
		}
		if err := reportFile.Close(); err != nil {
			pluginLogger.LogExecution(slogger.INFO, "Error closing file: %v", err)
		}
		errChan <- SendJSONResults(taskConfig, pluginLogger, pluginCom, results)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of attach results command")
		return nil
	}
}
// Implementation of Execute, to unpack the archive.
func (self *TarGzUnpackCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	if err := plugin.ExpandValues(self, conf.Expansions); err != nil {
		return fmt.Errorf("error expanding params: %v", err)
	}

	errChan := make(chan error)
	go func() {
		errChan <- self.UnpackArchive()
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of targz unpack command")
		return nil
	}
}
// Execute carries out the AttachResultsCommand command - this is required
// to satisfy the 'Command' interface
func (self *AttachXUnitResultsCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	taskConfig *model.TaskConfig,
	stop chan bool) error {

	if err := self.expandParams(taskConfig); err != nil {
		return err
	}

	errChan := make(chan error)
	go func() {
		errChan <- self.parseAndUploadResults(taskConfig, pluginLogger, pluginCom)
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Received signal to terminate"+
			" execution of attach xunit results command")
		return nil
	}
}
Example #20
0
// Wrapper around the Put() function to retry it
func (self *S3PutCommand) PutWithRetry(pluginLogger plugin.Logger,
	pluginComm plugin.PluginCommunicator) error {
	retriablePut := util.RetriableFunc(
		func() error {
			err := self.Put()
			if err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Error putting to s3"+
					" bucket: %v", err)
				return util.RetriableError{err}
			}
			return nil
		},
	)

	retryFail, err := util.RetryArithmeticBackoff(retriablePut,
		maxS3PutAttempts, s3PutSleep)
	if retryFail {
		pluginLogger.LogExecution(slogger.ERROR, "S3 put failed with error: %v",
			err)
		return err
	}
	return self.AttachTaskFiles(pluginLogger, pluginComm)
}
Example #21
0
// AttachTaskFiles is responsible for sending the
// specified file to the API Server
func (s3pc *S3PutCommand) AttachTaskFiles(log plugin.Logger,
	com plugin.PluginCommunicator) error {

	remoteFile := filepath.ToSlash(s3pc.RemoteFile)
	fileLink := s3baseURL + s3pc.Bucket + "/" + remoteFile

	displayName := s3pc.DisplayName
	if displayName == "" {
		displayName = filepath.Base(s3pc.LocalFile)
	}
	file := &artifact.File{
		Name:       displayName,
		Link:       fileLink,
		Visibility: s3pc.Visibility,
	}

	err := com.PostTaskFiles([]*artifact.File{file})
	if err != nil {
		return fmt.Errorf("Attach files failed: %v", err)
	}
	log.LogExecution(slogger.INFO, "API attach files call succeeded")
	return nil
}
Example #22
0
// Wrapper around the Get() function to retry it
func (self *S3GetCommand) GetWithRetry(pluginLogger plugin.Logger) error {
	retriableGet := util.RetriableFunc(
		func() error {
			pluginLogger.LogTask(slogger.INFO, "Fetching %v from"+
				" s3 bucket %v", self.RemoteFile, self.Bucket)
			err := self.Get()
			if err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Error getting from"+
					" s3 bucket: %v", err)
				return util.RetriableError{err}
			}
			return nil
		},
	)

	retryFail, err := util.RetryArithmeticBackoff(retriableGet,
		MaxS3GetAttempts, S3GetSleep)
	if retryFail {
		pluginLogger.LogExecution(slogger.ERROR, "S3 get failed with error: %v",
			err)
		return err
	}
	return nil
}
Example #23
0
// Load performs a GET on /manifest/load
func (mfc *ManifestLoadCommand) Load(log plugin.Logger, pluginCom plugin.PluginCommunicator, conf *model.TaskConfig) error {
	var loadedManifest *manifest.Manifest
	var err error

	retriableGet := util.RetriableFunc(
		func() error {
			resp, err := pluginCom.TaskGetJSON(ManifestLoadAPIEndpoint)
			if resp != nil {
				defer resp.Body.Close()
			}
			if err != nil {
				//Some generic error trying to connect - try again
				log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
				return util.RetriableError{err}
			}
			if resp != nil && resp.StatusCode != http.StatusOK {
				log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode)
				return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)}
			}
			err = util.ReadJSONInto(resp.Body, &loadedManifest)
			if err != nil {
				return err
			}
			return nil
		})

	_, err = util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second)
	if err != nil {
		return err
	}
	if loadedManifest == nil {
		return fmt.Errorf("Manifest is empty")
	}
	mfc.updateExpansions(loadedManifest, conf)
	return nil
}
Example #24
0
// Execute pulls the task's patch and then applies it
func (gapc *GitApplyPatchCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator, conf *model.TaskConfig, stop chan bool) error {

	errChan := make(chan error)

	go func() {
		//Apply patches only if necessary
		if conf.Task.Requester == evergreen.PatchVersionRequester {
			pluginLogger.LogExecution(slogger.INFO, "Fetching patch.")
			patch, err := gapc.GetPatch(conf, pluginCom, pluginLogger)
			if err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Failed to get patch: %v", err)
				errChan <- fmt.Errorf("Failed to get patch: %v", err)
			}

			err = gapc.getPatchContents(conf, pluginCom, pluginLogger, patch)
			if err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Failed to get patch contents: %v", err)
				errChan <- fmt.Errorf("Failed to get patch contents: %v", err)
			}

			err = gapc.applyPatch(conf, patch, pluginLogger)
			if err != nil {
				pluginLogger.LogExecution(slogger.INFO, "Failed to apply patch: %v", err)
				errChan <- fmt.Errorf("Failed to apply patch: %v", err)
			}
		}
		errChan <- nil
	}()

	select {
	case err := <-errChan:
		return err
	case <-stop:
		return fmt.Errorf("Patch command interrupted.")
	}
}
// getPatchContents() dereferences any patch files that are stored externally, fetching them from
// the API server, and setting them into the patch object.
func (gapc GitApplyPatchCommand) getPatchContents(conf *model.TaskConfig, com plugin.PluginCommunicator, log plugin.Logger, p *patch.Patch) error {
	for i, patchPart := range p.Patches {
		// If the patch isn't stored externally, no need to do anything.
		if patchPart.PatchSet.PatchFileId == "" {
			continue
		}
		// otherwise, fetch the contents and load it into the patch object
		log.LogExecution(slogger.INFO, "Fetching patch contents for %v", patchPart.PatchSet.PatchFileId)
		var result []byte
		retriableGet := util.RetriableFunc(
			func() error {
				resp, err := com.TaskGetJSON(fmt.Sprintf("%s/%s", GitPatchFilePath, patchPart.PatchSet.PatchFileId))
				if resp != nil {
					defer resp.Body.Close()
				}
				if err != nil {
					//Some generic error trying to connect - try again
					log.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
					return util.RetriableError{err}
				}
				if resp != nil && resp.StatusCode != http.StatusOK {
					log.LogExecution(slogger.WARN, "Unexpected status code %v, retrying", resp.StatusCode)
					resp.Body.Close()
					return util.RetriableError{fmt.Errorf("Unexpected status code %v", resp.StatusCode)}
				}
				result, err = ioutil.ReadAll(resp.Body)
				if err != nil {
					return err
				}
				return nil
			})

		_, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second)
		if err != nil {
			return err
		}
		p.Patches[i].PatchSet.Patch = string(result)
	}
	return nil
}
Example #26
0
// SendJSONResults is responsible for sending the
// specified file to the API Server
func SendJSONResults(taskConfig *model.TaskConfig,
	pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator,
	results *task.TestResults) error {
	for i, res := range results.Results {

		if res.LogRaw != "" {
			pluginLogger.LogExecution(slogger.INFO, "Attaching raw test logs")
			testLogs := &model.TestLog{
				Name:          res.TestFile,
				Task:          taskConfig.Task.Id,
				TaskExecution: taskConfig.Task.Execution,
				Lines:         []string{res.LogRaw},
			}

			id, err := pluginCom.TaskPostTestLog(testLogs)
			if err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Error posting raw logs from results: %v", err)
			} else {
				results.Results[i].LogId = id
			}

			// clear the logs from the TestResult struct after it has been saved in the test logs. Since they are
			// being saved in the test_logs collection, we can clear them to prevent them from being saved in the task
			// collection.
			results.Results[i].LogRaw = ""

		}
	}

	pluginLogger.LogExecution(slogger.INFO, "Attaching test results")
	err := pluginCom.TaskPostResults(results)
	if err != nil {
		return err
	}

	pluginLogger.LogTask(slogger.INFO, "Attach test results succeeded")
	return nil
}
Example #27
0
// Execute gets the source code required by the project
func (self *GitGetProjectCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {
	location, err := conf.ProjectRef.Location()

	if err != nil {
		return err
	}

	gitCommands := []string{
		fmt.Sprintf("set -o errexit"),
		fmt.Sprintf("set -o verbose"),
		fmt.Sprintf("rm -rf %v", self.Directory),
		fmt.Sprintf("git clone %v '%v'", location, self.Directory),
		fmt.Sprintf("cd %v; git checkout %v", self.Directory, conf.Task.Revision),
	}

	cmdsJoined := strings.Join(gitCommands, "\n")

	fetchSourceCmd := &command.LocalCommand{
		CmdString:        cmdsJoined,
		WorkingDirectory: conf.WorkDir,
		Stdout:           pluginLogger.GetTaskLogWriter(slogger.INFO),
		Stderr:           pluginLogger.GetTaskLogWriter(slogger.ERROR),
		ScriptMode:       true,
	}

	pluginLogger.LogExecution(slogger.INFO, "Fetching source from git...")
	if err := fetchSourceCmd.Run(); err != nil {
		return err
	}
	pluginLogger.Flush()

	// Fetch source for the modules
	for _, moduleName := range conf.BuildVariant.Modules {
		pluginLogger.LogExecution(slogger.INFO, "Fetching module: %v", moduleName)
		module, err := conf.Project.GetModuleByName(moduleName)
		if err != nil {
			pluginLogger.LogExecution(slogger.ERROR, "Couldn't get module %v: %v",
				moduleName, err)
			continue
		}
		if module == nil {
			pluginLogger.LogExecution(slogger.ERROR, "No module found for %v",
				moduleName)
			continue
		}

		moduleBase := filepath.Join(module.Prefix, module.Name)
		moduleDir := filepath.Join(conf.WorkDir, moduleBase, "/_")

		err = os.MkdirAll(moduleDir, 0755)
		if err != nil {
			return err
		}
		// clear the destination
		err = os.RemoveAll(moduleDir)
		if err != nil {
			return err
		}

		moduleCmds := []string{
			fmt.Sprintf("set -o errexit"),
			fmt.Sprintf("set -o verbose"),
			fmt.Sprintf("git clone %v '%v'", module.Repo, filepath.ToSlash(moduleBase)),
			fmt.Sprintf("cd %v; git checkout '%v'", filepath.ToSlash(moduleBase), module.Branch),
		}

		moduleFetchCmd := &command.LocalCommand{
			CmdString:        strings.Join(moduleCmds, "\n"),
			WorkingDirectory: filepath.ToSlash(filepath.Join(conf.WorkDir, self.Directory)),
			Stdout:           pluginLogger.GetTaskLogWriter(slogger.INFO),
			Stderr:           pluginLogger.GetTaskLogWriter(slogger.ERROR),
			ScriptMode:       true,
		}
		err = moduleFetchCmd.Run()
		if err != nil {
			return err
		}
		pluginLogger.Flush()
	}

	return nil

}
Example #28
0
// Execute gets the source code required by the project
func (ggpc *GitGetProjectCommand) Execute(pluginLogger plugin.Logger,
	pluginCom plugin.PluginCommunicator,
	conf *model.TaskConfig,
	stop chan bool) error {

	// expand the github parameters before running the task
	if err := plugin.ExpandValues(&ggpc.Revisions, conf.Expansions); err != nil {
		return err
	}

	location, err := conf.ProjectRef.Location()

	if err != nil {
		return err
	}

	gitCommands := []string{
		fmt.Sprintf("set -o errexit"),
		fmt.Sprintf("set -o verbose"),
		fmt.Sprintf("rm -rf %v", ggpc.Directory),
		fmt.Sprintf("git clone %v '%v'", location, ggpc.Directory),
		fmt.Sprintf("cd %v; git checkout %v", ggpc.Directory, conf.Task.Revision),
	}

	cmdsJoined := strings.Join(gitCommands, "\n")

	fetchSourceCmd := &command.LocalCommand{
		CmdString:        cmdsJoined,
		WorkingDirectory: conf.WorkDir,
		Stdout:           pluginLogger.GetTaskLogWriter(slogger.INFO),
		Stderr:           pluginLogger.GetTaskLogWriter(slogger.ERROR),
		ScriptMode:       true,
	}

	errChan := make(chan error)
	go func() {
		pluginLogger.LogExecution(slogger.INFO, "Fetching source from git...")
		errChan <- fetchSourceCmd.Run()
		pluginLogger.Flush()
	}()

	// wait until the command finishes or the stop channel is tripped
	select {
	case err := <-errChan:
		if err != nil {
			return err
		}
	case <-stop:
		pluginLogger.LogExecution(slogger.INFO, "Got kill signal")
		if fetchSourceCmd.Cmd != nil {
			pluginLogger.LogExecution(slogger.INFO, "Stopping process: %v", fetchSourceCmd.Cmd.Process.Pid)
			if err := fetchSourceCmd.Stop(); err != nil {
				pluginLogger.LogExecution(slogger.ERROR, "Error occurred stopping process: %v", err)
			}
		}
		return fmt.Errorf("Fetch command interrupted.")
	}

	// Fetch source for the modules
	for _, moduleName := range conf.BuildVariant.Modules {
		pluginLogger.LogExecution(slogger.INFO, "Fetching module: %v", moduleName)
		module, err := conf.Project.GetModuleByName(moduleName)
		if err != nil {
			pluginLogger.LogExecution(slogger.ERROR, "Couldn't get module %v: %v",
				moduleName, err)
			continue
		}
		if module == nil {
			pluginLogger.LogExecution(slogger.ERROR, "No module found for %v",
				moduleName)
			continue
		}

		moduleBase := filepath.Join(module.Prefix, module.Name)
		moduleDir := filepath.Join(conf.WorkDir, moduleBase, "/_")

		err = os.MkdirAll(moduleDir, 0755)
		if err != nil {
			return err
		}
		// clear the destination
		err = os.RemoveAll(moduleDir)
		if err != nil {
			return err
		}

		revision := ggpc.Revisions[moduleName]

		// if there is no revision, then use the branch name
		if revision == "" {
			revision = module.Branch
		}

		moduleCmds := []string{
			fmt.Sprintf("set -o errexit"),
			fmt.Sprintf("set -o verbose"),
			fmt.Sprintf("git clone %v '%v'", module.Repo, filepath.ToSlash(moduleBase)),
			fmt.Sprintf("cd %v; git checkout '%v'", filepath.ToSlash(moduleBase), revision),
		}

		moduleFetchCmd := &command.LocalCommand{
			CmdString:        strings.Join(moduleCmds, "\n"),
			WorkingDirectory: filepath.ToSlash(filepath.Join(conf.WorkDir, ggpc.Directory)),
			Stdout:           pluginLogger.GetTaskLogWriter(slogger.INFO),
			Stderr:           pluginLogger.GetTaskLogWriter(slogger.ERROR),
			ScriptMode:       true,
		}

		go func() {
			errChan <- moduleFetchCmd.Run()
			pluginLogger.Flush()
		}()

		// wait until the command finishes or the stop channel is tripped
		select {
		case err := <-errChan:
			if err != nil {
				return err
			}
		case <-stop:
			pluginLogger.LogExecution(slogger.INFO, "Got kill signal")
			if moduleFetchCmd.Cmd != nil {
				pluginLogger.LogExecution(slogger.INFO, "Stopping process: %v", moduleFetchCmd.Cmd.Process.Pid)
				if err := moduleFetchCmd.Stop(); err != nil {
					pluginLogger.LogExecution(slogger.ERROR, "Error occurred stopping process: %v", err)
				}
			}
			return fmt.Errorf("Fetch module command interrupted.")
		}
	}

	return nil

}
// applyPatch is used by the agent to copy patch data onto disk
// and then call the necessary git commands to apply the patch file
func (gapc *GitApplyPatchCommand) applyPatch(conf *model.TaskConfig,
	patch *patch.Patch, pluginLogger plugin.Logger) error {

	// patch sets and contain multiple patches, some of them for modules
	for _, patchPart := range patch.Patches {
		var dir string
		if patchPart.ModuleName == "" {
			// if patch is not part of a module, just apply patch against src root
			dir = gapc.Directory
			pluginLogger.LogExecution(slogger.INFO, "Applying patch with git...")
		} else {
			// if patch is part of a module, apply patch in module root
			module, err := conf.Project.GetModuleByName(patchPart.ModuleName)
			if err != nil {
				return fmt.Errorf("Error getting module: %v", err)
			}
			if module == nil {
				return fmt.Errorf("Module not found: %v", patchPart.ModuleName)
			}

			// skip the module if this build variant does not use it
			if !util.SliceContains(conf.BuildVariant.Modules, module.Name) {
				pluginLogger.LogExecution(slogger.INFO, "Skipping patch for"+
					" module %v, since the current build variant does not"+
					" use it", module.Name)
				continue
			}

			dir = filepath.Join(gapc.Directory, module.Prefix, module.Name)
			pluginLogger.LogExecution(slogger.INFO, "Applying module patch with git...")
		}

		// create a temporary folder and store patch files on disk,
		// for later use in shell script
		tempFile, err := ioutil.TempFile("", "mcipatch_")
		if err != nil {
			return err
		}
		defer tempFile.Close()
		_, err = io.WriteString(tempFile, patchPart.PatchSet.Patch)
		if err != nil {
			return err
		}
		tempAbsPath := tempFile.Name()

		// this applies the patch using the patch files in the temp directory
		patchCommandStrings := []string{
			fmt.Sprintf("set -o verbose"),
			fmt.Sprintf("set -o errexit"),
			fmt.Sprintf("ls"),
			fmt.Sprintf("cd '%v'", dir),
			fmt.Sprintf("git checkout '%v'", patchPart.Githash),
			fmt.Sprintf("git apply --check --whitespace=fix '%v'", tempAbsPath),
			fmt.Sprintf("git apply --stat '%v'", tempAbsPath),
			fmt.Sprintf("git apply --whitespace=fix < '%v'", tempAbsPath),
		}

		cmdsJoined := strings.Join(patchCommandStrings, "\n")
		patchCmd := &command.LocalCommand{
			CmdString:        cmdsJoined,
			WorkingDirectory: conf.WorkDir,
			Stdout:           pluginLogger.GetTaskLogWriter(slogger.INFO),
			Stderr:           pluginLogger.GetTaskLogWriter(slogger.ERROR),
			ScriptMode:       true,
		}

		err = patchCmd.Run()
		if err != nil {
			return err
		}
		pluginLogger.Flush()
	}
	return nil
}
// GetPatch tries to get the patch data from the server in json format,
// and unmarhals it into a patch struct. The GET request is attempted
// multiple times upon failure.
func (gapc GitApplyPatchCommand) GetPatch(conf *model.TaskConfig,
	pluginCom plugin.PluginCommunicator, pluginLogger plugin.Logger) (*patch.Patch, error) {
	patch := &patch.Patch{}
	retriableGet := util.RetriableFunc(
		func() error {
			resp, err := pluginCom.TaskGetJSON(GitPatchPath)
			if resp != nil {
				defer resp.Body.Close()
			}
			if err != nil {
				//Some generic error trying to connect - try again
				pluginLogger.LogExecution(slogger.WARN, "Error connecting to API server: %v", err)
				return util.RetriableError{err}
			}
			if resp != nil && resp.StatusCode == http.StatusNotFound {
				//nothing broke, but no patch was found for task Id - no retry
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					pluginLogger.LogExecution(slogger.ERROR, "Error reading response body")
				}
				msg := fmt.Sprintf("no patch found for task: %v", string(body))
				pluginLogger.LogExecution(slogger.WARN, msg)
				return fmt.Errorf(msg)
			}
			if resp != nil && resp.StatusCode == http.StatusInternalServerError {
				//something went wrong in api server
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					pluginLogger.LogExecution(slogger.ERROR, "Error reading response body")
				}
				msg := fmt.Sprintf("error fetching patch from server: %v", string(body))
				pluginLogger.LogExecution(slogger.WARN, msg)
				return util.RetriableError{
					fmt.Errorf(msg),
				}
			}
			if resp != nil && resp.StatusCode == http.StatusConflict {
				//wrong secret
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil {
					pluginLogger.LogExecution(slogger.ERROR, "Error reading response body")
				}
				msg := fmt.Sprintf("secret conflict: %v", string(body))
				pluginLogger.LogExecution(slogger.ERROR, msg)
				return fmt.Errorf(msg)
			}
			if resp == nil {
				pluginLogger.LogExecution(slogger.WARN, "Empty response from API server")
				return util.RetriableError{fmt.Errorf("empty response")}
			} else {
				err = util.ReadJSONInto(resp.Body, patch)
				if err != nil {
					pluginLogger.LogExecution(slogger.ERROR,
						"Error reading json into patch struct: %v", err)
					return util.RetriableError{err}
				}
				return nil
			}
		},
	)

	retryFail, err := util.RetryArithmeticBackoff(retriableGet, 5, 5*time.Second)
	if retryFail {
		return nil, fmt.Errorf("getting patch failed after %v tries: %v", 10, err)
	}
	if err != nil {
		return nil, fmt.Errorf("getting patch failed: %v", err)
	}
	return patch, nil
}