// MoveCommandState moves the CommandState object
func MoveCommandState(log log.T, commandID, instanceID, srcLocationFolder, dstLocationFolder string) {
//get a lock for documentID specific lock
lockDocument(commandID)
absoluteSource := path.Join(appconfig.DefaultDataStorePath,
instanceID,
appconfig.DefaultCommandRootDirName,
appconfig.DefaultLocationOfState,
srcLocationFolder)
absoluteDestination := path.Join(appconfig.DefaultDataStorePath,
instanceID,
appconfig.DefaultCommandRootDirName,
appconfig.DefaultLocationOfState,
dstLocationFolder)
if s, err := fileutil.MoveFile(commandID, absoluteSource, absoluteDestination); s && err == nil {
log.Debugf("moved file %v from %v to %v successfully", commandID, srcLocationFolder, dstLocationFolder)
} else {
log.Debugf("moving file %v from %v to %v failed with error %v", commandID, srcLocationFolder, dstLocationFolder, err)
}
//release documentID specific lock - before deleting the entry from the map
unlockDocument(commandID)
//delete documentID specific lock if document has finished executing. This is to avoid documentLock growing too much in memory.
//This is done by ensuring that as soon as document finishes executing it is removed from documentLock
//Its safe to assume that document has finished executing if it is being moved to appconfig.DefaultLocationOfCompleted
if dstLocationFolder == appconfig.DefaultLocationOfCompleted {
deleteLock(commandID)
}
}
// process launches one job in a separate go routine and waits
// for either the job to finish or for a cancel to be requested.
// If cancel is requested, this function waits for some time to allow the
// job to complete. If the job does not complete by the timeout, the go routine
// of the job is abandoned, and this function returns.
func process(log log.T, job Job, cancelFlag *ChanneledCancelFlag, cancelWait time.Duration, clock times.Clock) {
// Make a buffered channel to avoid blocking on send. This helps
// in case the job fails to cancel on time and we give up on it.
// If the job finally ends, it will succeed to send a signal
// on the channel and then it will terminate. This will allow
// the garbage collector to collect the go routine's resources
// and the channel.
doneChan := make(chan struct{}, 1)
go runJob(log, func() { job(cancelFlag) }, doneChan)
done := waitEither(doneChan, cancelFlag.ch)
if done {
// task done, set the flag to wake up waiting routines
cancelFlag.Set(Completed)
return
}
log.Debugf("Execution has been canceled, waiting up to %v to finish", cancelWait)
done = waitEither(doneChan, clock.After(cancelWait))
if done {
// job completed within cancel waiting window
cancelFlag.Set(Completed)
return
}
log.Debugf("Job failed to terminate within %v!", cancelWait)
}
// HandleAwsError logs an AWS error.
func HandleAwsError(log log.T, err error, stopPolicy *StopPolicy) {
if err != nil {
// notice that we're using 1, so it will actually log the where
// the error happened, 0 = this function, we don't want that.
pc, fn, line, _ := runtime.Caller(1)
log.Debugf("error in %s[%s:%d] %v", runtime.FuncForPC(pc).Name(), fn, line, err)
// In case this is aws error, update the stop policy as well.
if aErr, ok := err.(awserr.Error); ok {
// Generic AWS Error with Code, Message, and original error (if any)
log.Debugf("AWS error. Code: %v, Message: %v, origerror: %v ", aErr.Code(), aErr.Message(), aErr.OrigErr())
// special treatment for Timeout exception - as this is expected
if aErr.Code() == "RequestError" && aErr.OrigErr() != nil && strings.Contains(aErr.OrigErr().Error(), "Client.Timeout") {
// resetting the error count to 0 - as these exceptions are all expected
if stopPolicy != nil {
resetStopPolicy(stopPolicy)
}
return
}
}
log.Errorf("error when calling AWS APIs. error details - %v", err)
if stopPolicy != nil {
log.Infof("increasing error count by 1")
stopPolicy.AddErrorCount(1)
}
} else {
// there is no error,
resetStopPolicy(stopPolicy)
}
}
// LatestVersion returns latest version for specific package
func (m *Manifest) LatestVersion(log log.T, context *updateutil.InstanceContext, packageName string) (result string, err error) {
var version = minimumVersion
var compareResult = 0
for _, p := range m.Packages {
if p.Name == packageName {
for _, f := range p.Files {
if f.Name == context.FileName(packageName) {
for _, v := range f.AvailableVersions {
if compareResult, err = updateutil.VersionCompare(v.Version, version); err != nil {
return version, err
}
if compareResult > 0 {
version = v.Version
}
}
}
}
}
}
if version == minimumVersion {
log.Debugf("Filename: %v", context.FileName(packageName))
log.Debugf("Package Name: %v", packageName)
log.Debugf("Manifest: %v", m)
return version, fmt.Errorf("cannot find the latest version for package %v", packageName)
}
return version, nil
}
// LoadUpdateContext loads update context info from local storage, set current update with new update detail
func LoadUpdateContext(log log.T, source string) (context *UpdateContext, err error) {
log.Debugf("file %v", source)
if _, err := os.Stat(source); os.IsNotExist(err) {
log.Debugf("UpdateContext file doesn't exist, creating new one")
context = &UpdateContext{}
} else {
log.Debugf("UpdateContext file exists")
if context, err = parseContext(log, source); err != nil {
return context, err
}
}
return context, nil
}
func getPlatformDetail(log log.T, param string) (value string, err error) {
var contentsBytes []byte
value = notAvailableMessage
log.Debugf(gettingPlatformDetailsMessage)
if contentsBytes, err = exec.Command(platformDetailsCommand, param).Output(); err != nil {
log.Debugf(errorOccurredMessage, platformDetailsCommand, err)
return
}
value = strings.TrimSpace(string(contentsBytes))
log.Debugf(commandOutputMessage, value)
return
}
// uploadOutput uploads the stdout and stderr file to S3
func (c *contextManager) uploadOutput(log log.T, context *UpdateContext) (err error) {
awsConfig := sdkutil.AwsConfig()
var config appconfig.SsmagentConfig
config, err = appconfig.Config(false)
if err != nil {
return fmt.Errorf("could not load config file: %v", err)
}
// If customers have provided override in app config, honor that.
if config.S3.Region != "" {
awsConfig.Region = &config.S3.Region
}
log.Infof("Uploading output files to region: %v", *awsConfig.Region)
s3 := s3.New(session.New(awsConfig))
// upload outputs (if any) to s3
uploader := s3util.NewManager(s3)
uploadOutputsToS3 := func() {
// delete temp outputDir once we're done
defer pluginutil.DeleteDirectory(log, updateutil.UpdateOutputDirectory(context.Current.UpdateRoot))
// get stdout file path
stdoutPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StdoutFileName)
s3Key := path.Join(context.Current.OutputS3KeyPrefix, context.Current.StdoutFileName)
log.Debugf("Uploading %v to s3://%v/%v", stdoutPath, context.Current.OutputS3BucketName, s3Key)
err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stdoutPath)
if err != nil {
log.Errorf("failed uploading %v to s3://%v/%v \n err:%v",
stdoutPath,
context.Current.OutputS3BucketName,
s3Key,
err)
}
// get stderr file path
stderrPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StderrFileName)
s3Key = path.Join(context.Current.OutputS3KeyPrefix, context.Current.StderrFileName)
log.Debugf("Uploading %v to s3://%v/%v", stderrPath, context.Current.OutputS3BucketName, s3Key)
err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stderrPath)
if err != nil {
log.Errorf("failed uploading %v to s3://%v/%v \n err:%v", stderrPath, context.Current.StderrFileName, s3Key, err)
}
}
uploadOutputsToS3()
return nil
}
// setMsiExecStatus sets the exit status and output to be returned to the user based on exit code
func setMsiExecStatus(log log.T, pluginInput ApplicationPluginInput, cancelFlag task.CancelFlag, out *ApplicationPluginOutput) {
out.Stdout = pluginInput.Source
out.Stderr = ""
out.Status = contracts.ResultStatusFailed
isUnKnownError := false
switch out.ExitCode {
case appconfig.SuccessExitCode:
out.Status = contracts.ResultStatusSuccess
case ErrorUnknownProduct:
if pluginInput.Action == UNINSTALL {
// Uninstall will skip, if product is not currently installed.
// This is needed to support idempotent behavior.
out.Status = contracts.ResultStatusSuccess
}
case ErrorSuccessRebootInitiated:
fallthrough
case appconfig.RebootExitCode:
out.Status = contracts.ResultStatusSuccessAndReboot
case pluginutil.CommandStoppedPreemptivelyExitCode:
if cancelFlag.ShutDown() {
out.Status = contracts.ResultStatusFailed
}
if cancelFlag.Canceled() {
out.Status = contracts.ResultStatusCancelled
}
out.Status = contracts.ResultStatusTimedOut
default:
isUnKnownError = true
}
if isUnKnownError {
// Note: Sample Stderr:
// Action:{Installed}; Status:{Failed};
// ErrorCode:{1620}; ErrorMsg:{ERROR_INSTALL_PACKAGE_INVALID};
// Description:{This installation package could not be opened. Contact the application vendor to verify that this is a valid Windows Installer package.};
// Source:{https:///}
// Construct stderr in above format using StandardMsiErrorCodes
out.Stderr = fmt.Sprintf("Action:{%v}; Status:{%v}; ErrorCode:{%v}; %v Source:{%v};", pluginInput.Action, out.Status, out.ExitCode, getExitCodeDescription(out.ExitCode), pluginInput.Source)
}
// Logging msiexec.Result
log.Debug("logging stdouts & errors after setting final status for msiexec")
log.Debugf("resultCode: %v", out.ExitCode)
log.Debugf("stdout: %v", out.Stdout)
log.Debugf("stderr: %v", out.Stderr)
}
// setCmdState persists given commandState
func setCmdState(log log.T, commandState message.CommandState, absoluteFileName, locationFolder string) {
content, err := jsonutil.Marshal(commandState)
if err != nil {
log.Errorf("encountered error with message %v while marshalling %v to string", err, commandState)
} else {
if fileutil.Exists(absoluteFileName) {
log.Debugf("overwriting contents of %v", absoluteFileName)
}
log.Tracef("persisting interim state %v in file %v", jsonutil.Indent(content), absoluteFileName)
if s, err := fileutil.WriteIntoFileWithPermissions(absoluteFileName, jsonutil.Indent(content), os.FileMode(int(appconfig.ReadWriteAccess))); s && err == nil {
log.Debugf("successfully persisted interim state in %v", locationFolder)
} else {
log.Debugf("persisting interim state in %v failed with error %v", locationFolder, err)
}
}
}
// GetS3BucketRegionFromErrorMsg gets the expected region from the error message
func (m *Manager) GetS3BucketRegionFromErrorMsg(log log.T, errMsg string) string {
var expectedBucketRegion = ""
if errMsg != "" && m.IsS3ErrorRelatedToWrongBucketRegion(errMsg) {
//Sample error message:
//AuthorizationHeaderMalformed: The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'us-west-2' status code: 400, request id: []
splitResult := strings.Split(errMsg, ";")
furtherSplitResult := strings.Split(splitResult[len(splitResult)-1], "'")
expectedBucketRegion = furtherSplitResult[1]
log.Debugf("expected region according to error message = %v", expectedBucketRegion)
if expectedBucketRegion == "" {
log.Debugf("Setting expected region = us-east-1 as per http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGETlocation.html")
expectedBucketRegion = "us-east-1"
}
}
return expectedBucketRegion
}
// DownloadFileFromSource downloads file from source
func DownloadFileFromSource(log log.T, source string, sourceHash string, sourceHashType string) (artifact.DownloadOutput, error) {
// download source and verify its integrity
downloadInput := artifact.DownloadInput{
SourceURL: source,
SourceHashValue: sourceHash,
SourceHashType: sourceHashType,
}
log.Debugf("Downloading file %v", downloadInput)
return artifact.Download(log, downloadInput)
}
// RemoveData deletes the fileName from locationFolder under defaultLogDir/instanceID
func RemoveData(log log.T, commandID, instanceID, locationFolder string) {
absoluteFileName := getCmdStateFileName(commandID, instanceID, locationFolder)
err := fileutil.DeleteFile(absoluteFileName)
if err != nil {
log.Errorf("encountered error %v while deleting file %v", err, absoluteFileName)
} else {
log.Debugf("successfully deleted file %v", absoluteFileName)
}
}
// runCommandsRawInput executes one set of commands and returns their output.
// The input is in the default json unmarshal format (e.g. map[string]interface{}).
func (p *Plugin) runCommandsRawInput(log log.T, rawPluginInput interface{}, orchestrationDirectory string, cancelFlag task.CancelFlag, outputS3BucketName string, outputS3KeyPrefix string) (out ApplicationPluginOutput) {
var pluginInput ApplicationPluginInput
err := jsonutil.Remarshal(rawPluginInput, &pluginInput)
log.Debugf("Plugin input %v", pluginInput)
if err != nil {
errorString := fmt.Errorf("Invalid format in plugin properties %v;\nerror %v", rawPluginInput, err)
out.MarkAsFailed(log, errorString)
return
}
return p.runCommands(log, pluginInput, orchestrationDirectory, cancelFlag, outputS3BucketName, outputS3KeyPrefix)
}
// PersistData stores the given object in the file-system in pretty Json indented format
// This will override the contents of an already existing file
func PersistData(log log.T, commandID, instanceID, locationFolder string, object interface{}) {
lockDocument(commandID)
defer unlockDocument(commandID)
absoluteFileName := getCmdStateFileName(commandID, instanceID, locationFolder)
content, err := jsonutil.Marshal(object)
if err != nil {
log.Errorf("encountered error with message %v while marshalling %v to string", err, object)
} else {
if fileutil.Exists(absoluteFileName) {
log.Debugf("overwriting contents of %v", absoluteFileName)
}
log.Tracef("persisting interim state %v in file %v", jsonutil.Indent(content), absoluteFileName)
if s, err := fileutil.WriteIntoFileWithPermissions(absoluteFileName, jsonutil.Indent(content), os.FileMode(int(appconfig.ReadWriteAccess))); s && err == nil {
log.Debugf("successfully persisted interim state in %v", locationFolder)
} else {
log.Debugf("persisting interim state in %v failed with error %v", locationFolder, err)
}
}
}
// isErrorUnexpected processes GetMessages errors and determines if its unexpected error
func isErrorUnexpected(log log.T, err error, requestTime, responseTime time.Time) bool {
//determine the time it took for the api to respond
timeDiff := responseTime.Sub(requestTime).Seconds()
//check if response isn't coming too quick & if error is unexpected
if timeDiff < QuickResponseThreshold {
//response was too quick - this is unexpected
return true
}
//response wasn't too quick
//checking if the class of errors are expected
if isServerBasedError(err.Error()) {
log.Debugf("server terminated connection after %v seconds - this is expected in long polling api calls.", timeDiff)
return false
} else if isClientBasedError(err.Error()) {
log.Debugf("client terminated connection after %v seconds - this is expected in long polling api calls.", timeDiff)
return false
} else {
//errors are truly unexpected
return true
}
}
// UploadS3TestFile uploads a test S3 file (with current datetime) to given s3 bucket and key
func (m *Manager) UploadS3TestFile(log log.T, bucketName, key string) error {
var err error
//create a test content
testData := time.Now().String()
log.Debugf("Data being written in S3 - %v", testData)
content := bytes.NewReader([]byte(testData))
//objectName to be uploaded in S3
var objectKey = path.Join(key, "ssmaccesstext.txt")
params := &s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
Body: content,
ContentType: aws.String("text/plain"),
}
_, err = m.S3.PutObject(params)
return err
}
// Md5HashValue gets the md5 hash value
func Md5HashValue(log log.T, filePath string) (hash string, err error) {
var exists = false
exists, err = fileutil.LocalFileExist(filePath)
if err != nil || exists == false {
return
}
var f *os.File
f, err = os.Open(filePath)
if err != nil {
log.Error(err)
}
defer f.Close()
hasher := md5.New()
if _, err = io.Copy(hasher, f); err != nil {
log.Error(err)
}
hash = hex.EncodeToString(hasher.Sum(nil))
log.Debugf("Hash=%v, FilePath=%v", hash, filePath)
return
}
// finalizeUpdateAndSendReply completes the update and send reply to message service
func (u *updateManager) finalizeUpdateAndSendReply(log log.T, context *UpdateContext, errorCode string) (err error) {
update := context.Current
update.EndDateTime = time.Now().UTC()
// resolve context location base on the UpdateRoot
contextLocation := updateutil.UpdateContextFilePath(update.UpdateRoot)
if err = u.ctxMgr.saveUpdateContext(log, context, contextLocation); err != nil {
return err
}
// send reply
if update.HasMessageID() {
if err = u.svc.SendReply(log, update); err != nil {
log.Errorf(err.Error())
}
if err = u.svc.DeleteMessage(log, update); err != nil {
log.Errorf(err.Error())
}
}
// update health information
if err = u.svc.UpdateHealthCheck(log, update, errorCode); err != nil {
log.Errorf(err.Error())
}
// upload output to s3 bucket
log.Debugf("output s3 bucket name is %v", update.OutputS3BucketName)
if update.OutputS3BucketName != "" {
u.ctxMgr.uploadOutput(log, context)
}
context.cleanUpdate()
if err = u.ctxMgr.saveUpdateContext(log, context, contextLocation); err != nil {
return err
}
return nil
}
// UploadOutputToS3Bucket uploads outputs (if any) to s3
func (p *DefaultPlugin) UploadOutputToS3Bucket(log log.T, pluginID string, orchestrationDir string, outputS3BucketName string, outputS3KeyPrefix string, useTempDirectory bool, tempDir string, Stdout string, Stderr string) []string {
var uploadOutputToS3BucketErrors []string
if outputS3BucketName != "" {
uploadOutputsToS3 := func() {
uploadToS3 := true
var testUploadError error
if region, err := platform.Region(); err == nil && region != s3Bjs {
p.Uploader.SetS3ClientRegion(S3RegionUSStandard)
}
log.Infof("uploading a test file to s3 bucket - %v , s3 key - %v with S3Client using region endpoint - %v",
outputS3BucketName,
outputS3KeyPrefix,
p.Uploader.GetS3ClientRegion())
testUploadError = p.Uploader.UploadS3TestFile(log, outputS3BucketName, outputS3KeyPrefix)
if testUploadError != nil {
//Check if the error is related to Access Denied - i.e missing permissions
if p.Uploader.IsS3ErrorRelatedToAccessDenied(testUploadError.Error()) {
log.Debugf("encountered access denied related error - can't upload to S3 due to missing permissions -%v", testUploadError.Error())
uploadToS3 = false
//since we don't have permissions - no S3 calls will go through no matter what
} else if p.Uploader.IsS3ErrorRelatedToWrongBucketRegion(testUploadError.Error()) { //check if error is related to different bucket region
log.Debugf("encountered error related to wrong bucket region while uploading test file to S3 - %v. parsing the message to get expected region",
testUploadError.Error())
expectedBucketRegion := p.Uploader.GetS3BucketRegionFromErrorMsg(log, testUploadError.Error())
//set the region to expectedBucketRegion
p.Uploader.SetS3ClientRegion(expectedBucketRegion)
} else {
log.Debugf("encountered unexpected error while uploading test file to S3 - %v, no need to modify s3client", testUploadError.Error())
}
} else { //there were no errors while uploading a test file to S3 - our s3client should continue to use "us-east-1"
log.Debugf("there were no errors while uploading a test file to S3 in region - %v. S3 client will continue to use region - %v",
S3RegionUSStandard,
p.Uploader.GetS3ClientRegion())
}
if uploadToS3 {
log.Infof("uploading logs to S3 with client configured to use region - %v", p.Uploader.GetS3ClientRegion())
if useTempDirectory {
// delete temp directory once we're done
defer DeleteDirectory(log, tempDir)
}
if Stdout != "" {
localPath := filepath.Join(orchestrationDir, p.StdoutFileName)
s3Key := path.Join(outputS3KeyPrefix, pluginID, p.StdoutFileName)
log.Debugf("Uploading %v to s3://%v/%v", localPath, outputS3BucketName, s3Key)
err := p.Uploader.S3Upload(outputS3BucketName, s3Key, localPath)
if err != nil {
log.Errorf("failed uploading %v to s3://%v/%v err:%v", localPath, outputS3BucketName, s3Key, err)
if p.UploadToS3Sync {
// if we are in synchronous mode, we can also return the error
uploadOutputToS3BucketErrors = append(uploadOutputToS3BucketErrors, err.Error())
}
}
}
if Stderr != "" {
localPath := filepath.Join(orchestrationDir, p.StderrFileName)
s3Key := path.Join(outputS3KeyPrefix, pluginID, p.StderrFileName)
log.Debugf("Uploading %v to s3://%v/%v", localPath, outputS3BucketName, s3Key)
err := p.Uploader.S3Upload(outputS3BucketName, s3Key, localPath)
if err != nil {
log.Errorf("failed uploading %v to s3://%v/%v err:%v", localPath, outputS3BucketName, s3Key, err)
if p.UploadToS3Sync {
// if we are in synchronous mode, we can also return the error
uploadOutputToS3BucketErrors = append(uploadOutputToS3BucketErrors, err.Error())
}
}
}
} else {
//TODO:Bubble this up to engine - so that document level status reply can be sent stating no permissions to perform S3 upload - similar to ec2config
}
}
if p.UploadToS3Sync {
uploadOutputsToS3()
} else {
go uploadOutputsToS3()
}
}
//return out.Errors
return uploadOutputToS3BucketErrors
}
// runCommands executes one set of commands and returns their output.
func (p *Plugin) runCommands(log log.T, pluginInput ApplicationPluginInput, orchestrationDirectory string, cancelFlag task.CancelFlag, outputS3BucketName string, outputS3KeyPrefix string) (out ApplicationPluginOutput) {
var err error
// if no orchestration directory specified, create temp directory
var useTempDirectory = (orchestrationDirectory == "")
var tempDir string
if useTempDirectory {
if tempDir, err = ioutil.TempDir("", "Ec2RunCommand"); err != nil {
out.Errors = append(out.Errors, err.Error())
log.Error(err)
return
}
orchestrationDirectory = tempDir
}
orchestrationDir := fileutil.RemoveInvalidChars(filepath.Join(orchestrationDirectory, pluginInput.ID))
log.Debugf("OrchestrationDir %v ", orchestrationDir)
// create orchestration dir if needed
if err = fileutil.MakeDirs(orchestrationDir); err != nil {
log.Debug("failed to create orchestrationDir directory", orchestrationDir, err)
out.Errors = append(out.Errors, err.Error())
return
}
// Get application mode
mode, err := getMsiApplicationMode(pluginInput)
if err != nil {
out.MarkAsFailed(log, err)
return
}
log.Debugf("mode is %v", mode)
// Download file from source if available
downloadOutput, err := pluginutil.DownloadFileFromSource(log, pluginInput.Source, pluginInput.SourceHash, pluginInput.SourceHashType)
if err != nil || downloadOutput.IsHashMatched == false || downloadOutput.LocalFilePath == "" {
errorString := fmt.Errorf("failed to download file reliably %v", pluginInput.Source)
out.MarkAsFailed(log, errorString)
return
}
log.Debugf("local path to file is %v", downloadOutput.LocalFilePath)
// Create msi related log file
localSourceLogFilePath := downloadOutput.LocalFilePath + ".msiexec.log.txt"
log.Debugf("log path is %v", localSourceLogFilePath)
// TODO: This needs to be pulled out of this function as it runs multiple times getting initialized with the same values
// Create output file paths
stdoutFilePath := filepath.Join(orchestrationDir, p.StdoutFileName)
stderrFilePath := filepath.Join(orchestrationDir, p.StderrFileName)
log.Debugf("stdout file %v, stderr file %v", stdoutFilePath, stderrFilePath)
// Construct Command Name and Arguments
commandName := msiExecCommand
commandArguments := []string{mode, downloadOutput.LocalFilePath, "/quiet", "/norestart", "/log", localSourceLogFilePath}
if pluginInput.Parameters != "" {
log.Debugf("Got Parameters \"%v\"", pluginInput.Parameters)
params := processParams(log, pluginInput.Parameters)
commandArguments = append(commandArguments, params...)
}
// Execute Command
_, _, exitCode, errs := p.ExecuteCommand(log, defaultWorkingDirectory, stdoutFilePath, stderrFilePath, cancelFlag, defaultApplicationExecutionTimeoutInSeconds, commandName, commandArguments)
// Set output status
out.ExitCode = exitCode
setMsiExecStatus(log, pluginInput, cancelFlag, &out)
if len(errs) > 0 {
for _, err := range errs {
out.Errors = append(out.Errors, err.Error())
log.Error("failed to run commands: ", err)
out.Status = contracts.ResultStatusFailed
}
return
}
// Upload output to S3
uploadOutputToS3BucketErrors := p.ExecuteUploadOutputToS3Bucket(log, pluginInput.ID, orchestrationDir, outputS3BucketName, outputS3KeyPrefix, useTempDirectory, tempDir, out.Stdout, out.Stderr)
out.Errors = append(out.Errors, uploadOutputToS3BucketErrors...)
// Return Json indented response
responseContent, _ := jsonutil.Marshal(out)
log.Debug("Returning response:\n", jsonutil.Indent(responseContent))
return
}
// updateAgent downloads the installation packages and update the agent
func runUpdateAgent(
p *Plugin,
config contracts.Configuration,
log log.T,
manager pluginHelper,
util updateutil.T,
rawPluginInput interface{},
cancelFlag task.CancelFlag,
outputS3BucketName string,
outputS3KeyPrefix string,
startTime time.Time) (out UpdatePluginOutput) {
var pluginInput UpdatePluginInput
var err error
var context *updateutil.InstanceContext
if isUpdateSupported, err := util.IsPlatformSupportedForUpdate(log); err == nil && !isUpdateSupported {
out.Failed(log, fmt.Errorf("Unsupported platform for update"))
return
}
if err = jsonutil.Remarshal(rawPluginInput, &pluginInput); err != nil {
out.Failed(log,
fmt.Errorf("invalid format in plugin properties %v;\nerror %v", rawPluginInput, err))
return
}
if context, err = util.CreateInstanceContext(log); err != nil {
out.Failed(log, err)
return
}
//Use default manifest location is the override is not present
if len(pluginInput.Source) == 0 {
pluginInput.Source = p.ManifestLocation
}
//Calculate manifest location base on current instance's region
pluginInput.Source = strings.Replace(pluginInput.Source, updateutil.RegionHolder, context.Region, -1)
//Calculate updater package name base on agent name
pluginInput.UpdaterName = pluginInput.AgentName + updateutil.UpdaterPackageNamePrefix
//Generate update output
targetVersion := pluginInput.TargetVersion
if len(targetVersion) == 0 {
targetVersion = "latest"
}
out.AppendInfo(log, "Updating %v from %v to %v",
pluginInput.AgentName,
version.Version,
targetVersion)
//Download manifest file
manifest, downloadErr := manager.downloadManifest(log, util, &pluginInput, context, &out)
if downloadErr != nil {
out.Failed(log, downloadErr)
return
}
//Validate update details
noNeedToUpdate := false
if noNeedToUpdate, err = manager.validateUpdate(log, &pluginInput, context, manifest, &out); noNeedToUpdate {
if err != nil {
out.Failed(log, err)
}
return
}
//Download updater and retrieve the version number
updaterVersion := ""
if updaterVersion, err = manager.downloadUpdater(
log, util, pluginInput.UpdaterName, manifest, &out, context); err != nil {
out.Failed(log, err)
return
}
//Generate update command base on the update detail
cmd := ""
if cmd, err = manager.generateUpdateCmd(log,
manifest,
&pluginInput,
context,
updateutil.UpdaterFilePath(appconfig.UpdaterArtifactsRoot, pluginInput.UpdaterName, updaterVersion),
config.MessageId,
p.StdoutFileName,
p.StderrFileName,
outputS3KeyPrefix,
outputS3BucketName); err != nil {
out.Failed(log, err)
return
}
log.Debugf("Update command %v", cmd)
//Save update plugin result to local file, updater will read it during agent update
updatePluginResult := &updateutil.UpdatePluginResult{
StandOut: out.Stdout,
StartDateTime: startTime,
}
if err = util.SaveUpdatePluginResult(log, appconfig.UpdaterArtifactsRoot, updatePluginResult); err != nil {
out.Failed(log, err)
return
}
// If disk space is not sufficient, fail the update to prevent installation and notify user in output
// If loading disk space fails, continue to update (agent update is backed by rollback handler)
log.Infof("Checking available disk space ...")
if isDiskSpaceSufficient, err := util.IsDiskSpaceSufficientForUpdate(log); err == nil && !isDiskSpaceSufficient {
out.Failed(log, errors.New("Insufficient available disk space"))
return
}
log.Infof("Start Installation")
log.Infof("Hand over update process to %v", pluginInput.UpdaterName)
//Execute updater, hand over the update process
workDir := updateutil.UpdateArtifactFolder(
appconfig.UpdaterArtifactsRoot, pluginInput.UpdaterName, updaterVersion)
if err = util.ExeCommand(
log,
cmd,
workDir,
appconfig.UpdaterArtifactsRoot,
p.StdoutFileName,
p.StderrFileName,
true); err != nil {
out.Failed(log, err)
return
}
out.Pending()
return
}
// runCommands executes one set of commands and returns their output.
func (p *Plugin) runCommands(log log.T, pluginInput PSModulePluginInput, orchestrationDirectory string, cancelFlag task.CancelFlag, outputS3BucketName string, outputS3KeyPrefix string) (out PSModulePluginOutput) {
var err error
// if no orchestration directory specified, create temp directory
var useTempDirectory = (orchestrationDirectory == "")
var tempDir string
if useTempDirectory {
if tempDir, err = ioutil.TempDir("", "Ec2RunCommand"); err != nil {
out.Errors = append(out.Errors, err.Error())
log.Error(err)
return
}
orchestrationDirectory = tempDir
}
orchestrationDir := fileutil.RemoveInvalidChars(filepath.Join(orchestrationDirectory, pluginInput.ID))
log.Debugf("Running commands %v in workingDirectory %v; orchestrationDir %v ", pluginInput.RunCommand, pluginInput.WorkingDirectory, orchestrationDir)
// create orchestration dir if needed
if err = fileutil.MakeDirsWithExecuteAccess(orchestrationDir); err != nil {
log.Debug("failed to create orchestrationDir directory", orchestrationDir)
out.Errors = append(out.Errors, err.Error())
return
}
// Create script file path
scriptPath := filepath.Join(orchestrationDir, pluginutil.RunCommandScriptName)
log.Debugf("Writing commands %v to file %v", pluginInput, scriptPath)
// Create script file
if err = pluginutil.CreateScriptFile(log, scriptPath, pluginInput.RunCommand); err != nil {
out.Errors = append(out.Errors, err.Error())
log.Errorf("failed to create script file. %v", err)
return
}
// Download file from source if available
downloadOutput, err := pluginutil.DownloadFileFromSource(log, pluginInput.Source, pluginInput.SourceHash, pluginInput.SourceHashType)
if err != nil || downloadOutput.IsHashMatched == false || downloadOutput.LocalFilePath == "" {
errorString := fmt.Errorf("failed to download file reliably %v", pluginInput.Source)
out.MarkAsFailed(log, errorString)
return
} else {
// Uncompress the zip file received
fileutil.Uncompress(downloadOutput.LocalFilePath, PowerShellModulesDirectory)
}
// Set execution time
executionTimeout := pluginutil.ValidateExecutionTimeout(log, pluginInput.TimeoutSeconds)
// Create output file paths
stdoutFilePath := filepath.Join(orchestrationDir, p.StdoutFileName)
stderrFilePath := filepath.Join(orchestrationDir, p.StderrFileName)
log.Debugf("stdout file %v, stderr file %v", stdoutFilePath, stderrFilePath)
// Construct Command Name and Arguments
commandName := pluginutil.GetShellCommand()
commandArguments := append(pluginutil.GetShellArguments(), scriptPath, pluginutil.ExitCodeTrap)
// Execute Command
stdout, stderr, exitCode, errs := p.ExecuteCommand(log, pluginInput.WorkingDirectory, stdoutFilePath, stderrFilePath, cancelFlag, executionTimeout, commandName, commandArguments)
// Set output status
out.ExitCode = exitCode
out.Status = pluginutil.GetStatus(out.ExitCode, cancelFlag)
if len(errs) > 0 {
for _, err := range errs {
out.Errors = append(out.Errors, err.Error())
if out.Status != contracts.ResultStatusCancelled &&
out.Status != contracts.ResultStatusTimedOut &&
out.Status != contracts.ResultStatusSuccessAndReboot {
log.Error("failed to run commands: ", err)
out.Status = contracts.ResultStatusFailed
}
}
}
// read (a prefix of) the standard output/error
out.Stdout, err = pluginutil.ReadPrefix(stdout, p.MaxStdoutLength, p.OutputTruncatedSuffix)
if err != nil {
out.Errors = append(out.Errors, err.Error())
log.Error(err)
}
out.Stderr, err = pluginutil.ReadPrefix(stderr, p.MaxStderrLength, p.OutputTruncatedSuffix)
if err != nil {
out.Errors = append(out.Errors, err.Error())
log.Error(err)
}
// Upload output to S3
uploadOutputToS3BucketErrors := p.ExecuteUploadOutputToS3Bucket(log, pluginInput.ID, orchestrationDir, outputS3BucketName, outputS3KeyPrefix, useTempDirectory, tempDir, out.Stdout, out.Stderr)
out.Errors = append(out.Errors, uploadOutputToS3BucketErrors...)
// Return Json indented response
responseContent, _ := jsonutil.Marshal(out)
log.Debug("Returning response:\n", jsonutil.Indent(responseContent))
return
}
func getPlatformName(log log.T) (value string, err error) {
value, err = getPlatformDetail(log, "-productName")
log.Debugf("platform name: %v", value)
return
}
func getPlatformVersion(log log.T) (value string, err error) {
value, err = getPlatformDetail(log, "-productVersion")
log.Debugf("platform version: %v", value)
return
}
func getPlatformDetails(log log.T) (name string, version string, err error) {
log.Debugf(gettingPlatformDetailsMessage)
contents := ""
var contentsBytes []byte
name = notAvailableMessage
version = notAvailableMessage
if fileutil.Exists(systemReleaseCommand) {
log.Debugf(fetchingDetailsMessage, systemReleaseCommand)
contents, err = fileutil.ReadAllText(systemReleaseCommand)
log.Debugf(commandOutputMessage, contents)
if err != nil {
log.Debugf(errorOccurredMessage, systemReleaseCommand, err)
return
}
if strings.Contains(contents, "Amazon") {
data := strings.Split(contents, "release")
name = strings.TrimSpace(data[0])
version = strings.TrimSpace(data[1])
} else if strings.Contains(contents, "Red Hat") {
data := strings.Split(contents, "release")
name = strings.TrimSpace(data[0])
version = strings.TrimSpace(data[1])
} else if strings.Contains(contents, "CentOS") {
data := strings.Split(contents, "release")
name = strings.TrimSpace(data[0])
version = strings.TrimSpace(data[1])
}
} else if fileutil.Exists(redhatReleaseCommand) {
log.Debugf(fetchingDetailsMessage, redhatReleaseCommand)
contents, err = fileutil.ReadAllText(redhatReleaseCommand)
log.Debugf(commandOutputMessage, contents)
if err != nil {
log.Debugf(errorOccurredMessage, redhatReleaseCommand, err)
return
}
if strings.Contains(contents, "Red Hat") {
data := strings.Split(contents, "release")
name = strings.TrimSpace(data[0])
versionData := strings.Split(data[1], "(")
version = strings.TrimSpace(versionData[0])
}
} else {
log.Debugf(fetchingDetailsMessage, lsbReleaseCommand)
// platform name
if contentsBytes, err = exec.Command(lsbReleaseCommand, "-i").Output(); err != nil {
log.Debugf(fetchingDetailsMessage, lsbReleaseCommand, err)
return
}
name = strings.TrimSpace(string(contentsBytes))
log.Debugf(commandOutputMessage, name)
name = strings.TrimSpace(string(contentsBytes))
name = strings.TrimLeft(name, "Distributor ID:")
name = strings.TrimSpace(name)
log.Debugf("platform name %v", name)
// platform version
if contentsBytes, err = exec.Command(lsbReleaseCommand, "-r").Output(); err != nil {
log.Debugf(errorOccurredMessage, lsbReleaseCommand, err)
return
}
version = strings.TrimSpace(string(contentsBytes))
log.Debugf(commandOutputMessage, version)
version = strings.TrimLeft(version, "Release:")
version = strings.TrimSpace(version)
log.Debugf("platform version %v", version)
}
return
}
// httpDownload attempts to download a file via http/s call
func httpDownload(log log.T, fileURL string, destFile string) (output DownloadOutput, err error) {
log.Debugf("attempting to download as http/https download %v", destFile)
eTagFile := destFile + ".etag"
var check http.Client
var request *http.Request
request, err = http.NewRequest("GET", fileURL, nil)
if err != nil {
return
}
if fileutil.Exists(destFile) == true && fileutil.Exists(eTagFile) == true {
var existingETag string
existingETag, err = fileutil.ReadAllText(eTagFile)
request.Header.Add("If-None-Match", existingETag)
}
check = http.Client{
CheckRedirect: func(r *http.Request, via []*http.Request) error {
r.URL.Opaque = r.URL.Path
return nil
},
}
var resp *http.Response
resp, err = check.Do(request)
if err != nil {
log.Debug("failed to download from http/https, ", err)
fileutil.DeleteFile(destFile)
fileutil.DeleteFile(eTagFile)
return
}
if resp.StatusCode == http.StatusNotModified {
log.Debugf("Unchanged file.")
output.IsUpdated = false
output.LocalFilePath = destFile
return output, nil
} else if resp.StatusCode != http.StatusOK {
log.Debug("failed to download from http/https, ", err)
fileutil.DeleteFile(destFile)
fileutil.DeleteFile(eTagFile)
err = fmt.Errorf("http request failed. status:%v statuscode:%v", resp.Status, resp.StatusCode)
return
}
defer resp.Body.Close()
eTagValue := resp.Header.Get("Etag")
if eTagValue != "" {
log.Debug("file eTagValue is ", eTagValue)
err = fileutil.WriteAllText(eTagFile, eTagValue)
if err != nil {
log.Errorf("failed to write eTagfile %v, %v ", eTagFile, err)
return
}
}
_, err = FileCopy(log, destFile, resp.Body)
if err == nil {
output.LocalFilePath = destFile
output.IsUpdated = true
} else {
log.Errorf("failed to write destFile %v, %v ", destFile, err)
}
return
}
// s3Download attempts to download a file via the aws sdk.
func s3Download(log log.T, amazonS3URL s3util.AmazonS3URL, destFile string) (output DownloadOutput, err error) {
log.Debugf("attempting to download as s3 download %v", destFile)
eTagFile := destFile + ".etag"
config := &aws.Config{}
var appConfig appconfig.SsmagentConfig
appConfig, err = appconfig.Config(false)
if err != nil {
log.Error("failed to read appconfig.")
} else {
creds, err1 := appConfig.ProfileCredentials()
if err1 != nil {
config.Credentials = creds
}
}
config.S3ForcePathStyle = aws.Bool(amazonS3URL.IsPathStyle)
config.Region = aws.String(amazonS3URL.Region)
params := &s3.GetObjectInput{
Bucket: aws.String(amazonS3URL.Bucket),
Key: aws.String(amazonS3URL.Key),
}
if fileutil.Exists(destFile) == true && fileutil.Exists(eTagFile) == true {
var existingETag string
existingETag, err = fileutil.ReadAllText(eTagFile)
if err != nil {
log.Debugf("failed to read etag file %v, %v", eTagFile, err)
return
}
params.IfNoneMatch = aws.String(existingETag)
}
s3client := s3.New(session.New(config))
req, resp := s3client.GetObjectRequest(params)
err = req.Send()
if err != nil {
if req.HTTPResponse == nil || req.HTTPResponse.StatusCode != http.StatusNotModified {
log.Debug("failed to download from s3, ", err)
fileutil.DeleteFile(destFile)
fileutil.DeleteFile(eTagFile)
return
}
log.Debugf("Unchanged file.")
output.IsUpdated = false
output.LocalFilePath = destFile
return output, nil
}
if *resp.ETag != "" {
log.Debug("files etag is ", *resp.ETag)
err = fileutil.WriteAllText(eTagFile, *resp.ETag)
if err != nil {
log.Errorf("failed to write eTagfile %v, %v ", eTagFile, err)
return
}
}
defer resp.Body.Close()
_, err = FileCopy(log, destFile, resp.Body)
if err == nil {
output.LocalFilePath = destFile
output.IsUpdated = true
} else {
log.Errorf("failed to write destFile %v, %v ", destFile, err)
}
return
}
// ExeCommand executes shell command
func (util *Utility) ExeCommand(
log log.T,
cmd string,
workingDir string,
updaterRoot string,
stdOut string,
stdErr string,
isAsync bool) (err error) {
parts := strings.Fields(cmd)
if isAsync {
command := execCommand(parts[0], parts[1:]...)
command.Dir = workingDir
prepareProcess(command)
// Start command asynchronously
err = cmdStart(command)
if err != nil {
return
}
} else {
tempCmd := setPlatformSpecificCommand(parts)
command := execCommand(tempCmd[0], tempCmd[1:]...)
command.Dir = workingDir
stdoutWriter, stderrWriter, exeErr := setExeOutErr(updaterRoot, stdOut, stdErr)
if exeErr != nil {
return exeErr
}
defer stdoutWriter.Close()
defer stderrWriter.Close()
command.Stdout = stdoutWriter
command.Stderr = stderrWriter
err = cmdStart(command)
if err != nil {
return
}
timer := time.NewTimer(time.Duration(DefaultUpdateExecutionTimeoutInSeconds) * time.Second)
go killProcessOnTimeout(log, command, timer)
err = command.Wait()
timedOut := !timer.Stop()
if err != nil {
log.Debugf("command failed to run %v", err)
if exitErr, ok := err.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
exitCode := status.ExitStatus()
if exitCode == -1 && timedOut {
// set appropriate exit code based on cancel or timeout
exitCode = pluginutil.CommandStoppedPreemptivelyExitCode
log.Infof("The execution of command was timedout.")
}
err = fmt.Errorf("The execution of command returned Exit Status: %d \n %v", exitCode, err.Error())
}
}
return err
}
}
return nil
}
// RunCommand runs the given commands using the given working directory.
// Standard output and standard error are sent to the given writers.
func RunCommand(log log.T,
cancelFlag task.CancelFlag,
workingDir string,
stdoutWriter io.Writer,
stderrWriter io.Writer,
executionTimeout int,
commandName string,
commandArguments []string,
) (exitCode int, err error) {
command := exec.Command(commandName, commandArguments...)
command.Dir = workingDir
command.Stdout = stdoutWriter
command.Stderr = stderrWriter
exitCode = 0
// configure OS-specific process settings
prepareProcess(command)
log.Debug()
log.Debugf("Running in directory %v, command: %v %v.", workingDir, commandName, commandArguments)
log.Debug()
if err = command.Start(); err != nil {
log.Error("error occurred starting the command", err)
exitCode = 1
return
}
go killProcessOnCancel(log, command, cancelFlag)
timer := time.NewTimer(time.Duration(executionTimeout) * time.Second)
go killProcessOnTimeout(log, command, timer)
err = command.Wait()
timedOut := !timer.Stop() // returns false if called previously - indicates timedOut.
if err != nil {
exitCode = 1
log.Debugf("command failed to run %v", err)
if exiterr, ok := err.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
exitCode = status.ExitStatus()
// First try to handle Cancel and Timeout scenarios
// SIGKILL will result in an exitcode of -1
if exitCode == -1 {
if cancelFlag.Canceled() {
// set appropriate exit code based on cancel or timeout
exitCode = pluginutil.CommandStoppedPreemptivelyExitCode
log.Infof("The execution of command was cancelled.")
} else if timedOut {
// set appropriate exit code based on cancel or timeout
exitCode = pluginutil.CommandStoppedPreemptivelyExitCode
log.Infof("The execution of command was timedout.")
}
} else {
log.Infof("The execution of command returned Exit Status: %d", exitCode)
}
}
}
} else {
// check if cancellation or timeout failed to kill the process
// This will not occur as we do a SIGKILL, which is not recoverable.
if cancelFlag.Canceled() {
// This is when the cancellation failed and the command completed successfully
log.Errorf("the cancellation failed to stop the process.")
// do not return as the command could have been cancelled and also timedout
}
if timedOut {
// This is when the timeout failed and the command completed successfully
log.Errorf("the timeout failed to stop the process.")
}
}
log.Debug("Done waiting!")
return
}
// ParseAmazonS3URL parses a URL and returns AmazonS3URL object
func ParseAmazonS3URL(log log.T, s3URL *url.URL) (output AmazonS3URL) {
output = AmazonS3URL{
IsValidS3URI: false,
IsPathStyle: false,
Bucket: "",
Key: "",
Region: "",
}
match, _ := regexp.MatchString(EndpointPattern, s3URL.Host)
if match == false {
// Invalid S3 URI - hostname does not appear to be a valid S3 endpoint
output.IsValidS3URI = match
return
}
log.Debugf("%v is valid s3 url", s3URL.String())
endpointRegEx, err := regexp.Compile(EndpointPattern)
if err != nil {
output.IsValidS3URI = false
return
}
output.IsValidS3URI = true
// for host style urls:
// group 0 is bucketname plus 's3' prefix and possible region code
// group 1 is bucket name
// group 2 will be region or 'amazonaws' if US Classic region
// for path style urls:
// group 0 will be s3 prefix plus possible region code
// group 1 will be empty
// group 2 will be region or 'amazonaws' if US Classic region
result := endpointRegEx.FindStringSubmatch(s3URL.Host)
bucketNameGroup := ""
if len(result) > 1 {
bucketNameGroup = result[1]
}
path := s3URL.Path
//log.Debugf("endpointRegEx.FindStringSubmatch =%v, path=%v" , result,path)
if bucketNameGroup == "" {
// no bucket name in the authority, parse it from the path
output.IsPathStyle = true
// grab the encoded path so we don't run afoul of '/'s in the bucket name
if path == "/" {
} else {
path = path[1:]
index := strings.Index(path, "/")
if index == -1 {
// https://s3.amazonaws.com/bucket
output.Bucket = path
output.Key = ""
} else if index == (len(path) - 1) {
// https://s3.amazonaws.com/bucket/
output.Bucket = strings.TrimRight(path, "/")
output.Key = ""
} else {
// https://s3.amazonaws.com/bucket/key
output.Bucket = path[:index]
output.Key = path[index+1:]
}
}
} else {
// bucket name in the host, path is the object key
output.IsPathStyle = false
output.Bucket = strings.TrimRight(bucketNameGroup, ".")
if path == "/" {
output.Key = ""
} else {
output.Key = path[1:]
}
}
if len(result) > 2 {
bucketNameGroup = result[1]
regionGroupValue := result[2]
if strings.EqualFold(regionGroupValue, "external-1") {
output.Region = "us-east-1"
} else if !strings.EqualFold(regionGroupValue, "amazonaws") {
output.Region = regionGroupValue
}
}
// Temporary fix to allow pipeline tests to work
// Bucket ssmagent-alpha, ssmagent-beta, ssmagent are for internal testing
if (output.Bucket == "ssmagent-alpha" || output.Bucket == "ssmagent-beta" || output.Bucket == "ssmagent") && output.Region == "" {
output.Region = "us-east-1"
}
return
}