// GetS3Config returns the S3 config used for uploading output files to S3 func GetS3Config() *s3util.Manager { //There are multiple ways of supporting the cross-region upload to S3 bucket: //1) We can specify the url https://s3.amazonaws.com and not specify region in our s3 client. This approach only works in java & .net but not in golang //since it enforces to use region in our client. //2) We can make use of GetBucketLocation API to find the location of S3 bucket. This is a better way to handle this, however it has its own disadvantages: //-> We will have to update the managed policy of AmazonEC2RoleforSSM so that agent will have permissions to make that call. //-> We will still have to notify our customers regarding the change in our IAM policy - such that customers using inline policy will also make the change accordingly. //3) Special behavior for S3 PutObject API for IAD region which is described in detail below. //We have taken the 3rd option - until the changes for the 2nd option is in place. //In our current implementation, we upload a test S3 file and use the error message to determine the bucket's region, //but we do this with region set as "us-east-1". This is because of special behavior of S3 PutObject API: //Only for the endpoint "us-east-1", if the bucket is present in any other region (i.e non IAD bucket) PutObject API will throw an //error of type - AuthorizationHeaderMalformed with a message stating which region is the bucket present. A sample error message looks like: //AuthorizationHeaderMalformed: The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'us-west-2' status code: 400, request id: [] //We leverage the above error message to determine the bucket's region, and if there is no error - that means the bucket is indeed in IAD. //Note: The above behavior only exists for IAD endpoint (special endpoint for S3) - not just any other region. //For other region endpoints, you get a BucketRegionError which is not useful for us in determining where the bucket is present. //Revisit this if S3 ensures the PutObject API behavior consistent over all endpoints - in which case - instead of using IAD endpoint, //we can then pick the endpoint from meta-data instead. awsConfig := sdkutil.AwsConfig() if region, err := platform.Region(); err == nil && region == s3Bjs { awsConfig.Endpoint = &s3BjsEndpoint awsConfig.Region = &s3Bjs } else { awsConfig.Endpoint = &s3StandardEndpoint awsConfig.Region = &S3RegionUSStandard } s3 := s3.New(session.New(awsConfig)) return s3util.NewManager(s3) }
// uploadOutput uploads the stdout and stderr file to S3 func (c *contextManager) uploadOutput(log log.T, context *UpdateContext) (err error) { awsConfig := sdkutil.AwsConfig() var config appconfig.SsmagentConfig config, err = appconfig.Config(false) if err != nil { return fmt.Errorf("could not load config file: %v", err) } // If customers have provided override in app config, honor that. if config.S3.Region != "" { awsConfig.Region = &config.S3.Region } log.Infof("Uploading output files to region: %v", *awsConfig.Region) s3 := s3.New(session.New(awsConfig)) // upload outputs (if any) to s3 uploader := s3util.NewManager(s3) uploadOutputsToS3 := func() { // delete temp outputDir once we're done defer pluginutil.DeleteDirectory(log, updateutil.UpdateOutputDirectory(context.Current.UpdateRoot)) // get stdout file path stdoutPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StdoutFileName) s3Key := path.Join(context.Current.OutputS3KeyPrefix, context.Current.StdoutFileName) log.Debugf("Uploading %v to s3://%v/%v", stdoutPath, context.Current.OutputS3BucketName, s3Key) err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stdoutPath) if err != nil { log.Errorf("failed uploading %v to s3://%v/%v \n err:%v", stdoutPath, context.Current.OutputS3BucketName, s3Key, err) } // get stderr file path stderrPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StderrFileName) s3Key = path.Join(context.Current.OutputS3KeyPrefix, context.Current.StderrFileName) log.Debugf("Uploading %v to s3://%v/%v", stderrPath, context.Current.OutputS3BucketName, s3Key) err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stderrPath) if err != nil { log.Errorf("failed uploading %v to s3://%v/%v \n err:%v", stderrPath, context.Current.StderrFileName, s3Key, err) } } uploadOutputsToS3() return nil }