Example #1
0
// GetS3Config returns the S3 config used for uploading output files to S3
func GetS3Config() *s3util.Manager {
	//There are multiple ways of supporting the cross-region upload to S3 bucket:
	//1) We can specify the url https://s3.amazonaws.com and not specify region in our s3 client. This approach only works in java & .net but not in golang
	//since it enforces to use region in our client.
	//2) We can make use of GetBucketLocation API to find the location of S3 bucket. This is a better way to handle this, however it has its own disadvantages:
	//-> We will have to update the managed policy of AmazonEC2RoleforSSM so that agent will have permissions to make that call.
	//-> We will still have to notify our customers regarding the change in our IAM policy - such that customers using inline policy will also make the change accordingly.
	//3) Special behavior for S3 PutObject API for IAD region which is described in detail below.
	//We have taken the 3rd option - until the changes for the 2nd option is in place.

	//In our current implementation, we upload a test S3 file and use the error message to determine the bucket's region,
	//but we do this with region set as "us-east-1". This is because of special behavior of S3 PutObject API:
	//Only for the endpoint "us-east-1", if the bucket is present in any other region (i.e non IAD bucket) PutObject API will throw an
	//error of type - AuthorizationHeaderMalformed with a message stating which region is the bucket present. A sample error message looks like:
	//AuthorizationHeaderMalformed: The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'us-west-2' status code: 400, request id: []

	//We leverage the above error message to determine the bucket's region, and if there is no error - that means the bucket is indeed in IAD.

	//Note: The above behavior only exists for IAD endpoint (special endpoint for S3) - not just any other region.
	//For other region endpoints, you get a BucketRegionError which is not useful for us in determining where the bucket is present.
	//Revisit this if S3 ensures the PutObject API behavior consistent over all endpoints - in which case - instead of using IAD endpoint,
	//we can then pick the endpoint from meta-data instead.

	awsConfig := sdkutil.AwsConfig()

	if region, err := platform.Region(); err == nil && region == s3Bjs {
		awsConfig.Endpoint = &s3BjsEndpoint
		awsConfig.Region = &s3Bjs
	} else {
		awsConfig.Endpoint = &s3StandardEndpoint
		awsConfig.Region = &S3RegionUSStandard
	}
	s3 := s3.New(session.New(awsConfig))
	return s3util.NewManager(s3)
}
Example #2
0
// uploadOutput uploads the stdout and stderr file to S3
func (c *contextManager) uploadOutput(log log.T, context *UpdateContext) (err error) {

	awsConfig := sdkutil.AwsConfig()
	var config appconfig.SsmagentConfig
	config, err = appconfig.Config(false)

	if err != nil {
		return fmt.Errorf("could not load config file: %v", err)
	}
	// If customers have provided override in app config, honor that.
	if config.S3.Region != "" {
		awsConfig.Region = &config.S3.Region
	}
	log.Infof("Uploading output files to region: %v", *awsConfig.Region)

	s3 := s3.New(session.New(awsConfig))

	// upload outputs (if any) to s3
	uploader := s3util.NewManager(s3)
	uploadOutputsToS3 := func() {
		// delete temp outputDir once we're done
		defer pluginutil.DeleteDirectory(log, updateutil.UpdateOutputDirectory(context.Current.UpdateRoot))

		// get stdout file path
		stdoutPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StdoutFileName)
		s3Key := path.Join(context.Current.OutputS3KeyPrefix, context.Current.StdoutFileName)
		log.Debugf("Uploading %v to s3://%v/%v", stdoutPath, context.Current.OutputS3BucketName, s3Key)
		err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stdoutPath)
		if err != nil {
			log.Errorf("failed uploading %v to s3://%v/%v \n err:%v",
				stdoutPath,
				context.Current.OutputS3BucketName,
				s3Key,
				err)
		}

		// get stderr file path
		stderrPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StderrFileName)
		s3Key = path.Join(context.Current.OutputS3KeyPrefix, context.Current.StderrFileName)
		log.Debugf("Uploading %v to s3://%v/%v", stderrPath, context.Current.OutputS3BucketName, s3Key)
		err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stderrPath)
		if err != nil {
			log.Errorf("failed uploading %v to s3://%v/%v \n err:%v", stderrPath, context.Current.StderrFileName, s3Key, err)
		}
	}

	uploadOutputsToS3()

	return nil
}