// GetS3Config returns the S3 config used for uploading output files to S3
func GetS3Config() *s3util.Manager {
//There are multiple ways of supporting the cross-region upload to S3 bucket:
//1) We can specify the url https://s3.amazonaws.com and not specify region in our s3 client. This approach only works in java & .net but not in golang
//since it enforces to use region in our client.
//2) We can make use of GetBucketLocation API to find the location of S3 bucket. This is a better way to handle this, however it has its own disadvantages:
//-> We will have to update the managed policy of AmazonEC2RoleforSSM so that agent will have permissions to make that call.
//-> We will still have to notify our customers regarding the change in our IAM policy - such that customers using inline policy will also make the change accordingly.
//3) Special behavior for S3 PutObject API for IAD region which is described in detail below.
//We have taken the 3rd option - until the changes for the 2nd option is in place.
//In our current implementation, we upload a test S3 file and use the error message to determine the bucket's region,
//but we do this with region set as "us-east-1". This is because of special behavior of S3 PutObject API:
//Only for the endpoint "us-east-1", if the bucket is present in any other region (i.e non IAD bucket) PutObject API will throw an
//error of type - AuthorizationHeaderMalformed with a message stating which region is the bucket present. A sample error message looks like:
//AuthorizationHeaderMalformed: The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'us-west-2' status code: 400, request id: []
//We leverage the above error message to determine the bucket's region, and if there is no error - that means the bucket is indeed in IAD.
//Note: The above behavior only exists for IAD endpoint (special endpoint for S3) - not just any other region.
//For other region endpoints, you get a BucketRegionError which is not useful for us in determining where the bucket is present.
//Revisit this if S3 ensures the PutObject API behavior consistent over all endpoints - in which case - instead of using IAD endpoint,
//we can then pick the endpoint from meta-data instead.
awsConfig := sdkutil.AwsConfig()
if region, err := platform.Region(); err == nil && region == s3Bjs {
awsConfig.Endpoint = &s3BjsEndpoint
awsConfig.Region = &s3Bjs
} else {
awsConfig.Endpoint = &s3StandardEndpoint
awsConfig.Region = &S3RegionUSStandard
}
s3 := s3.New(session.New(awsConfig))
return s3util.NewManager(s3)
}
// NewService creates a new SSM service instance.
func NewService() Service {
if ssmStopPolicy == nil {
// create a stop policy where we will stop after 10 consecutive errors and if time period expires.
ssmStopPolicy = sdkutil.NewStopPolicy("ssmService", 10)
}
awsConfig := sdkutil.AwsConfig()
// parse appConfig overrides
appConfig, err := appconfig.Config(false)
if err == nil {
if appConfig.Ssm.Endpoint != "" {
awsConfig.Endpoint = &appConfig.Ssm.Endpoint
}
// TODO: test hook, can be removed before release
// this is to skip ssl verification for the beta self signed certs
if appConfig.Ssm.InsecureSkipVerify {
tr := &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
awsConfig.HTTPClient = &http.Client{Transport: tr}
}
}
ssmService := ssm.New(session.New(awsConfig))
return &sdkService{sdk: ssmService}
}
// uploadOutput uploads the stdout and stderr file to S3
func (c *contextManager) uploadOutput(log log.T, context *UpdateContext) (err error) {
awsConfig := sdkutil.AwsConfig()
var config appconfig.SsmagentConfig
config, err = appconfig.Config(false)
if err != nil {
return fmt.Errorf("could not load config file: %v", err)
}
// If customers have provided override in app config, honor that.
if config.S3.Region != "" {
awsConfig.Region = &config.S3.Region
}
log.Infof("Uploading output files to region: %v", *awsConfig.Region)
s3 := s3.New(session.New(awsConfig))
// upload outputs (if any) to s3
uploader := s3util.NewManager(s3)
uploadOutputsToS3 := func() {
// delete temp outputDir once we're done
defer pluginutil.DeleteDirectory(log, updateutil.UpdateOutputDirectory(context.Current.UpdateRoot))
// get stdout file path
stdoutPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StdoutFileName)
s3Key := path.Join(context.Current.OutputS3KeyPrefix, context.Current.StdoutFileName)
log.Debugf("Uploading %v to s3://%v/%v", stdoutPath, context.Current.OutputS3BucketName, s3Key)
err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stdoutPath)
if err != nil {
log.Errorf("failed uploading %v to s3://%v/%v \n err:%v",
stdoutPath,
context.Current.OutputS3BucketName,
s3Key,
err)
}
// get stderr file path
stderrPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StderrFileName)
s3Key = path.Join(context.Current.OutputS3KeyPrefix, context.Current.StderrFileName)
log.Debugf("Uploading %v to s3://%v/%v", stderrPath, context.Current.OutputS3BucketName, s3Key)
err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stderrPath)
if err != nil {
log.Errorf("failed uploading %v to s3://%v/%v \n err:%v", stderrPath, context.Current.StderrFileName, s3Key, err)
}
}
uploadOutputsToS3()
return nil
}
// NewService creates a new MDS service instance.
func NewService(region string, endpoint string, creds *credentials.Credentials, connectionTimeout time.Duration) Service {
config := sdkutil.AwsConfig()
if region != "" {
config.Region = ®ion
}
if endpoint != "" {
config.Endpoint = &endpoint
}
if creds != nil {
config.Credentials = creds
}
// capture Transport so we can use it to cancel requests
tr := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: connectionTimeout,
KeepAlive: 0,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
}
config.HTTPClient = &http.Client{Transport: tr, Timeout: connectionTimeout}
msgSvc := ssmmds.New(session.New(config))
//adding server based expected error messages
serverBasedErrorMessages = make([]string, 2)
serverBasedErrorMessages = append(serverBasedErrorMessages, "use of closed network connection")
serverBasedErrorMessages = append(serverBasedErrorMessages, "connection reset by peer")
//adding client based expected error messages
clientBasedErrorMessages = make([]string, 1)
clientBasedErrorMessages = append(clientBasedErrorMessages, "Client.Timeout exceeded while awaiting headers")
return &sdkService{sdk: msgSvc, tr: tr}
}