Example #1
0
// GetS3Config returns the S3 config used for uploading output files to S3
func GetS3Config() *s3util.Manager {
	//There are multiple ways of supporting the cross-region upload to S3 bucket:
	//1) We can specify the url https://s3.amazonaws.com and not specify region in our s3 client. This approach only works in java & .net but not in golang
	//since it enforces to use region in our client.
	//2) We can make use of GetBucketLocation API to find the location of S3 bucket. This is a better way to handle this, however it has its own disadvantages:
	//-> We will have to update the managed policy of AmazonEC2RoleforSSM so that agent will have permissions to make that call.
	//-> We will still have to notify our customers regarding the change in our IAM policy - such that customers using inline policy will also make the change accordingly.
	//3) Special behavior for S3 PutObject API for IAD region which is described in detail below.
	//We have taken the 3rd option - until the changes for the 2nd option is in place.

	//In our current implementation, we upload a test S3 file and use the error message to determine the bucket's region,
	//but we do this with region set as "us-east-1". This is because of special behavior of S3 PutObject API:
	//Only for the endpoint "us-east-1", if the bucket is present in any other region (i.e non IAD bucket) PutObject API will throw an
	//error of type - AuthorizationHeaderMalformed with a message stating which region is the bucket present. A sample error message looks like:
	//AuthorizationHeaderMalformed: The authorization header is malformed; the region 'us-east-1' is wrong; expecting 'us-west-2' status code: 400, request id: []

	//We leverage the above error message to determine the bucket's region, and if there is no error - that means the bucket is indeed in IAD.

	//Note: The above behavior only exists for IAD endpoint (special endpoint for S3) - not just any other region.
	//For other region endpoints, you get a BucketRegionError which is not useful for us in determining where the bucket is present.
	//Revisit this if S3 ensures the PutObject API behavior consistent over all endpoints - in which case - instead of using IAD endpoint,
	//we can then pick the endpoint from meta-data instead.

	awsConfig := sdkutil.AwsConfig()

	if region, err := platform.Region(); err == nil && region == s3Bjs {
		awsConfig.Endpoint = &s3BjsEndpoint
		awsConfig.Region = &s3Bjs
	} else {
		awsConfig.Endpoint = &s3StandardEndpoint
		awsConfig.Region = &S3RegionUSStandard
	}
	s3 := s3.New(session.New(awsConfig))
	return s3util.NewManager(s3)
}
Example #2
0
// NewService creates a new SSM service instance.
func NewService() Service {
	if ssmStopPolicy == nil {
		// create a stop policy where we will stop after 10 consecutive errors and if time period expires.
		ssmStopPolicy = sdkutil.NewStopPolicy("ssmService", 10)
	}

	awsConfig := sdkutil.AwsConfig()

	// parse appConfig overrides
	appConfig, err := appconfig.Config(false)
	if err == nil {
		if appConfig.Ssm.Endpoint != "" {
			awsConfig.Endpoint = &appConfig.Ssm.Endpoint
		}
		// TODO: test hook, can be removed before release
		// this is to skip ssl verification for the beta self signed certs
		if appConfig.Ssm.InsecureSkipVerify {
			tr := &http.Transport{
				TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			}
			awsConfig.HTTPClient = &http.Client{Transport: tr}
		}
	}

	ssmService := ssm.New(session.New(awsConfig))
	return &sdkService{sdk: ssmService}
}
Example #3
0
// uploadOutput uploads the stdout and stderr file to S3
func (c *contextManager) uploadOutput(log log.T, context *UpdateContext) (err error) {

	awsConfig := sdkutil.AwsConfig()
	var config appconfig.SsmagentConfig
	config, err = appconfig.Config(false)

	if err != nil {
		return fmt.Errorf("could not load config file: %v", err)
	}
	// If customers have provided override in app config, honor that.
	if config.S3.Region != "" {
		awsConfig.Region = &config.S3.Region
	}
	log.Infof("Uploading output files to region: %v", *awsConfig.Region)

	s3 := s3.New(session.New(awsConfig))

	// upload outputs (if any) to s3
	uploader := s3util.NewManager(s3)
	uploadOutputsToS3 := func() {
		// delete temp outputDir once we're done
		defer pluginutil.DeleteDirectory(log, updateutil.UpdateOutputDirectory(context.Current.UpdateRoot))

		// get stdout file path
		stdoutPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StdoutFileName)
		s3Key := path.Join(context.Current.OutputS3KeyPrefix, context.Current.StdoutFileName)
		log.Debugf("Uploading %v to s3://%v/%v", stdoutPath, context.Current.OutputS3BucketName, s3Key)
		err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stdoutPath)
		if err != nil {
			log.Errorf("failed uploading %v to s3://%v/%v \n err:%v",
				stdoutPath,
				context.Current.OutputS3BucketName,
				s3Key,
				err)
		}

		// get stderr file path
		stderrPath := updateutil.UpdateStandOutPath(context.Current.UpdateRoot, context.Current.StderrFileName)
		s3Key = path.Join(context.Current.OutputS3KeyPrefix, context.Current.StderrFileName)
		log.Debugf("Uploading %v to s3://%v/%v", stderrPath, context.Current.OutputS3BucketName, s3Key)
		err = uploader.S3Upload(context.Current.OutputS3BucketName, s3Key, stderrPath)
		if err != nil {
			log.Errorf("failed uploading %v to s3://%v/%v \n err:%v", stderrPath, context.Current.StderrFileName, s3Key, err)
		}
	}

	uploadOutputsToS3()

	return nil
}
Example #4
0
// NewService creates a new MDS service instance.
func NewService(region string, endpoint string, creds *credentials.Credentials, connectionTimeout time.Duration) Service {

	config := sdkutil.AwsConfig()

	if region != "" {
		config.Region = &region
	}

	if endpoint != "" {
		config.Endpoint = &endpoint
	}

	if creds != nil {
		config.Credentials = creds
	}

	// capture Transport so we can use it to cancel requests
	tr := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   connectionTimeout,
			KeepAlive: 0,
		}).Dial,
		TLSHandshakeTimeout: 10 * time.Second,
	}
	config.HTTPClient = &http.Client{Transport: tr, Timeout: connectionTimeout}

	msgSvc := ssmmds.New(session.New(config))

	//adding server based expected error messages
	serverBasedErrorMessages = make([]string, 2)
	serverBasedErrorMessages = append(serverBasedErrorMessages, "use of closed network connection")
	serverBasedErrorMessages = append(serverBasedErrorMessages, "connection reset by peer")

	//adding client based expected error messages
	clientBasedErrorMessages = make([]string, 1)
	clientBasedErrorMessages = append(clientBasedErrorMessages, "Client.Timeout exceeded while awaiting headers")

	return &sdkService{sdk: msgSvc, tr: tr}
}