Beispiel #1
0
func init() {
	if os.Getenv("DEBUG") != "" {
		aws.DefaultConfig.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	if os.Getenv("DEBUG_SIGNING") != "" {
		aws.DefaultConfig.LogLevel = aws.LogLevel(aws.LogDebugWithSigning)
	}
	if os.Getenv("DEBUG_BODY") != "" {
		aws.DefaultConfig.LogLevel = aws.LogLevel(aws.LogDebugWithSigning | aws.LogDebugWithHTTPBody)
	}

	if aws.StringValue(aws.DefaultConfig.Region) == "" {
		panic("AWS_REGION must be configured to run integration tests")
	}
}
func copyConfig(config *Config) *aws.Config {
	if config == nil {
		config = &Config{}
	}
	c := &aws.Config{
		Credentials: credentials.AnonymousCredentials,
		Endpoint:    config.Endpoint,
		HTTPClient:  config.HTTPClient,
		Logger:      config.Logger,
		LogLevel:    config.LogLevel,
		MaxRetries:  config.MaxRetries,
	}

	if c.HTTPClient == nil {
		c.HTTPClient = http.DefaultClient
	}
	if c.Logger == nil {
		c.Logger = aws.NewDefaultLogger()
	}
	if c.LogLevel == nil {
		c.LogLevel = aws.LogLevel(aws.LogOff)
	}
	if c.MaxRetries == nil {
		c.MaxRetries = aws.Int(DefaultRetries)
	}

	return c
}
Beispiel #3
0
func config() *aws.Config {
	log := aws.LogLevel(aws.LogOff)
	cfg := app.NewConfig()
	if cfg.AwsLog {
		log = aws.LogLevel(aws.LogDebug)
	}
	return &aws.Config{
		Credentials: credentials.NewChainCredentials(
			[]credentials.Provider{
				&credentials.EnvProvider{},
				&ec2rolecreds.EC2RoleProvider{ExpiryWindow: cfg.AwsRoleExpiry * time.Minute},
			}),
		Region:   aws.String(os.Getenv("AWS_REGION")),
		LogLevel: log,
	}
}
Beispiel #4
0
func GetFile(region, bucketName, path, keyId, secretKey, token string) (io.ReadCloser, int64, error) {
	creds := credentials.NewStaticCredentials(keyId, secretKey, token)

	if _, err := creds.Get(); err != nil {
		return nil, 0, err
	}

	awsconfig := &aws.Config{
		Region:           aws.String(region),
		Endpoint:         aws.String("s3.amazonaws.com"),
		S3ForcePathStyle: aws.Bool(true),
		Credentials:      creds,
		LogLevel:         aws.LogLevel(0),
	}

	sess := session.New(awsconfig)
	svc := s3.New(sess)

	params := &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(path),
	}

	resp, err := svc.GetObject(params)
	if err != nil {
		return nil, 0, err
	}

	// log.Println(resp)
	return resp.Body, *resp.ContentLength, nil
}
Beispiel #5
0
// NewSessionWithLevel returns an AWS Session (https://github.com/aws/aws-sdk-go/wiki/Getting-Started-Configuration)
// object that attaches a debug level handler to all AWS requests from services
// sharing the session value.
func NewSessionWithLevel(level aws.LogLevelType, logger *logrus.Logger) *session.Session {
	awsConfig := &aws.Config{
		CredentialsChainVerboseErrors: aws.Bool(true),
	}
	// Log AWS calls if needed
	switch logger.Level {
	case logrus.DebugLevel:
		awsConfig.LogLevel = aws.LogLevel(level)
	}
	awsConfig.Logger = &logrusProxy{logger}
	sess := session.New(awsConfig)
	sess.Handlers.Send.PushFront(func(r *request.Request) {
		logger.WithFields(logrus.Fields{
			"Service":   r.ClientInfo.ServiceName,
			"Operation": r.Operation.Name,
			"Method":    r.Operation.HTTPMethod,
			"Path":      r.Operation.HTTPPath,
			"Payload":   r.Params,
		}).Debug("AWS Request")
	})

	logger.WithFields(logrus.Fields{
		"Name":    aws.SDKName,
		"Version": aws.SDKVersion,
	}).Debug("AWS SDK Info")

	return sess
}
func New(debug bool) MetadataFetcher {
	sess := session.New()
	if debug {
		sess.Config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return ec2metadata.New(sess)
}
func New(debug bool) MetadataFetcher {
	c := ec2metadata.Config{}
	if debug {
		c.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return ec2metadata.New(&c)
}
Beispiel #8
0
func TestMain(m *testing.M) {
	flag.Parse()
	if !*integration {
		fmt.Fprintln(os.Stderr, "Skipping integration tests")
		os.Exit(0)
	}
	cfg = &aws.Config{
		Region:      aws.String("us-west-2"),
		Endpoint:    aws.String("http://localhost:8000"),
		Credentials: credentials.NewSharedCredentials("", *awsprofile),
	}
	sess = session.New(cfg)
	if *dynamodebug {
		sess.Config.LogLevel = aws.LogLevel(aws.LogDebug)
	}

	if err := loadUserFixtures(sess); err != nil {
		fmt.Fprintf(os.Stderr, "Error loading 'user' integration fixtures: %s", err)
		os.Exit(1)
	}
	if err := loadPostFixtures(sess); err != nil {
		fmt.Fprintf(os.Stderr, "Error loading 'post' integration fixtures: %s", err)
		os.Exit(1)
	}
	os.Exit(m.Run())
}
Beispiel #9
0
func (app *App) roleHandler(w http.ResponseWriter, r *http.Request) {
	svc := sts.New(session.New(), &aws.Config{LogLevel: aws.LogLevel(2)})
	resp, err := svc.AssumeRole(&sts.AssumeRoleInput{
		RoleArn:         aws.String(app.RoleArn),
		RoleSessionName: aws.String("aws-mock-metadata"),
	})
	if err != nil {
		log.Errorf("Error assuming role %+v", err)
		http.Error(w, err.Error(), 500)
		return
	}
	log.Debugf("STS response %+v", resp)
	credentials := Credentials{
		AccessKeyID:     *resp.Credentials.AccessKeyId,
		Code:            "Success",
		Expiration:      resp.Credentials.Expiration.Format("2006-01-02T15:04:05Z"),
		LastUpdated:     time.Now().Format("2006-01-02T15:04:05Z"),
		SecretAccessKey: *resp.Credentials.SecretAccessKey,
		Token:           *resp.Credentials.SessionToken,
		Type:            "AWS-HMAC",
	}
	if err := json.NewEncoder(w).Encode(credentials); err != nil {
		log.Errorf("Error sending json %+v", err)
		http.Error(w, err.Error(), 500)
	}
}
Beispiel #10
0
func (gs GlacierStorage) getStorageClient() *glacier.Glacier {
	creds := credentials.NewStaticCredentials(gs.aws_access_key_id, gs.aws_secret_access_key, "")

	return glacier.New(&aws.Config{
		Region:      aws.String(gs.region),
		Credentials: creds,
		LogLevel:    aws.LogLevel(1),
	})
}
Beispiel #11
0
func getService(debug bool) *route53.Route53 {
	config := aws.Config{}
	// ensures throttled requests are retried
	config.MaxRetries = aws.Int(100)
	if debug {
		config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return route53.New(&config)
}
Beispiel #12
0
// GetECRAuth requests AWS ECR API to get docker.AuthConfiguration token
func GetECRAuth(registry, region string) (result docker.AuthConfiguration, err error) {
	_ecrAuthCache.mu.Lock()
	defer _ecrAuthCache.mu.Unlock()

	if token, ok := _ecrAuthCache.tokens[registry]; ok {
		return token, nil
	}

	defer func() {
		_ecrAuthCache.tokens[registry] = result
	}()

	cfg := &aws.Config{
		Region: aws.String(region),
	}

	if log.StandardLogger().Level >= log.DebugLevel {
		cfg.LogLevel = aws.LogLevel(aws.LogDebugWithRequestErrors)
	}

	split := strings.Split(registry, ".")

	svc := ecr.New(session.New(), cfg)
	params := &ecr.GetAuthorizationTokenInput{
		RegistryIds: []*string{aws.String(split[0])},
	}

	res, err := svc.GetAuthorizationToken(params)
	if err != nil {
		return result, err
	}

	if len(res.AuthorizationData) == 0 {
		return result, nil
	}

	data, err := base64.StdEncoding.DecodeString(*res.AuthorizationData[0].AuthorizationToken)
	if err != nil {
		return result, err
	}

	userpass := strings.Split(string(data), ":")
	if len(userpass) != 2 {
		return result, fmt.Errorf("Cannot parse token got from ECR: %s", string(data))
	}

	result = docker.AuthConfiguration{
		Username:      userpass[0],
		Password:      userpass[1],
		ServerAddress: *res.AuthorizationData[0].ProxyEndpoint,
	}

	return
}
Beispiel #13
0
func NewGoofys(bucket string, awsConfig *aws.Config, flags *FlagStorage) *Goofys {
	// Set up the basic struct.
	fs := &Goofys{
		bucket: bucket,
		flags:  flags,
		umask:  0122,
	}

	if flags.DebugS3 {
		awsConfig.LogLevel = aws.LogLevel(aws.LogDebug | aws.LogDebugWithRequestErrors)
		s3Log.Level = logrus.DebugLevel
	}

	fs.awsConfig = awsConfig
	fs.sess = session.New(awsConfig)
	fs.s3 = fs.newS3()

	err := fs.detectBucketLocation()
	if err != nil {
		return nil
	}

	now := time.Now()
	fs.rootAttrs = fuseops.InodeAttributes{
		Size:   4096,
		Nlink:  2,
		Mode:   flags.DirMode | os.ModeDir,
		Atime:  now,
		Mtime:  now,
		Ctime:  now,
		Crtime: now,
		Uid:    fs.flags.Uid,
		Gid:    fs.flags.Gid,
	}

	fs.bufferPool = BufferPool{}.Init()

	fs.nextInodeID = fuseops.RootInodeID + 1
	fs.inodes = make(map[fuseops.InodeID]*Inode)
	root := NewInode(aws.String(""), aws.String(""), flags)
	root.Id = fuseops.RootInodeID
	root.Attributes = &fs.rootAttrs

	fs.inodes[fuseops.RootInodeID] = root
	fs.inodesCache = make(map[string]*Inode)

	fs.nextHandleID = 1
	fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)

	fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)

	return fs
}
Beispiel #14
0
func getService(debug bool, profile string) *route53.Route53 {
	config := aws.Config{}
	if profile != "" {
		config.Credentials = credentials.NewSharedCredentials("", profile)
	}
	// ensures throttled requests are retried
	config.MaxRetries = aws.Int(100)
	if debug {
		config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return route53.New(session.New(), &config)
}
Beispiel #15
0
func getAWSConfig(region string, debug bool) *aws.Config {
	if debug {
		logging.SetLogging("DEBUG")
		return &aws.Config{
			Region:   aws.String(region),
			LogLevel: aws.LogLevel(aws.LogDebugWithHTTPBody),
		}
	}
	logging.SetLogging("INFO")
	return &aws.Config{
		Region: aws.String(region),
	}
}
Beispiel #16
0
func getConfig(c *cli.Context) *aws.Config {
	debug := c.Bool("debug")
	profile := c.String("profile")
	config := aws.Config{}
	if profile != "" {
		config.Credentials = credentials.NewSharedCredentials("", profile)
	}
	// ensures throttled requests are retried
	config.MaxRetries = aws.Int(100)
	if debug {
		config.LogLevel = aws.LogLevel(aws.LogDebug)
	}
	return &config
}
Beispiel #17
0
func doAction(c *cli.Context) {
	awsKeyID := c.String("aws-key-id")
	awsSecKey := c.String("aws-secret-key")
	mackerelAPIKey := c.String("mackerel-api-key")

	client := mkr.NewClient(mackerelAPIKey)

	sess := session.New(&aws.Config{
		Credentials: credentials.NewStaticCredentials(awsKeyID, awsSecKey, ""),
		Region:      aws.String("ap-northeast-1"),
	})
	awsSession := NewAWSSession(sess)

	if c.Bool("debug") {
		sess.Config.LogLevel = aws.LogLevel(aws.LogDebug)
	}

	rdss := awsSession.fetchRDSList()
	awsSession.updateAWSElementList(rdss, client)

	elbs := awsSession.fetchLoadBalancerList()
	awsSession.updateAWSElementList(elbs, client)

	tickChan := time.NewTicker(60 * time.Second)
	quit := make(chan struct{})

	awsSession.crawlRDSMetrics(client, rdss)

	for {
		select {
		case <-tickChan.C:
			awsSession.crawlELBMetrics(client, elbs)
			awsSession.crawlRDSMetrics(client, rdss)
		case <-quit:
			tickChan.Stop()
			return
		}
	}

	//listMetric(sess)
}
Beispiel #18
0
// New makes an instance of StorageS3 storage driver
func New(client *docker.Client, cacheRoot string) *StorageS3 {
	retryer := NewRetryer(400, 6)

	// TODO: configure region?
	cfg := &aws.Config{
		Region:  aws.String("us-east-1"),
		Retryer: retryer,
		Logger:  &Logger{},
	}

	if log.StandardLogger().Level >= log.DebugLevel {
		cfg.LogLevel = aws.LogLevel(aws.LogDebugWithRequestErrors)
	}

	return &StorageS3{
		client:    client,
		cacheRoot: cacheRoot,
		s3:        s3.New(session.New(), cfg),
		retryer:   retryer,
	}
}
Beispiel #19
0
func copyConfig(config *Config) *aws.Config {
	if config == nil {
		config = &Config{}
	}
	c := &aws.Config{
		Credentials: credentials.AnonymousCredentials,
		Endpoint:    config.Endpoint,
		HTTPClient:  config.HTTPClient,
		Logger:      config.Logger,
		LogLevel:    config.LogLevel,
		MaxRetries:  config.MaxRetries,
	}

	if c.HTTPClient == nil {
		c.HTTPClient = &http.Client{
			Transport: &http.Transport{
				Proxy: http.ProxyFromEnvironment,
				Dial: (&net.Dialer{
					// use a shorter timeout than default because the metadata
					// service is local if it is running, and to fail faster
					// if not running on an ec2 instance.
					Timeout:   5 * time.Second,
					KeepAlive: 30 * time.Second,
				}).Dial,
				TLSHandshakeTimeout: 10 * time.Second,
			},
		}
	}
	if c.Logger == nil {
		c.Logger = aws.NewDefaultLogger()
	}
	if c.LogLevel == nil {
		c.LogLevel = aws.LogLevel(aws.LogOff)
	}
	if c.MaxRetries == nil {
		c.MaxRetries = aws.Int(DefaultRetries)
	}

	return c
}
Beispiel #20
0
func (s *GoofysTest) SetUpSuite(t *C) {
	//addr := "play.minio.io:9000"
	const LOCAL_TEST = true

	if LOCAL_TEST {
		addr := "127.0.0.1:8080"

		err := s.waitFor(t, addr)
		t.Assert(err, IsNil)

		s.awsConfig = &aws.Config{
			//Credentials: credentials.AnonymousCredentials,
			Credentials:      credentials.NewStaticCredentials("foo", "bar", ""),
			Region:           aws.String("us-west-2"),
			Endpoint:         aws.String(addr),
			DisableSSL:       aws.Bool(true),
			S3ForcePathStyle: aws.Bool(true),
			MaxRetries:       aws.Int(0),
			//Logger: t,
			//LogLevel: aws.LogLevel(aws.LogDebug),
			//LogLevel: aws.LogLevel(aws.LogDebug | aws.LogDebugWithHTTPBody),
		}
	} else {
		s.awsConfig = &aws.Config{
			Region:           aws.String("us-west-2"),
			DisableSSL:       aws.Bool(true),
			LogLevel:         aws.LogLevel(aws.LogDebug | aws.LogDebugWithSigning),
			S3ForcePathStyle: aws.Bool(true),
		}
	}
	s.sess = session.New(s.awsConfig)
	s.s3 = s3.New(s.sess)
	s.s3.Handlers.Sign.Clear()
	s.s3.Handlers.Sign.PushBack(SignV2)
	s.s3.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
	_, err := s.s3.ListBuckets(nil)
	t.Assert(err, IsNil)
}
Beispiel #21
0
func (s *GoofysTest) SetUpSuite(t *C) {
	//addr := "play.minio.io:9000"
	addr := "127.0.0.1:9000"

	accessKey, secretKey := s.setupMinio(t, addr)

	s.awsConfig = &aws.Config{
		//Credentials: credentials.AnonymousCredentials,
		Credentials:      credentials.NewStaticCredentials(accessKey, secretKey, ""),
		Region:           aws.String("milkyway"), //aws.String("us-west-2"),
		Endpoint:         aws.String(addr),
		DisableSSL:       aws.Bool(true),
		S3ForcePathStyle: aws.Bool(true),
		MaxRetries:       aws.Int(0),
		Logger:           t,
		LogLevel:         aws.LogLevel(aws.LogDebug),
		//LogLevel: aws.LogLevel(aws.LogDebug | aws.LogDebugWithHTTPBody),
	}
	s.s3 = s3.New(s.awsConfig)

	_, err := s.s3.ListBuckets(nil)
	t.Assert(err, IsNil)
}
Beispiel #22
0
func GetFileIAM(region, bucketName, path string) (io.ReadCloser, int64, error) {
	awsconfig := &aws.Config{
		Region:           aws.String(region),
		Endpoint:         aws.String("s3.amazonaws.com"),
		S3ForcePathStyle: aws.Bool(true),
		LogLevel:         aws.LogLevel(0),
	}

	sess := session.New(awsconfig)
	svc := s3.New(sess)

	params := &s3.GetObjectInput{
		Bucket: aws.String(bucketName),
		Key:    aws.String(path),
	}

	resp, err := svc.GetObject(params)
	if err != nil {
		return nil, 0, err
	}

	// log.Println(resp)
	return resp.Body, *resp.ContentLength, nil
}
func ConfigureAws(region string) {
	logLevelType := func(logLevel uint) aws.LogLevelType {
		var awsLogLevel aws.LogLevelType
		switch logLevel {
		case 0:
			awsLogLevel = aws.LogOff
		case 1:
			awsLogLevel = aws.LogDebug
		case 2:
			awsLogLevel = aws.LogDebugWithSigning
		case 3:
			awsLogLevel = aws.LogDebugWithHTTPBody
		case 4:
			awsLogLevel = aws.LogDebugWithRequestRetries
		case 5:
			awsLogLevel = aws.LogDebugWithRequestErrors
		}
		return awsLogLevel
	}

	defaults.DefaultConfig.Credentials = credentials.NewStaticCredentials(Cfg.BackupSet.AccessKey, Cfg.BackupSet.SecretKey, "")
	defaults.DefaultConfig.Region = &region
	defaults.DefaultConfig.LogLevel = aws.LogLevel(logLevelType(uint(Cfg.AwsLogLevel)))
}
Beispiel #24
0
// Client configures and returns a fully initialized AWSClient
func (c *Config) Client() (interface{}, error) {
	// Get the auth and region. This can fail if keys/regions were not
	// specified and we're attempting to use the environment.
	var errs []error

	log.Println("[INFO] Building AWS region structure")
	err := c.ValidateRegion()
	if err != nil {
		errs = append(errs, err)
	}

	var client AWSClient
	if len(errs) == 0 {
		// store AWS region in client struct, for region specific operations such as
		// bucket storage in S3
		client.region = c.Region

		log.Println("[INFO] Building AWS auth structure")
		creds := GetCredentials(c.AccessKey, c.SecretKey, c.Token, c.Profile, c.CredsFilename)
		// Call Get to check for credential provider. If nothing found, we'll get an
		// error, and we can present it nicely to the user
		cp, err := creds.Get()
		if err != nil {
			if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
				errs = append(errs, fmt.Errorf(`No valid credential sources found for AWS Provider.
  Please see https://terraform.io/docs/providers/aws/index.html for more information on
  providing credentials for the AWS Provider`))
			} else {
				errs = append(errs, fmt.Errorf("Error loading credentials for AWS Provider: %s", err))
			}
			return nil, &multierror.Error{Errors: errs}
		}

		log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)

		awsConfig := &aws.Config{
			Credentials: creds,
			Region:      aws.String(c.Region),
			MaxRetries:  aws.Int(c.MaxRetries),
			HTTPClient:  cleanhttp.DefaultClient(),
		}

		if logging.IsDebugOrHigher() {
			awsConfig.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
			awsConfig.Logger = awsLogger{}
		}

		if c.Insecure {
			transport := awsConfig.HTTPClient.Transport.(*http.Transport)
			transport.TLSClientConfig = &tls.Config{
				InsecureSkipVerify: true,
			}
		}

		// Set up base session
		sess := session.New(awsConfig)
		sess.Handlers.Build.PushFrontNamed(addTerraformVersionToUserAgent)

		// Some services exist only in us-east-1, e.g. because they manage
		// resources that can span across multiple regions, or because
		// signature format v4 requires region to be us-east-1 for global
		// endpoints:
		// http://docs.aws.amazon.com/general/latest/gr/sigv4_changes.html
		usEast1Sess := sess.Copy(&aws.Config{Region: aws.String("us-east-1")})

		// Some services have user-configurable endpoints
		awsEc2Sess := sess.Copy(&aws.Config{Endpoint: aws.String(c.Ec2Endpoint)})
		awsElbSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.ElbEndpoint)})
		awsIamSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.IamEndpoint)})
		dynamoSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.DynamoDBEndpoint)})
		kinesisSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.KinesisEndpoint)})

		// These two services need to be set up early so we can check on AccountID
		client.iamconn = iam.New(awsIamSess)
		client.stsconn = sts.New(sess)

		err = c.ValidateCredentials(client.stsconn)
		if err != nil {
			errs = append(errs, err)
			return nil, &multierror.Error{Errors: errs}
		}
		accountId, err := GetAccountId(client.iamconn, client.stsconn, cp.ProviderName)
		if err == nil {
			client.accountid = accountId
		}

		authErr := c.ValidateAccountId(client.accountid)
		if authErr != nil {
			errs = append(errs, authErr)
		}

		client.apigateway = apigateway.New(sess)
		client.autoscalingconn = autoscaling.New(sess)
		client.cfconn = cloudformation.New(sess)
		client.cloudfrontconn = cloudfront.New(sess)
		client.cloudtrailconn = cloudtrail.New(sess)
		client.cloudwatchconn = cloudwatch.New(sess)
		client.cloudwatcheventsconn = cloudwatchevents.New(sess)
		client.cloudwatchlogsconn = cloudwatchlogs.New(sess)
		client.codecommitconn = codecommit.New(usEast1Sess)
		client.codedeployconn = codedeploy.New(sess)
		client.dsconn = directoryservice.New(sess)
		client.dynamodbconn = dynamodb.New(dynamoSess)
		client.ec2conn = ec2.New(awsEc2Sess)
		client.ecrconn = ecr.New(sess)
		client.ecsconn = ecs.New(sess)
		client.efsconn = efs.New(sess)
		client.elasticacheconn = elasticache.New(sess)
		client.elasticbeanstalkconn = elasticbeanstalk.New(sess)
		client.elastictranscoderconn = elastictranscoder.New(sess)
		client.elbconn = elb.New(awsElbSess)
		client.emrconn = emr.New(sess)
		client.esconn = elasticsearch.New(sess)
		client.firehoseconn = firehose.New(sess)
		client.glacierconn = glacier.New(sess)
		client.kinesisconn = kinesis.New(kinesisSess)
		client.kmsconn = kms.New(sess)
		client.lambdaconn = lambda.New(sess)
		client.opsworksconn = opsworks.New(usEast1Sess)
		client.r53conn = route53.New(usEast1Sess)
		client.rdsconn = rds.New(sess)
		client.redshiftconn = redshift.New(sess)
		client.simpledbconn = simpledb.New(sess)
		client.s3conn = s3.New(sess)
		client.sesConn = ses.New(sess)
		client.snsconn = sns.New(sess)
		client.sqsconn = sqs.New(sess)
	}

	if len(errs) > 0 {
		return nil, &multierror.Error{Errors: errs}
	}

	return &client, nil
}
Beispiel #25
0
// Client configures and returns a fully initialized AWSClient
func (c *Config) Client() (interface{}, error) {
	// Get the auth and region. This can fail if keys/regions were not
	// specified and we're attempting to use the environment.
	log.Println("[INFO] Building AWS region structure")
	err := c.ValidateRegion()
	if err != nil {
		return nil, err
	}

	var client AWSClient
	// store AWS region in client struct, for region specific operations such as
	// bucket storage in S3
	client.region = c.Region

	log.Println("[INFO] Building AWS auth structure")
	creds, err := GetCredentials(c)
	if err != nil {
		return nil, err
	}
	// Call Get to check for credential provider. If nothing found, we'll get an
	// error, and we can present it nicely to the user
	cp, err := creds.Get()
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "NoCredentialProviders" {
			return nil, errors.New(`No valid credential sources found for AWS Provider.
  Please see https://terraform.io/docs/providers/aws/index.html for more information on
  providing credentials for the AWS Provider`)
		}

		return nil, fmt.Errorf("Error loading credentials for AWS Provider: %s", err)
	}

	log.Printf("[INFO] AWS Auth provider used: %q", cp.ProviderName)

	awsConfig := &aws.Config{
		Credentials:      creds,
		Region:           aws.String(c.Region),
		MaxRetries:       aws.Int(c.MaxRetries),
		HTTPClient:       cleanhttp.DefaultClient(),
		S3ForcePathStyle: aws.Bool(c.S3ForcePathStyle),
	}

	if logging.IsDebugOrHigher() {
		awsConfig.LogLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
		awsConfig.Logger = awsLogger{}
	}

	if c.Insecure {
		transport := awsConfig.HTTPClient.Transport.(*http.Transport)
		transport.TLSClientConfig = &tls.Config{
			InsecureSkipVerify: true,
		}
	}

	// Set up base session
	sess, err := session.NewSession(awsConfig)
	if err != nil {
		return nil, errwrap.Wrapf("Error creating AWS session: {{err}}", err)
	}

	// Removes the SDK Version handler, so we only have the provider User-Agent
	// Ex: "User-Agent: APN/1.0 HashiCorp/1.0 Terraform/0.7.9-dev"
	sess.Handlers.Build.Remove(request.NamedHandler{Name: "core.SDKVersionUserAgentHandler"})
	sess.Handlers.Build.PushFrontNamed(addTerraformVersionToUserAgent)

	if extraDebug := os.Getenv("TERRAFORM_AWS_AUTHFAILURE_DEBUG"); extraDebug != "" {
		sess.Handlers.UnmarshalError.PushFrontNamed(debugAuthFailure)
	}

	// Some services exist only in us-east-1, e.g. because they manage
	// resources that can span across multiple regions, or because
	// signature format v4 requires region to be us-east-1 for global
	// endpoints:
	// http://docs.aws.amazon.com/general/latest/gr/sigv4_changes.html
	usEast1Sess := sess.Copy(&aws.Config{Region: aws.String("us-east-1")})

	// Some services have user-configurable endpoints
	awsEc2Sess := sess.Copy(&aws.Config{Endpoint: aws.String(c.Ec2Endpoint)})
	awsElbSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.ElbEndpoint)})
	awsIamSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.IamEndpoint)})
	awsS3Sess := sess.Copy(&aws.Config{Endpoint: aws.String(c.S3Endpoint)})
	dynamoSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.DynamoDBEndpoint)})
	kinesisSess := sess.Copy(&aws.Config{Endpoint: aws.String(c.KinesisEndpoint)})

	// These two services need to be set up early so we can check on AccountID
	client.iamconn = iam.New(awsIamSess)
	client.stsconn = sts.New(sess)

	if !c.SkipCredsValidation {
		err = c.ValidateCredentials(client.stsconn)
		if err != nil {
			return nil, err
		}
	}

	if !c.SkipRequestingAccountId {
		partition, accountId, err := GetAccountInfo(client.iamconn, client.stsconn, cp.ProviderName)
		if err == nil {
			client.partition = partition
			client.accountid = accountId
		}
	}

	authErr := c.ValidateAccountId(client.accountid)
	if authErr != nil {
		return nil, authErr
	}

	client.acmconn = acm.New(sess)
	client.apigateway = apigateway.New(sess)
	client.appautoscalingconn = applicationautoscaling.New(sess)
	client.autoscalingconn = autoscaling.New(sess)
	client.cfconn = cloudformation.New(sess)
	client.cloudfrontconn = cloudfront.New(sess)
	client.cloudtrailconn = cloudtrail.New(sess)
	client.cloudwatchconn = cloudwatch.New(sess)
	client.cloudwatcheventsconn = cloudwatchevents.New(sess)
	client.cloudwatchlogsconn = cloudwatchlogs.New(sess)
	client.codecommitconn = codecommit.New(usEast1Sess)
	client.codedeployconn = codedeploy.New(sess)
	client.dsconn = directoryservice.New(sess)
	client.dynamodbconn = dynamodb.New(dynamoSess)
	client.ec2conn = ec2.New(awsEc2Sess)
	client.ecrconn = ecr.New(sess)
	client.ecsconn = ecs.New(sess)
	client.efsconn = efs.New(sess)
	client.elasticacheconn = elasticache.New(sess)
	client.elasticbeanstalkconn = elasticbeanstalk.New(sess)
	client.elastictranscoderconn = elastictranscoder.New(sess)
	client.elbconn = elb.New(awsElbSess)
	client.elbv2conn = elbv2.New(awsElbSess)
	client.emrconn = emr.New(sess)
	client.esconn = elasticsearch.New(sess)
	client.firehoseconn = firehose.New(sess)
	client.glacierconn = glacier.New(sess)
	client.kinesisconn = kinesis.New(kinesisSess)
	client.kmsconn = kms.New(sess)
	client.lambdaconn = lambda.New(sess)
	client.lightsailconn = lightsail.New(usEast1Sess)
	client.opsworksconn = opsworks.New(usEast1Sess)
	client.r53conn = route53.New(usEast1Sess)
	client.rdsconn = rds.New(sess)
	client.redshiftconn = redshift.New(sess)
	client.simpledbconn = simpledb.New(sess)
	client.s3conn = s3.New(awsS3Sess)
	client.sesConn = ses.New(sess)
	client.snsconn = sns.New(sess)
	client.sqsconn = sqs.New(sess)
	client.ssmconn = ssm.New(sess)
	client.wafconn = waf.New(sess)

	return &client, nil
}
Beispiel #26
0
func init() {
	logLevel := Session.Config.LogLevel
	if os.Getenv("DEBUG") != "" {
		logLevel = aws.LogLevel(aws.LogDebug)
	}
	if os.Getenv("DEBUG_SIGNING") != "" {
		logLevel = aws.LogLevel(aws.LogDebugWithSigning)
	}
	if os.Getenv("DEBUG_BODY") != "" {
		logLevel = aws.LogLevel(aws.LogDebugWithHTTPBody)
	}
	Session.Config.LogLevel = logLevel

	When(`^I call the "(.+?)" API$`, func(op string) {
		call(op, nil, false)
	})

	When(`^I call the "(.+?)" API with:$`, func(op string, args [][]string) {
		call(op, args, false)
	})

	Then(`^the value at "(.+?)" should be a list$`, func(member string) {
		vals := awsutil.ValuesAtAnyPath(World["response"], member)
		assert.NotNil(T, vals)
	})

	Then(`^the response should contain a "(.+?)"$`, func(member string) {
		vals := awsutil.ValuesAtAnyPath(World["response"], member)
		assert.NotEmpty(T, vals)
	})

	When(`^I attempt to call the "(.+?)" API with:$`, func(op string, args [][]string) {
		call(op, args, true)
	})

	Then(`^I expect the response error code to be "(.+?)"$`, func(code string) {
		err, ok := World["error"].(awserr.Error)
		assert.True(T, ok, "no error returned")
		if ok {
			assert.Equal(T, code, err.Code(), "Error: %v", err)
		}
	})

	And(`^I expect the response error message to include:$`, func(data string) {
		err, ok := World["error"].(awserr.Error)
		assert.True(T, ok, "no error returned")
		if ok {
			assert.Contains(T, err.Message(), data)
		}
	})

	And(`^I expect the response error message to include one of:$`, func(table [][]string) {
		err, ok := World["error"].(awserr.Error)
		assert.True(T, ok, "no error returned")
		if ok {
			found := false
			for _, row := range table {
				if strings.Contains(err.Message(), row[0]) {
					found = true
					break
				}
			}

			assert.True(T, found, fmt.Sprintf("no error messages matched: \"%s\"", err.Message()))
		}
	})

	When(`^I call the "(.+?)" API with JSON:$`, func(s1 string, data string) {
		callWithJSON(s1, data, false)
	})

	When(`^I attempt to call the "(.+?)" API with JSON:$`, func(s1 string, data string) {
		callWithJSON(s1, data, true)
	})

	Then(`^the error code should be "(.+?)"$`, func(s1 string) {
		err, ok := World["error"].(awserr.Error)
		assert.True(T, ok, "no error returned")
		assert.Equal(T, s1, err.Code())
	})

	And(`^the error message should contain:$`, func(data string) {
		err, ok := World["error"].(awserr.Error)
		assert.True(T, ok, "no error returned")
		assert.Contains(T, err.Error(), data)
	})

	Then(`^the request should fail$`, func() {
		err, ok := World["error"].(awserr.Error)
		assert.True(T, ok, "no error returned")
		assert.Error(T, err)
	})

	Then(`^the request should be successful$`, func() {
		err, ok := World["error"].(awserr.Error)
		assert.False(T, ok, "error returned")
		assert.NoError(T, err)
	})
}
Beispiel #27
0
func NewGoofys(bucket string, awsConfig *aws.Config, flags *flagStorage) *Goofys {
	// Set up the basic struct.
	fs := &Goofys{
		bucket: bucket,
		flags:  flags,
		umask:  0122,
	}

	if flags.DebugS3 {
		awsConfig.LogLevel = aws.LogLevel(aws.LogDebug)
	}

	fs.awsConfig = awsConfig
	fs.s3 = s3.New(awsConfig)

	params := &s3.GetBucketLocationInput{Bucket: &bucket}
	resp, err := fs.s3.GetBucketLocation(params)
	var fromRegion, toRegion string
	if err != nil {
		if mapAwsError(err) == fuse.ENOENT {
			log.Printf("bucket %v does not exist", bucket)
			return nil
		}
		fromRegion, toRegion = parseRegionError(err)
	} else {
		fs.logS3(resp)

		if resp.LocationConstraint == nil {
			toRegion = "us-east-1"
		} else {
			toRegion = *resp.LocationConstraint
		}

		fromRegion = *awsConfig.Region
	}

	if len(toRegion) != 0 && fromRegion != toRegion {
		log.Printf("Switching from region '%v' to '%v'", fromRegion, toRegion)
		awsConfig.Region = &toRegion
		fs.s3 = s3.New(awsConfig)
		_, err = fs.s3.GetBucketLocation(params)
		if err != nil {
			log.Println(err)
			return nil
		}
	} else if len(toRegion) == 0 && *awsConfig.Region != "milkyway" {
		log.Printf("Unable to detect bucket region, staying at '%v'", *awsConfig.Region)
	}

	now := time.Now()
	fs.rootAttrs = fuseops.InodeAttributes{
		Size:   4096,
		Nlink:  2,
		Mode:   flags.DirMode | os.ModeDir,
		Atime:  now,
		Mtime:  now,
		Ctime:  now,
		Crtime: now,
		Uid:    fs.flags.Uid,
		Gid:    fs.flags.Gid,
	}

	fs.bufferPool = NewBufferPool(100*1024*1024, 20*1024*1024)

	fs.nextInodeID = fuseops.RootInodeID + 1
	fs.inodes = make(map[fuseops.InodeID]*Inode)
	root := NewInode(aws.String(""), aws.String(""), flags)
	root.Id = fuseops.RootInodeID
	root.Attributes = &fs.rootAttrs

	fs.inodes[fuseops.RootInodeID] = root
	fs.inodesCache = make(map[string]*Inode)

	fs.nextHandleID = 1
	fs.dirHandles = make(map[fuseops.HandleID]*DirHandle)

	fs.fileHandles = make(map[fuseops.HandleID]*FileHandle)

	return fs
}
Beispiel #28
0
func ListFolder(region, bucketName, path, keyId, secretKey, token string, iam bool) (FileList, error) {
	var f FileList
	var awsconfig *aws.Config
	if !iam {
		creds := credentials.NewStaticCredentials(keyId, secretKey, token)
		var f FileList

		if _, err := creds.Get(); err != nil {
			return f, err
		}

		awsconfig = &aws.Config{
			Region:           aws.String(region),
			Endpoint:         aws.String("s3.amazonaws.com"),
			S3ForcePathStyle: aws.Bool(true),
			Credentials:      creds,
			LogLevel:         aws.LogLevel(0),
		}
	} else {
		awsconfig = &aws.Config{
			Region:           aws.String(region),
			Endpoint:         aws.String("s3.amazonaws.com"),
			S3ForcePathStyle: aws.Bool(true),
			LogLevel:         aws.LogLevel(0),
		}
	}
	sess := session.New(awsconfig)
	svc := s3.New(sess)

	inparams := &s3.ListObjectsInput{
		Bucket:    aws.String(bucketName),
		Prefix:    aws.String(path),
		Delimiter: aws.String("/"),
	}

	err := svc.ListObjectsPages(inparams, func(p *s3.ListObjectsOutput, lastPage bool) bool {
		for _, o := range p.Contents {
			switch *o.StorageClass {
			case s3.ObjectStorageClassGlacier:
				log.Printf("GLACIER  %s\n", *o.Key)
				continue
			case s3.ObjectStorageClassReducedRedundancy:
				log.Printf("REDUCED  %s\n", *o.Key)
				continue
			case s3.ObjectStorageClassStandard:
				log.Printf("STANDARD %s\n", *o.Key)
			}
			loc, err := time.LoadLocation("America/New_York")
			if err != nil {
				log.Fatal(err)
			}
			keyString := *o.Key
			fileString := strings.TrimPrefix(keyString, *p.Prefix)
			lm := o.LastModified.In(loc)
			t := lm.Format("15:04:05")
			d := lm.Format("2006-01-02")

			f.Files = append(f.Files, &FileObject{keyString, fileString,
				*o.Size, t, d})
		}
		return !lastPage
	})
	if err != nil {
		log.Println(err)
	}
	log.Printf("Kept %d backup files.\n", len(f.Files))

	// log.Println(f)
	sort.Sort(sort.Reverse(f.Files))

	return f, nil
}
func (p *SigningRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
	glog.V(2).Infof("Got request: %s %s", req.Method, req.URL)

	// Fix the host header in case broken by proxy-rewrite
	if req.URL.Host != "" {
		req.Host = req.URL.Host
	}

	// I think the AWS authentication proxy does not like forwarded headers
	for k := range req.Header {
		lk := strings.ToLower(k)
		if lk == "x-forwarded-host" {
			delete(req.Header, k)
		}
		if lk == "x-forwarded-for" {
			delete(req.Header, k)
		}
		if lk == "x-forwarded-proto" {
			delete(req.Header, k)
		}
		if lk == "x-forward-for" {
			delete(req.Header, k)
		}
		if lk == "x-forward-proto" {
			delete(req.Header, k)
		}
		if lk == "x-forward-port" {
			delete(req.Header, k)
		}
		if lk == "x-forwarded-port" {
			delete(req.Header, k)
		}
		if lk == "x-forwarded-prefix" {
			delete(req.Header, k)
		}
		if lk == "x-netflix-httpclientname" {
			delete(req.Header, k)
		}
		if lk == "x-newrelic-id" {
			delete(req.Header, k)
		}
		if lk == "x-newrelic-transaction" {
			delete(req.Header, k)
		}
		if lk == "netflix.nfhttpclient.version" {
			delete(req.Header, k)
		}
	}

	// We're going to put our own auth headers on here
	delete(req.Header, "Authorization")

	var body []byte
	var err error

	if req.Body != nil {
		body, err = ioutil.ReadAll(req.Body)
		if err != nil {
			glog.Infof("error reading request body: %v", err)
			return nil, err
		}
	}

	if req.Method == "GET" || req.Method == "HEAD" {
		delete(req.Header, "Content-Length")
	}

	oldPath := req.URL.Path
	if oldPath != "" {
		// Escape the path before signing so that the path in the signature and
		// the path in the request match.
		req.URL.Path = req.URL.EscapedPath()
		glog.V(4).Infof("Path -> %q", req.URL.Path)
	}

	awsReq := &request.Request{}
	awsReq.Config.Credentials = p.credentials
	awsReq.Config.Region = aws.String(p.region)
	awsReq.ClientInfo.ServiceName = SERVICE_NAME
	awsReq.HTTPRequest = req
	awsReq.Time = time.Now()
	awsReq.ExpireTime = 0
	if body != nil {
		awsReq.Body = bytes.NewReader(body)
	}

	if glog.V(4) {
		awsReq.Config.LogLevel = aws.LogLevel(aws.LogDebugWithSigning)
		awsReq.Config.Logger = aws.NewDefaultLogger()
	}

	v4.Sign(awsReq)

	if awsReq.Error != nil {
		glog.Warningf("error signing request: %v", awsReq.Error)
		return nil, awsReq.Error
	}

	req.URL.Path = oldPath

	if body != nil {
		req.Body = ioutil.NopCloser(bytes.NewReader(body))
	}

	response, err := p.inner.RoundTrip(req)

	if err != nil {
		glog.Warning("Request error: ", err)
		return nil, err
	} else {
		glog.V(2).Infof("response %s", response.Status)
		return response, err
	}
}
Beispiel #30
0
func (d *driver) Login(ctx types.Context) (interface{}, error) {
	sessionsL.Lock()
	defer sessionsL.Unlock()

	var (
		endpoint *string
		ckey     string
		hkey     = md5.New()
		akey     = d.accessKey
		region   = d.mustRegion(ctx)
	)

	if region != nil && d.endpointFormat != "" {
		szEndpoint := fmt.Sprintf(d.endpointFormat, *region)
		endpoint = &szEndpoint
	} else {
		endpoint = d.endpoint
	}

	if !d.disableSessionCache {
		writeHkey(hkey, region)
		writeHkey(hkey, endpoint)
		writeHkey(hkey, &akey)
		ckey = fmt.Sprintf("%x", hkey.Sum(nil))

		// if the session is cached then return it
		if svc, ok := sessions[ckey]; ok {
			ctx.WithField(cacheKeyC, ckey).Debug("using cached efs service")
			return svc, nil
		}
	}

	var (
		skey   = d.getSecretKey()
		fields = map[string]interface{}{
			efs.AccessKey: akey,
			efs.Tag:       d.tag,
			cacheKeyC:     ckey,
		}
	)

	if skey == "" {
		fields[efs.SecretKey] = ""
	} else {
		fields[efs.SecretKey] = "******"
	}
	if region != nil {
		fields[efs.Region] = *region
	}
	if endpoint != nil {
		fields[efs.Endpoint] = *endpoint
	}

	ctx.WithFields(fields).Debug("efs service connetion attempt")
	sess := session.New()

	var (
		awsLogger   = &awsLogger{ctx: ctx}
		awsLogLevel = aws.LogOff
	)
	if ll, ok := context.GetLogLevel(ctx); ok {
		switch ll {
		case log.DebugLevel:
			awsLogger.lvl = log.DebugLevel
			awsLogLevel = aws.LogDebugWithHTTPBody
		case log.InfoLevel:
			awsLogger.lvl = log.InfoLevel
			awsLogLevel = aws.LogDebug
		}
	}

	svc := awsefs.New(sess, &aws.Config{
		Region:     region,
		Endpoint:   endpoint,
		MaxRetries: d.maxRetries,
		Credentials: credentials.NewChainCredentials(
			[]credentials.Provider{
				&credentials.StaticProvider{
					Value: credentials.Value{
						AccessKeyID:     akey,
						SecretAccessKey: skey,
					},
				},
				&credentials.EnvProvider{},
				&credentials.SharedCredentialsProvider{},
				&ec2rolecreds.EC2RoleProvider{
					Client: ec2metadata.New(sess),
				},
			},
		),
		Logger:   awsLogger,
		LogLevel: aws.LogLevel(awsLogLevel),
	})

	ctx.WithFields(fields).Info("efs service connection created")

	if !d.disableSessionCache {
		sessions[ckey] = svc
		ctx.WithFields(fields).Info("efs service connection cached")
	}

	return svc, nil
}