func TestDynamoDBHABackend(t *testing.T) { if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" { t.SkipNow() } creds, err := credentials.NewEnvCredentials().Get() if err != nil { t.Fatalf("err: %v", err) } // If the variable is empty or doesn't exist, the default // AWS endpoints will be used endpoint := os.Getenv("AWS_DYNAMODB_ENDPOINT") region := os.Getenv("AWS_DEFAULT_REGION") if region == "" { region = "us-east-1" } conn := dynamodb.New(session.New(&aws.Config{ Credentials: credentials.NewEnvCredentials(), Endpoint: aws.String(endpoint), Region: aws.String(region), })) var randInt = rand.New(rand.NewSource(time.Now().UnixNano())).Int() table := fmt.Sprintf("vault-dynamodb-testacc-%d", randInt) defer func() { conn.DeleteTable(&dynamodb.DeleteTableInput{ TableName: aws.String(table), }) }() logger := logformat.NewVaultLogger(log.LevelTrace) b, err := NewBackend("dynamodb", logger, map[string]string{ "access_key": creds.AccessKeyID, "secret_key": creds.SecretAccessKey, "session_token": creds.SessionToken, "table": table, }) if err != nil { t.Fatalf("err: %s", err) } ha, ok := b.(HABackend) if !ok { t.Fatalf("dynamodb does not implement HABackend") } testHABackend(t, ha, ha) }
func elbDNSName(c *conf) string { //weirdness around how aws handles credentials os.Setenv("AWS_ACCESS_KEY_ID", c.awsAccessKey) os.Setenv("AWS_SECRET_ACCESS_KEY", c.awsSecretKey) svc := elb.New( session.New( &aws.Config{ Region: aws.String(c.awsRegion), Credentials: credentials.NewEnvCredentials(), }, ), ) params := &elb.DescribeLoadBalancersInput{ LoadBalancerNames: []*string{ aws.String(c.elbName), // Required }, } resp, err := svc.DescribeLoadBalancers(params) if err != nil { log.Fatal(err) } return *resp.LoadBalancerDescriptions[0].DNSName }
func main() { flag.StringVar(&streamName, "stream", "mystream", "what stream to tail") flag.StringVar(&shardName, "shard", "shardId-000000000000", "what shard to tail") flag.IntVar(&delayTime, "delay", 5, "time to sleep between GetRecord calls") flag.Parse() println(fmt.Sprintf("Working on kinesis stream: %s:%s", streamName, shardName)) creds := credentials.NewEnvCredentials() config = aws.Config{} config.Credentials = creds config.Region = aws.String("us-east-1") // TODO, ensure we can load the region from ENV variables. // Bypass the error now and just use us-east-1 if *defaults.DefaultConfig.Region == "" { println("Could not find AWS Region in ENV. Please configure your ENV for AWS access") //os.Exit(1) } listStreams() describeStream(streamName) watchStream(streamName, shardName) }
// GetAWSCreds returns the appropriate value as the need arises. // // evaluated in the following order // 1. input variable // 2. Environment variable // 3. IAM Role // // "/.aws/credentials" necessary item increased about that, so it isn't used. func (c *Config) GetAWSCreds() (*credentials.Credentials, error) { var creds *credentials.Credentials var err error err = nil // 1. input variable used if c.Aws.Accesskey != "" && c.Aws.SecretKey != "" { creds = credentials.NewStaticCredentials(c.Aws.Accesskey, c.Aws.SecretKey, "") creds.Expire() _, err = creds.Get() } if err != nil { // 2. Environment variable used creds = credentials.NewEnvCredentials() creds.Expire() _, err = creds.Get() if err != nil { // 3. IAM Role used creds = credentials.NewCredentials(&ec2rolecreds.EC2RoleProvider{}) creds.Expire() _, err = creds.Get() } } return creds, err }
// NewSNSPublisher will initiate the SNS client. // If no credentials are passed in with the config, // the publisher is instantiated with the AWS_ACCESS_KEY // and the AWS_SECRET_KEY environment variables. func NewSNSPublisher(cfg *config.SNS) (*SNSPublisher, error) { p := &SNSPublisher{} if cfg.Topic == "" { return p, errors.New("SNS topic name is required") } p.topic = cfg.Topic if cfg.Region == "" { return p, errors.New("SNS region is required") } var creds *credentials.Credentials if cfg.AccessKey != "" { creds = credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, "") } else { creds = credentials.NewEnvCredentials() } p.sns = sns.New(session.New(&aws.Config{ Credentials: creds, Region: &cfg.Region, })) return p, nil }
// Svc configures the DynamoDB service to use func Svc(opts config.Options) *dynamodb.DynamoDB { awsConfig := &aws.Config{Region: aws.String(opts.Storage.AWS.Region)} // If a session was passed... (AWS Lambda does this) if opts.Storage.AWS.SessionToken != "" { os.Setenv("AWS_SESSION_TOKEN", opts.Storage.AWS.SessionToken) } // Look in a variety of places for AWS credentials. First, try the credentials file set by AWS CLI tool. // Note the empty string instructs to look under default file path (different based on OS). // This file can have multiple profiles and a default profile will be used unless otherwise configured. // See: https://godoc.org/github.com/aws/aws-sdk-go/aws/credentials#SharedCredentialsProvider creds := credentials.NewSharedCredentials("", opts.Storage.AWS.CredProfile) _, err := creds.Get() // If that failed, try environment variables. if err != nil { // The following are checked: // Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY // Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY creds = credentials.NewEnvCredentials() } // If credentials were passed via config, then use those. They will take priority over other methods. if opts.Storage.AWS.AccessKeyID != "" && opts.Storage.AWS.SecretAccessKey != "" { creds = credentials.NewStaticCredentials(opts.Storage.AWS.AccessKeyID, opts.Storage.AWS.SecretAccessKey, "") } awsConfig.Credentials = creds return dynamodb.New(session.New(awsConfig)) }
func TestMigrateStreams(t *testing.T) { if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" { Log.Printf("WARNING: NO AWS CREDS SPECIFIED, SKIPPING MIGRATIONS TEST") return } config := &aws.Config{ Credentials: credentials.NewEnvCredentials(), Region: aws.String("us-east-1"), } client := kinesis.New(session.New(config)) sm := StreamMigrator{ Streams: []kinesis.CreateStreamInput{ kinesis.CreateStreamInput{ StreamName: aws.String(testDomain), ShardCount: aws.Int64(1), }, }, Client: client, Timeout: 30, } sm.Migrate() sm.Migrate() }
// MustClient will use the cache cluster ID to describe // the cache cluster and instantiate a memcache.Client // with the cache nodes returned from AWS. func (e *ElastiCache) MustClient() *memcache.Client { var creds *credentials.Credentials if e.AccessKey != "" { creds = credentials.NewStaticCredentials(e.AccessKey, e.SecretKey, "") } else { creds = credentials.NewEnvCredentials() } ecclient := elasticache.New(session.New(&aws.Config{ Credentials: creds, Region: &e.Region, })) resp, err := ecclient.DescribeCacheClusters(&elasticache.DescribeCacheClustersInput{ CacheClusterId: &e.ClusterID, ShowCacheNodeInfo: aws.Bool(true), }) if err != nil { log.Fatalf("unable to describe cache cluster: %s", err) } var nodes []string for _, cluster := range resp.CacheClusters { for _, cnode := range cluster.CacheNodes { addr := fmt.Sprintf("%s:%d", *cnode.Endpoint.Address, *cnode.Endpoint.Port) nodes = append(nodes, addr) } } return memcache.New(nodes...) }
func main() { var err error // connect to the database dbconn, err = sql.Open("postgres", PACKAGEBUG_DB) if err != nil { log.Fatal(err) } // make sure the database up err = dbconn.Ping() if err != nil { log.Fatal(err) } // set up aws SDK credentials & config cred := credentials.NewEnvCredentials() _, err = cred.Get() if err != nil { log.Fatal(err) } config := aws.NewConfig() config.Credentials = cred config.Endpoint = &PACKAGEBUG_SQS_ENDPOINT config.Region = &PACKAGEBUG_SQS_REGION sqsconn = sqs.New(config) // dispatch jobs once a day for { <-time.After(24 * time.Hour) go dispatchJobs() } }
func addToLambda(dir string) error { desc, err := util.ReadTestDescription(dir) if err != nil { return err } var zipContents []byte if desc.Runtime == "java8" { zipContents, err = ioutil.ReadFile(filepath.Join(dir, "test-build.jar")) if err != nil { return err } } else { zipContents, err = makeZip(dir) if err != nil { return err } } s := session.New(&aws.Config{Region: aws.String("us-east-1"), Credentials: credentials.NewEnvCredentials()}) l := lambda.New(s) err = createLambdaFunction(l, zipContents, desc.Runtime, lambdaRole, desc.Name, desc.Handler, desc.Timeout) return err }
func (s *S3) Init() error { if s.Bucket == "" { return errors.New("S3 bucket not set") } else if s.Key == "" { return errors.New("S3 key not set") } if s.Region == "" { s.Region = "ap-southeast-2" } creds := credentials.AnonymousCredentials if s.Access != "" { creds = credentials.NewStaticCredentials(s.Access, s.Secret, "") } else if os.Getenv("AWS_ACCESS_KEY") != "" { creds = credentials.NewEnvCredentials() } config := &aws.Config{ Credentials: creds, Region: &s.Region, } s.client = s3.New(session.New(config)) //TODO include this? maybe given access to bucket after init // resp, err := s.client.HeadBucketRequest(&s3.HeadBucketInput{Bucket: &s.Bucket}) // if err != nil {} //apply defaults if s.Interval == 0 { s.Interval = 5 * time.Minute } return nil }
// NewKinesisRelay gets the relay started. func NewKinesisRelay(l Logger, c ConfigValues) *KinesisRelay { l.Info.Println("Beginning to start kinesis") // New KinesisRelay setup. kr := new(KinesisRelay) kr.logger = l kr.config = c // Setup the AWS config. creds := credentials.NewEnvCredentials() kr.awsConfig = aws.Config{} kr.awsConfig.Credentials = creds // TODO - handle this via ENV variables as well. kr.awsConfig.Region = aws.String(kr.config.Kinesis.Region) // TODO configure the channel better in the future. kr.Pipe = make(chan string, 100) // Prep the stream. l.Trace.Printf("We want to use stream: %s", kr.config.Kinesis.StreamName) kr.createStreamIfNotExists(kr.config.Kinesis.StreamName) l.Info.Println("Done starting kinesis relay") return kr }
func main() { if len(os.Args) != 3 { log.Fatal("wrong number of arguments") } src, _ := os.Args[1], os.Args[2] creds := credentials.NewEnvCredentials() if _, err := creds.Get(); err != nil { log.Fatal(err) } svc := s3.New(&aws.Config{ Credentials: creds, Region: "us-east-2", Endpoint: "s3.amazonaws.com", S3ForcePathStyle: true, }) uploader := s3manager.NewUploader(&s3manager.UploadOptions{S3: svc}) watcher, err := fsnotify.NewWatcher() if err != nil { log.Fatal(err) } defer watcher.Close() done := make(chan bool) go func() { for { select { case event := <-watcher.Events: log.Println("event:", event) switch event.Op { case fsnotify.Create: handleCreate(watcher, uploader, event.Name) case fsnotify.Write: log.Println("modified file:", event.Name) } case err := <-watcher.Errors: log.Println("error:", err) } } }() err = filepath.Walk(src, func(path string, info os.FileInfo, err error) error { if info.IsDir() { log.Println(path) return watcher.Add(path) } return nil }) if err != nil { log.Fatal(err) } <-done }
func loadConfig() *aws.Config { region := os.Getenv("AWS_REGION") if region == "" { region = "eu-west-1" } creds := credentials.NewEnvCredentials() return aws.NewConfig().WithCredentials(creds).WithRegion(region) }
func createConfig() *aws.Config { config := aws.NewConfig() config.WithCredentials(credentials.NewEnvCredentials()) config.WithRegion(*region) if *logging { config.WithLogLevel(aws.LogDebugWithHTTPBody) } return config }
func main() { flag.Parse() log.Println("Total:", *total) log.Println("Gophers:", *gophers) log.Println("ID Top:", *idTop) if *debug { *verbose = true } db := dynamodb.New(session.New( aws.NewConfig(). WithEndpoint(*endpoint). WithRegion(*region). WithCredentials(credentials.NewEnvCredentials()), )) jobChans := make(chan struct{}, *gophers) start := time.Now() var totalDuration int64 var wg sync.WaitGroup rand.Seed(time.Now().Unix()) for i := 0; i <= *total; i++ { jobChans <- struct{}{} wg.Add(1) if i%((*total)/100) == 0 { fmt.Printf("\r%s Queried %d%%", time.Now().Format("2006-01-02 15:04:05"), i/((*total)/100)) } go func() { start := time.Now() if resp, err := db.GetItem(&dynamodb.GetItemInput{ TableName: aws.String(*table), Key: map[string]*dynamodb.AttributeValue{ // "bench_area": &dynamodb.AttributeValue{S: aws.String("KingsLanding")}, "id": &dynamodb.AttributeValue{S: aws.String(fmt.Sprintf("%s%d", *idPrefix, rand.Intn(*idTop)))}, }, }); err != nil { fmt.Println(err) } else { if *debug { log.Println(resp) } } atomic.AddInt64(&totalDuration, int64(time.Now().Sub(start))) wg.Done() <-jobChans }() } wg.Wait() fmt.Println("") log.Println("Took:", time.Now().Sub(start)) log.Println("Total Duration:", time.Duration(totalDuration)) log.Println("TPQ:", time.Duration(totalDuration/int64(*total))) }
func New(region, keyID string) (*KMS, error) { config := aws.NewConfig().WithCredentials(credentials.NewEnvCredentials()).WithRegion(region) session := session.New(config) kms := &KMS{ kms: kms.New(session), keyID: keyID, } return kms, nil }
func main() { bucketFlag := flag.String("bucket", "my-s3-bucket", "s3 bucket to upload to") subfolderFlag := flag.String("subfolder", "", "subfolder in s3 bucket, can be blank") num_workersFlag := flag.Int("workers", 100, "number of upload workers to use") regionFlag := flag.String("region", "eu-west-1", "aws region") aclFlag := flag.String("acl", "private", "s3 upload acl - use either private or public") sourceDirFlag := flag.String("sourcedir", "files/", "source directory") destDirFlag := flag.String("destdir", "files-uploaded/", "dest dir for uploaded files (on local box)") flag.Parse() bucket := *bucketFlag subfolder := *subfolderFlag num_workers := *num_workersFlag region := *regionFlag acl := *aclFlag sourceDir := *sourceDirFlag destDir := *destDirFlag fmt.Println("Using options:") fmt.Println("bucket:", bucket) fmt.Println("subfolder:", subfolder) fmt.Println("num_workers:", num_workers) fmt.Println("region:", region) fmt.Println("acl:", acl) fmt.Println("sourceDir:", sourceDir) fmt.Println("destDir:", destDir) var wg sync.WaitGroup wg.Add(num_workers + 1) // add 1 to account for the get_file_list thread! // file channel and thread to get the files file_channel := make(chan string, 0) go get_file_list(sourceDir, file_channel, num_workers, &wg) // set up s3 credentials from environment variables // these are shared to every worker creds := credentials.NewEnvCredentials() fmt.Println("Starting " + strconv.Itoa(num_workers) + " workers...") // create the desired number of workers for i := 1; i <= num_workers; i++ { // make a new worker sess := session.New(&aws.Config{Region: aws.String(region), Credentials: creds, LogLevel: aws.LogLevel(1)}) svc := s3.New(sess) worker := &Worker{Acl: acl, Bucket: bucket, Subfolder: subfolder, Svc: svc, File_channel: file_channel, Wg: &wg, SourceDir: sourceDir, DestDir: destDir, Id: i} go worker.doUploads() } // wait for all workers to finish // (1x file worker and all uploader workers) wg.Wait() }
func checkCreds() (err error) { creds := credentials.NewEnvCredentials() _, err = creds.Get() if err == nil { // If ENV credentials are present, I don't need to check shared credentials on fs return } creds = credentials.NewSharedCredentials("", "") _, err = creds.Get() return }
func New(region, keyID string) (*KMS, error) { config := &aws.Config{ Credentials: credentials.NewEnvCredentials(), Region: ®ion, } kms := &KMS{ kms: kms.New(config), keyID: keyID, } return kms, nil }
// GetClient configure and return initialized client. func GetClient() *Client { if client != nil { return client } var c Client fmt.Println("[INFO] Initializing ECS Connection") c.ecs = ecs.New(&aws.Config{ Credentials: credentials.NewEnvCredentials(), }) client = &c return client }
func main() { // load env vars creds := credentials.NewEnvCredentials() credValue, err := creds.Get() if err != nil { fmt.Println(err) os.Exit(1) } fmt.Println("Value of credentials : ", credValue) fmt.Println("----------------------------------------------") fmt.Println("Raw credentials : ", creds) sess := session.New(&aws.Config{Region: aws.String("eu-west-1")}) s3client := s3.New(sess) bucketName := "eddysstuff" key := "TestFile.txt" result, err := s3client.CreateBucket(&s3.CreateBucketInput{ Bucket: &bucketName, }) if err != nil { fmt.Println("Failed to create bucket", err) return } fmt.Println("CreateBucket result : ", result.GoString()) uploadResult, err2 := s3client.PutObject(&s3.PutObjectInput{ Body: strings.NewReader("Hello World!"), Bucket: &bucketName, Key: &key, }) if err2 != nil { fmt.Printf("Failed to upload data to %s/%s, %s\n", bucketName, key, err) return } fmt.Println("CreateBucket result : ", uploadResult.GoString()) var params2 *s3.ListBucketsInput resp2, err := s3client.ListBuckets(params2) fmt.Println(resp2) }
func setupIntegrationTest() (rc *RoleSet) { client := sts.New( session.New(), &aws.Config{Credentials: credentials.NewEnvCredentials()}, ) rc = NewRoleSet(client) rc.SetRole("valid", os.Getenv("FINTO_VALID_ARN")) rc.SetRole("invalid", os.Getenv("FINTO_INVALID_ARN")) return }
func NewS3Info(region, bucketName, s3Host string) *S3Info { conf := &aws.Config{ Credentials: credentials.NewEnvCredentials(), Region: aws.String(region), Endpoint: aws.String(s3Host), } return &S3Info{ region: region, bucketName: bucketName, conf: conf, } }
func setupAmazon() *s3.S3 { // the file location and load default profile creds := credentials.NewEnvCredentials() _, err := creds.Get() if err != nil { fmt.Println(err) os.Exit(1) } svc := s3.New(session.New(), &aws.Config{Region: aws.String("us-west-1"), Credentials: creds, LogLevel: aws.LogLevel(1)}) return svc }
// Registers public docker image named `imageNameVersion` as a IronWorker called `imageName`. // For example, // RegisterWithIron("foo/myimage:1", credentials.NewEnvCredentials()) will register a worker called "foo/myimage" that will use Docker Image "foo/myimage:1". func RegisterWithIron(imageNameVersion string) error { tokens := strings.Split(imageNameVersion, ":") if len(tokens) != 2 || tokens[0] == "" || tokens[1] == "" { return errors.New("Invalid image name. Should be of the form \"name:version\".") } imageName := tokens[0] // Worker API doesn't have support for register yet, but we use it to extract the configuration. w := worker.New() url := fmt.Sprintf("https://%s/2/projects/%s/codes?oauth=%s", w.Settings.Host, w.Settings.ProjectId, w.Settings.Token) registerOpts := map[string]interface{}{ "name": imageName, "image": imageNameVersion, "env_vars": map[string]string{ "AWS_LAMBDA_FUNCTION_NAME": imageName, "AWS_LAMBDA_FUNCTION_VERSION": "1", // FIXME: swapi does not allow $ right now. }, } // Try to forward AWS credentials. { creds := credentials.NewEnvCredentials() v, err := creds.Get() if err == nil { registerOpts["env_vars"].(map[string]string)["AWS_ACCESS_KEY_ID"] = v.AccessKeyID registerOpts["env_vars"].(map[string]string)["AWS_SECRET_ACCESS_KEY"] = v.SecretAccessKey } } marshal, err := json.Marshal(registerOpts) var body bytes.Buffer mw := multipart.NewWriter(&body) jsonWriter, err := mw.CreateFormField("data") if err != nil { log.Fatalf("This should never fail") } jsonWriter.Write(marshal) mw.Close() resp, err := http.Post(url, mw.FormDataContentType(), &body) if err == nil { b, err := ioutil.ReadAll(resp.Body) if err != nil { return fmt.Errorf("%s readall %s", imageName, err) } log.Println("Register", imageName, "with iron, response:", string(b)) } return err }
func Init(params volume.DriverParams) (volume.VolumeDriver, error) { // Initialize the EC2 interface. creds := credentials.NewEnvCredentials() inst := &awsDriver{ ec2: ec2.New(&aws.Config{ Region: "us-west-1", Credentials: creds, }), } inst.db = kvdb.Instance() return inst, nil }
// Returns an S3 session for this objectList. func GetS3Session(awsRegion string) (*session.Session, error) { if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" { return nil, fmt.Errorf("AWS_ACCESS_KEY_ID and/or " + "AWS_SECRET_ACCESS_KEY not set in environment") } creds := credentials.NewEnvCredentials() _session := session.New(&aws.Config{ Region: aws.String(awsRegion), Credentials: creds, }) if _session == nil { return nil, fmt.Errorf("AWS Session returned nil") } return _session, nil }
// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets. func NewEC2Discovery(conf *config.EC2SDConfig) *EC2Discovery { creds := credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, "") if conf.AccessKey == "" && conf.SecretKey == "" { creds = credentials.NewEnvCredentials() } return &EC2Discovery{ aws: &aws.Config{ Region: &conf.Region, Credentials: creds, }, done: make(chan struct{}), interval: time.Duration(conf.RefreshInterval), port: conf.Port, } }
func Main(do func(*aws.Config) error) { common.Main(func() error { var awsEnv awsEnv if err := env.Populate(&awsEnv, env.PopulateOptions{}); err != nil { return err } return do( &aws.Config{ Credentials: credentials.NewEnvCredentials(), HTTPClient: cacerts.NewHTTPClient(), Region: awsEnv.AwsRegion, }, ) }) }