func CreateBucket() *s3.Bucket {
auth, err := aws.EnvAuth()
if err != nil {
println("store -- error connecting to AWS\n")
panic(err)
}
fmt.Printf("store -- connected to AWS\n")
awsRegion := os.Getenv("STORE_AWSREGION")
if awsRegion == "" {
println("store -- error: environment variable STORE_AWSREGION not set")
panic("store -- STORE_AWSREGION not set")
}
region := aws.Regions[awsRegion]
if region.Name == "" {
panic("store -- region '" + awsRegion + "' is invalid")
}
println("store -- AWS region: " + region.Name)
S3 := s3.New(auth, aws.Regions["eu-west-1"])
bucketName := os.Getenv("STORE_BUCKETNAME")
if bucketName == "" {
println("store -- error: environment variable STORE_BUCKETNAME not set")
panic("store -- STORE_BUCKETNAME not set")
}
bucket := S3.Bucket(bucketName)
println("store -- Bucket name: " + bucket.Name)
return bucket
}
func (gozo Gozo) SendImage(filename string) (url string, err error) {
data, err := ioutil.ReadFile(filename)
if err != nil {
return
}
auth, err := aws.GetAuth(
gozo.accessKey,
gozo.secretAccessKey,
)
if err != nil {
return
}
s3client := s3.New(auth, gozo.region)
bucket := s3client.Bucket(gozo.bucketName)
path := "images/" + hexdigest(fmt.Sprintf("%s-%d", filename, time.Now().Unix())) + ".png"
err = bucket.Put(path, data, "image/png", s3.PublicRead)
if err != nil {
return
}
url = gozo.rootURL + path
return
}
func (p *PostProcessor) Configure(raws ...interface{}) error {
err := config.Decode(&p.config, &config.DecodeOpts{
Interpolate: true,
InterpolateContext: &p.config.ctx,
InterpolateFilter: &interpolate.RenderFilter{
Exclude: []string{"output"},
},
}, raws...)
if err != nil {
return err
}
errs := new(packer.MultiError)
// required configuration
templates := map[string]*string{
"region": &p.config.Region,
"bucket": &p.config.Bucket,
"manifest": &p.config.ManifestPath,
"box_name": &p.config.BoxName,
"box_dir": &p.config.BoxDir,
"version": &p.config.Version,
}
for key, ptr := range templates {
if *ptr == "" {
errs = packer.MultiErrorAppend(errs, fmt.Errorf("vagrant-s3 %s must be set", key))
}
}
// Template process
for key, ptr := range templates {
if err = interpolate.Validate(*ptr, &p.config.ctx); err != nil {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("Error parsing %s template: %s", key, err))
}
}
auth, err := aws.GetAuth(p.config.AccessKey, p.config.SecretKey)
if err != nil {
errs = packer.MultiErrorAppend(errs, fmt.Errorf("Unable to create Aws Authentication. Try providing keys 'access_key_id' and 'secret_key'"))
}
// determine region
region, valid := aws.Regions[p.config.Region]
if valid {
p.s3 = s3.New(auth, region).Bucket(p.config.Bucket)
} else {
errs = packer.MultiErrorAppend(errs, fmt.Errorf("Invalid region specified: %s", p.config.Region))
}
if p.config.ACL == "" {
p.config.ACL = "public-read"
}
if len(errs.Errors) > 0 {
return errs
}
return nil
}
func (p *PostProcessor) Configure(raws ...interface{}) error {
err := config.Decode(&p.config, &config.DecodeOpts{
Interpolate: true,
InterpolateContext: &p.config.ctx,
InterpolateFilter: &interpolate.RenderFilter{
Exclude: []string{},
},
}, raws...)
if err != nil {
return err
}
errs := &packer.MultiError{}
errs = packer.MultiErrorAppend(errs, p.config.AccessConfig.Prepare(&p.config.ctx)...)
// required configuration
templates := map[string]*string{
"region": &p.config.Region,
"bucket": &p.config.Bucket,
"manifest": &p.config.ManifestPath,
"box_name": &p.config.BoxName,
"box_dir": &p.config.BoxDir,
"version": &p.config.Version,
}
// Template process
for key, ptr := range templates {
if *ptr == "" {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("%s must be set", key))
}
*ptr, err = interpolate.Render(*ptr, &p.config.ctx)
if err != nil {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("Error processing %s: %s", key, err))
}
}
// setup the s3 bucket
auth, err := aws.GetAuth(p.config.AccessConfig.AccessKey, p.config.AccessConfig.SecretKey)
if err != nil {
errs = packer.MultiErrorAppend(errs, err)
}
// determine region
region, valid := aws.Regions[p.config.Region]
if valid {
p.s3 = s3.New(auth, region).Bucket(p.config.Bucket)
} else {
errs = packer.MultiErrorAppend(errs, fmt.Errorf("Invalid region specified: %s", p.config.Region))
}
if len(errs.Errors) > 0 {
return errs
}
return nil
}
func UploadToS3(file multipart.File, s3Key string) (err error) {
auth, err := aws.EnvAuth()
if err != nil {
log.Fatal("Incorrect AWS Auth Details ", err)
}
connection := s3.New(auth, region)
bucket := connection.Bucket(os.Getenv("S3_BUCKET"))
buffer := new(bytes.Buffer)
_, err = io.Copy(buffer, file)
if err != nil {
log.Fatal("Cannot create file ", err)
}
filetype := http.DetectContentType(buffer.Bytes())
err = bucket.Put(
s3Key,
buffer.Bytes(),
filetype,
s3.ACL("public-read"),
)
return err
}
func FromSource(source models.Source) (Driver, error) {
var initialVersion semver.Version
if source.InitialVersion != "" {
version, err := semver.Parse(source.InitialVersion)
if err != nil {
return nil, fmt.Errorf("invalid initial version (%s): %s", source.InitialVersion, err)
}
initialVersion = version
} else {
initialVersion = semver.Version{Major: 0, Minor: 0, Patch: 0}
}
switch source.Driver {
case models.DriverUnspecified, models.DriverS3:
auth := aws.Auth{
AccessKey: source.AccessKeyID,
SecretKey: source.SecretAccessKey,
}
regionName := source.RegionName
if len(regionName) == 0 {
regionName = aws.USEast.Name
}
region, ok := aws.Regions[regionName]
if !ok {
return nil, errors.New(fmt.Sprintf("no such region '%s'", regionName))
}
if len(source.Endpoint) != 0 {
region = aws.Region{
S3Endpoint: fmt.Sprintf("https://%s", source.Endpoint),
}
}
client := s3.New(auth, region)
bucket := client.Bucket(source.Bucket)
return &S3Driver{
InitialVersion: initialVersion,
Bucket: bucket,
Key: source.Key,
}, nil
case models.DriverGit:
return &GitDriver{
InitialVersion: initialVersion,
URI: source.URI,
Branch: source.Branch,
PrivateKey: source.PrivateKey,
File: source.File,
}, nil
default:
return nil, fmt.Errorf("unknown driver: %s", source.Driver)
}
}
// Communicate with all endpoints to see if they are alive.
func (s *ClientTests) TestRegions(c *C) {
errs := make(chan error, len(aws.Regions))
for _, region := range aws.Regions {
go func(r aws.Region) {
s := s3.New(s.s3.Auth, r)
b := s.Bucket("goamz-" + s.Auth.AccessKey)
_, err := b.Get("non-existent")
errs <- err
}(region)
}
for _ = range aws.Regions {
err := <-errs
if err != nil {
s3_err, ok := err.(*s3.Error)
if ok {
c.Check(s3_err.Code, Matches, "NoSuchBucket")
} else if _, ok = err.(*net.DNSError); ok {
// Okay as well.
} else {
c.Errorf("Non-S3 error: %s", err)
}
} else {
c.Errorf("Test should have errored but it seems to have succeeded")
}
}
}
func PutS3File(pushAuth *aws.Auth, localFilePath, s3URL, contentType string) error {
urlParsed, err := url.Parse(s3URL)
if err != nil {
return err
}
if urlParsed.Scheme != "s3" {
return fmt.Errorf("Don't know how to use URL with scheme %v", urlParsed.Scheme)
}
localFileReader, err := os.Open(localFilePath)
if err != nil {
return err
}
fi, err := os.Stat(localFilePath)
if err != nil {
return err
}
session := s3.New(*pushAuth, aws.USEast)
bucket := session.Bucket(urlParsed.Host)
err = bucket.PutReader(urlParsed.Path, localFileReader, fi.Size(), contentType, s3.PublicRead)
if err != nil {
return err
}
return nil
}
// Put the specified resource to s3.
func (self *S3PutCommand) Put() error {
fi, err := os.Stat(self.LocalFile)
if err != nil {
return err
}
fileReader, err := os.Open(self.LocalFile)
if err != nil {
return err
}
defer fileReader.Close()
// get the appropriate session and bucket
auth := &aws.Auth{
AccessKey: self.AwsKey,
SecretKey: self.AwsSecret,
}
session := s3.New(*auth, aws.USEast)
bucket := session.Bucket(self.Bucket)
// put the data
return bucket.PutReader(
self.RemoteFile,
fileReader,
fi.Size(),
self.ContentType,
s3.ACL(self.Permissions),
)
}
func awsConnectForTest() *s3.Bucket {
os.Setenv("AWS_ACCESS_KEY_ID", Config.Aws.AccessKeyId)
os.Setenv("AWS_SECRET_ACCESS_KEY", Config.Aws.SecretAccessKey)
auth, err := aws.EnvAuth()
perror(err)
return s3.New(auth, aws.Regions[Config.Aws.Region]).Bucket(Config.Aws.BucketName)
}
func uploadToS3(file string, s3_bucket string, awsRegion aws.Region) error {
log.Println("Uploading to S3...")
auth, err := aws.EnvAuth()
if err != nil {
return err
}
client := s3.New(auth, awsRegion)
bucket := client.Bucket(s3_bucket)
r, errF := os.Open(file)
fi, _ := r.Stat()
length := fi.Size()
if errF != nil {
return errF
}
path := file
perm := s3.ACL("private")
contType := "application/x-compressed"
err = bucket.PutReader(path, r, length, contType, perm)
if err != nil {
return err
}
log.Println("Upload successful")
return nil
}
// Store puts an `ExportResult` struct to an S3 bucket within the specified directory
func (x *S3) Store(result *ExportResult, directory string) *Error {
if result.Error != nil {
return result.Error
}
file, err := os.Open(result.Path)
if err != nil {
return makeErr(err, "")
}
defer file.Close()
buffy := bufio.NewReader(file)
stat, err := file.Stat()
if err != nil {
return makeErr(err, "")
}
size := stat.Size()
auth := aws.Auth{
AccessKey: x.AccessKey,
SecretKey: x.ClientSecret,
}
s := s3.New(auth, aws.Regions[x.Region])
bucket := s.Bucket(x.Bucket)
err = bucket.PutReader(directory+result.Filename(), buffy, size, result.MIME, s3.BucketOwnerFull)
return makeErr(err, "")
}
func getS3Bucket(accessKey, secretKey, bucket string) *s3.Bucket {
auth := aws.Auth{
AccessKey: accessKey,
SecretKey: secretKey,
}
return s3.New(auth, aws.USEast).Bucket(bucket)
}
func main() {
var err error
var match = flag.String("match", "", "pattern to match when listing keys")
flag.Parse()
args := flag.Args()
auth, err := aws.SharedAuth()
if err != nil {
fmt.Fprintf(os.Stderr, "unable to authenticate: %s\n", err.Error())
os.Exit(1)
}
client := s3.New(auth, aws.USEast)
if len(args) == 0 {
err = listBuckets(client, *match)
} else {
err = listObjectsInBuckets(client, args, *match)
}
if err != nil {
fmt.Fprintf(os.Stderr, err.Error())
os.Exit(1)
}
}
func setupS3Backend(t *testing.T) *bes3.S3Backend {
s.config = &s3test.Config{
Send409Conflict: true,
}
srv, err := s3test.NewServer(s.config)
OK(t, err)
s.srv = srv
s.region = aws.Region{
Name: "faux-region-1",
S3Endpoint: srv.URL(),
S3LocationConstraint: true, // s3test server requires a LocationConstraint
}
s.auth = aws.Auth{"abc", "123", ""}
service := s3.New(s.auth, s.region)
bucket := service.Bucket("testbucket")
err = bucket.PutBucket("private")
OK(t, err)
t.Logf("created s3 backend locally")
return bes3.OpenS3Bucket(bucket, "testbucket")
}
// NewPublishedStorageRaw creates published storage from raw aws credentials
func NewPublishedStorageRaw(auth aws.Auth, region aws.Region, bucket, defaultACL, prefix,
storageClass, encryptionMethod string, plusWorkaround, disabledMultiDel bool) (*PublishedStorage, error) {
if defaultACL == "" {
defaultACL = "private"
}
if storageClass == "STANDARD" {
storageClass = ""
}
result := &PublishedStorage{
s3: s3.New(auth, region),
acl: s3.ACL(defaultACL),
prefix: prefix,
storageClass: storageClass,
encryptionMethod: encryptionMethod,
plusWorkaround: plusWorkaround,
disableMultiDel: disabledMultiDel,
}
result.s3.HTTPClient = func() *http.Client {
return RetryingClient
}
result.bucket = result.s3.Bucket(bucket)
return result, nil
}
func (s3p *s3Provider) getConn(auth aws.Auth) *s3.S3 {
if s3p.overrideConn != nil {
s3p.log.WithField("conn", s3p.overrideConn).Debug("using override connection")
return s3p.overrideConn
}
return s3.New(auth, s3p.getRegion())
}
func (pl *Plugin) Init() error {
auth, err := aws.EnvAuth()
if err != nil {
return err
}
pl.s3 = s3.New(auth, aws.APNortheast)
return nil
}
func init() {
s3srv.SetUp()
testS3 = s3.New(s3srv.Auth, s3srv.Region)
err = testS3.Bucket("bucket").PutBucket(s3.ACL("public-read"))
if err != nil {
panic(err)
}
}
func (s *LocalServerSuite) SetUpSuite(c *C) {
s.srv.SetUp(c)
s.clientTests.s3 = s3.New(s.srv.auth, s.srv.region)
// TODO Sadly the fake server ignores auth completely right now. :-(
s.clientTests.authIsBroken = true
s.clientTests.Cleanup()
}
func GetS3Bucket() (bucket *s3.Bucket) {
auth, err := aws.EnvAuth()
ErrorHandler(err)
client := s3.New(auth, aws.USEast)
bucket = client.Bucket(os.Getenv("AWS_STORAGE_BUCKET_NAME"))
return
}
func S3Bucket() *s3.Bucket {
auth, err := aws.EnvAuth()
ErrorHandler(err)
client := s3.New(auth, aws.USEast)
b := client.Bucket("cc-gifgroup-api")
return b
}
// S3BucketFromOS returns the bucket from the environment variables (cf. README.md).
func S3BucketFromOS() *s3.Bucket {
// Prepare AWS S3 connection.
s3auth, err := aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
client := s3.New(s3auth, aws.USEast)
return client.Bucket(os.Getenv("AWS_STORAGE_BUCKET_NAME"))
}
func GetS3File(auth *aws.Auth, s3URL string) (io.ReadCloser, error) {
urlParsed, err := url.Parse(s3URL)
if err != nil {
return nil, err
}
session := s3.New(*auth, aws.USEast)
bucket := session.Bucket(urlParsed.Host)
return bucket.GetReader(urlParsed.Path)
}
func (h *FileHandler) getBucket() (*s3.Bucket, error) {
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
client := s3.New(auth, aws.APSoutheast2)
bucket := client.Bucket(h.BucketName)
return bucket, nil
}
// Client returns a S3 instance
func (s *S3Storage) Client() (*s3.S3, error) {
auth, err := s.Auth()
if err != nil {
return nil, err
}
return s3.New(auth, s.Region), nil
}
func init() {
log.Println("Initializing AWS infomation...")
auth, err := aws.EnvAuth() //TODO setenv auths
if err != nil {
log.Fatalln(err)
}
s3 := s3.New(auth, aws.APNortheast)
LgtmBucket = s3.Bucket("lgtm-images")
}
func (s *AmazonClientSuite) SetUpSuite(c *C) {
if !testutil.Amazon {
c.Skip("live tests against AWS disabled (no -amazon)")
}
s.srv.SetUp(c)
s.s3 = s3.New(s.srv.auth, s.Region)
// In case tests were interrupted in the middle before.
s.ClientTests.Cleanup()
}
func getS3FileBytes(auth *aws.Auth, bucket string, path string) ([]byte, error) {
session := s3.New(*auth, aws.USEast)
s3bucket := session.Bucket(bucket)
reader, err := s3bucket.GetReader(path)
if err != nil {
return nil, err
}
return ioutil.ReadAll(reader)
}
func NewListing(user *User, jsonReader io.Reader) (*Listing, error) {
var rL RequestListing
if err := json.NewDecoder(jsonReader).Decode(&rL); err != nil {
return nil, err
}
if err := rL.validate(); err != nil {
return nil, err
}
address, err := geo.Geocode(rL.Location.Address)
if err != nil {
return nil, err
}
balancedClient := balanced.NewClient(nil, os.Getenv("BALANCED_SECRET"))
customer := balanced.Customer{Uri: user.CustomerUri}
_, err = balancedClient.Customers.AddBankAccount(&customer, rL.BankAccountUri)
if err != nil {
return nil, err
}
path := generatePath()
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
s3Client := s3.New(auth, aws.USWest)
bucket := s3Client.Bucket(os.Getenv("S3_BUCKET_NAME"))
data, err := base64.StdEncoding.DecodeString(rL.Image)
if err != nil {
return nil, err
}
err = bucket.Put(path, data, "image/jpeg", s3.PublicRead)
if err != nil {
return nil, err
}
listing := Listing{
Id: bson.NewObjectId(),
Title: rL.Title,
Description: rL.Description,
Location: Location{
Address: rL.Location.Address,
Geometry: Geometry{
Type: "Point",
Coordinates: []float64{address.Lng, address.Lat},
},
},
Price: rL.Price,
Image: bucket.URL(path),
UserId: user.Id,
}
return &listing, nil
}
