func uploadToS3(file string, s3_bucket string, awsRegion aws.Region) error {
log.Println("Uploading to S3...")
auth, err := aws.EnvAuth()
if err != nil {
return err
}
client := s3.New(auth, awsRegion)
bucket := client.Bucket(s3_bucket)
r, errF := os.Open(file)
fi, _ := r.Stat()
length := fi.Size()
if errF != nil {
return errF
}
path := file
perm := s3.ACL("private")
contType := "application/x-compressed"
err = bucket.PutReader(path, r, length, contType, perm)
if err != nil {
return err
}
log.Println("Upload successful")
return nil
}
func main() {
kingpin.Version("0.0.1")
kingpin.CommandLine.Help = "AWS deployment tools."
kingpin.Parse()
auth, err := aws.EnvAuth()
Check(err)
state := new(multistep.BasicStateBag)
// This allows us to share our client connections while in each of the steps.
state.Put("client_elb", elb.New(auth, aws.USWest2))
state.Put("client_ec2", ec2.New(auth, aws.USWest2))
// Standard configuration that has been passed in via the CLI.
state.Put("elb", *elbId)
state.Put("ami", *amiId)
state.Put("key", *key)
state.Put("size", *size)
state.Put("region", aws.Regions[*region])
state.Put("security", *security)
state.Put("tags", *tags)
steps := []multistep.Step{
&StepDestroy{}, // Remove the existing hosts from the Load balancer.
&StepCreate{}, // Create some EC2 instances and ensure they are ready to be deployed.
}
runner := &multistep.BasicRunner{Steps: steps}
runner.Run(state)
}
func NewEc2Discoverer(c *cli.Context) Ec2Discoverer {
auth, err := aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
tag := c.String("ec2-tag")
if tag == "" {
log.Fatal(errors.New("You must specify --ec2-tag option"))
}
t := strings.Split(tag, ":")
tagKey, tagValue := t[0], t[1]
regionStr := c.String("ec2-region")
region, ok := aws.Regions[regionStr]
if !ok {
log.Fatal(fmt.Errorf("%s region is not valid", regionStr))
}
client := ec2.New(auth, region)
return Ec2Discoverer{
Client: client,
TagKey: tagKey,
TagValue: tagValue,
}
}
func CreateBucket() *s3.Bucket {
auth, err := aws.EnvAuth()
if err != nil {
println("store -- error connecting to AWS\n")
panic(err)
}
fmt.Printf("store -- connected to AWS\n")
awsRegion := os.Getenv("STORE_AWSREGION")
if awsRegion == "" {
println("store -- error: environment variable STORE_AWSREGION not set")
panic("store -- STORE_AWSREGION not set")
}
region := aws.Regions[awsRegion]
if region.Name == "" {
panic("store -- region '" + awsRegion + "' is invalid")
}
println("store -- AWS region: " + region.Name)
S3 := s3.New(auth, aws.Regions["eu-west-1"])
bucketName := os.Getenv("STORE_BUCKETNAME")
if bucketName == "" {
println("store -- error: environment variable STORE_BUCKETNAME not set")
panic("store -- STORE_BUCKETNAME not set")
}
bucket := S3.Bucket(bucketName)
println("store -- Bucket name: " + bucket.Name)
return bucket
}
func awsConnectForTest() *s3.Bucket {
os.Setenv("AWS_ACCESS_KEY_ID", Config.Aws.AccessKeyId)
os.Setenv("AWS_SECRET_ACCESS_KEY", Config.Aws.SecretAccessKey)
auth, err := aws.EnvAuth()
perror(err)
return s3.New(auth, aws.Regions[Config.Aws.Region]).Bucket(Config.Aws.BucketName)
}
func UploadToS3(file multipart.File, s3Key string) (err error) {
auth, err := aws.EnvAuth()
if err != nil {
log.Fatal("Incorrect AWS Auth Details ", err)
}
connection := s3.New(auth, region)
bucket := connection.Bucket(os.Getenv("S3_BUCKET"))
buffer := new(bytes.Buffer)
_, err = io.Copy(buffer, file)
if err != nil {
log.Fatal("Cannot create file ", err)
}
filetype := http.DetectContentType(buffer.Bytes())
err = bucket.Put(
s3Key,
buffer.Bytes(),
filetype,
s3.ACL("public-read"),
)
return err
}
func (s *AmazonServer) SetUp(c *C) {
auth, err := aws.EnvAuth()
if err != nil {
c.Fatal(err.Error())
}
s.auth = auth
}
func main() {
var regionStr string
flag.StringVar(®ionStr, "r", "us-east-1", "Region")
flag.Parse()
region, err := strToRegion(regionStr)
if err == nil {
log.Fatal(err)
}
auth, err := aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
client := ec2.New(auth, *region)
resp, err := client.Instances(nil, nil)
if err != nil {
log.Fatal(err)
}
instances := []ec2.Instance{}
for _, reservation := range resp.Reservations {
instances = append(instances, reservation.Instances...)
}
b, err := json.Marshal(instances)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", b)
}
func (pl *Plugin) Init() error {
auth, err := aws.EnvAuth()
if err != nil {
return err
}
pl.s3 = s3.New(auth, aws.APNortheast)
return nil
}
func S3Bucket() *s3.Bucket {
auth, err := aws.EnvAuth()
ErrorHandler(err)
client := s3.New(auth, aws.USEast)
b := client.Bucket("cc-gifgroup-api")
return b
}
func GetS3Bucket() (bucket *s3.Bucket) {
auth, err := aws.EnvAuth()
ErrorHandler(err)
client := s3.New(auth, aws.USEast)
bucket = client.Bucket(os.Getenv("AWS_STORAGE_BUCKET_NAME"))
return
}
func (s *S) TestEnvAuthAlt(c *C) {
os.Clearenv()
os.Setenv("AWS_SECRET_KEY", "secret")
os.Setenv("AWS_ACCESS_KEY", "access")
auth, err := aws.EnvAuth()
c.Assert(err, IsNil)
c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access"})
}
func (h *FileHandler) getBucket() (*s3.Bucket, error) {
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
client := s3.New(auth, aws.APSoutheast2)
bucket := client.Bucket(h.BucketName)
return bucket, nil
}
func init() {
log.Println("Initializing AWS infomation...")
auth, err := aws.EnvAuth() //TODO setenv auths
if err != nil {
log.Fatalln(err)
}
s3 := s3.New(auth, aws.APNortheast)
LgtmBucket = s3.Bucket("lgtm-images")
}
func (s *S) TestEnvAuthWithToken(c *C) {
os.Clearenv()
os.Setenv("AWS_SECRET_ACCESS_KEY", "secret")
os.Setenv("AWS_ACCESS_KEY_ID", "access")
os.Setenv("AWS_SECURITY_TOKEN", "token")
auth, err := aws.EnvAuth()
c.Assert(err, IsNil)
c.Assert(auth, Equals, aws.Auth{SecretKey: "secret", AccessKey: "access", Token: "token"})
}
// S3BucketFromOS returns the bucket from the environment variables (cf. README.md).
func S3BucketFromOS() *s3.Bucket {
// Prepare AWS S3 connection.
s3auth, err := aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
client := s3.New(s3auth, aws.USEast)
return client.Bucket(os.Getenv("AWS_STORAGE_BUCKET_NAME"))
}
func NewListing(user *User, jsonReader io.Reader) (*Listing, error) {
var rL RequestListing
if err := json.NewDecoder(jsonReader).Decode(&rL); err != nil {
return nil, err
}
if err := rL.validate(); err != nil {
return nil, err
}
address, err := geo.Geocode(rL.Location.Address)
if err != nil {
return nil, err
}
balancedClient := balanced.NewClient(nil, os.Getenv("BALANCED_SECRET"))
customer := balanced.Customer{Uri: user.CustomerUri}
_, err = balancedClient.Customers.AddBankAccount(&customer, rL.BankAccountUri)
if err != nil {
return nil, err
}
path := generatePath()
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
s3Client := s3.New(auth, aws.USWest)
bucket := s3Client.Bucket(os.Getenv("S3_BUCKET_NAME"))
data, err := base64.StdEncoding.DecodeString(rL.Image)
if err != nil {
return nil, err
}
err = bucket.Put(path, data, "image/jpeg", s3.PublicRead)
if err != nil {
return nil, err
}
listing := Listing{
Id: bson.NewObjectId(),
Title: rL.Title,
Description: rL.Description,
Location: Location{
Address: rL.Location.Address,
Geometry: Geometry{
Type: "Point",
Coordinates: []float64{address.Lng, address.Lat},
},
},
Price: rL.Price,
Image: bucket.URL(path),
UserId: user.Id,
}
return &listing, nil
}
// Open opens the S3 backend at bucket and region.
func Open(regionname, bucketname string) (backend.Backend, error) {
auth, err := aws.EnvAuth()
if err != nil {
return nil, err
}
client := s3.New(auth, aws.Regions[regionname])
return OpenS3Bucket(client.Bucket(bucketname), bucketname), nil
}
func (s *LiveSuite) SetUpSuite(c *C) {
if !Amazon {
c.Skip("amazon tests not enabled (-amazon flag)")
}
auth, err := aws.EnvAuth()
if err != nil {
c.Fatal(err.Error())
}
s.auth = auth
}
func init() {
// BASE_URL, AUTH_USER and AUTH_PASS, AWS_S3_BASE_URL are not required or else wercker tests would fail
baseURL = os.Getenv("BASE_URL")
authUser = os.Getenv("AUTH_USER")
authPass = os.Getenv("AUTH_PASS")
s3BaseURL = os.Getenv("AWS_S3_BASE_URL")
if awsAuth, err := aws.EnvAuth(); err != nil {
// not required or else wercker tests would fail
log.Println(err)
} else {
// TODO(jrubin) allow region to be chosen by env variable
s3Data := s3.New(awsAuth, aws.USWest2)
s3Bucket = s3Data.Bucket(os.Getenv("AWS_S3_BUCKET_NAME"))
}
m = martini.Classic()
m.Use(gzip.All())
m.Use(render.Renderer())
m.Get("/", func() string {
return "hello, world"
})
m.Post(
"/v1/transcribe",
auth.Basic(authUser, authPass),
strict.Accept("application/json"),
strict.ContentType("application/x-www-form-urlencoded"),
binding.Bind(transcribeData{}),
binding.ErrorHandler,
handleTranscribe,
)
m.Post(
"/v1/transcribe/process",
strict.ContentType("application/x-www-form-urlencoded"),
binding.Bind(telapi.TranscribeCallbackData{}),
binding.ErrorHandler,
handleTranscribeProcess,
)
m.Post(
"/v1/transcribe/upload",
auth.Basic(authUser, authPass),
strict.Accept("application/json"),
binding.MultipartForm(transcribeUploadData{}),
binding.ErrorHandler,
handleTranscribeUpload,
)
m.Router.NotFound(strict.MethodNotAllowed, strict.NotFound)
}
func New() *Conn {
c := new(Conn)
// this is looking for keys in env
auth, err := aws.EnvAuth() // TODO(mleone896): maybe make a switch to use from config ?
if err != nil {
log.Fatal(err)
}
c.r53 = route53.New(auth, aws.USWest)
return c
}
//UploadToS3 sends the snapshot to S3 storage and returns a public link
func uploadToS3(data urlbox.ShotData) (s3Link string) {
if data.URL != "" && data.Width > 0 && data.Height > 0 {
auth, _ := aws.EnvAuth()
client := s3.New(auth, aws.USEast)
demoBucket := client.Bucket(BUCKETNAME)
fileName := urlbox.GetFileName(data)
bytes, filetype := getFile(fileName)
demoBucket.Put(fileName, bytes, filetype, s3.PublicRead)
s3Link = demoBucket.URL(fileName)
os.Remove(fileName)
}
return
}
func main() {
dirnamePtr := flag.String("folder", "", "the older where the assets reside")
versionPtr := flag.String("version", "", "the version of the styleguide")
flag.Parse()
dirname := *dirnamePtr
version := *versionPtr
if dirname == "" {
fmt.Println("`folder` must be specified")
return
}
if !versionRegex.MatchString(version) {
fmt.Println("`version` must be specified in the format major-minor-patch")
return
}
var files []file
prepareFilesInDir(dirname, version, &files)
requestNumber := len(files)
uploads := make(chan bool, requestNumber)
auth, err := aws.EnvAuth()
if err != nil {
panic(err)
os.Exit(1)
}
client := s3.New(auth, aws.USEast)
bucket := client.Bucket("testington")
for _, _file := range files {
go upload(_file, uploads, client, bucket)
}
for u := 1; u <= requestNumber; u++ {
<-uploads
}
close(uploads)
fmt.Println("DONE")
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
awsAuth, err := aws.EnvAuth()
if err != nil {
log.Fatalf(err.Error())
}
s3conn := s3.New(awsAuth, getRegion())
storage = store.NewS3Store(s3conn)
peers = peer.DebugPool()
peers.SetContext(func(r *http.Request) groupcache.Context {
return fetch.RequestContext(r)
})
cache = groupcache.NewGroup("ImageProxyCache", 64<<20, groupcache.GetterFunc(
func(c groupcache.Context, key string, dest groupcache.Sink) error {
log.Printf("Cache MISS for key -> %s", key)
// Get image data from S3
b, err := fetch.ImageData(storage, c)
if err != nil {
return err
}
return dest.SetBytes(b)
}))
if !*verbose {
logwriter, err := syslog.Dial("udp", "app_syslog:514", syslog.LOG_NOTICE, "vip")
if err != nil {
log.Println(err.Error())
log.Println("using default logger")
} else {
log.SetOutput(logwriter)
}
}
go peers.Listen()
go listenHttp()
go Queue.Start(4)
log.Println("Cache listening on port :" + peers.Port())
s := &http.Server{
Addr: ":" + peers.Port(),
Handler: peers,
}
s.ListenAndServe()
}
// NewContentStore creates a ContentStore at the base directory.
func NewAwsContentStore() (*AwsContentStore, error) {
os.Setenv("AWS_ACCESS_KEY_ID", Config.Aws.AccessKeyId)
os.Setenv("AWS_SECRET_ACCESS_KEY", Config.Aws.SecretAccessKey)
auth, err := aws.EnvAuth()
if err != nil {
logger.Log(kv{"fn": "AwsContentStore.NewAwsContentStore", "err": ": " + err.Error()})
return &AwsContentStore{}, err
}
client := s3.New(auth, aws.Regions[Config.Aws.Region])
bucket := client.Bucket(Config.Aws.BucketName)
self := &AwsContentStore{bucket: bucket, client: client}
self.makeBucket()
self.setAcl()
return self, nil
}
func s3Upload(uploadName string, bucketName string, content *os.File) error {
auth, err := aws.EnvAuth()
if err != nil {
return err
}
client := s3.New(auth, aws.USEast)
bucket := client.Bucket(bucketName)
info, err := content.Stat()
if err != nil {
return err
}
bucket.PutReader(uploadName, content, info.Size(), "application/octet-stream", s3.PublicRead)
return nil
}
func s3init(region aws.Region) S3Client {
auth, err := aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
client := s3.New(auth, region)
buckets, bucketerr := client.ListBuckets()
if bucketerr != nil {
log.Fatal(bucketerr)
}
return S3Client{
client: client,
buckets: buckets.Buckets,
}
}
func setRegion() error {
regionName := os.Getenv("AWS_REGION")
r, ok := aws.Regions[regionName]
if !ok {
return fmt.Errorf("Could not find region by name %s", regionName)
}
region = r
auth, err := aws.EnvAuth()
if err != nil {
logrus.Fatal("AWS failed to authenticate: %v", err)
}
client = route53.New(auth, region)
return nil
}
func (s *s3Storage) init() error {
auth, err := aws.EnvAuth()
if err != nil {
return err
}
bucketName := os.Getenv(s3BucketEnvVar)
if bucketName == "" {
return fmt.Errorf("%s not set", s3BucketEnvVar)
}
conn := s3.New(auth, aws.EUWest)
s.bucket = conn.Bucket(bucketName)
return nil
}
func GETS3Policy(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
auth, err := aws.EnvAuth()
if err != nil {
log.Fatal(err)
}
// s3Connection := s3.New(auth, aws.EUCentral)
conditions := make([]interface{}, 0)
conditions = append(conditions, map[string]string{"bucket": "openpot1"})
conditions = append(conditions, map[string]s3.ACL{"acl": s3.PublicRead})
conditions = append(conditions, []string{"starts-with", "$key", ""})
conditions = append(conditions, []string{"starts-with", "$Content-Type", "image/"})
conditions = append(conditions, []string{"starts-with", "$filename", ""})
s3Policy, err := json.Marshal(S3Policy{
Expiration: time.Now().Add(time.Hour * 48).UTC().Format(time.RFC3339),
Conditions: conditions,
})
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Print(err)
return
}
s3PolicyBase64 := base64.StdEncoding.EncodeToString(s3Policy)
mac := hmac.New(sha1.New, []byte(auth.SecretKey))
mac.Write([]byte(s3PolicyBase64))
policy := S3Object{
Policy: s3PolicyBase64,
Signature: base64.StdEncoding.EncodeToString(mac.Sum(nil)),
Key: auth.AccessKey,
URL: "https://openpot1.s3-eu-west-1.amazonaws.com/",
CDN: "https://d1f0mp28tfo8uk.cloudfront.net/",
}
jsonPolicy, err := json.Marshal(policy)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
log.Print(err)
return
}
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, string(jsonPolicy))
}
