func GetImagesInGallery(bucket *s3.Bucket, g *Gallery) ([]Image, error) { pre := fmt.Sprintf("%s%s", g.S3Prefix, "orig-") res, err := bucket.List(pre, "/", "", 1000) if err != nil { return nil, tracederror.New(err) } paths := []Image{} for _, v := range res.Contents { if v.Key[len(v.Key)-1] != '/' { raw := strings.TrimPrefix(v.Key, pre) t, err := time.Parse("2006-01-02T15:04:05.000Z", v.LastModified) if err != nil { log.Print("Couldn't parse time from amazon, assuming 'now' for upload date.") log.Print(err) t = time.Now() } paths = append(paths, Image{ Thumb: fmt.Sprintf("%sthumb-%s", g.S3Prefix, raw), Orig: fmt.Sprintf("%sorig-%s", g.S3Prefix, raw), Hero: fmt.Sprintf("%shero-%s", g.S3Prefix, raw), ETag: v.ETag, LastModified: t, }) } } sort.Sort(ByDate(paths)) return paths, nil }
//Uploads only if the given key does not exist func uploads3fileifnotexists(binpath, binfile, contenttype string, bucket *s3.Bucket) error { k, _ := bucket.GetKey(binpath) if k == nil { //Binary does not exist on s3.. upload it now... return uploads3file(binpath, binfile, contenttype, bucket) } return nil }
func upload(_file file, uploads chan<- bool, client *s3.S3, bucket *s3.Bucket) { err := bucket.Put(_file.path, _file.data, _file.contentType, permissions) if err != nil { fmt.Printf("UPLOAD ERROR: %+v\n", err) panic(err) } uploads <- true fmt.Printf("Uploaded %s!\n", _file.path) }
func s3Upload(bucket *s3.Bucket, path string, im *Image) (string, error) { var url string if len(im.Data) == 0 { return "", fmt.Errorf("No image data found for %s", path) } err := bucket.Put(path, im.Data, im.MimeType(), s3.PublicRead) if err != nil { return url, err } url = bucket.URL(path) return url, nil }
//Uploads only if the given key does not exist func uploads3fileifnotexists(binpath, binfile, contenttype string, bucket *s3.Bucket) error { k, _ := bucket.GetKey(binpath) if k == nil { //Binary does not exist on s3.. gzip and upload it now... finalfile, err := gziptotempfile(binfile) if err != nil { return err } return uploads3file(binpath, finalfile, "application/x-gzip", bucket) } return nil }
func writeS3FileToPath(file string, bucket *s3.Bucket, path string) error { data, err := bucket.Get(path) if err != nil { return err } perms := os.FileMode(0644) err = ioutil.WriteFile(file, data, perms) if err != nil { return err } return nil }
func killBucket(b *s3.Bucket) { var err error for attempt := attempts.Start(); attempt.Next(); { err = b.DelBucket() if err == nil { return } if _, ok := err.(*net.DNSError); ok { return } e, ok := err.(*s3.Error) if ok && e.Code == "NoSuchBucket" { return } if ok && e.Code == "BucketNotEmpty" { // Errors are ignored here. Just retry. resp, err := b.List("", "", "", 1000) if err == nil { for _, key := range resp.Contents { _ = b.Del(key.Key) } } multis, _, _ := b.ListMulti("", "") for _, m := range multis { _ = m.Abort() } } } message := "cannot delete test bucket" if err != nil { message += ": " + err.Error() } panic(message) }
func writeLocalFileToS3(bucket *s3.Bucket, path string, file string) error { contType := mime.TypeByExtension(filepath.Ext(file)) Perms := s3.ACL("private") data, err := ioutil.ReadFile(file) if err != nil { return err } if err := bucket.Put(path, data, contType, Perms); err != nil { return err } return nil }
func writeS3FileToS3(sourceBucket, targetBucket *s3.Bucket, sourceKeyPath, targetKeyPath string) error { data, err := sourceBucket.Get(sourceKeyPath) if err != nil { return err } contType := mime.TypeByExtension(filepath.Ext(sourceKeyPath)) Perms := s3.ACL("private") if err := targetBucket.Put(targetKeyPath, data, contType, Perms); err != nil { return err } return nil }
func uploads3file(path, file, contenttype string, bucket *s3.Bucket) error { f, err := os.Open(file) if err != nil { return err } defer f.Close() info, err := f.Stat() if err != nil { return err } err = bucket.PutReader(path, f, info.Size(), contenttype, s3.Private) if err != nil { return err } return nil }
func (s3p *s3Provider) rawUpload(opts *Options, b *s3.Bucket, a *artifact.Artifact) error { dest := a.FullDest() reader, err := a.Reader() if err != nil { return err } ctype := a.ContentType() size, err := a.Size() if err != nil { return err } downloadHost := s3p.getRegion().S3BucketEndpoint if downloadHost == "" { downloadHost = fmt.Sprintf("https://%s.s3.amazonaws.com", b.Name) } s3p.log.WithFields(logrus.Fields{ "download_url": fmt.Sprintf("%s/%s", downloadHost, dest), }).Info(fmt.Sprintf("uploading: %s (size: %s)", a.Source, humanize.Bytes(size))) s3p.log.WithFields(logrus.Fields{ "percent_max_size": pctMax(size, opts.MaxSize), "max_size": humanize.Bytes(opts.MaxSize), "source": a.Source, "dest": dest, "bucket": b.Name, "content_type": ctype, "cache_control": opts.CacheControl, }).Debug("more artifact details") err = b.PutReaderHeader(dest, reader, int64(size), map[string][]string{ "Content-Type": []string{ctype}, "Cache-Control": []string{opts.CacheControl}, }, a.Perm) if err != nil { return err } return nil }
// listFiles lists the files in a specific s3 bucket func listFiles(prefix, delimiter, marker string, maxKeys int, b *s3.Bucket) (files []s3.Key, err error) { resp, err := b.List(prefix, delimiter, marker, maxKeys) if err != nil { return nil, err } // append to files files = append(files, resp.Contents...) // recursion for the recursion god if resp.IsTruncated && resp.NextMarker != "" { f, err := listFiles(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, b) if err != nil { return nil, err } // append to files files = append(files, f...) } return files, nil }
func loadS3Files(bucket *s3.Bucket, path string, files map[string]string, marker string) (map[string]string, error) { log.Debugf("Loading files from 's3://%s/%s'.", bucket.Name, path) data, err := bucket.List(path, "", marker, 0) if err != nil { return files, err } for _, key := range data.Contents { md5sum := strings.Trim(key.ETag, "\"") files[key.Key] = md5sum } // Continue to call loadS3files and add // Files to map if next marker set if data.IsTruncated { lastKey := data.Contents[(len(data.Contents) - 1)].Key log.Infof("Results truncated, loading additional files via previous last key '%s'.", lastKey) loadS3Files(bucket, path, files, lastKey) } log.Debugf("Loaded '%d' files from 's3://%s/%s' succesfully.", len(files), bucket.Name, path) return files, nil }
Ω(err).ShouldNot(HaveOccurred()) destination = path.Join(tmpdir, "in-dir") inCmd = exec.Command(inPath, destination) }) AfterEach(func() { os.RemoveAll(tmpdir) }) Context("when executed", func() { var request models.InRequest var response models.InResponse var bucket *s3.Bucket BeforeEach(func() { guid, err := uuid.NewV4() Ω(err).ShouldNot(HaveOccurred()) key = guid.String() auth := aws.Auth{ AccessKey: accessKeyID, SecretKey: secretAccessKey, } region, ok := aws.Regions[regionName] Ω(ok).Should(BeTrue())
Ω(err).ShouldNot(HaveOccurred()) destination = path.Join(tmpdir, "in-dir") checkCmd = exec.Command(checkPath, destination) }) AfterEach(func() { os.RemoveAll(tmpdir) }) Context("when executed", func() { var request models.CheckRequest var response models.CheckResponse var bucket *s3.Bucket BeforeEach(func() { guid, err := uuid.NewV4() Ω(err).ShouldNot(HaveOccurred()) key = guid.String() auth := aws.Auth{ AccessKey: accessKeyID, SecretKey: secretAccessKey, } region, ok := aws.Regions[regionName] Ω(ok).Should(BeTrue())
source, err = ioutil.TempDir("", "out-source") Ω(err).ShouldNot(HaveOccurred()) outCmd = exec.Command(outPath, source) }) AfterEach(func() { os.RemoveAll(source) }) Context("when executed", func() { var request models.OutRequest var response models.OutResponse var bucket *s3.Bucket BeforeEach(func() { guid, err := uuid.NewV4() Ω(err).ShouldNot(HaveOccurred()) key = guid.String() auth := aws.Auth{ AccessKey: accessKeyID, SecretKey: secretAccessKey, } region, ok := aws.Regions[regionName] Ω(ok).Should(BeTrue())
"github.com/mitchellh/goamz/aws" goamz "github.com/mitchellh/goamz/s3" "github.com/pivotal-cf/cf-redis-broker/s3" "github.com/pivotal-golang/lager" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" ) var _ = Describe("Client", func() { var ( fakeRegion aws.Region goamzBucketClient *goamz.Bucket bucketName string log *gbytes.Buffer logger lager.Logger ) BeforeEach(func() { bucketName = "i_am_bucket" logger = lager.NewLogger("logger") log = gbytes.NewBuffer() logger.RegisterSink(lager.NewWriterSink(log, lager.INFO)) fakeRegion = aws.Region{ Name: "fake_region", S3Endpoint: fakeS3EndpointURL, S3LocationConstraint: true,