func GetImagesInGallery(bucket *s3.Bucket, g *Gallery) ([]Image, error) { pre := fmt.Sprintf("%s%s", g.S3Prefix, "orig-") res, err := bucket.List(pre, "/", "", 1000) if err != nil { return nil, tracederror.New(err) } paths := []Image{} for _, v := range res.Contents { if v.Key[len(v.Key)-1] != '/' { raw := strings.TrimPrefix(v.Key, pre) t, err := time.Parse("2006-01-02T15:04:05.000Z", v.LastModified) if err != nil { log.Print("Couldn't parse time from amazon, assuming 'now' for upload date.") log.Print(err) t = time.Now() } paths = append(paths, Image{ Thumb: fmt.Sprintf("%sthumb-%s", g.S3Prefix, raw), Orig: fmt.Sprintf("%sorig-%s", g.S3Prefix, raw), Hero: fmt.Sprintf("%shero-%s", g.S3Prefix, raw), ETag: v.ETag, LastModified: t, }) } } sort.Sort(ByDate(paths)) return paths, nil }
func killBucket(b *s3.Bucket) { var err error for attempt := attempts.Start(); attempt.Next(); { err = b.DelBucket() if err == nil { return } if _, ok := err.(*net.DNSError); ok { return } e, ok := err.(*s3.Error) if ok && e.Code == "NoSuchBucket" { return } if ok && e.Code == "BucketNotEmpty" { // Errors are ignored here. Just retry. resp, err := b.List("", "", "", 1000) if err == nil { for _, key := range resp.Contents { _ = b.Del(key.Key) } } multis, _, _ := b.ListMulti("", "") for _, m := range multis { _ = m.Abort() } } } message := "cannot delete test bucket" if err != nil { message += ": " + err.Error() } panic(message) }
// listFiles lists the files in a specific s3 bucket func listFiles(prefix, delimiter, marker string, maxKeys int, b *s3.Bucket) (files []s3.Key, err error) { resp, err := b.List(prefix, delimiter, marker, maxKeys) if err != nil { return nil, err } // append to files files = append(files, resp.Contents...) // recursion for the recursion god if resp.IsTruncated && resp.NextMarker != "" { f, err := listFiles(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, b) if err != nil { return nil, err } // append to files files = append(files, f...) } return files, nil }
func loadS3Files(bucket *s3.Bucket, path string, files map[string]string, marker string) (map[string]string, error) { log.Debugf("Loading files from 's3://%s/%s'.", bucket.Name, path) data, err := bucket.List(path, "", marker, 0) if err != nil { return files, err } for _, key := range data.Contents { md5sum := strings.Trim(key.ETag, "\"") files[key.Key] = md5sum } // Continue to call loadS3files and add // Files to map if next marker set if data.IsTruncated { lastKey := data.Contents[(len(data.Contents) - 1)].Key log.Infof("Results truncated, loading additional files via previous last key '%s'.", lastKey) loadS3Files(bucket, path, files, lastKey) } log.Debugf("Loaded '%d' files from 's3://%s/%s' succesfully.", len(files), bucket.Name, path) return files, nil }