func killBucket(b *s3.Bucket) {
var err error
for attempt := attempts.Start(); attempt.Next(); {
err = b.DelBucket()
if err == nil {
return
}
if _, ok := err.(*net.DNSError); ok {
return
}
e, ok := err.(*s3.Error)
if ok && e.Code == "NoSuchBucket" {
return
}
if ok && e.Code == "BucketNotEmpty" {
// Errors are ignored here. Just retry.
resp, err := b.List("", "", "", 1000)
if err == nil {
for _, key := range resp.Contents {
_ = b.Del(key.Key)
}
}
multis, _, _ := b.ListMulti("", "")
for _, m := range multis {
_ = m.Abort()
}
}
}
message := "cannot delete test bucket"
if err != nil {
message += ": " + err.Error()
}
panic(message)
}
// listFiles lists the files in a specific s3 bucket.
func listFiles(prefix, delimiter, marker string, maxKeys int, b *s3.Bucket) (files []s3.Key, err error) {
resp, err := b.List(prefix, delimiter, marker, maxKeys)
if err != nil {
return nil, err
}
// append to files
for _, fl := range resp.Contents {
if strings.Contains(fl.Key, "index.html") || strings.Contains(fl.Key, "static") || strings.Contains(fl.Key, "logs") {
continue
}
files = append(files, fl)
}
// recursion for the recursion god
if resp.IsTruncated && resp.NextMarker != "" {
f, err := listFiles(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, b)
if err != nil {
return nil, err
}
// append to files
files = append(files, f...)
}
return files, nil
}
func LastS3KeyWithPrefix(bucket *s3.Bucket, prefix *string) (string, error) {
narrowedPrefix := *prefix
keyMarker := ""
// First, do a few checks for shortcuts for checking backwards: focus in on the 14 days.
// Otherwise just loop forward until there aren't any more results
currentDay := time.Now()
for i := 0; i < S3_REWIND_IN_DAYS_BEFORE_LONG_LOOP; i++ {
testPrefix := fmt.Sprintf("%s%s", *prefix, S3DatePrefix(¤tDay))
results, err := bucket.List(narrowedPrefix, "", keyMarker, 0)
if err != nil && len(results.Contents) > 0 {
narrowedPrefix = testPrefix
break
}
currentDay = currentDay.Add(-1 * time.Duration(DAY_IN_SECONDS) * time.Second)
}
lastKey := ""
moreResults := true
for moreResults {
results, err := bucket.List(narrowedPrefix, "", keyMarker, 0)
if err != nil {
return lastKey, err
}
if len(results.Contents) == 0 { // empty request, return last found lastKey
return lastKey, nil
}
lastKey = results.Contents[len(results.Contents)-1].Key
keyMarker = lastKey
moreResults = results.IsTruncated
}
return lastKey, nil
}
func getS3FileHash(bucket *s3.Bucket, filePath string) (string, error) {
list, err := bucket.List(filePath, "", "", 1)
if err != nil {
return "", err
}
if len(list.Contents) < 1 {
return "", ErrFileDoesNotExistS3
}
md5sum := strings.Trim(list.Contents[0].ETag, "\"")
return md5sum, nil
}