func killBucket(b *s3.Bucket) {
var err error
for attempt := attempts.Start(); attempt.Next(); {
err = b.DelBucket()
if err == nil {
return
}
if _, ok := err.(*net.DNSError); ok {
return
}
e, ok := err.(*s3.Error)
if ok && e.Code == "NoSuchBucket" {
return
}
if ok && e.Code == "BucketNotEmpty" {
// Errors are ignored here. Just retry.
resp, err := b.List("", "", "", 1000)
if err == nil {
for _, key := range resp.Contents {
_ = b.Del(key.Key)
}
}
multis, _, _ := b.ListMulti("", "")
for _, m := range multis {
_ = m.Abort()
}
}
}
message := "cannot delete test bucket"
if err != nil {
message += ": " + err.Error()
}
panic(message)
}
func getFileList(date string, bucket *s3.Bucket) []string {
resultsSlice := make([]string,0)
response, err := bucket.List(date, "", "", 500)
if err != nil {
log.Fatal(err)
}
for _, objects := range response.Contents {
resultsSlice = append(resultsSlice, objects.Key)
}
return resultsSlice
}
//ApplyToMultiList applies the action a to all the keys that match prefix
//To select ALL contents of the bucket use prefix, delim = ""
func ApplyToMultiList(b *s3.Bucket, prefix, delim string, a Action) {
//prints stats
go func() {
for {
select {
case <-time.After(3 * time.Second):
r := atomic.LoadUint64(&donecounter)
fmt.Printf("read %v keys\n", r)
r = atomic.LoadUint64(&counter)
fmt.Printf("processed %v keys\n", r)
}
}
}()
resp, err := b.List(prefix, delim, "", 1000)
if err != nil {
panic(err)
}
if len(resp.Contents) < 1 {
log.Infof("got no Contents")
return
}
lastSeen := resp.Contents[len(resp.Contents)-1]
for _, obj := range resp.Contents {
atomic.AddUint64(&counter, 1)
go a(KeyBucket{b, obj})
}
for {
if resp.IsTruncated {
resp, err = b.List(prefix, delim, lastSeen.Key, 1000)
if err != nil {
panic(err)
}
lastSeen = resp.Contents[len(resp.Contents)-1]
fmt.Printf("------ \n %v \n-----", lastSeen.Key)
// TODO allow setting a max number of workers
for _, obj := range resp.Contents {
atomic.AddUint64(&counter, 1)
go func() {
wg.Add(1)
a(KeyBucket{b, obj})
wg.Done()
}()
}
} else {
break
}
}
wg.Wait()
}
// List get list from bucket.
func List(bucket *s3.Bucket, prefix, delim, marker string, max int) (*s3.ListResp, error) {
return bucket.List(prefix, delim, marker, max)
}
