Example #1
0
func killBucket(b *s3.Bucket) {
	var err error
	for attempt := attempts.Start(); attempt.Next(); {
		err = b.DelBucket()
		if err == nil {
			return
		}
		if _, ok := err.(*net.DNSError); ok {
			return
		}
		e, ok := err.(*s3.Error)
		if ok && e.Code == "NoSuchBucket" {
			return
		}
		if ok && e.Code == "BucketNotEmpty" {
			// Errors are ignored here. Just retry.
			resp, err := b.List("", "", "", 1000)
			if err == nil {
				for _, key := range resp.Contents {
					_ = b.Del(key.Key)
				}
			}
			multis, _, _ := b.ListMulti("", "")
			for _, m := range multis {
				_ = m.Abort()
			}
		}
	}
	message := "cannot delete test bucket"
	if err != nil {
		message += ": " + err.Error()
	}
	panic(message)
}
Example #2
0
func getFileList(date string, bucket *s3.Bucket) []string {
	resultsSlice := make([]string,0)

	response, err := bucket.List(date, "", "", 500)

	if err != nil {
		log.Fatal(err)
	}

	for _, objects := range response.Contents {
		resultsSlice = append(resultsSlice, objects.Key)
	}
	return resultsSlice
}
Example #3
0
//ApplyToMultiList applies the action a to all the keys that match prefix
//To select ALL contents of the bucket use prefix, delim = ""
func ApplyToMultiList(b *s3.Bucket, prefix, delim string, a Action) {
	//prints stats
	go func() {
		for {
			select {
			case <-time.After(3 * time.Second):
				r := atomic.LoadUint64(&donecounter)
				fmt.Printf("read %v keys\n", r)
				r = atomic.LoadUint64(&counter)
				fmt.Printf("processed %v keys\n", r)
			}
		}
	}()
	resp, err := b.List(prefix, delim, "", 1000)
	if err != nil {
		panic(err)
	}
	if len(resp.Contents) < 1 {
		log.Infof("got no Contents")
		return
	}
	lastSeen := resp.Contents[len(resp.Contents)-1]
	for _, obj := range resp.Contents {
		atomic.AddUint64(&counter, 1)
		go a(KeyBucket{b, obj})
	}
	for {
		if resp.IsTruncated {
			resp, err = b.List(prefix, delim, lastSeen.Key, 1000)
			if err != nil {
				panic(err)
			}
			lastSeen = resp.Contents[len(resp.Contents)-1]
			fmt.Printf("------ \n %v \n-----", lastSeen.Key)
			// TODO allow setting a max number of workers
			for _, obj := range resp.Contents {
				atomic.AddUint64(&counter, 1)
				go func() {
					wg.Add(1)
					a(KeyBucket{b, obj})
					wg.Done()
				}()
			}
		} else {
			break
		}
	}
	wg.Wait()
}
Example #4
0
// List get list from bucket.
func List(bucket *s3.Bucket, prefix, delim, marker string, max int) (*s3.ListResp, error) {
	return bucket.List(prefix, delim, marker, max)
}