Пример #1
0
func GetImagesInGallery(bucket *s3.Bucket, g *Gallery) ([]Image, error) {
	pre := fmt.Sprintf("%s%s", g.S3Prefix, "orig-")
	res, err := bucket.List(pre, "/", "", 1000)
	if err != nil {
		return nil, tracederror.New(err)
	}

	paths := []Image{}
	for _, v := range res.Contents {
		if v.Key[len(v.Key)-1] != '/' {
			raw := strings.TrimPrefix(v.Key, pre)

			t, err := time.Parse("2006-01-02T15:04:05.000Z", v.LastModified)
			if err != nil {
				log.Print("Couldn't parse time from amazon, assuming 'now' for upload date.")
				log.Print(err)

				t = time.Now()
			}

			paths = append(paths, Image{
				Thumb: fmt.Sprintf("%sthumb-%s", g.S3Prefix, raw),
				Orig:  fmt.Sprintf("%sorig-%s", g.S3Prefix, raw),
				Hero:  fmt.Sprintf("%shero-%s", g.S3Prefix, raw),

				ETag:         v.ETag,
				LastModified: t,
			})
		}
	}

	sort.Sort(ByDate(paths))
	return paths, nil
}
Пример #2
0
func killBucket(b *s3.Bucket) {
	var err error
	for attempt := attempts.Start(); attempt.Next(); {
		err = b.DelBucket()
		if err == nil {
			return
		}
		if _, ok := err.(*net.DNSError); ok {
			return
		}
		e, ok := err.(*s3.Error)
		if ok && e.Code == "NoSuchBucket" {
			return
		}
		if ok && e.Code == "BucketNotEmpty" {
			// Errors are ignored here. Just retry.
			resp, err := b.List("", "", "", 1000)
			if err == nil {
				for _, key := range resp.Contents {
					_ = b.Del(key.Key)
				}
			}
			multis, _, _ := b.ListMulti("", "")
			for _, m := range multis {
				_ = m.Abort()
			}
		}
	}
	message := "cannot delete test bucket"
	if err != nil {
		message += ": " + err.Error()
	}
	panic(message)
}
Пример #3
0
// listFiles lists the files in a specific s3 bucket
func listFiles(prefix, delimiter, marker string, maxKeys int, b *s3.Bucket) (files []s3.Key, err error) {
	resp, err := b.List(prefix, delimiter, marker, maxKeys)
	if err != nil {
		return nil, err
	}

	// append to files
	files = append(files, resp.Contents...)

	// recursion for the recursion god
	if resp.IsTruncated && resp.NextMarker != "" {
		f, err := listFiles(resp.Prefix, resp.Delimiter, resp.NextMarker, resp.MaxKeys, b)
		if err != nil {
			return nil, err
		}

		// append to files
		files = append(files, f...)
	}

	return files, nil
}
Пример #4
0
func loadS3Files(bucket *s3.Bucket, path string, files map[string]string, marker string) (map[string]string, error) {
	log.Debugf("Loading files from 's3://%s/%s'.", bucket.Name, path)
	data, err := bucket.List(path, "", marker, 0)
	if err != nil {
		return files, err
	}

	for _, key := range data.Contents {
		md5sum := strings.Trim(key.ETag, "\"")
		files[key.Key] = md5sum
	}

	// Continue to call loadS3files and add
	// Files to map if next marker set
	if data.IsTruncated {
		lastKey := data.Contents[(len(data.Contents) - 1)].Key
		log.Infof("Results truncated, loading additional files via previous last key '%s'.", lastKey)
		loadS3Files(bucket, path, files, lastKey)
	}

	log.Debugf("Loaded '%d' files from 's3://%s/%s' succesfully.", len(files), bucket.Name, path)
	return files, nil
}