Exemple #1
0
func worker(id int, jobs <-chan *s3.Object, downloader *s3manager.Downloader, awsBucket string, destDir string) {
	for object := range jobs {
		os.MkdirAll(path.Dir(path.Join(destDir, *object.Key)), 0777)
		file, err := os.Create(path.Join(destDir, *object.Key))
		if err != nil {
			log.Fatal("Failed to create file", err)
		}
		defer file.Close()

		numBytes, err := downloader.Download(file,
			&s3.GetObjectInput{
				Bucket: aws.String(awsBucket),
				Key:    object.Key,
			})

		file.Close()

		if err != nil {
			fmt.Println("Failed to download file", err)
			return
		}

		fmt.Println("worker", id, "downloaded", file.Name(), numBytes, "bytes", objectsCount)
	}
}
Exemple #2
0
//Download a GZipped item from S3 and return the non-gzipped version of the item
func (billingObject *bucketContents) downloadGzippedItem(downloader *s3manager.Downloader, awsBilling *awsBillingConfig) (*[]byte, error) {
	//Get a temporary file to dump this into
	tempFile, err := ioutil.TempFile("", "scollector-aws-billing-")
	if err != nil {
		return nil, err
	}
	defer tempFile.Close()
	if _, err = downloader.Download(tempFile,
		&s3.GetObjectInput{
			Bucket: aws.String(awsBilling.bucketName),
			Key:    billingObject.origS3Item.Key,
		}); err != nil {
		return nil, err
	}
	unzippedFile, err := readGzFile(tempFile)
	if err != nil {
		return nil, err
	}
	tempFile.Close()
	err = os.Remove(tempFile.Name())
	if err != nil {
		slog.Warningf("Could not remove temporary file", tempFile.Name())
	}
	return &unzippedFile, err
}
Exemple #3
0
func GetLogLines(client *s3.S3, downloader *s3manager.Downloader, logFiles []string) ([]string, error) {
	buffer := &aws.WriteAtBuffer{}
	for i, f := range logFiles {
		if i > 0 {
			fmt.Printf("\033[1A")
		}
		fmt.Printf("Downloading log %d of %d...\n", i+1, len(logFiles))
		_, err := downloader.Download(buffer, &s3.GetObjectInput{
			Bucket: aws.String("lattice-logs"),
			Key:    aws.String(f),
		})
		if err != nil {
			return []string{}, err
		}
	}

	return strings.Split(string(buffer.Bytes()), "\n"), nil
}