func worker(id int, jobs <-chan *s3.Object, downloader *s3manager.Downloader, awsBucket string, destDir string) {
for object := range jobs {
os.MkdirAll(path.Dir(path.Join(destDir, *object.Key)), 0777)
file, err := os.Create(path.Join(destDir, *object.Key))
if err != nil {
log.Fatal("Failed to create file", err)
}
defer file.Close()
numBytes, err := downloader.Download(file,
&s3.GetObjectInput{
Bucket: aws.String(awsBucket),
Key: object.Key,
})
file.Close()
if err != nil {
fmt.Println("Failed to download file", err)
return
}
fmt.Println("worker", id, "downloaded", file.Name(), numBytes, "bytes", objectsCount)
}
}
//Download a GZipped item from S3 and return the non-gzipped version of the item
func (billingObject *bucketContents) downloadGzippedItem(downloader *s3manager.Downloader, awsBilling *awsBillingConfig) (*[]byte, error) {
//Get a temporary file to dump this into
tempFile, err := ioutil.TempFile("", "scollector-aws-billing-")
if err != nil {
return nil, err
}
defer tempFile.Close()
if _, err = downloader.Download(tempFile,
&s3.GetObjectInput{
Bucket: aws.String(awsBilling.bucketName),
Key: billingObject.origS3Item.Key,
}); err != nil {
return nil, err
}
unzippedFile, err := readGzFile(tempFile)
if err != nil {
return nil, err
}
tempFile.Close()
err = os.Remove(tempFile.Name())
if err != nil {
slog.Warningf("Could not remove temporary file", tempFile.Name())
}
return &unzippedFile, err
}
func GetLogLines(client *s3.S3, downloader *s3manager.Downloader, logFiles []string) ([]string, error) {
buffer := &aws.WriteAtBuffer{}
for i, f := range logFiles {
if i > 0 {
fmt.Printf("\033[1A")
}
fmt.Printf("Downloading log %d of %d...\n", i+1, len(logFiles))
_, err := downloader.Download(buffer, &s3.GetObjectInput{
Bucket: aws.String("lattice-logs"),
Key: aws.String(f),
})
if err != nil {
return []string{}, err
}
}
return strings.Split(string(buffer.Bytes()), "\n"), nil
}