func hashRemote(bucket *s3.Bucket, path string) []byte {
file, err := bucket.GetReader(path)
if err != nil {
return nil
}
return hashFile(file)
}
func GetReader(b *s3.Bucket, path string) (reader *TweetReader, err error) {
body, err := b.GetReader(path)
if err != nil {
return nil, err
}
gr, err := gzip.NewReader(body)
if err != nil {
return nil, err
}
reader = NewTweetReaderSerial(bufio.NewReaderSize(gr, 16*4096))
return reader, err
}
func getFiles(bucket *s3.Bucket, c <-chan *Item) {
var wg sync.WaitGroup
wg.Add(options.Concurrency)
for i := 0; i < options.Concurrency; i++ {
go func() {
for item := range c {
func() {
itempath := item.Path[len(item.Prefix):]
dirname, fname := filepath.Split(itempath)
dirname = filepath.Join(options.Remainder[0], dirname)
err := os.MkdirAll(dirname, os.FileMode(0755))
if err != nil {
log.Printf("Could not create target folder %s: %s", dirname, err)
return
}
f, err := os.Create(filepath.Join(dirname, fname))
if err != nil {
log.Printf("Opening %s failed: %s", item.Path, err)
return
}
defer f.Close()
rc, err := bucket.GetReader(item.Path)
if err != nil {
log.Printf("Downloading %s failed: %s", item.Path, err)
return
}
defer rc.Close()
io.Copy(f, rc)
log.Printf("Downloading %s done", item.Path)
}()
}
wg.Done()
}()
}
wg.Wait()
}
