func hashRemote(bucket *s3.Bucket, path string) []byte { file, err := bucket.GetReader(path) if err != nil { return nil } return hashFile(file) }
func GetReader(b *s3.Bucket, path string) (reader *TweetReader, err error) { body, err := b.GetReader(path) if err != nil { return nil, err } gr, err := gzip.NewReader(body) if err != nil { return nil, err } reader = NewTweetReaderSerial(bufio.NewReaderSize(gr, 16*4096)) return reader, err }
func getFiles(bucket *s3.Bucket, c <-chan *Item) { var wg sync.WaitGroup wg.Add(options.Concurrency) for i := 0; i < options.Concurrency; i++ { go func() { for item := range c { func() { itempath := item.Path[len(item.Prefix):] dirname, fname := filepath.Split(itempath) dirname = filepath.Join(options.Remainder[0], dirname) err := os.MkdirAll(dirname, os.FileMode(0755)) if err != nil { log.Printf("Could not create target folder %s: %s", dirname, err) return } f, err := os.Create(filepath.Join(dirname, fname)) if err != nil { log.Printf("Opening %s failed: %s", item.Path, err) return } defer f.Close() rc, err := bucket.GetReader(item.Path) if err != nil { log.Printf("Downloading %s failed: %s", item.Path, err) return } defer rc.Close() io.Copy(f, rc) log.Printf("Downloading %s done", item.Path) }() } wg.Done() }() } wg.Wait() }