func (r *s3Replica) Pull(from string, target Pusher) error { client := s3utils.NewClient(false) err := s3utils.ForEachFile(r.uri, false, from, func(path string, modtime time.Time) (retErr error) { response, err := client.GetObject(&s3.GetObjectInput{ Bucket: &r.uri, Key: &path, }) f := response.Body if f == nil { return fmt.Errorf("Nil file returned.") } if err != nil { return err } defer func() { if err := f.Close(); err != nil && retErr == nil { retErr = err } }() err = target.Push(f) if err != nil { return err } return nil }) if err != nil { return err } return nil }
// pushToOutputs should be called after `finish` to push the outputted data to s3 func (p *pipeline) pushToOutputs() error { if p.externalOutput == "" { return nil } client := s3utils.NewClient(false) bucket, err := s3utils.GetBucket(p.externalOutput) if err != nil { return err } pathPrefix, err := s3utils.GetPath(p.externalOutput) if err != nil { return err } files, err := btrfs.NewIn(p.outRepo, p.commit) if err != nil { return err } var wg sync.WaitGroup for _, file := range files { wg.Add(1) go func(file string) { defer wg.Done() key := path.Join(pathPrefix, file) f, err := btrfs.Open(path.Join(p.outRepo, p.commit, file)) if err != nil { log.Print(err) return } acl := "private" defer f.Close() if _, err = client.PutObject(&s3.PutObjectInput{ Bucket: &bucket, Key: &key, Body: f, ACL: &acl, }); err != nil { log.Print(err) return } }(file) } wg.Wait() return nil }
// inject injects data from an external source into the output directory func (p *pipeline) inject(name string, public bool) error { switch { case strings.HasPrefix(name, "s3://"): bucket, err := s3utils.GetBucket(name) if err != nil { return err } client := s3utils.NewClient(public) var wg sync.WaitGroup s3utils.ForEachFile(name, public, "", func(file string, modtime time.Time) error { // Grab the path, it's handy later _path, err := s3utils.GetPath(name) if err != nil { return err } if err != nil { return err } // Check if the file belongs on shit shard match, err := route.Match(file, p.shard) if err != nil { return err } if !match { return nil } // Check if the file has changed changed, err := btrfs.Changed(path.Join(p.outRepo, p.branch, strings.TrimPrefix(file, _path)), modtime) if err != nil { return err } if !changed { return nil } // TODO match the on disk timestamps to s3's timestamps and make // sure we only pull data that has changed wg.Add(1) go func() { defer wg.Done() response, err := client.GetObject(&s3.GetObjectInput{ Bucket: &bucket, Key: &file, }) if err != nil { return } src := response.Body dst, err := btrfs.CreateAll(path.Join(p.outRepo, p.branch, strings.TrimPrefix(file, _path))) if err != nil { return } defer dst.Close() _, err = io.Copy(dst, src) if err != nil { return } err = btrfs.Chtimes(path.Join(p.outRepo, p.branch, strings.TrimPrefix(file, _path)), modtime, modtime) if err != nil { return } }() return nil }) wg.Wait() default: log.Print("Unknown protocol: ", name) return ErrUnknownProtocol } return nil }