Exemple #1
0
func (r *s3Replica) Pull(from string, target Pusher) error {
	bucket, err := s3utils.NewBucket(r.uri)
	if err != nil {
		return err
	}
	err = s3utils.ForEachFile(r.uri, from, func(path string, modtime time.Time) (retErr error) {
		f, err := bucket.GetReader(path)
		if f == nil {
			return fmt.Errorf("Nil file returned.")
		}
		if err != nil {
			return err
		}
		defer func() {
			if err := f.Close(); err != nil && retErr == nil {
				retErr = err
			}
		}()

		err = target.Push(f)
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	return nil
}
Exemple #2
0
func (r *s3Replica) Push(diff io.Reader) error {
	bucket, err := s3utils.NewBucket(r.uri)
	if err != nil {
		return err
	}
	key := fmt.Sprintf("%.10d", r.count)
	r.count++

	p, err := s3utils.GetPath(r.uri)
	if err != nil {
		return err
	}

	return s3utils.PutMulti(bucket, path.Join(p, key), diff, "application/octet-stream", s3.BucketOwnerFull)
}
Exemple #3
0
// inject injects data from an external source into the output directory
func (p *pipeline) inject(name string) error {
	switch {
	case strings.HasPrefix(name, "s3://"):
		bucket, err := s3utils.NewBucket(name)
		if err != nil {
			return err
		}
		var wg sync.WaitGroup
		s3utils.ForEachFile(name, "", func(file string, modtime time.Time) error {
			// Grab the path, it's handy later
			_path, err := s3utils.GetPath(name)
			if err != nil {
				return err
			}
			if err != nil {
				return err
			}
			// Check if the file belongs on shit shard
			match, err := route.Match(file, p.shard)
			if err != nil {
				return err
			}
			if !match {
				return nil
			}
			// Check if the file has changed
			changed, err := btrfs.Changed(path.Join(p.outRepo, p.branch,
				strings.TrimPrefix(file, _path)), modtime)
			log.Print("Changed: ", changed)
			if err != nil {
				return err
			}
			if !changed {
				return nil
			}
			// TODO match the on disk timestamps to s3's timestamps and make
			// sure we only pull data that has changed
			wg.Add(1)
			go func() {
				defer wg.Done()
				src, err := bucket.GetReader(file)
				if err != nil {
					return
				}
				dst, err := btrfs.CreateAll(path.Join(p.outRepo, p.branch, strings.TrimPrefix(file, _path)))
				if err != nil {
					return
				}
				defer dst.Close()
				_, err = io.Copy(dst, src)
				if err != nil {
					return
				}
				err = btrfs.Chtimes(path.Join(p.outRepo, p.branch, strings.TrimPrefix(file, _path)), modtime, modtime)
				if err != nil {
					return
				}
			}()
			return nil
		})
		wg.Wait()
	default:
		log.Print("Unknown protocol: ", name)
		return ErrUnknownProtocol
	}
	return nil
}