// Shuffle rehashes an output directory. // If 2 shards each have a copy of the file `foo` with the content: `bar`. // Then after shuffling 1 of those nodes will have a file `foo` with content // `barbar` and the other will have no file foo. func (p *pipeline) shuffle(dir string) error { // this function always increments counter defer func() { p.counter++ }() // First we clear the directory, notice that the previous commit from // which we're pulling has already been made so this doesn't destroy the // data that others are trying to pull. // TODO(jd) #performance this is a seriously unperformant part of the code // since it messes up our ability to do incremental results. We should do // something smarter here. if err := btrfs.RemoveAll(path.Join(p.outRepo, p.branch, dir)); err != nil { return err } if err := btrfs.MkdirAll(path.Join(p.outRepo, p.branch, dir)); err != nil { return err } // We want to pull files from the previous commit commit := fmt.Sprintf("%s-%d", p.commit, p.counter-1) // Notice we're just passing "host" here. Multicast will fill in the host // field so we don't actually need to specify it. req, err := http.NewRequest("GET", "http://host/"+path.Join("pipeline", p.name, "file", dir, "*")+"?commit="+commit+"&shard="+p.shard, nil) if err != nil { return err } // Dispatch the request resps, err := route.Multicast(p.cache, req, "/pfs/master") if err != nil { return err } // Set up some concurrency structures. errors := make(chan error, len(resps)) var wg sync.WaitGroup wg.Add(len(resps)) lock := util.NewPathLock() // for _, resp := range resps { // We used to iterate like the above but it exhibited racy behavior. I // don't fully understand why this was. Something to look in to. for _, resp := range resps { go func(resp *http.Response) { defer wg.Done() reader := multipart.NewReader(resp.Body, resp.Header.Get("Boundary")) for part, err := reader.NextPart(); err != io.EOF; part, err = reader.NextPart() { lock.Lock(part.FileName()) _, err := btrfs.Append(path.Join(p.outRepo, p.branch, part.FileName()), part) lock.Unlock(part.FileName()) if err != nil { errors <- err return } } }(resp) } wg.Wait() close(errors) // Check for errors for err := range errors { if err != nil { return err } } return btrfs.Commit(p.outRepo, p.runCommit(), p.branch) }
// Run runs all of the pipelines it finds in pipelineDir. Returns the // first error it encounters. func (r *Runner) Run() error { if err := btrfs.MkdirAll(r.outPrefix); err != nil { return err } if err := r.startInputPipelines(); err != nil { return err } pipelines, err := btrfs.ReadDir(path.Join(r.inRepo, r.commit, r.pipelineDir)) if err != nil { // Notice we don't return this error but instead no-op. It's fine to not // have a pipeline dir. return nil } // A chanel for the errors, notice that it's capacity is the same as the // number of pipelines. The below code should make sure that each pipeline only // sends 1 error otherwise deadlock may occur. errors := make(chan error, len(pipelines)) // Make sure we don't race with cancel this is held while we add pipelines. r.lock.Lock() if r.cancelled { // we were cancelled before we even started r.lock.Unlock() return ErrCancelled } for _, pInfo := range pipelines { if err := r.makeOutRepo(pInfo.Name()); err != nil { return err } p := newPipeline(pInfo.Name(), r.inRepo, path.Join(r.outPrefix, pInfo.Name()), r.commit, r.branch, r.shard, r.outPrefix, r.cache) r.pipelines = append(r.pipelines, p) } // unlocker lets us defer unlocking and explicitly unlock var unlocker sync.Once defer unlocker.Do(r.lock.Unlock) for _, p := range r.pipelines { p := p r.wait.Add(1) go func() { defer r.wait.Done() f, err := btrfs.Open(path.Join(r.inRepo, r.commit, r.pipelineDir, p.name)) if err != nil { errors <- err return } defer f.Close() err = p.runPachFile(f) if err != nil { errors <- err return } }() } // We're done adding pipelines so unlock unlocker.Do(r.lock.Unlock) // Wait for the pipelines to finish r.wait.Wait() close(errors) if r.cancelled { // Pipelines finished because we were cancelled return ErrCancelled } for err := range errors { return err } return nil }
func (s *shard) FileCreate(name string, content io.Reader, branch string) error { filePath := path.Join(s.dataRepo, branch, name) btrfs.MkdirAll(path.Dir(filePath)) _, err := btrfs.CreateFromReader(filePath, content) return err }