// Store chunk in assemble func (a *Assembler) StoreChunk(r io.Reader, id proto.ID) (err error) { lock, err := a.model.FdLocks.Take() if err != nil { return } defer lock.Release() caOpts := contentaddressable.DefaultOptions() caOpts.Hasher = sha3.New256() w, err := contentaddressable.NewFileWithOptions( lists.OSFromSlash(lists.OSJoin(a.Where, id.String())), caOpts) if os.IsExist(err) { err = nil return } if err != nil { return } defer w.Close() if _, err = io.Copy(w, r); err != nil { return } err = w.Accept() return }
// Abort diversion func (d *Divert) Cleanup(spec DivertSpec) (err error) { // Remove orphan diverted files orphans := map[string]struct{}{} for _, f := range spec.TargetFiles { orphans[f] = struct{}{} } for _, f := range spec.ToRecover { _, ok := orphans[f] if ok { delete(orphans, f) } } for f, _ := range orphans { os.Remove(lists.OSFromSlash(lists.OSJoin(d.Git.Root, f))) logx.Debugf("removed orphan %s", f) } // Reset to Head if err = d.Git.Reset(spec.Head); err != nil { return } if err = d.Git.Checkout(spec.Head, spec.ToRecover...); err != nil { return } logx.Info("cleanup finished") return }
func (m *Model) getManifest(name string, blobs, manifests bool) (res *proto.Manifest, err error) { lock, err := m.FdLocks.Take() if err != nil { return } defer lock.Release() logx.Tracef("feeding manifest from %s", lists.OSFromSlash(lists.OSJoin(m.WD, name))) f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, name))) if err != nil { logx.Errorf("!!! %s", err) return } defer f.Close() var r io.Reader var isManifest bool if r, isManifest, err = proto.PeekManifest(f); err != nil { return } if (isManifest && !manifests) || (!isManifest && !blobs) { return } if isManifest { res, err = proto.NewFromManifest(r) return } // Hard way. First - try git var sideR io.Reader if m.Git != nil { if sideR = m.getGitReader(name); sideR != nil { res, err = proto.NewFromAny(sideR, m.chunkSize) return } } // No git - make from blob res, err = proto.NewFromBLOB(r, m.chunkSize) return }
func (a *Assembler) commitBlob(name string, id proto.ID) (err error) { dst := lists.OSFromSlash(lists.OSJoin(a.model.WD, name)) src := dst + "-" + id.String() bak := dst + "-bak" os.Rename(dst, bak) if err = os.Rename(src, dst); err != nil { os.Remove(dst) os.Rename(bak, dst) return } os.Remove(src) os.Remove(bak) return }
func (m *Model) ReadChunk(name string, chunk proto.Chunk, res []byte) (err error) { lock, err := m.FdLocks.Take() if err != nil { return } defer lock.Release() f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, name))) if err != nil { return } defer f.Close() _, err = f.ReadAt(res, chunk.Offset) return }
func (m *Model) SquashBlobs(blobs lists.BlobMap) (err error) { logx.Tracef("squashing blobs %s", blobs.IDMap()) var req, res []interface{} for _, v := range blobs.ToSlice() { req = append(req, v) } err = m.BatchPool.Do( func(ctx context.Context, in interface{}) (out interface{}, err error) { r := in.(lists.BlobLink) lock, err := m.FdLocks.Take() if err != nil { return } defer lock.Release() absname := lists.OSFromSlash(lists.OSJoin(m.WD, r.Name)) backName := absname + ".bar-backup" os.Rename(absname, absname+".bar-backup") os.MkdirAll(filepath.Dir(absname), 0755) w, err := os.Create(absname) if err != nil { return } err = r.Manifest.Serialize(w) if err != nil { os.Remove(absname) os.Rename(backName, absname) return } defer os.Remove(backName) logx.Debugf("squashed %s", r.Name) return }, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(), ) if err != nil { return } logx.Infof("blob %s squashed successfully", blobs.Names()) return }
func (a *Assembler) writeChunkTo(w io.Writer, id proto.ID) (err error) { lock, err := a.model.FdLocks.Take() if err != nil { return } defer lock.Release() name := lists.OSFromSlash(lists.OSJoin(a.Where, id.String())) r, err := os.Open(name) if err != nil { return } defer r.Close() _, err = io.Copy(w, r) return }
func (m *Model) IsBlobs(names ...string) (res map[string]bool, err error) { res = map[string]bool{} var req, res1 []interface{} for _, n := range names { req = append(req, n) } err = m.BatchPool.Do( func(ctx context.Context, in interface{}) (out interface{}, err error) { lock, err := m.FdLocks.Take() if err != nil { return } defer lock.Release() f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, in.(string)))) if err != nil { return } defer f.Close() _, isManifest, err := proto.PeekManifest(f) if err != nil { return } out = struct { name string isBlob bool }{in.(string), !isManifest} return }, &req, &res1, concurrency.DefaultBatchOptions(), ) for _, r := range res1 { r1 := r.(struct { name string isBlob bool }) res[r1.name] = r1.isBlob } return }
// Assemble target files from stored chunks func (a *Assembler) Done(what lists.BlobMap) (err error) { logx.Tracef("assembling %s", what.Names()) var req, res []interface{} for k, v := range what { req = append(req, lists.BlobLink{v, k}) } err = a.model.BatchPool.Do( func(ctx context.Context, in interface{}) (out interface{}, err error) { r := in.(lists.BlobLink) lock, err := a.model.FdLocks.Take() if err != nil { return } defer lock.Release() w, err := os.Create(lists.OSFromSlash(lists.OSJoin(a.model.WD, r.Name+"-"+r.Manifest.ID.String()))) if err != nil { return } for _, chunk := range r.Manifest.Chunks { if err = a.writeChunkTo(w, chunk.ID); err != nil { return } } w.Close() err = a.commitBlob(r.Name, r.Manifest.ID) return }, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(), ) if err != nil { return } defer a.Close() return }
func (c *SpecExportCmd) Run(args ...string) (err error) { var mod *model.Model if mod, err = model.New(c.WD, c.UseGit, c.ChunkSize, c.PoolSize); err != nil { return } feed := lists.NewFileList(args...).ListDir(c.WD) isDirty, dirty, err := mod.Check(feed...) if err != nil { return } if isDirty { err = fmt.Errorf("dirty files in working tree %s", dirty) return } if c.UseGit { // filter by attrs feed, err = mod.Git.FilterByAttr("bar", feed...) } blobs, err := mod.FeedManifests(true, true, true, feed...) if err != nil { return } // make specmap nameMap := map[string]proto.ID{} for name, m := range blobs { nameMap[name] = m.ID } spec, err := proto.NewSpec(time.Now().UnixNano(), nameMap, []string{}) if err != nil { return } if c.DoCC { ccName := fmt.Sprintf("bar-spec-%d-%s.json", time.Now().UnixNano(), spec.ID) logx.Infof("storing carbon copy to %s", ccName) ccf, err := os.Create(lists.OSFromSlash(lists.OSJoin(c.WD, ccName))) if err != nil { return err } defer ccf.Close() if err = json.NewEncoder(ccf).Encode(&spec); err != nil { return err } } if !c.Upload { err = json.NewEncoder(c.Stdout).Encode(&spec) return } trans := transport.NewTransport(mod, "", c.Endpoint, c.PoolSize) if err = trans.UploadSpec(spec); err != nil { return } fmt.Fprint(c.Stdout, spec.ID) return }