// Store chunk in assemble
func (a *Assembler) StoreChunk(r io.Reader, id proto.ID) (err error) {
lock, err := a.model.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
caOpts := contentaddressable.DefaultOptions()
caOpts.Hasher = sha3.New256()
w, err := contentaddressable.NewFileWithOptions(
lists.OSFromSlash(lists.OSJoin(a.Where, id.String())), caOpts)
if os.IsExist(err) {
err = nil
return
}
if err != nil {
return
}
defer w.Close()
if _, err = io.Copy(w, r); err != nil {
return
}
err = w.Accept()
return
}
// Abort diversion
func (d *Divert) Cleanup(spec DivertSpec) (err error) {
// Remove orphan diverted files
orphans := map[string]struct{}{}
for _, f := range spec.TargetFiles {
orphans[f] = struct{}{}
}
for _, f := range spec.ToRecover {
_, ok := orphans[f]
if ok {
delete(orphans, f)
}
}
for f, _ := range orphans {
os.Remove(lists.OSFromSlash(lists.OSJoin(d.Git.Root, f)))
logx.Debugf("removed orphan %s", f)
}
// Reset to Head
if err = d.Git.Reset(spec.Head); err != nil {
return
}
if err = d.Git.Checkout(spec.Head, spec.ToRecover...); err != nil {
return
}
logx.Info("cleanup finished")
return
}
func (m *Model) getManifest(name string, blobs, manifests bool) (res *proto.Manifest, err error) {
lock, err := m.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
logx.Tracef("feeding manifest from %s",
lists.OSFromSlash(lists.OSJoin(m.WD, name)))
f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, name)))
if err != nil {
logx.Errorf("!!! %s", err)
return
}
defer f.Close()
var r io.Reader
var isManifest bool
if r, isManifest, err = proto.PeekManifest(f); err != nil {
return
}
if (isManifest && !manifests) || (!isManifest && !blobs) {
return
}
if isManifest {
res, err = proto.NewFromManifest(r)
return
}
// Hard way. First - try git
var sideR io.Reader
if m.Git != nil {
if sideR = m.getGitReader(name); sideR != nil {
res, err = proto.NewFromAny(sideR, m.chunkSize)
return
}
}
// No git - make from blob
res, err = proto.NewFromBLOB(r, m.chunkSize)
return
}
func (a *Assembler) commitBlob(name string, id proto.ID) (err error) {
dst := lists.OSFromSlash(lists.OSJoin(a.model.WD, name))
src := dst + "-" + id.String()
bak := dst + "-bak"
os.Rename(dst, bak)
if err = os.Rename(src, dst); err != nil {
os.Remove(dst)
os.Rename(bak, dst)
return
}
os.Remove(src)
os.Remove(bak)
return
}
func (m *Model) ReadChunk(name string, chunk proto.Chunk, res []byte) (err error) {
lock, err := m.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, name)))
if err != nil {
return
}
defer f.Close()
_, err = f.ReadAt(res, chunk.Offset)
return
}
func (m *Model) SquashBlobs(blobs lists.BlobMap) (err error) {
logx.Tracef("squashing blobs %s", blobs.IDMap())
var req, res []interface{}
for _, v := range blobs.ToSlice() {
req = append(req, v)
}
err = m.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
r := in.(lists.BlobLink)
lock, err := m.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
absname := lists.OSFromSlash(lists.OSJoin(m.WD, r.Name))
backName := absname + ".bar-backup"
os.Rename(absname, absname+".bar-backup")
os.MkdirAll(filepath.Dir(absname), 0755)
w, err := os.Create(absname)
if err != nil {
return
}
err = r.Manifest.Serialize(w)
if err != nil {
os.Remove(absname)
os.Rename(backName, absname)
return
}
defer os.Remove(backName)
logx.Debugf("squashed %s", r.Name)
return
},
&req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
)
if err != nil {
return
}
logx.Infof("blob %s squashed successfully", blobs.Names())
return
}
func (a *Assembler) writeChunkTo(w io.Writer, id proto.ID) (err error) {
lock, err := a.model.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
name := lists.OSFromSlash(lists.OSJoin(a.Where, id.String()))
r, err := os.Open(name)
if err != nil {
return
}
defer r.Close()
_, err = io.Copy(w, r)
return
}
func (m *Model) IsBlobs(names ...string) (res map[string]bool, err error) {
res = map[string]bool{}
var req, res1 []interface{}
for _, n := range names {
req = append(req, n)
}
err = m.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
lock, err := m.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, in.(string))))
if err != nil {
return
}
defer f.Close()
_, isManifest, err := proto.PeekManifest(f)
if err != nil {
return
}
out = struct {
name string
isBlob bool
}{in.(string), !isManifest}
return
},
&req, &res1, concurrency.DefaultBatchOptions(),
)
for _, r := range res1 {
r1 := r.(struct {
name string
isBlob bool
})
res[r1.name] = r1.isBlob
}
return
}
// Assemble target files from stored chunks
func (a *Assembler) Done(what lists.BlobMap) (err error) {
logx.Tracef("assembling %s", what.Names())
var req, res []interface{}
for k, v := range what {
req = append(req, lists.BlobLink{v, k})
}
err = a.model.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
r := in.(lists.BlobLink)
lock, err := a.model.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
w, err := os.Create(lists.OSFromSlash(lists.OSJoin(a.model.WD,
r.Name+"-"+r.Manifest.ID.String())))
if err != nil {
return
}
for _, chunk := range r.Manifest.Chunks {
if err = a.writeChunkTo(w, chunk.ID); err != nil {
return
}
}
w.Close()
err = a.commitBlob(r.Name, r.Manifest.ID)
return
}, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
)
if err != nil {
return
}
defer a.Close()
return
}
func (c *SpecExportCmd) Run(args ...string) (err error) {
var mod *model.Model
if mod, err = model.New(c.WD, c.UseGit, c.ChunkSize, c.PoolSize); err != nil {
return
}
feed := lists.NewFileList(args...).ListDir(c.WD)
isDirty, dirty, err := mod.Check(feed...)
if err != nil {
return
}
if isDirty {
err = fmt.Errorf("dirty files in working tree %s", dirty)
return
}
if c.UseGit {
// filter by attrs
feed, err = mod.Git.FilterByAttr("bar", feed...)
}
blobs, err := mod.FeedManifests(true, true, true, feed...)
if err != nil {
return
}
// make specmap
nameMap := map[string]proto.ID{}
for name, m := range blobs {
nameMap[name] = m.ID
}
spec, err := proto.NewSpec(time.Now().UnixNano(), nameMap, []string{})
if err != nil {
return
}
if c.DoCC {
ccName := fmt.Sprintf("bar-spec-%d-%s.json",
time.Now().UnixNano(), spec.ID)
logx.Infof("storing carbon copy to %s", ccName)
ccf, err := os.Create(lists.OSFromSlash(lists.OSJoin(c.WD, ccName)))
if err != nil {
return err
}
defer ccf.Close()
if err = json.NewEncoder(ccf).Encode(&spec); err != nil {
return err
}
}
if !c.Upload {
err = json.NewEncoder(c.Stdout).Encode(&spec)
return
}
trans := transport.NewTransport(mod, "", c.Endpoint, c.PoolSize)
if err = trans.UploadSpec(spec); err != nil {
return
}
fmt.Fprint(c.Stdout, spec.ID)
return
}
