func (m *Model) FeedManifests(blobs, manifests, strict bool, names ...string) (res lists.BlobMap, err error) {
var req, res1 []interface{}
for _, n := range names {
req = append(req, n)
}
err = m.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
res2, err := m.getManifest(in.(string), blobs, manifests)
if err != nil {
return nil, err
}
if res2 == nil {
return
}
out = struct {
name string
manifest *proto.Manifest
}{in.(string), res2}
return
},
&req, &res1, concurrency.DefaultBatchOptions().AllowErrors(),
)
res = lists.BlobMap{}
for _, r := range res1 {
r1 := r.(struct {
name string
manifest *proto.Manifest
})
res[r1.name] = *r1.manifest
}
return
}
func (m *Model) SquashBlobs(blobs lists.BlobMap) (err error) {
logx.Tracef("squashing blobs %s", blobs.IDMap())
var req, res []interface{}
for _, v := range blobs.ToSlice() {
req = append(req, v)
}
err = m.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
r := in.(lists.BlobLink)
lock, err := m.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
absname := filepath.Join(m.WD, r.Name)
backName := absname + ".bar-backup"
os.Rename(absname, absname+".bar-backup")
os.MkdirAll(filepath.Dir(absname), 0755)
w, err := os.Create(absname)
if err != nil {
return
}
err = r.Manifest.Serialize(w)
if err != nil {
os.Remove(absname)
os.Rename(backName, absname)
return
}
defer os.Remove(backName)
logx.Debugf("squashed %s", r.Name)
return
},
&req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
)
if err != nil {
return
}
logx.Infof("blob %s squashed successfully", blobs.Names())
return
}
func (m *Model) IsBlobs(names ...string) (res map[string]bool, err error) {
res = map[string]bool{}
var req, res1 []interface{}
for _, n := range names {
req = append(req, n)
}
err = m.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
lock, err := m.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
f, err := os.Open(filepath.Join(m.WD, in.(string)))
if err != nil {
return
}
defer f.Close()
_, isManifest, err := proto.PeekManifest(f)
if err != nil {
return
}
out = struct {
name string
isBlob bool
}{in.(string), !isManifest}
return
},
&req, &res1, concurrency.DefaultBatchOptions(),
)
for _, r := range res1 {
r1 := r.(struct {
name string
isBlob bool
})
res[r1.name] = r1.isBlob
}
return
}
// Assemble target files from stored chunks
func (a *Assembler) Done(what lists.BlobMap) (err error) {
logx.Tracef("assembling %s", what.Names())
var req, res []interface{}
for k, v := range what {
req = append(req, lists.BlobLink{v, k})
}
err = a.model.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
r := in.(lists.BlobLink)
lock, err := a.model.FdLocks.Take()
if err != nil {
return
}
defer lock.Release()
w, err := os.Create(filepath.Join(a.model.WD,
r.Name+r.Manifest.ID.String()))
if err != nil {
return
}
defer w.Close()
for _, chunk := range r.Manifest.Chunks {
if err = a.writeChunkTo(w, chunk.ID); err != nil {
return
}
}
err = a.commitBlob(r.Name, r.Manifest.ID)
return
}, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
)
if err != nil {
return
}
defer a.Close()
return
}
func (s *BlockStorage) GetManifests(ids []proto.ID) (res []proto.Manifest, err error) {
var req, res1 []interface{}
for _, i := range ids {
req = append(req, i)
}
if err = s.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
r := in.(proto.ID)
out, err = s.readManifest(s.idPath(manifests_ns, r) + ".json")
return
}, &req, &res1, concurrency.DefaultBatchOptions(),
); err != nil {
return
}
for _, v := range res1 {
res = append(res, v.(proto.Manifest))
}
return
}
func (s *BlockStorage) FinishUploadSession(uploadID uuid.UUID) (err error) {
hexid := proto.ID(hex.EncodeToString(uploadID[:]))
base := s.idPath(upload_ns, hexid)
defer os.RemoveAll(base)
// load manifests
manifests_base := filepath.Join(base, manifests_ns)
var manifests []proto.Manifest
if err = func() (err error) {
lock, err := s.FDLocks.Take()
if err != nil {
return
}
defer lock.Release()
err = filepath.Walk(manifests_base, func(path string, info os.FileInfo, ferr error) (err error) {
if strings.HasSuffix(path, "-manifest.json") {
var man proto.Manifest
if man, err = s.readManifest(path); err != nil {
return
}
manifests = append(manifests, man)
}
return
})
return
}(); err != nil {
return
}
// collect all manifests
var req, res []interface{}
for _, v := range manifests {
req = append(req, v)
}
err = s.BatchPool.Do(
func(ctx context.Context, in interface{}) (out interface{}, err error) {
lock, err := s.FDLocks.Take()
if err != nil {
return
}
defer lock.Release()
m := in.(proto.Manifest)
target := s.idPath(blob_ns, m.ID)
f, fErr := s.getCAFile(target)
if os.IsExist(fErr) {
return
} else if fErr != nil {
err = fErr
return
}
defer f.Close()
logx.Debugf("assembling %s", m.ID)
for _, chunk := range m.Chunks {
if err = func(chunk proto.Chunk) (err error) {
lock, err := s.FDLocks.Take()
if err != nil {
return
}
defer lock.Release()
r, err := os.Open(filepath.Join(base, chunk.ID.String()))
if err != nil {
return
}
defer r.Close()
_, err = io.Copy(f, r)
return
}(chunk); err != nil {
return
}
}
err = f.Accept()
// move manifest
manTarget := s.idPath(manifests_ns, m.ID) + ".json"
os.MkdirAll(filepath.Dir(manTarget), 0755)
err = os.Rename(filepath.Join(manifests_base, m.ID.String()+"-manifest.json"), manTarget)
return
}, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
)
return
}