Ejemplo n.º 1
0
func (m *Model) FeedManifests(blobs, manifests, strict bool, names ...string) (res lists.BlobMap, err error) {
	var req, res1 []interface{}
	for _, n := range names {
		req = append(req, n)
	}

	err = m.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			res2, err := m.getManifest(in.(string), blobs, manifests)
			if err != nil {
				return nil, err
			}
			if res2 == nil {
				return
			}
			out = struct {
				name     string
				manifest *proto.Manifest
			}{in.(string), res2}
			return
		},
		&req, &res1, concurrency.DefaultBatchOptions().AllowErrors(),
	)
	res = lists.BlobMap{}
	for _, r := range res1 {
		r1 := r.(struct {
			name     string
			manifest *proto.Manifest
		})
		res[r1.name] = *r1.manifest
	}
	return
}
Ejemplo n.º 2
0
func (m *Model) SquashBlobs(blobs lists.BlobMap) (err error) {
	logx.Tracef("squashing blobs %s", blobs.IDMap())

	var req, res []interface{}
	for _, v := range blobs.ToSlice() {
		req = append(req, v)
	}

	err = m.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			r := in.(lists.BlobLink)

			lock, err := m.FdLocks.Take()
			if err != nil {
				return
			}
			defer lock.Release()

			absname := filepath.Join(m.WD, r.Name)
			backName := absname + ".bar-backup"
			os.Rename(absname, absname+".bar-backup")
			os.MkdirAll(filepath.Dir(absname), 0755)

			w, err := os.Create(absname)
			if err != nil {
				return
			}
			err = r.Manifest.Serialize(w)
			if err != nil {
				os.Remove(absname)
				os.Rename(backName, absname)
				return
			}
			defer os.Remove(backName)
			logx.Debugf("squashed %s", r.Name)
			return
		},
		&req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
	)
	if err != nil {
		return
	}

	logx.Infof("blob %s squashed successfully", blobs.Names())
	return
}
Ejemplo n.º 3
0
func (m *Model) IsBlobs(names ...string) (res map[string]bool, err error) {
	res = map[string]bool{}
	var req, res1 []interface{}
	for _, n := range names {
		req = append(req, n)
	}

	err = m.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			lock, err := m.FdLocks.Take()
			if err != nil {
				return
			}
			defer lock.Release()

			f, err := os.Open(filepath.Join(m.WD, in.(string)))
			if err != nil {
				return
			}
			defer f.Close()
			_, isManifest, err := proto.PeekManifest(f)
			if err != nil {
				return
			}
			out = struct {
				name   string
				isBlob bool
			}{in.(string), !isManifest}
			return
		},
		&req, &res1, concurrency.DefaultBatchOptions(),
	)

	for _, r := range res1 {
		r1 := r.(struct {
			name   string
			isBlob bool
		})
		res[r1.name] = r1.isBlob
	}
	return
}
Ejemplo n.º 4
0
// Assemble target files from stored chunks
func (a *Assembler) Done(what lists.BlobMap) (err error) {
	logx.Tracef("assembling %s", what.Names())

	var req, res []interface{}
	for k, v := range what {
		req = append(req, lists.BlobLink{v, k})
	}

	err = a.model.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			r := in.(lists.BlobLink)

			lock, err := a.model.FdLocks.Take()
			if err != nil {
				return
			}
			defer lock.Release()

			w, err := os.Create(filepath.Join(a.model.WD,
				r.Name+r.Manifest.ID.String()))
			if err != nil {
				return
			}
			defer w.Close()

			for _, chunk := range r.Manifest.Chunks {
				if err = a.writeChunkTo(w, chunk.ID); err != nil {
					return
				}
			}
			err = a.commitBlob(r.Name, r.Manifest.ID)

			return
		}, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
	)
	if err != nil {
		return
	}
	defer a.Close()
	return
}
Ejemplo n.º 5
0
func (s *BlockStorage) GetManifests(ids []proto.ID) (res []proto.Manifest, err error) {
	var req, res1 []interface{}
	for _, i := range ids {
		req = append(req, i)
	}

	if err = s.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			r := in.(proto.ID)
			out, err = s.readManifest(s.idPath(manifests_ns, r) + ".json")
			return
		}, &req, &res1, concurrency.DefaultBatchOptions(),
	); err != nil {
		return
	}

	for _, v := range res1 {
		res = append(res, v.(proto.Manifest))
	}

	return
}
Ejemplo n.º 6
0
func (s *BlockStorage) FinishUploadSession(uploadID uuid.UUID) (err error) {
	hexid := proto.ID(hex.EncodeToString(uploadID[:]))
	base := s.idPath(upload_ns, hexid)
	defer os.RemoveAll(base)

	// load manifests
	manifests_base := filepath.Join(base, manifests_ns)
	var manifests []proto.Manifest
	if err = func() (err error) {
		lock, err := s.FDLocks.Take()
		if err != nil {
			return
		}
		defer lock.Release()

		err = filepath.Walk(manifests_base, func(path string, info os.FileInfo, ferr error) (err error) {
			if strings.HasSuffix(path, "-manifest.json") {
				var man proto.Manifest
				if man, err = s.readManifest(path); err != nil {
					return
				}
				manifests = append(manifests, man)
			}
			return
		})
		return
	}(); err != nil {
		return
	}

	// collect all manifests
	var req, res []interface{}
	for _, v := range manifests {
		req = append(req, v)
	}

	err = s.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			lock, err := s.FDLocks.Take()
			if err != nil {
				return
			}
			defer lock.Release()

			m := in.(proto.Manifest)
			target := s.idPath(blob_ns, m.ID)

			f, fErr := s.getCAFile(target)
			if os.IsExist(fErr) {
				return
			} else if fErr != nil {
				err = fErr
				return
			}
			defer f.Close()
			logx.Debugf("assembling %s", m.ID)

			for _, chunk := range m.Chunks {
				if err = func(chunk proto.Chunk) (err error) {
					lock, err := s.FDLocks.Take()
					if err != nil {
						return
					}
					defer lock.Release()

					r, err := os.Open(filepath.Join(base, chunk.ID.String()))
					if err != nil {
						return
					}
					defer r.Close()

					_, err = io.Copy(f, r)
					return
				}(chunk); err != nil {
					return
				}
			}
			err = f.Accept()

			// move manifest
			manTarget := s.idPath(manifests_ns, m.ID) + ".json"
			os.MkdirAll(filepath.Dir(manTarget), 0755)
			err = os.Rename(filepath.Join(manifests_base, m.ID.String()+"-manifest.json"), manTarget)
			return
		}, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
	)
	return
}