Beispiel #1
0
// Store chunk in assemble
func (a *Assembler) StoreChunk(r io.Reader, id proto.ID) (err error) {
	lock, err := a.model.FdLocks.Take()
	if err != nil {
		return
	}
	defer lock.Release()

	caOpts := contentaddressable.DefaultOptions()
	caOpts.Hasher = sha3.New256()

	w, err := contentaddressable.NewFileWithOptions(
		lists.OSFromSlash(lists.OSJoin(a.Where, id.String())), caOpts)
	if os.IsExist(err) {
		err = nil
		return
	}
	if err != nil {
		return
	}
	defer w.Close()

	if _, err = io.Copy(w, r); err != nil {
		return
	}
	err = w.Accept()
	return
}
Beispiel #2
0
// Abort diversion
func (d *Divert) Cleanup(spec DivertSpec) (err error) {
	// Remove orphan diverted files
	orphans := map[string]struct{}{}
	for _, f := range spec.TargetFiles {
		orphans[f] = struct{}{}
	}
	for _, f := range spec.ToRecover {
		_, ok := orphans[f]
		if ok {
			delete(orphans, f)
		}
	}
	for f, _ := range orphans {
		os.Remove(lists.OSFromSlash(lists.OSJoin(d.Git.Root, f)))
		logx.Debugf("removed orphan %s", f)
	}

	// Reset to Head
	if err = d.Git.Reset(spec.Head); err != nil {
		return
	}

	if err = d.Git.Checkout(spec.Head, spec.ToRecover...); err != nil {
		return
	}

	logx.Info("cleanup finished")
	return
}
Beispiel #3
0
func (m *Model) getManifest(name string, blobs, manifests bool) (res *proto.Manifest, err error) {
	lock, err := m.FdLocks.Take()
	if err != nil {
		return
	}
	defer lock.Release()

	logx.Tracef("feeding manifest from %s",
		lists.OSFromSlash(lists.OSJoin(m.WD, name)))
	f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, name)))
	if err != nil {
		logx.Errorf("!!! %s", err)
		return
	}
	defer f.Close()

	var r io.Reader
	var isManifest bool
	if r, isManifest, err = proto.PeekManifest(f); err != nil {
		return
	}
	if (isManifest && !manifests) || (!isManifest && !blobs) {
		return
	}

	if isManifest {
		res, err = proto.NewFromManifest(r)
		return
	}
	// Hard way. First - try git
	var sideR io.Reader
	if m.Git != nil {
		if sideR = m.getGitReader(name); sideR != nil {
			res, err = proto.NewFromAny(sideR, m.chunkSize)
			return
		}
	}
	// No git - make from blob
	res, err = proto.NewFromBLOB(r, m.chunkSize)
	return
}
Beispiel #4
0
func (a *Assembler) commitBlob(name string, id proto.ID) (err error) {
	dst := lists.OSFromSlash(lists.OSJoin(a.model.WD, name))
	src := dst + "-" + id.String()
	bak := dst + "-bak"

	os.Rename(dst, bak)
	if err = os.Rename(src, dst); err != nil {
		os.Remove(dst)
		os.Rename(bak, dst)
		return
	}
	os.Remove(src)
	os.Remove(bak)
	return
}
Beispiel #5
0
func (m *Model) ReadChunk(name string, chunk proto.Chunk, res []byte) (err error) {
	lock, err := m.FdLocks.Take()
	if err != nil {
		return
	}
	defer lock.Release()

	f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, name)))
	if err != nil {
		return
	}
	defer f.Close()

	_, err = f.ReadAt(res, chunk.Offset)
	return
}
Beispiel #6
0
func (m *Model) SquashBlobs(blobs lists.BlobMap) (err error) {
	logx.Tracef("squashing blobs %s", blobs.IDMap())

	var req, res []interface{}
	for _, v := range blobs.ToSlice() {
		req = append(req, v)
	}

	err = m.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			r := in.(lists.BlobLink)

			lock, err := m.FdLocks.Take()
			if err != nil {
				return
			}
			defer lock.Release()

			absname := lists.OSFromSlash(lists.OSJoin(m.WD, r.Name))
			backName := absname + ".bar-backup"
			os.Rename(absname, absname+".bar-backup")
			os.MkdirAll(filepath.Dir(absname), 0755)

			w, err := os.Create(absname)
			if err != nil {
				return
			}
			err = r.Manifest.Serialize(w)
			if err != nil {
				os.Remove(absname)
				os.Rename(backName, absname)
				return
			}
			defer os.Remove(backName)
			logx.Debugf("squashed %s", r.Name)
			return
		},
		&req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
	)
	if err != nil {
		return
	}

	logx.Infof("blob %s squashed successfully", blobs.Names())
	return
}
Beispiel #7
0
func (a *Assembler) writeChunkTo(w io.Writer, id proto.ID) (err error) {
	lock, err := a.model.FdLocks.Take()
	if err != nil {
		return
	}
	defer lock.Release()

	name := lists.OSFromSlash(lists.OSJoin(a.Where, id.String()))
	r, err := os.Open(name)
	if err != nil {
		return
	}
	defer r.Close()

	_, err = io.Copy(w, r)
	return
}
Beispiel #8
0
func (m *Model) IsBlobs(names ...string) (res map[string]bool, err error) {
	res = map[string]bool{}
	var req, res1 []interface{}
	for _, n := range names {
		req = append(req, n)
	}

	err = m.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			lock, err := m.FdLocks.Take()
			if err != nil {
				return
			}
			defer lock.Release()

			f, err := os.Open(lists.OSFromSlash(lists.OSJoin(m.WD, in.(string))))
			if err != nil {
				return
			}
			defer f.Close()
			_, isManifest, err := proto.PeekManifest(f)
			if err != nil {
				return
			}
			out = struct {
				name   string
				isBlob bool
			}{in.(string), !isManifest}
			return
		},
		&req, &res1, concurrency.DefaultBatchOptions(),
	)

	for _, r := range res1 {
		r1 := r.(struct {
			name   string
			isBlob bool
		})
		res[r1.name] = r1.isBlob
	}
	return
}
Beispiel #9
0
// Assemble target files from stored chunks
func (a *Assembler) Done(what lists.BlobMap) (err error) {
	logx.Tracef("assembling %s", what.Names())

	var req, res []interface{}
	for k, v := range what {
		req = append(req, lists.BlobLink{v, k})
	}

	err = a.model.BatchPool.Do(
		func(ctx context.Context, in interface{}) (out interface{}, err error) {
			r := in.(lists.BlobLink)

			lock, err := a.model.FdLocks.Take()
			if err != nil {
				return
			}
			defer lock.Release()

			w, err := os.Create(lists.OSFromSlash(lists.OSJoin(a.model.WD,
				r.Name+"-"+r.Manifest.ID.String())))
			if err != nil {
				return
			}

			for _, chunk := range r.Manifest.Chunks {
				if err = a.writeChunkTo(w, chunk.ID); err != nil {
					return
				}
			}
			w.Close()
			err = a.commitBlob(r.Name, r.Manifest.ID)

			return
		}, &req, &res, concurrency.DefaultBatchOptions().AllowErrors(),
	)
	if err != nil {
		return
	}
	defer a.Close()
	return
}
Beispiel #10
0
func (c *SpecExportCmd) Run(args ...string) (err error) {
	var mod *model.Model
	if mod, err = model.New(c.WD, c.UseGit, c.ChunkSize, c.PoolSize); err != nil {
		return
	}

	feed := lists.NewFileList(args...).ListDir(c.WD)

	isDirty, dirty, err := mod.Check(feed...)
	if err != nil {
		return
	}
	if isDirty {
		err = fmt.Errorf("dirty files in working tree %s", dirty)
		return
	}

	if c.UseGit {
		// filter by attrs
		feed, err = mod.Git.FilterByAttr("bar", feed...)
	}

	blobs, err := mod.FeedManifests(true, true, true, feed...)
	if err != nil {
		return
	}

	// make specmap
	nameMap := map[string]proto.ID{}
	for name, m := range blobs {
		nameMap[name] = m.ID
	}

	spec, err := proto.NewSpec(time.Now().UnixNano(), nameMap, []string{})
	if err != nil {
		return
	}

	if c.DoCC {
		ccName := fmt.Sprintf("bar-spec-%d-%s.json",
			time.Now().UnixNano(), spec.ID)
		logx.Infof("storing carbon copy to %s", ccName)
		ccf, err := os.Create(lists.OSFromSlash(lists.OSJoin(c.WD, ccName)))
		if err != nil {
			return err
		}
		defer ccf.Close()

		if err = json.NewEncoder(ccf).Encode(&spec); err != nil {
			return err
		}
	}

	if !c.Upload {
		err = json.NewEncoder(c.Stdout).Encode(&spec)
		return
	}

	trans := transport.NewTransport(mod, "", c.Endpoint, c.PoolSize)
	if err = trans.UploadSpec(spec); err != nil {
		return
	}
	fmt.Fprint(c.Stdout, spec.ID)
	return
}