Exemple #1
0
// LoadSnap opens an index persister which stores pre-built index trees.
// When loading a index snapshot, only the top level of the tree is loaded.
// All other tree branches are loaded only when it's necessary (on request).
func LoadSnap(dir string) (s *Snap, err error) {
	segpathr := path.Join(dir, prefixsnaproot)
	segpathd := path.Join(dir, prefixsnapdata)

	rf, err := segfile.New(segpathr, segszsnap)
	if err != nil {
		return nil, err
	}

	root, branches, err := readSnapRoot(rf)
	if err != nil {
		return nil, err
	}

	if err := rf.Close(); err != nil {
		return nil, err
	}

	df, err := segfile.New(segpathd, segszsnap)
	if err != nil {
		return nil, err
	}

	s = &Snap{
		RootNode: root,
		branches: branches,
		dataFile: df,
	}

	return s, nil
}
Exemple #2
0
// writeSnapshot creates a snapshot on given path and returns created snapshot.
// This snapshot will have the complete index tree already loaded into ram.
func writeSnapshot(dir string, tree *TNode) (s *Snap, err error) {
	segpathr := path.Join(dir, prefixsnaproot)
	segpathd := path.Join(dir, prefixsnapdata)

	rf, err := segfile.New(segpathr, segszsnap)
	if err != nil {
		return nil, err
	}

	// can close this
	defer rf.Close()

	df, err := segfile.New(segpathd, segszsnap)
	if err != nil {
		return nil, err
	}

	brf := bufio.NewWriterSize(rf, 1e7)
	bdf := bufio.NewWriterSize(df, 1e7)
	branches := map[string]*Offset{}

	var offset int64
	var buffer []byte

	for name, tn := range tree.Children {
		size := tn.Size()
		sz64 := int64(size)

		if len(buffer) < size {
			buffer = make([]byte, size)
		}

		// slice to data size
		towrite := buffer[:size]

		_, err := tn.MarshalTo(towrite)
		if err != nil {
			return nil, err
		}

		for len(towrite) > 0 {
			n, err := bdf.Write(towrite)
			if err != nil {
				return nil, err
			}

			towrite = towrite[n:]
		}

		branches[name] = &Offset{offset, offset + sz64}
		offset += sz64
	}

	info := &SnapInfo{
		Branches: branches,
	}

	{
		size := info.Size()
		sz64 := int64(size)
		full := size + hybrid.SzInt64

		if len(buffer) < full {
			buffer = make([]byte, full)
		}

		towrite := buffer[:full]

		// prepend root info struct size to the buffer
		hybrid.EncodeInt64(towrite[:hybrid.SzInt64], &sz64)

		_, err := info.MarshalTo(towrite[hybrid.SzInt64:])
		if err != nil {
			return nil, err
		}

		for len(towrite) > 0 {
			n, err := brf.Write(towrite)
			if err != nil {
				return nil, err
			}

			towrite = towrite[n:]
		}
	}

	if err := bdf.Flush(); err != nil {
		return nil, err
	}
	if err := brf.Flush(); err != nil {
		return nil, err
	}

	s = &Snap{
		RootNode: tree,
		branches: branches,
		dataFile: df,
	}

	return s, nil
}