Example #1
0
// writeSnapshot creates a snapshot on given path and returns created snapshot.
// This snapshot will have the complete index tree already loaded into ram.
func writeSnapshot(dir string, tree *TNode) (s *Snap, err error) {
	segpathr := path.Join(dir, prefixsnaproot)
	segpathd := path.Join(dir, prefixsnapdata)

	rf, err := segfile.New(segpathr, segszsnap)
	if err != nil {
		return nil, err
	}

	// can close this
	defer rf.Close()

	df, err := segfile.New(segpathd, segszsnap)
	if err != nil {
		return nil, err
	}

	brf := bufio.NewWriterSize(rf, 1e7)
	bdf := bufio.NewWriterSize(df, 1e7)
	branches := map[string]*Offset{}

	var offset int64
	var buffer []byte

	for name, tn := range tree.Children {
		size := tn.Size()
		sz64 := int64(size)

		if len(buffer) < size {
			buffer = make([]byte, size)
		}

		// slice to data size
		towrite := buffer[:size]

		_, err := tn.MarshalTo(towrite)
		if err != nil {
			return nil, err
		}

		for len(towrite) > 0 {
			n, err := bdf.Write(towrite)
			if err != nil {
				return nil, err
			}

			towrite = towrite[n:]
		}

		branches[name] = &Offset{offset, offset + sz64}
		offset += sz64
	}

	info := &SnapInfo{
		Branches: branches,
	}

	{
		size := info.Size()
		sz64 := int64(size)
		full := size + hybrid.SzInt64

		if len(buffer) < full {
			buffer = make([]byte, full)
		}

		towrite := buffer[:full]

		// prepend root info struct size to the buffer
		hybrid.EncodeInt64(towrite[:hybrid.SzInt64], &sz64)

		_, err := info.MarshalTo(towrite[hybrid.SzInt64:])
		if err != nil {
			return nil, err
		}

		for len(towrite) > 0 {
			n, err := brf.Write(towrite)
			if err != nil {
				return nil, err
			}

			towrite = towrite[n:]
		}
	}

	if err := bdf.Flush(); err != nil {
		return nil, err
	}
	if err := brf.Flush(); err != nil {
		return nil, err
	}

	s = &Snap{
		RootNode: tree,
		branches: branches,
		dataFile: df,
	}

	return s, nil
}
Example #2
0
// Store appends a node to the index log file and updates ID and Offset fields.
func (l *Logs) Store(n *TNode) (err error) {
	l.iomutex.Lock()
	defer l.iomutex.Unlock()

	// If the index node can be written to a single segment file without breaking
	// its content, we can directly use a byte slice from the segment file.
	// Otherwise, we must write it to a temporary buffer and flush it later.
	var fast bool
	var buff []byte

	// protobuf size
	node := n.Node
	size := node.Size()
	sz64 := int64(size)
	full := sz64 + hybrid.SzInt64

	if err := l.logFile.Ensure(l.nextOff + full); err != nil {
		return err
	}

	p, err := l.logFile.SliceAt(full, l.nextOff)
	if err != nil {
		return err
	}

	if len(p) == int(full) {
		buff = p
		fast = true
	} else {
		buff = make([]byte, full)
	}

	// Write the node size to the buffer with hybrid
	hybrid.EncodeInt64(buff[:hybrid.SzInt64], &sz64)

	// Using protobuf MarshalTo for better performance
	if n, err := node.MarshalTo(buff[hybrid.SzInt64:]); err != nil {
		return err
	} else if n != size {
		panic("marshalled size is different from node size")
	}

	if !fast {
		// If we were using a temporary buffer to marshal data,
		// it's time for it to go to its final destination!
		towrite := buff[:]
		for len(towrite) > 0 {
			n, err := l.logFile.WriteAt(towrite, l.nextOff)
			if err != nil {
				return err
			}

			towrite = towrite[n:]
		}
	}

	// next item offset
	l.nextOff += full

	return nil
}