// writeSnapshot creates a snapshot on given path and returns created snapshot. // This snapshot will have the complete index tree already loaded into ram. func writeSnapshot(dir string, tree *TNode) (s *Snap, err error) { segpathr := path.Join(dir, prefixsnaproot) segpathd := path.Join(dir, prefixsnapdata) rf, err := segfile.New(segpathr, segszsnap) if err != nil { return nil, err } // can close this defer rf.Close() df, err := segfile.New(segpathd, segszsnap) if err != nil { return nil, err } brf := bufio.NewWriterSize(rf, 1e7) bdf := bufio.NewWriterSize(df, 1e7) branches := map[string]*Offset{} var offset int64 var buffer []byte for name, tn := range tree.Children { size := tn.Size() sz64 := int64(size) if len(buffer) < size { buffer = make([]byte, size) } // slice to data size towrite := buffer[:size] _, err := tn.MarshalTo(towrite) if err != nil { return nil, err } for len(towrite) > 0 { n, err := bdf.Write(towrite) if err != nil { return nil, err } towrite = towrite[n:] } branches[name] = &Offset{offset, offset + sz64} offset += sz64 } info := &SnapInfo{ Branches: branches, } { size := info.Size() sz64 := int64(size) full := size + hybrid.SzInt64 if len(buffer) < full { buffer = make([]byte, full) } towrite := buffer[:full] // prepend root info struct size to the buffer hybrid.EncodeInt64(towrite[:hybrid.SzInt64], &sz64) _, err := info.MarshalTo(towrite[hybrid.SzInt64:]) if err != nil { return nil, err } for len(towrite) > 0 { n, err := brf.Write(towrite) if err != nil { return nil, err } towrite = towrite[n:] } } if err := bdf.Flush(); err != nil { return nil, err } if err := brf.Flush(); err != nil { return nil, err } s = &Snap{ RootNode: tree, branches: branches, dataFile: df, } return s, nil }
// Store appends a node to the index log file and updates ID and Offset fields. func (l *Logs) Store(n *TNode) (err error) { l.iomutex.Lock() defer l.iomutex.Unlock() // If the index node can be written to a single segment file without breaking // its content, we can directly use a byte slice from the segment file. // Otherwise, we must write it to a temporary buffer and flush it later. var fast bool var buff []byte // protobuf size node := n.Node size := node.Size() sz64 := int64(size) full := sz64 + hybrid.SzInt64 if err := l.logFile.Ensure(l.nextOff + full); err != nil { return err } p, err := l.logFile.SliceAt(full, l.nextOff) if err != nil { return err } if len(p) == int(full) { buff = p fast = true } else { buff = make([]byte, full) } // Write the node size to the buffer with hybrid hybrid.EncodeInt64(buff[:hybrid.SzInt64], &sz64) // Using protobuf MarshalTo for better performance if n, err := node.MarshalTo(buff[hybrid.SzInt64:]); err != nil { return err } else if n != size { panic("marshalled size is different from node size") } if !fast { // If we were using a temporary buffer to marshal data, // it's time for it to go to its final destination! towrite := buff[:] for len(towrite) > 0 { n, err := l.logFile.WriteAt(towrite, l.nextOff) if err != nil { return err } towrite = towrite[n:] } } // next item offset l.nextOff += full return nil }