func (n *FileId) String() string { bytes := make([]byte, 12) util.Uint64toBytes(bytes[0:8], n.Key) util.Uint32toBytes(bytes[8:12], n.Hashcode) nonzero_index := 0 for ; bytes[nonzero_index] == 0; nonzero_index++ { } return n.VolumeId.String() + "," + hex.EncodeToString(bytes[nonzero_index:]) }
// dumps a NeedleMap into a cdb func DumpNeedleMapToCdb(cdbName string, nm *NeedleMap) error { tempnam := cdbName + "t" fnames := make([]string, 1, 2) adder, closer, err := openTempCdb(tempnam) if err != nil { return fmt.Errorf("error creating factory: %s", err) } fnames[0] = tempnam elt := cdb.Element{Key: make([]byte, 8), Data: make([]byte, 8)} fcount := uint64(0) walk := func(key uint64, offset, size uint32) error { //如果写入1失败则写入2中 if fcount >= maxCdbRecCount { if err = closer(); err != nil { return err } tempnam = cdbName + "t2" if adder, closer, err = openTempCdb(tempnam); err != nil { return fmt.Errorf("error creating second factory: %s", err) } fnames = append(fnames, tempnam) fcount = 0 } util.Uint64toBytes(elt.Key, key) util.Uint32toBytes(elt.Data[:4], offset) util.Uint32toBytes(elt.Data[4:], size) fcount++ return adder(elt) } // and write out the cdb from there err = nm.Visit(func(nv NeedleValue) error { return walk(uint64(nv.Key), nv.Offset, nv.Size) }) if err != nil { closer() return fmt.Errorf("error walking index %v: %s", nm, err) } // store fileBytes //写入metric数据 data, e := json.Marshal(nm.mapMetric) if e != nil { return fmt.Errorf("error marshaling metric %v: %s", nm.mapMetric, e) } if err = adder(cdb.Element{Key: []byte{'M'}, Data: data}); err != nil { return err } if err = closer(); err != nil { return err } os.Remove(cdbName) //如果只有一个文件就叫base.cdb如果有2个则叫base.1/2.cdb if len(fnames) == 1 { return os.Rename(fnames[0], cdbName) } bn, ext := baseFilename(cdbName) if err = os.Rename(fnames[0], bn+".1"+ext); err != nil { return err } return os.Rename(fnames[1], bn+".2"+ext) }
func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) { if s, ok := w.(io.Seeker); ok { if end, e := s.Seek(0, 1); e == nil { defer func(s io.Seeker, off int64) { if err != nil { if _, e = s.Seek(off, 0); e != nil { glog.V(0).Infof("Failed to seek %s back to %d with error: %v", w, off, e) } } }(s, end) } else { err = fmt.Errorf("Cannot Read Current Volume Position: %v", e) return } } switch version { case Version1: header := make([]byte, NeedleHeaderSize) util.Uint32toBytes(header[0:4], n.Cookie) util.Uint64toBytes(header[4:12], n.Id) n.Size = uint32(len(n.Data)) size = n.Size util.Uint32toBytes(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } if _, err = w.Write(n.Data); err != nil { return } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return case Version2: header := make([]byte, NeedleHeaderSize) util.Uint32toBytes(header[0:4], n.Cookie) util.Uint64toBytes(header[4:12], n.Id) n.DataSize, n.NameSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Name)), uint8(len(n.Mime)) if n.DataSize > 0 { n.Size = 4 + n.DataSize + 1 if n.HasName() { n.Size = n.Size + 1 + uint32(n.NameSize) } if n.HasMime() { n.Size = n.Size + 1 + uint32(n.MimeSize) } if n.HasLastModifiedDate() { n.Size = n.Size + LastModifiedBytesLength } if n.HasTtl() { n.Size = n.Size + TtlBytesLength } } size = n.DataSize util.Uint32toBytes(header[12:16], n.Size) if _, err = w.Write(header); err != nil { return } if n.DataSize > 0 { util.Uint32toBytes(header[0:4], n.DataSize) if _, err = w.Write(header[0:4]); err != nil { return } if _, err = w.Write(n.Data); err != nil { return } util.Uint8toBytes(header[0:1], n.Flags) if _, err = w.Write(header[0:1]); err != nil { return } if n.HasName() { util.Uint8toBytes(header[0:1], n.NameSize) if _, err = w.Write(header[0:1]); err != nil { return } if _, err = w.Write(n.Name); err != nil { return } } if n.HasMime() { util.Uint8toBytes(header[0:1], n.MimeSize) if _, err = w.Write(header[0:1]); err != nil { return } if _, err = w.Write(n.Mime); err != nil { return } } if n.HasLastModifiedDate() { util.Uint64toBytes(header[0:8], n.LastModified) if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil { return } } if n.HasTtl() { n.Ttl.ToBytes(header[0:TtlBytesLength]) if _, err = w.Write(header[0:TtlBytesLength]); err != nil { return } } } padding := NeedlePaddingSize - ((NeedleHeaderSize + n.Size + NeedleChecksumSize) % NeedlePaddingSize) util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value()) _, err = w.Write(header[0 : NeedleChecksumSize+padding]) return n.DataSize, err } return 0, fmt.Errorf("Unsupported Version! (%d)", version) }
//此处的etag即为Needle的checksum func (n *Needle) Etag() string { bits := make([]byte, 4) util.Uint32toBytes(bits, uint32(n.Checksum)) return fmt.Sprintf("\"%x\"", bits) }