func TestNeedle(t *testing.T) {
var (
err error
n, tn *Needle
br *bufio.Reader
data1 = []byte("tes1")
checksum1 = crc32.Update(0, _crc32Table, data1)
data2 = []byte("tes2")
checksum2 = crc32.Update(0, _crc32Table, data2)
buf = &bytes.Buffer{}
)
// WriteFrom
if _, err = buf.Write(data1); err != nil {
t.Error(err)
t.FailNow()
}
n = NewWriter(3, 3, 4)
defer n.Close()
if err = n.ReadFrom(buf); err != nil {
t.Error(err)
t.FailNow()
}
t.Log(n)
tn = new(Needle)
tn.buffer = n.Buffer()
// Parse
if err = tn.Parse(); err != nil {
t.Error(err)
t.FailNow()
}
t.Log(n)
compareNeedle(t, tn, 3, 3, data1, FlagOK, checksum1)
buf.Write(data2)
n = NewWriter(4, 4, 4)
defer n.Close()
if err = n.ReadFrom(buf); err != nil {
t.Error(err)
t.FailNow()
}
tn = new(Needle)
tn.buffer = n.Buffer()
if err = tn.Parse(); err != nil {
t.Error(err)
t.FailNow()
}
compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2)
// ParseFrom
if _, err = buf.Write(n.Buffer()); err != nil {
t.Error(err)
t.FailNow()
}
br = bufio.NewReader(buf)
tn = new(Needle)
if err = tn.ParseFrom(br); err != nil {
t.Error(err)
t.FailNow()
}
t.Log(tn)
compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2)
}
func (w *Writer) finishBlock() (blockHandle, error) {
if w.nEntries == 0 {
w.restarts = w.restarts[:1]
w.restarts[0] = 0
}
tmp4 := w.tmp[:4]
for _, x := range w.restarts {
binary.LittleEndian.PutUint32(tmp4, x)
w.buf.Write(tmp4)
}
binary.LittleEndian.PutUint32(tmp4, uint32(len(w.restarts)))
w.buf.Write(tmp4)
b := w.buf.Bytes()
w.tmp[0] = 0
checksum := crc32.Update(0, table, b)
checksum = crc32.Update(checksum, table, w.tmp[:1])
checksum = uint32(checksum>>15|checksum<<17) + 0xa282ead8
binary.LittleEndian.PutUint32(w.tmp[1:5], checksum)
if _, err := w.writer.Write(b); err != nil {
return blockHandle{}, err
}
if _, err := w.writer.Write(w.tmp[:5]); err != nil {
return blockHandle{}, err
}
bh := blockHandle{int64(w.offset), int64(len(b))}
w.offset += uint64(len(b)) + BlockTrailerLen
w.buf.Reset()
w.nEntries = 0
w.restarts = w.restarts[:0]
return bh, nil
}
func calcNewCRC32(path string, newHeader *BinaryHeader) (uint32, error) {
buf := &bytes.Buffer{}
err := binary.Write(buf, binary.BigEndian, newHeader)
if err != nil {
return 0, err
}
header := buf.Bytes()
crc := crc32.Checksum(header[:28], crc32.IEEETable) // before crc32
crc = crc32.Update(crc, crc32.IEEETable, header[32:])
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
if _, err := f.Seek(int64(len(header)), os.SEEK_SET); err != nil {
return 0, err
}
data := make([]byte, 4*1024*1024)
for {
n, err := f.Read(data)
crc = crc32.Update(crc, crc32.IEEETable, data[:n])
if err != nil {
if err != io.EOF {
return 0, err
}
break
}
}
return crc, nil
}
// FillNeedleBuf fill needle buf with photo.
func FillNeedleBuf(key, cookie int64, data, buf []byte) (size int32) {
var (
n int
padding int32
checksum = crc32.Update(0, crc32Table, data)
)
size = int32(NeedleHeaderSize + len(data) + NeedleFooterSize)
padding = NeedlePaddingSize - (size % NeedlePaddingSize)
size += padding
// header
copy(buf[:needleMagicSize], needleHeaderMagic)
n += needleMagicSize
BigEndian.PutInt64(buf[n:], cookie)
n += needleCookieSize
BigEndian.PutInt64(buf[n:], key)
n += needleKeySize
buf[n] = NeedleStatusOK
n += needleFlagSize
BigEndian.PutInt32(buf[n:], int32(len(data)))
n += needleSizeSize
// data
copy(buf[n:], data)
n += len(data)
// footer
copy(buf[n:], needleFooterMagic)
n += needleMagicSize
BigEndian.PutUint32(buf[n:], checksum)
n += needleChecksumSize
copy(buf[n:], needlePadding[padding])
return
}
// ParseNeedleData parse a needle data part.
func ParseNeedleData(buf []byte, n *Needle) (err error) {
var (
bn int32
checksum uint32
)
n.Data = buf[:n.Size]
bn += n.Size
n.FooterMagic = buf[bn : bn+needleMagicSize]
if bytes.Compare(n.FooterMagic, needleFooterMagic) != 0 {
err = ErrNeedleFooterMagic
return
}
bn += needleMagicSize
checksum = crc32.Update(0, crc32Table, n.Data)
n.Checksum = BigEndian.Uint32(buf[bn : bn+needleChecksumSize])
if n.Checksum != checksum {
err = ErrNeedleChecksum
return
}
bn += needleChecksumSize
n.Padding = buf[bn : bn+n.PaddingSize]
log.Printf("padding: %d, %v vs %v\n", n.PaddingSize, n.Padding, needlePadding[n.PaddingSize])
if bytes.Compare(n.Padding, needlePadding[n.PaddingSize]) != 0 {
err = ErrNeedlePaddingNotMatch
return
}
return
}
func (r *Receiver) handleTransfer(conn io.ReadWriter) (
transfer.TransferResults, error,
) {
if _, err := conn.Write([]byte("ok")); err != nil {
return transfer.TransferResults{}, err
}
res := transfer.TransferResults{}
buffer := make([]byte, 1024)
startTime := time.Now()
for {
n, err := conn.Read(buffer)
if err != nil { // done reading
break
}
res.BytesSent += uint32(n)
res.Checksum = crc32.Update(res.Checksum, crc32.IEEETable, buffer)
}
endTime := time.Now()
res.Duration = endTime.Sub(startTime)
return res, nil
}
// WriteFrom Write needle from io.Reader into buffer.
func (ns *Needles) WriteFrom(key int64, cookie int32, size int32, rd io.Reader) (err error) {
var (
n *Needle
data []byte
headerOffset int32
dataOffset int32
footerOffset int32
endOffset int32
)
if ns.wn >= ns.Num {
return errors.ErrNeedleFull
}
n = &ns.needles[ns.wn]
n.initSize(key, cookie, size)
headerOffset = ns.ws
dataOffset = headerOffset + _headerSize
footerOffset = dataOffset + n.Size
endOffset = footerOffset + n.FooterSize
data = ns.buffer[dataOffset:footerOffset]
// write into buffer header->data->footer
if err = n.WriteHeader(ns.buffer[headerOffset:dataOffset]); err == nil {
if _, err = rd.Read(data); err == nil {
n.Data = data
n.Checksum = crc32.Update(0, _crc32Table, data)
err = n.WriteFooter(ns.buffer[footerOffset:endOffset])
}
}
ns.wn++
ns.ws += n.TotalSize
return
}
// FillNeedle fill needle buffer.
func FillNeedle(padding, size int32, key, cookie int64, data, buf []byte) {
var (
n int
checksum = crc32.Update(0, crc32Table, data)
)
// --- header ---
// magic
copy(buf[:needleMagicSize], needleHeaderMagic)
n += needleMagicSize
// cookie
BigEndian.PutInt64(buf[n:], cookie)
n += needleCookieSize
// key
BigEndian.PutInt64(buf[n:], key)
n += needleKeySize
// flag
buf[n] = NeedleStatusOK
n += needleFlagSize
// size
BigEndian.PutInt32(buf[n:], size)
n += needleSizeSize
// data
copy(buf[n:], data)
n += len(data)
// --- footer ---
// magic
copy(buf[n:], needleFooterMagic)
n += needleMagicSize
// checksum
BigEndian.PutUint32(buf[n:], checksum)
n += needleChecksumSize
// padding
copy(buf[n:], needlePadding[padding])
return
}
func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
start := time.Now()
fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
b := pbutil.MustMarshal(snapshot)
crc := crc32.Update(0, crcTable, b)
snap := snappb.Snapshot{Crc: crc, Data: b}
d, err := snap.Marshal()
if err != nil {
return err
} else {
marshallingDurations.Observe(float64(time.Since(start)) / float64(time.Second))
}
err = pioutil.WriteAndSyncFile(path.Join(s.dir, fname), d, 0666)
if err == nil {
saveDurations.Observe(float64(time.Since(start)) / float64(time.Second))
} else {
err1 := os.Remove(path.Join(s.dir, fname))
if err1 != nil {
plog.Errorf("failed to remove broken snapshot file %s", path.Join(s.dir, fname))
}
}
return err
}
// Read reads the snapshot named by snapname and returns the snapshot.
func Read(snapname string) (*raftpb.Snapshot, error) {
b, err := ioutil.ReadFile(snapname)
if err != nil {
log.Printf("snap: snapshotter cannot read file %v: %v", snapname, err)
return nil, err
}
var serializedSnap snappb.Snapshot
if err = serializedSnap.Unmarshal(b); err != nil {
log.Printf("snap: corrupted snapshot file %v: %v", snapname, err)
return nil, err
}
if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
log.Printf("snap: unexpected empty snapshot")
return nil, ErrEmptySnapshot
}
crc := crc32.Update(0, crcTable, serializedSnap.Data)
if crc != serializedSnap.Crc {
log.Printf("snap: corrupted snapshot file %v: crc mismatch", snapname)
return nil, ErrCRCMismatch
}
var snap raftpb.Snapshot
if err = snap.Unmarshal(serializedSnap.Data); err != nil {
log.Printf("snap: corrupted snapshot file %v: %v", snapname, err)
return nil, err
}
return &snap, nil
}
// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed.
func (z *Writer) Write(p []byte) (int, error) {
if z.err != nil {
return 0, z.err
}
var n int
// Write the GZIP header lazily.
if !z.wroteHeader {
z.wroteHeader = true
z.buf = [10]byte{0: gzipID1, 1: gzipID2, 2: gzipDeflate}
if z.Extra != nil {
z.buf[3] |= 0x04
}
if z.Name != "" {
z.buf[3] |= 0x08
}
if z.Comment != "" {
z.buf[3] |= 0x10
}
if z.ModTime.After(time.Unix(0, 0)) {
// Section 2.3.1, the zero value for MTIME means that the
// modified time is not set.
le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix()))
}
if z.level == BestCompression {
z.buf[8] = 2
} else if z.level == BestSpeed {
z.buf[8] = 4
}
z.buf[9] = z.OS
n, z.err = z.w.Write(z.buf[:10])
if z.err != nil {
return n, z.err
}
if z.Extra != nil {
z.err = z.writeBytes(z.Extra)
if z.err != nil {
return n, z.err
}
}
if z.Name != "" {
z.err = z.writeString(z.Name)
if z.err != nil {
return n, z.err
}
}
if z.Comment != "" {
z.err = z.writeString(z.Comment)
if z.err != nil {
return n, z.err
}
}
if z.compressor == nil {
z.compressor, _ = flate.NewWriter(z.w, z.level)
}
}
z.size += uint32(len(p))
z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
n, z.err = z.compressor.Write(p)
return n, z.err
}
// ParseNeedleData parse a needle data part.
func (n *Needle) ParseData(buf []byte) (err error) {
var (
bn int32
checksum uint32
)
n.Data = buf[:n.Size]
bn += n.Size
n.FooterMagic = buf[bn : bn+magicSize]
if !bytes.Equal(n.FooterMagic, footerMagic) {
err = errors.ErrNeedleFooterMagic
return
}
bn += magicSize
checksum = crc32.Update(0, crc32Table, n.Data)
n.Checksum = binary.BigEndian.Uint32(buf[bn : bn+checksumSize])
if n.Checksum != checksum {
err = errors.ErrNeedleChecksum
return
}
bn += checksumSize
n.Padding = buf[bn : bn+n.PaddingSize]
if !bytes.Equal(n.Padding, padding[n.PaddingSize]) {
err = errors.ErrNeedlePadding
}
return
}
// readString reads a NUL-terminated string from z.r.
// It treats the bytes read as being encoded as ISO 8859-1 (Latin-1) and
// will output a string encoded using UTF-8.
// This method always updates z.digest with the data read.
func (z *Reader) readString() (string, error) {
var err error
needConv := false
for i := 0; ; i++ {
if i >= len(z.buf) {
return "", ErrHeader
}
z.buf[i], err = z.r.ReadByte()
if err != nil {
return "", err
}
if z.buf[i] > 0x7f {
needConv = true
}
if z.buf[i] == 0 {
// Digest covers the NUL terminator.
z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:i+1])
// Strings are ISO 8859-1, Latin-1 (RFC 1952, section 2.3.1).
if needConv {
s := make([]rune, 0, i)
for _, v := range z.buf[:i] {
s = append(s, rune(v))
}
return string(s), nil
}
return string(z.buf[:i]), nil
}
}
}
// PushRow is part of the RowReceiver interface.
func (hr *hashRouter) PushRow(row sqlbase.EncDatumRow) bool {
if hr.err != nil {
return false
}
hr.buffer = hr.buffer[:0]
for _, col := range hr.hashCols {
if int(col) >= len(row) {
hr.err = errors.Errorf("hash column %d, stream with only %d columns", col, len(row))
return false
}
// TODO(radu): we should choose an encoding that is already available as
// much as possible. However, we cannot decide this locally as multiple
// nodes may be doing the same hashing and the encodings need to match. The
// encoding needs to be determined at planning time.
hr.buffer, hr.err = row[col].Encode(&hr.alloc, preferredEncoding, hr.buffer)
if hr.err != nil {
return false
}
}
// We use CRC32-C because it makes for a decent hash function and is faster
// than most hashing algorithms (on recent x86 platforms where it is hardware
// accelerated).
streamIdx := crc32.Update(0, crc32Table, hr.buffer) % uint32(len(hr.streams))
// We can't return false if this stream happened to not need any more rows. We
// could only return false once all streams returned false, but that seems of
// limited benefit.
_ = hr.streams[streamIdx].PushRow(row)
return true
}
func TestNeedles(t *testing.T) {
var (
err error
tn *Needle
data1 = []byte("tes1")
checksum1 = crc32.Update(0, _crc32Table, data1)
data2 = []byte("tes2")
checksum2 = crc32.Update(0, _crc32Table, data2)
ns = NewNeedles(2)
buf = &bytes.Buffer{}
)
if _, err = buf.Write(data1); err != nil {
t.FailNow()
}
if err = ns.ReadFrom(1, 1, 4, buf); err != nil {
t.FailNow()
}
if _, err = buf.Write(data2); err != nil {
t.FailNow()
}
if err = ns.ReadFrom(2, 2, 4, buf); err != nil {
t.FailNow()
}
tn = new(Needle)
tn.buffer = ns.Next().Buffer()
if err = tn.Parse(); err != nil {
t.FailNow()
}
t.Log(tn)
compareNeedle(t, tn, 1, 1, data1, FlagOK, checksum1)
tn = new(Needle)
tn.buffer = ns.Next().Buffer()
if err = tn.Parse(); err != nil {
t.FailNow()
}
t.Log(tn)
compareNeedle(t, tn, 2, 2, data2, FlagOK, checksum2)
if err = ns.ReadFrom(3, 3, 4, buf); err != errors.ErrNeedleFull {
t.FailNow()
}
if tn = ns.Next(); tn != nil {
t.FailNow()
}
}
// ParseData parse a needle data part.
func (n *Needle) ParseData(buf []byte) (err error) {
if len(buf) != int(n.Size) {
return errors.ErrNeedleDataSize
}
// data
n.Data = buf
// checksum
n.Checksum = crc32.Update(0, _crc32Table, n.Data)
return
}
func (n *Needle) Init(key int64, cookie int64, data []byte) {
var dataSize = int64(len(data))
n.Header = headerMagic
n.Key = key
n.Cookie = cookie
n.Size = dataSize
n.Data = data
n.Footer = footerMagic
n.DataChecksum = crc32.Update(0, crc32Table, data)
}
func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
b := pbutil.MustMarshal(snapshot)
crc := crc32.Update(0, crcTable, b)
snap := snappb.Snapshot{Crc: crc, Data: b}
d, err := snap.Marshal()
if err != nil {
return err
}
return ioutil.WriteFile(path.Join(s.dir, fname), d, 0666)
}
// WriteFrom Write needle from io.Reader into buffer.
func (n *Needle) WriteFrom(rd io.Reader) (err error) {
var (
dataOffset = _headerSize + n.Size
data = n.Buffer[_headerSize:dataOffset]
)
if err = n.WriteHeader(n.Buffer[:_headerSize]); err == nil {
if _, err = rd.Read(data); err == nil {
n.Data = data
n.Checksum = crc32.Update(0, _crc32Table, data)
err = n.WriteFooter(n.Buffer[dataOffset:n.TotalSize])
}
}
return
}
// private native static int updateBytes(int crc, byte[] b, int off, int len);
// (I[BII)I
func updateBytes(frame *rtda.Frame) {
vars := frame.LocalVars()
crc := uint32(vars.GetInt(0))
byteArr := vars.GetRef(1)
off := vars.GetInt(2)
_len := vars.GetInt(3)
goBytes := byteArr.GoBytes()
goBytes = goBytes[off : off+_len]
// func Update(crc uint32, tab *Table, p []byte) uint32
crc = crc32.Update(crc, crc32.IEEETable, goBytes)
stack := frame.OperandStack()
stack.PushInt(int32(crc))
}
// Parse parse needle from data.
func (n *Needle) Parse(key int64, cookie int32, data []byte) {
var dataSize = int32(len(data))
n.TotalSize = int32(HeaderSize + dataSize + FooterSize)
n.PaddingSize = align(n.TotalSize) - n.TotalSize
n.TotalSize += n.PaddingSize
n.HeaderMagic = headerMagic
n.Key = key
n.Cookie = cookie
n.Size = dataSize
n.Data = data
n.FooterMagic = footerMagic
n.Checksum = crc32.Update(0, crc32Table, data)
n.Padding = padding[n.PaddingSize]
return
}
// updateCRC returns the result of adding the bytes in buf to the crc.
func updateCRC(crc uint32, buf []byte) uint32 {
// The CRC-32 computation in bzip2 treats bytes as having bits in big-endian
// order. That is, the MSB is read before the LSB. Thus, we can use the
// standard library version of CRC-32 IEEE with some minor adjustments.
crc = internal.ReverseUint32(crc)
var arr [4096]byte
for len(buf) > 0 {
cnt := copy(arr[:], buf)
buf = buf[cnt:]
for i, b := range arr[:cnt] {
arr[i] = internal.ReverseLUT[b]
}
crc = crc32.Update(crc, crc32.IEEETable, arr[:cnt])
}
return internal.ReverseUint32(crc)
}
func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
start := time.Now()
fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
b := pbutil.MustMarshal(snapshot)
crc := crc32.Update(0, crcTable, b)
snap := snappb.Snapshot{Crc: crc, Data: b}
d, err := snap.Marshal()
if err != nil {
return err
}
err = ioutil.WriteFile(path.Join(s.dir, fname), d, 0666)
if err == nil {
saveDurations.Observe(float64(time.Since(start).Nanoseconds() / int64(time.Microsecond)))
}
return err
}
// WriteFrom read from io.Reader and write into needle buffer.
func (n *Needle) WriteFrom(key int64, cookie, size int32, rd io.Reader) (err error) {
var (
dataOffset int32
data []byte
)
n.initSize(key, cookie, size)
dataOffset = _headerSize + n.Size
data = n.buffer[_headerSize:dataOffset]
if err = n.writeHeader(n.buffer[:_headerSize]); err == nil {
if _, err = rd.Read(data); err == nil {
n.Data = data
n.Checksum = crc32.Update(0, _crc32Table, data)
err = n.writeFooter(n.buffer[dataOffset:n.TotalSize])
}
}
return
}
// Parse parse needle from data.
func (n *Needle) Parse(key, cookie int64, data []byte) (err error) {
var dataSize = int32(len(data))
n.TotalSize = int32(NeedleHeaderSize + dataSize + NeedleFooterSize)
n.PaddingSize = NeedlePaddingSize - (n.TotalSize % NeedlePaddingSize)
n.TotalSize += n.PaddingSize
if n.TotalSize > NeedleMaxSize {
err = ErrNeedleTooLarge
return
}
n.Key = key
n.Cookie = cookie
n.Size = dataSize
n.Data = data
n.Checksum = crc32.Update(0, crc32Table, data)
n.Padding = needlePadding[n.PaddingSize]
return
}
// WriteFrom Write needle from io.Reader into buffer.
func (ns *Needles) WriteFrom(n *Needle, rd io.Reader) (err error) {
var (
headerOffset = ns.TotalSize
dataOffset = headerOffset + _headerSize
footerOffset = dataOffset + n.Size
endOffset = footerOffset + n.FooterSize
data = ns.Buffer[dataOffset:footerOffset]
)
if err = n.WriteHeader(ns.Buffer[headerOffset:dataOffset]); err == nil {
if _, err = rd.Read(data); err == nil {
n.Data = data
n.Checksum = crc32.Update(0, _crc32Table, data)
err = n.WriteFooter(ns.Buffer[footerOffset:endOffset])
}
}
ns.TotalSize += n.TotalSize
return
}
func (r *Ring) FindMultiple(key []byte, count int) ([]string, bool) {
r.mutex.RLock()
defer r.mutex.RUnlock()
if count <= 0 {
return nil, false
}
c, n := crc32.Update(0, crcTable, key), len(r.buckets)
if n == 0 {
return nil, false
}
i, j := 0, n
for i < j {
p := i + (j-i)/2
if r.buckets[p].id < c {
i = p + 1
} else {
j = p
}
}
if i == n {
i = 0
}
first := i
found := []string{r.buckets[i].node}
outer:
for count > len(found) {
i += 1
if i == n {
i = 0
}
if i == first {
break
}
next := r.buckets[i].node
for _, node := range found {
if node == next {
continue outer
}
}
found = append(found, next)
}
return found, true
}
func (z *Reader) Read(p []byte) (n int, err error) {
if z.err != nil {
return 0, z.err
}
n, z.err = z.decompressor.Read(p)
z.digest = crc32.Update(z.digest, crc32.IEEETable, p[:n])
z.size += uint32(n)
if z.err != io.EOF {
// In the normal case we return here.
return n, z.err
}
// Finished file; check checksum and size.
if _, err := io.ReadFull(z.r, z.buf[:8]); err != nil {
z.err = noEOF(err)
return n, z.err
}
digest := le.Uint32(z.buf[:4])
size := le.Uint32(z.buf[4:8])
if digest != z.digest || size != z.size {
z.err = ErrChecksum
return n, z.err
}
z.digest, z.size = 0, 0
// File is ok; check if there is another.
if !z.multistream {
return n, io.EOF
}
z.err = nil // Remove io.EOF
if _, z.err = z.readHeader(); z.err != nil {
return n, z.err
}
// Read from next file, if necessary.
if n > 0 {
return n, nil
}
return z.Read(p)
}
func TestCPUProfileMultithreaded(t *testing.T) {
buf := make([]byte, 100000)
defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2))
testCPUProfile(t, []string{"crc32.update"}, func() {
c := make(chan int)
go func() {
for i := 0; i < 2000; i++ {
crc32.Update(0, crc32.IEEETable, buf)
}
c <- 1
}()
// This loop takes about a quarter second on a 2 GHz laptop.
// We only need to get one 100 Hz clock tick, so we've got
// a 25x safety buffer.
for i := 0; i < 2000; i++ {
crc32.ChecksumIEEE(buf)
}
<-c
})
}
func loadSnap(dir, name string) (*raftpb.Snapshot, error) {
var err error
var b []byte
fpath := path.Join(dir, name)
defer func() {
if err != nil {
renameBroken(fpath)
}
}()
b, err = ioutil.ReadFile(fpath)
if err != nil {
log.Printf("snap: snapshotter cannot read file %v: %v", name, err)
return nil, err
}
var serializedSnap snappb.Snapshot
if err = serializedSnap.Unmarshal(b); err != nil {
log.Printf("snap: corrupted snapshot file %v: %v", name, err)
return nil, err
}
if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
log.Printf("snap: unexpected empty snapshot")
return nil, ErrEmptySnapshot
}
crc := crc32.Update(0, crcTable, serializedSnap.Data)
if crc != serializedSnap.Crc {
log.Printf("snap: corrupted snapshot file %v: crc mismatch", name)
return nil, ErrCRCMismatch
}
var snap raftpb.Snapshot
if err = snap.Unmarshal(serializedSnap.Data); err != nil {
log.Printf("snap: corrupted snapshot file %v: %v", name, err)
return nil, err
}
return &snap, nil
}