Esempio n. 1
0
func TestNeedle(t *testing.T) {
	var (
		err       error
		n, tn     *Needle
		br        *bufio.Reader
		data1     = []byte("tes1")
		checksum1 = crc32.Update(0, _crc32Table, data1)
		data2     = []byte("tes2")
		checksum2 = crc32.Update(0, _crc32Table, data2)
		buf       = &bytes.Buffer{}
	)
	// WriteFrom
	if _, err = buf.Write(data1); err != nil {
		t.Error(err)
		t.FailNow()
	}
	n = NewWriter(3, 3, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Error(err)
		t.FailNow()
	}
	t.Log(n)
	tn = new(Needle)
	tn.buffer = n.Buffer()
	// Parse
	if err = tn.Parse(); err != nil {
		t.Error(err)
		t.FailNow()
	}
	t.Log(n)
	compareNeedle(t, tn, 3, 3, data1, FlagOK, checksum1)
	buf.Write(data2)
	n = NewWriter(4, 4, 4)
	defer n.Close()
	if err = n.ReadFrom(buf); err != nil {
		t.Error(err)
		t.FailNow()
	}
	tn = new(Needle)
	tn.buffer = n.Buffer()
	if err = tn.Parse(); err != nil {
		t.Error(err)
		t.FailNow()
	}
	compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2)
	// ParseFrom
	if _, err = buf.Write(n.Buffer()); err != nil {
		t.Error(err)
		t.FailNow()
	}
	br = bufio.NewReader(buf)
	tn = new(Needle)
	if err = tn.ParseFrom(br); err != nil {
		t.Error(err)
		t.FailNow()
	}
	t.Log(tn)
	compareNeedle(t, tn, 4, 4, data2, FlagOK, checksum2)
}
Esempio n. 2
0
func (w *Writer) finishBlock() (blockHandle, error) {
	if w.nEntries == 0 {
		w.restarts = w.restarts[:1]
		w.restarts[0] = 0
	}
	tmp4 := w.tmp[:4]
	for _, x := range w.restarts {
		binary.LittleEndian.PutUint32(tmp4, x)
		w.buf.Write(tmp4)
	}
	binary.LittleEndian.PutUint32(tmp4, uint32(len(w.restarts)))
	w.buf.Write(tmp4)

	b := w.buf.Bytes()
	w.tmp[0] = 0

	checksum := crc32.Update(0, table, b)
	checksum = crc32.Update(checksum, table, w.tmp[:1])
	checksum = uint32(checksum>>15|checksum<<17) + 0xa282ead8
	binary.LittleEndian.PutUint32(w.tmp[1:5], checksum)

	if _, err := w.writer.Write(b); err != nil {
		return blockHandle{}, err
	}
	if _, err := w.writer.Write(w.tmp[:5]); err != nil {
		return blockHandle{}, err
	}
	bh := blockHandle{int64(w.offset), int64(len(b))}
	w.offset += uint64(len(b)) + BlockTrailerLen

	w.buf.Reset()
	w.nEntries = 0
	w.restarts = w.restarts[:0]
	return bh, nil
}
Esempio n. 3
0
func calcNewCRC32(path string, newHeader *BinaryHeader) (uint32, error) {
	buf := &bytes.Buffer{}
	err := binary.Write(buf, binary.BigEndian, newHeader)
	if err != nil {
		return 0, err
	}

	header := buf.Bytes()
	crc := crc32.Checksum(header[:28], crc32.IEEETable) // before crc32
	crc = crc32.Update(crc, crc32.IEEETable, header[32:])

	f, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	if _, err := f.Seek(int64(len(header)), os.SEEK_SET); err != nil {
		return 0, err
	}

	data := make([]byte, 4*1024*1024)
	for {
		n, err := f.Read(data)
		crc = crc32.Update(crc, crc32.IEEETable, data[:n])
		if err != nil {
			if err != io.EOF {
				return 0, err
			}
			break
		}
	}
	return crc, nil
}
Esempio n. 4
0
// FillNeedleBuf fill needle buf with photo.
func FillNeedleBuf(key, cookie int64, data, buf []byte) (size int32) {
	var (
		n        int
		padding  int32
		checksum = crc32.Update(0, crc32Table, data)
	)
	size = int32(NeedleHeaderSize + len(data) + NeedleFooterSize)
	padding = NeedlePaddingSize - (size % NeedlePaddingSize)
	size += padding
	// header
	copy(buf[:needleMagicSize], needleHeaderMagic)
	n += needleMagicSize
	BigEndian.PutInt64(buf[n:], cookie)
	n += needleCookieSize
	BigEndian.PutInt64(buf[n:], key)
	n += needleKeySize
	buf[n] = NeedleStatusOK
	n += needleFlagSize
	BigEndian.PutInt32(buf[n:], int32(len(data)))
	n += needleSizeSize
	// data
	copy(buf[n:], data)
	n += len(data)
	// footer
	copy(buf[n:], needleFooterMagic)
	n += needleMagicSize
	BigEndian.PutUint32(buf[n:], checksum)
	n += needleChecksumSize
	copy(buf[n:], needlePadding[padding])
	return
}
Esempio n. 5
0
// ParseNeedleData parse a needle data part.
func ParseNeedleData(buf []byte, n *Needle) (err error) {
	var (
		bn       int32
		checksum uint32
	)
	n.Data = buf[:n.Size]
	bn += n.Size
	n.FooterMagic = buf[bn : bn+needleMagicSize]
	if bytes.Compare(n.FooterMagic, needleFooterMagic) != 0 {
		err = ErrNeedleFooterMagic
		return
	}
	bn += needleMagicSize
	checksum = crc32.Update(0, crc32Table, n.Data)
	n.Checksum = BigEndian.Uint32(buf[bn : bn+needleChecksumSize])
	if n.Checksum != checksum {
		err = ErrNeedleChecksum
		return
	}
	bn += needleChecksumSize
	n.Padding = buf[bn : bn+n.PaddingSize]
	log.Printf("padding: %d, %v vs %v\n", n.PaddingSize, n.Padding, needlePadding[n.PaddingSize])
	if bytes.Compare(n.Padding, needlePadding[n.PaddingSize]) != 0 {
		err = ErrNeedlePaddingNotMatch
		return
	}
	return
}
Esempio n. 6
0
func (r *Receiver) handleTransfer(conn io.ReadWriter) (
	transfer.TransferResults, error,
) {
	if _, err := conn.Write([]byte("ok")); err != nil {
		return transfer.TransferResults{}, err
	}

	res := transfer.TransferResults{}
	buffer := make([]byte, 1024)

	startTime := time.Now()
	for {
		n, err := conn.Read(buffer)
		if err != nil { // done reading
			break
		}

		res.BytesSent += uint32(n)
		res.Checksum = crc32.Update(res.Checksum, crc32.IEEETable, buffer)
	}
	endTime := time.Now()

	res.Duration = endTime.Sub(startTime)

	return res, nil
}
Esempio n. 7
0
// WriteFrom Write needle from io.Reader into buffer.
func (ns *Needles) WriteFrom(key int64, cookie int32, size int32, rd io.Reader) (err error) {
	var (
		n            *Needle
		data         []byte
		headerOffset int32
		dataOffset   int32
		footerOffset int32
		endOffset    int32
	)
	if ns.wn >= ns.Num {
		return errors.ErrNeedleFull
	}
	n = &ns.needles[ns.wn]
	n.initSize(key, cookie, size)
	headerOffset = ns.ws
	dataOffset = headerOffset + _headerSize
	footerOffset = dataOffset + n.Size
	endOffset = footerOffset + n.FooterSize
	data = ns.buffer[dataOffset:footerOffset]
	// write into buffer header->data->footer
	if err = n.WriteHeader(ns.buffer[headerOffset:dataOffset]); err == nil {
		if _, err = rd.Read(data); err == nil {
			n.Data = data
			n.Checksum = crc32.Update(0, _crc32Table, data)
			err = n.WriteFooter(ns.buffer[footerOffset:endOffset])
		}
	}
	ns.wn++
	ns.ws += n.TotalSize
	return
}
Esempio n. 8
0
// FillNeedle fill needle buffer.
func FillNeedle(padding, size int32, key, cookie int64, data, buf []byte) {
	var (
		n        int
		checksum = crc32.Update(0, crc32Table, data)
	)
	// --- header ---
	// magic
	copy(buf[:needleMagicSize], needleHeaderMagic)
	n += needleMagicSize
	// cookie
	BigEndian.PutInt64(buf[n:], cookie)
	n += needleCookieSize
	// key
	BigEndian.PutInt64(buf[n:], key)
	n += needleKeySize
	// flag
	buf[n] = NeedleStatusOK
	n += needleFlagSize
	// size
	BigEndian.PutInt32(buf[n:], size)
	n += needleSizeSize
	// data
	copy(buf[n:], data)
	n += len(data)
	// --- footer ---
	// magic
	copy(buf[n:], needleFooterMagic)
	n += needleMagicSize
	// checksum
	BigEndian.PutUint32(buf[n:], checksum)
	n += needleChecksumSize
	// padding
	copy(buf[n:], needlePadding[padding])
	return
}
Esempio n. 9
0
func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
	start := time.Now()

	fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
	b := pbutil.MustMarshal(snapshot)
	crc := crc32.Update(0, crcTable, b)
	snap := snappb.Snapshot{Crc: crc, Data: b}
	d, err := snap.Marshal()
	if err != nil {
		return err
	} else {
		marshallingDurations.Observe(float64(time.Since(start)) / float64(time.Second))
	}

	err = pioutil.WriteAndSyncFile(path.Join(s.dir, fname), d, 0666)
	if err == nil {
		saveDurations.Observe(float64(time.Since(start)) / float64(time.Second))
	} else {
		err1 := os.Remove(path.Join(s.dir, fname))
		if err1 != nil {
			plog.Errorf("failed to remove broken snapshot file %s", path.Join(s.dir, fname))
		}
	}
	return err
}
Esempio n. 10
0
// Read reads the snapshot named by snapname and returns the snapshot.
func Read(snapname string) (*raftpb.Snapshot, error) {
	b, err := ioutil.ReadFile(snapname)
	if err != nil {
		log.Printf("snap: snapshotter cannot read file %v: %v", snapname, err)
		return nil, err
	}

	var serializedSnap snappb.Snapshot
	if err = serializedSnap.Unmarshal(b); err != nil {
		log.Printf("snap: corrupted snapshot file %v: %v", snapname, err)
		return nil, err
	}

	if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
		log.Printf("snap: unexpected empty snapshot")
		return nil, ErrEmptySnapshot
	}

	crc := crc32.Update(0, crcTable, serializedSnap.Data)
	if crc != serializedSnap.Crc {
		log.Printf("snap: corrupted snapshot file %v: crc mismatch", snapname)
		return nil, ErrCRCMismatch
	}

	var snap raftpb.Snapshot
	if err = snap.Unmarshal(serializedSnap.Data); err != nil {
		log.Printf("snap: corrupted snapshot file %v: %v", snapname, err)
		return nil, err
	}
	return &snap, nil
}
Esempio n. 11
0
// Write writes a compressed form of p to the underlying io.Writer. The
// compressed bytes are not necessarily flushed until the Writer is closed.
func (z *Writer) Write(p []byte) (int, error) {
	if z.err != nil {
		return 0, z.err
	}
	var n int
	// Write the GZIP header lazily.
	if !z.wroteHeader {
		z.wroteHeader = true
		z.buf = [10]byte{0: gzipID1, 1: gzipID2, 2: gzipDeflate}
		if z.Extra != nil {
			z.buf[3] |= 0x04
		}
		if z.Name != "" {
			z.buf[3] |= 0x08
		}
		if z.Comment != "" {
			z.buf[3] |= 0x10
		}
		if z.ModTime.After(time.Unix(0, 0)) {
			// Section 2.3.1, the zero value for MTIME means that the
			// modified time is not set.
			le.PutUint32(z.buf[4:8], uint32(z.ModTime.Unix()))
		}
		if z.level == BestCompression {
			z.buf[8] = 2
		} else if z.level == BestSpeed {
			z.buf[8] = 4
		}
		z.buf[9] = z.OS
		n, z.err = z.w.Write(z.buf[:10])
		if z.err != nil {
			return n, z.err
		}
		if z.Extra != nil {
			z.err = z.writeBytes(z.Extra)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.Name != "" {
			z.err = z.writeString(z.Name)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.Comment != "" {
			z.err = z.writeString(z.Comment)
			if z.err != nil {
				return n, z.err
			}
		}
		if z.compressor == nil {
			z.compressor, _ = flate.NewWriter(z.w, z.level)
		}
	}
	z.size += uint32(len(p))
	z.digest = crc32.Update(z.digest, crc32.IEEETable, p)
	n, z.err = z.compressor.Write(p)
	return n, z.err
}
Esempio n. 12
0
File: needle.go Progetto: wtmmac/bfs
// ParseNeedleData parse a needle data part.
func (n *Needle) ParseData(buf []byte) (err error) {
	var (
		bn       int32
		checksum uint32
	)
	n.Data = buf[:n.Size]
	bn += n.Size
	n.FooterMagic = buf[bn : bn+magicSize]
	if !bytes.Equal(n.FooterMagic, footerMagic) {
		err = errors.ErrNeedleFooterMagic
		return
	}
	bn += magicSize
	checksum = crc32.Update(0, crc32Table, n.Data)
	n.Checksum = binary.BigEndian.Uint32(buf[bn : bn+checksumSize])
	if n.Checksum != checksum {
		err = errors.ErrNeedleChecksum
		return
	}
	bn += checksumSize
	n.Padding = buf[bn : bn+n.PaddingSize]
	if !bytes.Equal(n.Padding, padding[n.PaddingSize]) {
		err = errors.ErrNeedlePadding
	}
	return
}
Esempio n. 13
0
File: gunzip.go Progetto: 2thetop/go
// readString reads a NUL-terminated string from z.r.
// It treats the bytes read as being encoded as ISO 8859-1 (Latin-1) and
// will output a string encoded using UTF-8.
// This method always updates z.digest with the data read.
func (z *Reader) readString() (string, error) {
	var err error
	needConv := false
	for i := 0; ; i++ {
		if i >= len(z.buf) {
			return "", ErrHeader
		}
		z.buf[i], err = z.r.ReadByte()
		if err != nil {
			return "", err
		}
		if z.buf[i] > 0x7f {
			needConv = true
		}
		if z.buf[i] == 0 {
			// Digest covers the NUL terminator.
			z.digest = crc32.Update(z.digest, crc32.IEEETable, z.buf[:i+1])

			// Strings are ISO 8859-1, Latin-1 (RFC 1952, section 2.3.1).
			if needConv {
				s := make([]rune, 0, i)
				for _, v := range z.buf[:i] {
					s = append(s, rune(v))
				}
				return string(s), nil
			}
			return string(z.buf[:i]), nil
		}
	}
}
Esempio n. 14
0
// PushRow is part of the RowReceiver interface.
func (hr *hashRouter) PushRow(row sqlbase.EncDatumRow) bool {
	if hr.err != nil {
		return false
	}
	hr.buffer = hr.buffer[:0]
	for _, col := range hr.hashCols {
		if int(col) >= len(row) {
			hr.err = errors.Errorf("hash column %d, stream with only %d columns", col, len(row))
			return false
		}
		// TODO(radu): we should choose an encoding that is already available as
		// much as possible. However, we cannot decide this locally as multiple
		// nodes may be doing the same hashing and the encodings need to match. The
		// encoding needs to be determined at planning time.
		hr.buffer, hr.err = row[col].Encode(&hr.alloc, preferredEncoding, hr.buffer)
		if hr.err != nil {
			return false
		}
	}

	// We use CRC32-C because it makes for a decent hash function and is faster
	// than most hashing algorithms (on recent x86 platforms where it is hardware
	// accelerated).
	streamIdx := crc32.Update(0, crc32Table, hr.buffer) % uint32(len(hr.streams))

	// We can't return false if this stream happened to not need any more rows. We
	// could only return false once all streams returned false, but that seems of
	// limited benefit.
	_ = hr.streams[streamIdx].PushRow(row)
	return true
}
Esempio n. 15
0
func TestNeedles(t *testing.T) {
	var (
		err       error
		tn        *Needle
		data1     = []byte("tes1")
		checksum1 = crc32.Update(0, _crc32Table, data1)
		data2     = []byte("tes2")
		checksum2 = crc32.Update(0, _crc32Table, data2)
		ns        = NewNeedles(2)
		buf       = &bytes.Buffer{}
	)
	if _, err = buf.Write(data1); err != nil {
		t.FailNow()
	}
	if err = ns.ReadFrom(1, 1, 4, buf); err != nil {
		t.FailNow()
	}
	if _, err = buf.Write(data2); err != nil {
		t.FailNow()
	}
	if err = ns.ReadFrom(2, 2, 4, buf); err != nil {
		t.FailNow()
	}
	tn = new(Needle)
	tn.buffer = ns.Next().Buffer()
	if err = tn.Parse(); err != nil {
		t.FailNow()
	}
	t.Log(tn)
	compareNeedle(t, tn, 1, 1, data1, FlagOK, checksum1)
	tn = new(Needle)
	tn.buffer = ns.Next().Buffer()
	if err = tn.Parse(); err != nil {
		t.FailNow()
	}
	t.Log(tn)
	compareNeedle(t, tn, 2, 2, data2, FlagOK, checksum2)
	if err = ns.ReadFrom(3, 3, 4, buf); err != errors.ErrNeedleFull {
		t.FailNow()
	}
	if tn = ns.Next(); tn != nil {
		t.FailNow()
	}
}
Esempio n. 16
0
File: needle.go Progetto: 1d7500/bfs
// ParseData parse a needle data part.
func (n *Needle) ParseData(buf []byte) (err error) {
	if len(buf) != int(n.Size) {
		return errors.ErrNeedleDataSize
	}
	// data
	n.Data = buf
	// checksum
	n.Checksum = crc32.Update(0, _crc32Table, n.Data)
	return
}
Esempio n. 17
0
func (n *Needle) Init(key int64, cookie int64, data []byte) {
	var dataSize = int64(len(data))
	n.Header = headerMagic
	n.Key = key
	n.Cookie = cookie
	n.Size = dataSize
	n.Data = data
	n.Footer = footerMagic
	n.DataChecksum = crc32.Update(0, crc32Table, data)
}
Esempio n. 18
0
func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
	fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
	b := pbutil.MustMarshal(snapshot)
	crc := crc32.Update(0, crcTable, b)
	snap := snappb.Snapshot{Crc: crc, Data: b}
	d, err := snap.Marshal()
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path.Join(s.dir, fname), d, 0666)
}
Esempio n. 19
0
File: needle.go Progetto: 1d7500/bfs
// WriteFrom Write needle from io.Reader into buffer.
func (n *Needle) WriteFrom(rd io.Reader) (err error) {
	var (
		dataOffset = _headerSize + n.Size
		data       = n.Buffer[_headerSize:dataOffset]
	)
	if err = n.WriteHeader(n.Buffer[:_headerSize]); err == nil {
		if _, err = rd.Read(data); err == nil {
			n.Data = data
			n.Checksum = crc32.Update(0, _crc32Table, data)
			err = n.WriteFooter(n.Buffer[dataOffset:n.TotalSize])
		}
	}
	return
}
Esempio n. 20
0
// private native static int updateBytes(int crc, byte[] b, int off, int len);
// (I[BII)I
func updateBytes(frame *rtda.Frame) {
	vars := frame.LocalVars()
	crc := uint32(vars.GetInt(0))
	byteArr := vars.GetRef(1)
	off := vars.GetInt(2)
	_len := vars.GetInt(3)

	goBytes := byteArr.GoBytes()
	goBytes = goBytes[off : off+_len]
	// func Update(crc uint32, tab *Table, p []byte) uint32
	crc = crc32.Update(crc, crc32.IEEETable, goBytes)

	stack := frame.OperandStack()
	stack.PushInt(int32(crc))
}
Esempio n. 21
0
File: needle.go Progetto: wtmmac/bfs
// Parse parse needle from data.
func (n *Needle) Parse(key int64, cookie int32, data []byte) {
	var dataSize = int32(len(data))
	n.TotalSize = int32(HeaderSize + dataSize + FooterSize)
	n.PaddingSize = align(n.TotalSize) - n.TotalSize
	n.TotalSize += n.PaddingSize
	n.HeaderMagic = headerMagic
	n.Key = key
	n.Cookie = cookie
	n.Size = dataSize
	n.Data = data
	n.FooterMagic = footerMagic
	n.Checksum = crc32.Update(0, crc32Table, data)
	n.Padding = padding[n.PaddingSize]
	return
}
Esempio n. 22
0
// updateCRC returns the result of adding the bytes in buf to the crc.
func updateCRC(crc uint32, buf []byte) uint32 {
	// The CRC-32 computation in bzip2 treats bytes as having bits in big-endian
	// order. That is, the MSB is read before the LSB. Thus, we can use the
	// standard library version of CRC-32 IEEE with some minor adjustments.
	crc = internal.ReverseUint32(crc)
	var arr [4096]byte
	for len(buf) > 0 {
		cnt := copy(arr[:], buf)
		buf = buf[cnt:]
		for i, b := range arr[:cnt] {
			arr[i] = internal.ReverseLUT[b]
		}
		crc = crc32.Update(crc, crc32.IEEETable, arr[:cnt])
	}
	return internal.ReverseUint32(crc)
}
Esempio n. 23
0
func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
	start := time.Now()

	fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
	b := pbutil.MustMarshal(snapshot)
	crc := crc32.Update(0, crcTable, b)
	snap := snappb.Snapshot{Crc: crc, Data: b}
	d, err := snap.Marshal()
	if err != nil {
		return err
	}
	err = ioutil.WriteFile(path.Join(s.dir, fname), d, 0666)
	if err == nil {
		saveDurations.Observe(float64(time.Since(start).Nanoseconds() / int64(time.Microsecond)))
	}
	return err
}
Esempio n. 24
0
// WriteFrom read from io.Reader and write into needle buffer.
func (n *Needle) WriteFrom(key int64, cookie, size int32, rd io.Reader) (err error) {
	var (
		dataOffset int32
		data       []byte
	)
	n.initSize(key, cookie, size)
	dataOffset = _headerSize + n.Size
	data = n.buffer[_headerSize:dataOffset]
	if err = n.writeHeader(n.buffer[:_headerSize]); err == nil {
		if _, err = rd.Read(data); err == nil {
			n.Data = data
			n.Checksum = crc32.Update(0, _crc32Table, data)
			err = n.writeFooter(n.buffer[dataOffset:n.TotalSize])
		}
	}
	return
}
Esempio n. 25
0
// Parse parse needle from data.
func (n *Needle) Parse(key, cookie int64, data []byte) (err error) {
	var dataSize = int32(len(data))
	n.TotalSize = int32(NeedleHeaderSize + dataSize + NeedleFooterSize)
	n.PaddingSize = NeedlePaddingSize - (n.TotalSize % NeedlePaddingSize)
	n.TotalSize += n.PaddingSize
	if n.TotalSize > NeedleMaxSize {
		err = ErrNeedleTooLarge
		return
	}
	n.Key = key
	n.Cookie = cookie
	n.Size = dataSize
	n.Data = data
	n.Checksum = crc32.Update(0, crc32Table, data)
	n.Padding = needlePadding[n.PaddingSize]
	return
}
Esempio n. 26
0
File: needle.go Progetto: 1d7500/bfs
// WriteFrom Write needle from io.Reader into buffer.
func (ns *Needles) WriteFrom(n *Needle, rd io.Reader) (err error) {
	var (
		headerOffset = ns.TotalSize
		dataOffset   = headerOffset + _headerSize
		footerOffset = dataOffset + n.Size
		endOffset    = footerOffset + n.FooterSize
		data         = ns.Buffer[dataOffset:footerOffset]
	)
	if err = n.WriteHeader(ns.Buffer[headerOffset:dataOffset]); err == nil {
		if _, err = rd.Read(data); err == nil {
			n.Data = data
			n.Checksum = crc32.Update(0, _crc32Table, data)
			err = n.WriteFooter(ns.Buffer[footerOffset:endOffset])
		}
	}
	ns.TotalSize += n.TotalSize
	return
}
Esempio n. 27
0
func (r *Ring) FindMultiple(key []byte, count int) ([]string, bool) {
	r.mutex.RLock()
	defer r.mutex.RUnlock()
	if count <= 0 {
		return nil, false
	}
	c, n := crc32.Update(0, crcTable, key), len(r.buckets)
	if n == 0 {
		return nil, false
	}
	i, j := 0, n
	for i < j {
		p := i + (j-i)/2
		if r.buckets[p].id < c {
			i = p + 1
		} else {
			j = p
		}
	}
	if i == n {
		i = 0
	}
	first := i
	found := []string{r.buckets[i].node}
outer:
	for count > len(found) {
		i += 1
		if i == n {
			i = 0
		}
		if i == first {
			break
		}
		next := r.buckets[i].node
		for _, node := range found {
			if node == next {
				continue outer
			}
		}
		found = append(found, next)
	}
	return found, true
}
Esempio n. 28
0
File: gunzip.go Progetto: 2thetop/go
func (z *Reader) Read(p []byte) (n int, err error) {
	if z.err != nil {
		return 0, z.err
	}

	n, z.err = z.decompressor.Read(p)
	z.digest = crc32.Update(z.digest, crc32.IEEETable, p[:n])
	z.size += uint32(n)
	if z.err != io.EOF {
		// In the normal case we return here.
		return n, z.err
	}

	// Finished file; check checksum and size.
	if _, err := io.ReadFull(z.r, z.buf[:8]); err != nil {
		z.err = noEOF(err)
		return n, z.err
	}
	digest := le.Uint32(z.buf[:4])
	size := le.Uint32(z.buf[4:8])
	if digest != z.digest || size != z.size {
		z.err = ErrChecksum
		return n, z.err
	}
	z.digest, z.size = 0, 0

	// File is ok; check if there is another.
	if !z.multistream {
		return n, io.EOF
	}
	z.err = nil // Remove io.EOF

	if _, z.err = z.readHeader(); z.err != nil {
		return n, z.err
	}

	// Read from next file, if necessary.
	if n > 0 {
		return n, nil
	}
	return z.Read(p)
}
Esempio n. 29
0
func TestCPUProfileMultithreaded(t *testing.T) {
	buf := make([]byte, 100000)
	defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(2))
	testCPUProfile(t, []string{"crc32.update"}, func() {
		c := make(chan int)
		go func() {
			for i := 0; i < 2000; i++ {
				crc32.Update(0, crc32.IEEETable, buf)
			}
			c <- 1
		}()
		// This loop takes about a quarter second on a 2 GHz laptop.
		// We only need to get one 100 Hz clock tick, so we've got
		// a 25x safety buffer.
		for i := 0; i < 2000; i++ {
			crc32.ChecksumIEEE(buf)
		}
		<-c
	})
}
Esempio n. 30
0
func loadSnap(dir, name string) (*raftpb.Snapshot, error) {
	var err error
	var b []byte

	fpath := path.Join(dir, name)
	defer func() {
		if err != nil {
			renameBroken(fpath)
		}
	}()

	b, err = ioutil.ReadFile(fpath)
	if err != nil {
		log.Printf("snap: snapshotter cannot read file %v: %v", name, err)
		return nil, err
	}

	var serializedSnap snappb.Snapshot
	if err = serializedSnap.Unmarshal(b); err != nil {
		log.Printf("snap: corrupted snapshot file %v: %v", name, err)
		return nil, err
	}

	if len(serializedSnap.Data) == 0 || serializedSnap.Crc == 0 {
		log.Printf("snap: unexpected empty snapshot")
		return nil, ErrEmptySnapshot
	}

	crc := crc32.Update(0, crcTable, serializedSnap.Data)
	if crc != serializedSnap.Crc {
		log.Printf("snap: corrupted snapshot file %v: crc mismatch", name)
		return nil, ErrCRCMismatch
	}

	var snap raftpb.Snapshot
	if err = snap.Unmarshal(serializedSnap.Data); err != nil {
		log.Printf("snap: corrupted snapshot file %v: %v", name, err)
		return nil, err
	}
	return &snap, nil
}