Exemple #1
0
// readBlock reads a full or partial block into p from rs. len(p) must be smaller than (bs - crc32Len).
// The checksum of the block is calculated and verified.
// ErrBadCRC is returned if the checksum is invalid.
func readBlock(rs io.ReadSeeker, p []byte, index, bs int64) (int64, error) {
	payloadLen := int(bs - crc32Len)
	if len(p) > payloadLen {
		return 0, ErrPayloadSizeTooLarge
	}
	b := make([]byte, crc32Len)

	_, err := rs.Seek(index*bs, os.SEEK_SET)
	if err != nil {
		return 0, err
	}

	n, err := rs.Read(b)
	// Cannot read full crc
	if n > 0 && n < 4 {
		return 0, ErrBadCRC
	}
	if err != nil {
		return 0, err
	}

	crc := binary.BigEndian.Uint32(b)
	if len(p) == payloadLen {
		// fast path for reading into a buffer of payload size
		n, err = rs.Read(p)
		if err != nil {
			return int64(n), err
		}
		// Invalid crc
		if crc != crc32.Checksum(p[:n], crc32cTable) {
			return 0, ErrBadCRC
		}
		return int64(n), nil
	} else {
		// p is smaller than payload size
		// read into another buffer (so we can compare full block CRC) and copy into p
		buf := make([]byte, payloadLen)
		n, err = rs.Read(buf)
		// If there is an eof returned, two cases may happen
		// 1. n < len(p) -- we don't have enough data to
		// fill into p, we should return the eof
		// 2. n >= len(p) -- we have enough data to fill
		// into p, shoudln't return the eof
		if err == io.EOF && n >= len(p) {
			err = nil
		}
		if err != nil {
			return int64(copy(p, buf[:n])), err
		}
		if crc != crc32.Checksum(buf[:n], crc32cTable) {
			return 0, ErrBadCRC
		}
		return int64(copy(p, buf[:n])), nil
	}
}
Exemple #2
0
func (p fakePeers) PickPeer(key string) (peer ProtoGetter, ok bool) {
	if len(p) == 0 {
		return
	}
	n := crc32.Checksum([]byte(key), crc32.IEEETable) % uint32(len(p))
	return p[n], p[n] != nil
}
Exemple #3
0
//一个packet包括包头和包体,保证在接收到包头后两秒钟内接收到包体,否则线程会一直阻塞
//因此,引入了超时机制
func RecvPackage(conn net.Conn) (packet *GetPacket, e error, isClose bool) {
	var data []byte
	header := make([]byte, 12)
	if conn == nil {
		// e = errors.New("")
		fmt.Println("连接已经关闭")
		isClose = true
		return
	}
	n, err := io.ReadFull(conn, header)
	if n == 0 && err == io.EOF {
		// fmt.Println("客户端断开连接")
		isClose = true
		e = err
		return
	} else if err != nil {
		// fmt.Println("接收数据出错:", err)
		isClose = true
		e = err
		return
	}
	//数据包长度
	size := binary.BigEndian.Uint32(header)
	//crc值
	crc1 := binary.BigEndian.Uint32(header[4:8])
	msgID := binary.BigEndian.Uint32(header[8:12])

	data = make([]byte, size)

	timeout := NewTimeOut(func() {

		n, err = io.ReadFull(conn, data)
		if uint32(n) != size {
			log.Println("数据包长度不正确", n, "!=", size)
			e = errors.New(fmt.Sprint("数据包长度不正确:%d!=%d", n, size))
			return
		}
		if err != nil {
			log.Println("读取数据出错:", err)
			e = err
			return
		}
		crc2 := crc32.Checksum(data, crc32.IEEETable)
		if crc1 != crc2 {
			log.Println("crc 数据验证不正确: ", crc1, " != ", crc2)
			e = errors.New(fmt.Sprint("crc 数据验证不正确:%d!=%d", crc1, crc2))
			return
		}
		packet = new(GetPacket)
		packet.Date = data
		packet.MsgID = int32(msgID)
		packet.Size = uint32(len(data))
	})
	isTimeOut := timeout.Do(time.Second * 5)
	if isTimeOut {
		e = errors.New("数据包头和数据包体不完整")
		return
	}
	return
}
Exemple #4
0
// writeBlock writes a full or partial block into ws. len(p) must be smaller than (bs - crc32Len).
// Each block contains a crc32Len bytes crc32c of the payload and a (bs-crc32Len)
// bytes payload. Any error encountered is returned.
// It is caller's responsibility to ensure that only the last block in the file has
// len(p) < bs - crc32Len
func writeBlock(ws io.WriteSeeker, index, bs int64, p []byte) error {
	if int64(len(p)) > bs-crc32Len {
		return ErrPayloadSizeTooLarge
	}

	// seek to the beginning of the block
	_, err := ws.Seek(index*bs, os.SEEK_SET)
	if err != nil {
		return err
	}

	// write crc32c
	// TODO: reuse buffer
	b := make([]byte, crc32Len)
	binary.BigEndian.PutUint32(b, crc32.Checksum(p, crc32cTable))
	_, err = ws.Write(b)
	if err != nil {
		return err
	}

	// write payload
	_, err = ws.Write(p)
	if err != nil {
		return err
	}
	return nil
}
func get(key int64, circle map[uint32]string) string {
	keyStr := strconv.Itoa(int(key))
	if len(circle) == 0 {
		return ""
	} else {
		hash := crc32.Checksum([]byte(keyStr), crc32q)
		var keys []int
		for k := range circle {
			keys = append(keys, int(k))
		}
		sort.Ints(keys)
		var serverHash uint32
		foundHash := false
		for i := 0; i < len(keys); i++ {
			if keys[i] >= int(hash) {
				serverHash = uint32(keys[i])
				foundHash = true
				break
			}
		}
		if foundHash == true {
			return circle[serverHash]
		} else {
			return circle[uint32(keys[0])]
		}

	}
}
func (s *blockWriteStream) makePacket() outboundPacket {
	packetLength := outboundPacketSize
	if s.buf.Len() < outboundPacketSize {
		packetLength = s.buf.Len()
	}

	numChunks := int(math.Ceil(float64(packetLength) / float64(outboundChunkSize)))
	packet := outboundPacket{
		seqno:     s.seqno,
		offset:    s.offset,
		last:      false,
		checksums: make([]byte, numChunks*4),
		data:      make([]byte, packetLength),
	}

	// TODO: we shouldn't actually need this extra copy. We should also be able
	// to "reuse" packets.
	io.ReadFull(&s.buf, packet.data)

	// Fill in the checksum for each chunk of data.
	for i := 0; i < numChunks; i++ {
		chunkOff := i * outboundChunkSize
		chunkEnd := chunkOff + outboundChunkSize
		if chunkEnd >= len(packet.data) {
			chunkEnd = len(packet.data)
		}

		checksum := crc32.Checksum(packet.data[chunkOff:chunkEnd], crc32.IEEETable)
		binary.BigEndian.PutUint32(packet.checksums[i*4:], checksum)
	}

	return packet
}
Exemple #7
0
func PacketData(msgID uint32, data *[]byte) *[]byte {
	buf := bytes.NewBuffer([]byte{})
	binary.Write(buf, binary.BigEndian, uint32(uint32(len(*data))))
	crc32 := crc32.Checksum(*data, crc32.IEEETable)
	binary.Write(buf, binary.BigEndian, crc32)
	// binary.Write(msgID, binary.BigEndian, )
	buf.Write(*data)
	bs := buf.Bytes()
	return &bs

	// writer := packet.Writer()
	// //size uint32
	// writer.WriteU32(uint32(len(*data)))
	// //crc32 uint32
	// crc32 := crc32.Checksum(*data, crc32.IEEETable)
	// writer.WriteU32(crc32)

	// //msgID
	// writer.WriteU32(msgID)
	// //Data
	// writer.WriteRawBytes(*data)
	// bs := writer.Data()

	// return &bs
}
Exemple #8
0
func LockPartition(pg *sql.DB, ns string, max uint64) (uint64, error) {
	tab := crc32.MakeTable(crc32.IEEE)
	for {
		var p uint64
		for p = 0; p < max; p++ {
			pId := fmt.Sprintf("%s.%d", ns, p)
			check := crc32.Checksum([]byte(pId), tab)
			rows, err := pg.Query("select pg_try_advisory_lock($1)", check)
			if err != nil {
				continue
			}
			for rows.Next() {
				var result sql.NullBool
				rows.Scan(&result)
				if result.Valid && result.Bool {
					fmt.Printf("at=%q partition=%d max=%d\n",
						"acquired-lock", p, max)
					rows.Close()
					return p, nil
				}
			}
			rows.Close()
		}
		fmt.Printf("at=%q\n", "waiting-for-partition-lock")
		time.Sleep(time.Second * 10)
	}
	return 0, errors.New("Unable to lock partition.")
}
Exemple #9
0
func calcNewCRC32(path string, newHeader *BinaryHeader) (uint32, error) {
	buf := &bytes.Buffer{}
	err := binary.Write(buf, binary.BigEndian, newHeader)
	if err != nil {
		return 0, err
	}

	header := buf.Bytes()
	crc := crc32.Checksum(header[:28], crc32.IEEETable) // before crc32
	crc = crc32.Update(crc, crc32.IEEETable, header[32:])

	f, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	defer f.Close()

	if _, err := f.Seek(int64(len(header)), os.SEEK_SET); err != nil {
		return 0, err
	}

	data := make([]byte, 4*1024*1024)
	for {
		n, err := f.Read(data)
		crc = crc32.Update(crc, crc32.IEEETable, data[:n])
		if err != nil {
			if err != io.EOF {
				return 0, err
			}
			break
		}
	}
	return crc, nil
}
Exemple #10
0
func TestReadRecord(t *testing.T) {
	badInfoRecord := make([]byte, len(infoRecord))
	copy(badInfoRecord, infoRecord)
	badInfoRecord[len(badInfoRecord)-1] = 'a'

	tests := []struct {
		data []byte
		wr   *walpb.Record
		we   error
	}{
		{infoRecord, &walpb.Record{Type: 1, Crc: crc32.Checksum(infoData, crcTable), Data: infoData}, nil},
		{[]byte(""), &walpb.Record{}, io.EOF},
		{infoRecord[:8], &walpb.Record{}, io.ErrUnexpectedEOF},
		{infoRecord[:len(infoRecord)-len(infoData)-8], &walpb.Record{}, io.ErrUnexpectedEOF},
		{infoRecord[:len(infoRecord)-len(infoData)], &walpb.Record{}, io.ErrUnexpectedEOF},
		{infoRecord[:len(infoRecord)-8], &walpb.Record{}, io.ErrUnexpectedEOF},
		{badInfoRecord, &walpb.Record{}, walpb.ErrCRCMismatch},
	}

	rec := &walpb.Record{}
	for i, tt := range tests {
		buf := bytes.NewBuffer(tt.data)
		decoder := newDecoder(ioutil.NopCloser(buf))
		e := decoder.decode(rec)
		if !reflect.DeepEqual(rec, tt.wr) {
			t.Errorf("#%d: block = %v, want %v", i, rec, tt.wr)
		}
		if !reflect.DeepEqual(e, tt.we) {
			t.Errorf("#%d: err = %v, want %v", i, e, tt.we)
		}
		rec = &walpb.Record{}
	}
}
Exemple #11
0
func (c *Connection) parcelValidity(parcel Parcel) uint8 {
	verbose(c.peer.PeerIdent(), "Connection.isValidParcel(%s)", parcel.MessageType())
	crc := crc32.Checksum(parcel.Payload, CRCKoopmanTable)
	switch {
	case parcel.Header.NodeID == NodeID: // We are talking to ourselves!
		parcel.Trace("Connection.isValidParcel()-loopback", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to loopback!: %+v", parcel.Header)
		c.peer.QualityScore = MinumumQualityScore - 50 // Ban ourselves for a week
		return InvalidDisconnectPeer
	case parcel.Header.Network != CurrentNetwork:
		parcel.Trace("Connection.isValidParcel()-network", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong network. Remote: %0x Us: %0x", parcel.Header.Network, CurrentNetwork)
		return InvalidDisconnectPeer
	case parcel.Header.Version < ProtocolVersionMinimum:
		parcel.Trace("Connection.isValidParcel()-version", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong version: %+v", parcel.Header)
		return InvalidDisconnectPeer
	case parcel.Header.Length != uint32(len(parcel.Payload)):
		parcel.Trace("Connection.isValidParcel()-length", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong length: %+v", parcel.Header)
		return InvalidPeerDemerit
	case parcel.Header.Crc32 != crc:
		parcel.Trace("Connection.isValidParcel()-checksum", "H")
		significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to bad checksum: %+v", parcel.Header)
		return InvalidPeerDemerit
	default:
		parcel.Trace("Connection.isValidParcel()-ParcelValid", "H")
		return ParcelValid
	}
	return ParcelValid
}
Exemple #12
0
// Create an object struct for the given attributes and contents.
//
// LOCKS_REQUIRED(b.mu)
func (b *bucket) mintObject(
	req *gcs.CreateObjectRequest,
	contents []byte) (o fakeObject) {
	md5Sum := md5.Sum(contents)

	// Set up basic info.
	b.prevGeneration++
	o.metadata = gcs.Object{
		Name:            req.Name,
		ContentType:     req.ContentType,
		ContentLanguage: req.ContentLanguage,
		CacheControl:    req.CacheControl,
		Owner:           "user-fake",
		Size:            uint64(len(contents)),
		ContentEncoding: req.ContentEncoding,
		ComponentCount:  1,
		MD5:             &md5Sum,
		CRC32C:          crc32.Checksum(contents, crc32cTable),
		MediaLink:       "http://localhost/download/storage/fake/" + req.Name,
		Metadata:        copyMetadata(req.Metadata),
		Generation:      b.prevGeneration,
		MetaGeneration:  1,
		StorageClass:    "STANDARD",
		Updated:         b.clock.Now(),
	}

	// Set up data.
	o.data = contents

	return
}
Exemple #13
0
// sendSerializedData sets the leading CRC on buf and writes the data to conn.  Any errors on the
// write will be logged but otherwise ignored.
func sendSerializedData(buf []byte, conn io.Writer) {
	// Prefix buf with a crc of everything we've appended to it.
	AppendUint32(buf[0:0], crc32.Checksum(buf[4:], crcTable))
	_, err := conn.Write(buf)
	if err != nil {
		log.Printf("Failed to write %d bytes in BatchAndSend: %v", err)
	}
}
Exemple #14
0
// serializeWriteRow serialize the current block file and offset where new
// will be written into a format suitable for storage into the metadata.
func serializeWriteRow(curBlockFileNum, curFileOffset uint32) []byte {
	var serializedRow [12]byte
	byteOrder.PutUint32(serializedRow[0:4], curBlockFileNum)
	byteOrder.PutUint32(serializedRow[4:8], curFileOffset)
	checksum := crc32.Checksum(serializedRow[:8], castagnoli)
	byteOrder.PutUint32(serializedRow[8:12], checksum)
	return serializedRow[:]
}
Exemple #15
0
func Benchmark_crc32_6(b *testing.B) {
	buf, n := initSample(6)
	tab := crc32.MakeTable(crc32.Castagnoli)
	b.SetBytes(n)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		crc32.Checksum(buf, tab)
	}
}
Exemple #16
0
//接收客户端消息协程
func (this *ServerConn) recv() {
	//处理客户端主动断开连接的情况
	defer func(clientConn *ServerConn) {
		log.Println("客户端主动关闭连接")
	}(this)

	for !this.isClose {
		var data []byte
		header := make([]byte, 12)

		n, err := io.ReadFull(this.conn, header)
		if n == 0 && err == io.EOF {
			// temp := new(GetPacket)
			// temp.ConnId = this.session
			// temp.MsgID = int32(Close)
			// this.inPack <- temp
			fmt.Println("客户端断开连接")
			return
		} else if err != nil {
			fmt.Println("接收数据出错:", err)
			return
		}
		//数据包长度

		size := binary.BigEndian.Uint32(header)
		//crc值
		crc1 := binary.BigEndian.Uint32(header[4:8])
		// msgID := binary.BigEndian.Uint32(header[8:12])

		data = make([]byte, size)
		n, err = io.ReadFull(this.conn, data)
		if uint32(n) != size {
			log.Println("数据包长度不正确", n, "!=", size)
			return
		}
		if err != nil {
			log.Println("读取数据出错:", err)
			return
		}

		crc2 := crc32.Checksum(data, crc32.IEEETable)
		if crc1 != crc2 {
			log.Println("crc 数据验证不正确: ", crc1, " != ", crc2)
			return
		}

		// temp := new(GetPacket)
		// temp.ConnId = this.session
		// temp.Date = data
		// temp.MsgID = int32(msgID)
		// temp.Size = uint32(len(data))
		// this.inPack <- temp
	}
	//最后一个包接收了之后关闭chan
	//如果有超时包需要等超时了才关闭,目前未做处理
	close(this.outData)
}
Exemple #17
0
func (c *crc32Validator) Checksum(data []byte) (rawCRC []byte) {
	// Calculate the CRC checksum of data.
	crc := crc32.Checksum(data, c.table)

	// Transform to a byte slice.
	rawCRC = make([]byte, 4)
	binary.LittleEndian.PutUint32(rawCRC, crc)

	return rawCRC
}
Exemple #18
0
func (c *Context) Login(email string, admin bool) {
	c.req.Header.Add("X-AppEngine-Internal-User-Email", email)
	c.req.Header.Add("X-AppEngine-Internal-User-Id", strconv.Itoa(int(crc32.Checksum([]byte(email), crc32.IEEETable))))
	c.req.Header.Add("X-AppEngine-Internal-User-Federated-Identity", email)
	if admin {
		c.req.Header.Add("X-AppEngine-Internal-User-Is-Admin", "1")
	} else {
		c.req.Header.Add("X-AppEngine-Internal-User-Is-Admin", "0")
	}
}
Exemple #19
0
func (c *crc32Validator) Validate(data []byte, rawCRC []byte) bool {
	// Convert the raw CRC byte slice.
	origCRC := binary.LittleEndian.Uint32(rawCRC)

	// Calculate the CRC checksum of data.
	crc := crc32.Checksum(data, c.table)

	// Compare the checksums.
	return crc == origCRC
}
Exemple #20
0
func (this *Client) recv() {
	for !this.isClose {

		header := make([]byte, 12)

		n, err := io.ReadFull(this.conn, header)

		if n == 0 && err == io.EOF {
			fmt.Println("客户端断开连接")
			go this.reConnect()
			return
		} else if err != nil {
			fmt.Println("接受数据出错:", err)
			go this.reConnect()
			return
		}

		//数据包长度
		size := binary.BigEndian.Uint32(header)
		//crc值
		crc1 := binary.BigEndian.Uint32(header[4:8])

		// msgID := binary.BigEndian.Uint32(header[8:12])

		data := make([]byte, size)

		n, err = io.ReadFull(this.conn, data)
		if err != nil {
			log.Println("读取数据出错:", err)
			go this.reConnect()
			return
		}
		if uint32(n) != size {
			log.Println("数据包长度不正确", n, "!=", size)
			continue
		}

		crc2 := crc32.Checksum(data, crc32.IEEETable)

		if crc1 != crc2 {
			log.Println("crc 数据验证不正确: ", crc1, " != ", crc2)
			continue
		}

		// temp := new(GetPacket)
		// temp.ConnId = this.Session
		// temp.Date = data
		// temp.MsgID = int32(msgID)
		// temp.Size = uint32(len(data))
		// this.inPack <- temp
	}
	//最后一个包接收了之后关闭chan
	//如果有超时包需要等超时了才关闭,目前未做处理
	close(this.outData)
}
Exemple #21
0
// Generates the byte representation of the document, including the header.
// Empty v interpreted as tombstone.
func newDocument(k, v []byte) []byte {
	outsize := len(k) + len(v) + 12
	out := make([]byte, 12, outsize)
	uint32ToBytes(out, 4, uint32(len(k)))
	uint32ToBytes(out, 8, uint32(len(v)))
	out = append(out, k...)
	out = append(out, v...)
	hash := crc32.Checksum(out[4:], crcTable)
	uint32ToBytes(out, 0, hash)
	return out
}
Exemple #22
0
// Set if key not exists, returns whether already exists.
func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) bool {
	idx := int64(crc32.Checksum(key, sm.crcTable)) % sm.size
	k := string(key)
	_, exist := sm.maps[idx][k]
	if exist && !force {
		return exist
	}

	sm.maps[idx][k] = value
	return exist
}
Exemple #23
0
func (m Message) WriteTo(w io.Writer) (n int64, err error) {
	compression := CompressionTypeNone
	if compression != CompressionTypeNone {
		return -1, fmt.Errorf("Only support none compression for now")
	}

	totalLen := m.Len() - 4 // Subtract the size of the length
	checksum := uint32(crc32.Checksum(m, crc32.IEEETable))

	return binwrite(w, totalLen, MagicTypeWithCompression, compression, checksum, m)
}
Exemple #24
0
func MarshalPacket(msgID uint32, data *[]byte) *[]byte {
	buf := bytes.NewBuffer([]byte{})
	binary.Write(buf, binary.BigEndian, uint32(uint32(len(*data))))
	crc32 := crc32.Checksum(*data, crc32.IEEETable)
	binary.Write(buf, binary.BigEndian, crc32)
	binary.Write(buf, binary.BigEndian, msgID)
	buf.Write(*data)
	bs := buf.Bytes()

	return &bs
}
func (ea *EtcdAero) SetTTL(timerTTL time.Duration) {

	ea.timerTTL = timerTTL
	ea.etcdLockTTL = timerTTL * 3 / 2
	ea.AeroTTL = timerTTL * 5

	// randomize sleep ttl form 0.8 to 1.2 of original timerTTL / 2
	crc32q := crc32.MakeTable(0xD5828281)
	d := 1.0 + float64(int64(crc32.Checksum([]byte(ea.value), crc32q))%1000-500)/2500.0
	ea.sleepTTL = time.Duration(float64(timerTTL) * d / 2)
}
Exemple #26
0
func conformNodeID(ip net.IP, nodeID NodeID) NodeID {
	var b [20]byte
	copy(b[:], nodeID)

	// r is supposed to be random but let's make this deterministic and get it
	// from the last byte of the original NodeID.
	r := byte(b[19]) & 0x07

	ip4 := ip.To4()
	var y uint32
	if ip4 != nil {
		var x [4]byte
		copy(x[:], ip4)
		x[0] &= 0x03
		x[1] &= 0x0F
		x[2] &= 0x3F
		x[3] &= 0xFF
		x[0] |= (r << 5)
		y = crc32.Checksum(x[:], crc32c)
	} else {
		var x [16]byte
		copy(x[:], ip)
		x[0] &= 0x01
		x[1] &= 0x03
		x[2] &= 0x07
		x[3] &= 0x0f
		x[4] &= 0x1f
		x[5] &= 0x3f
		x[6] &= 0x7f
		x[7] &= 0xff
		x[0] |= (r << 5)
		y = crc32.Checksum(x[:], crc32c)
	}

	b[0] = byte((y >> 24) & 0xFF)
	b[1] = byte((y >> 16) & 0xFF)
	b[2] = byte((y>>8)&0xF8) | (b[2] & 0x07)
	b[19] = (b[19] & 0xF8) | r

	return NodeID(string(b[:]))
}
Exemple #27
0
func (s *blockReadStream) validateChecksum(b []byte) error {
	checksumOffset := 4 * s.chunkIndex
	checksumBytes := s.checksums.Bytes()[checksumOffset : checksumOffset+4]
	checksum := binary.BigEndian.Uint32(checksumBytes)

	crc := crc32.Checksum(b, s.checksumTab)
	if crc != checksum {
		return errInvalidChecksum
	}

	return nil
}
Exemple #28
0
// Generate the CRC used to make or validate secure node ID.
func crcIP(ip net.IP, rand uint8) uint32 {
	if ip4 := ip.To4(); ip4 != nil {
		ip = ip4
	}
	// Copy IP so we can make changes. Go sux at this.
	ip = append(make(net.IP, 0, len(ip)), ip...)
	mask := maskForIP(ip)
	for i := range mask {
		ip[i] &= mask[i]
	}
	r := rand & 7
	ip[0] |= r << 5
	return crc32.Checksum(ip[:len(mask)], crc32.MakeTable(crc32.Castagnoli))
}
Exemple #29
0
func (r *reader) nextFrame() error {
	for {
		_, err := io.ReadFull(r.reader, r.hdr)
		if err != nil {
			return err
		}

		buf, err := r.readBlock()
		if err != nil {
			return err
		}

		switch r.hdr[0] {
		case 0x00, 0x01:
			// compressed or uncompressed bytes

			// first 4 bytes are the little endian crc32 checksum
			checksum := unmaskChecksum(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
			b := buf[4:]

			if r.hdr[0] == 0x00 {
				// compressed bytes
				r.dst, err = snappy.Decode(r.dst, b)
				if err != nil {
					return err
				}
				b = r.dst
			}

			if r.verifyChecksum {
				actualChecksum := crc32.Checksum(b, crcTable)
				if checksum != actualChecksum {
					return errors.New(fmt.Sprintf("invalid checksum %x != %x", checksum, actualChecksum))
				}
			}

			_, err = r.buf.Write(b)
			return err
		case 0xff:
			// stream identifier
			if !bytes.Equal(buf, []byte{0x73, 0x4e, 0x61, 0x50, 0x70, 0x59}) {
				return errors.New("invalid stream ID")
			}
			// continue...
		default:
			return errors.New("invalid frame identifier")
		}
	}
	panic("should never happen")
}
Exemple #30
0
// hashOrder - hashes input key to return returns consistent
// hashed integer slice. Returned integer order is salted
// with an input key. This results in consistent order.
// NOTE: collisions are fine, we are not looking for uniqueness
// in the slices returned.
func hashOrder(key string, cardinality int) []int {
	if cardinality < 0 {
		// Returns an empty int slice for negative cardinality.
		return nil
	}
	nums := make([]int, cardinality)
	keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable)

	start := int(uint32(keyCrc)%uint32(cardinality)) | 1
	for i := 1; i <= cardinality; i++ {
		nums[i-1] = 1 + ((start + i) % cardinality)
	}
	return nums
}