// readBlock reads a full or partial block into p from rs. len(p) must be smaller than (bs - crc32Len).
// The checksum of the block is calculated and verified.
// ErrBadCRC is returned if the checksum is invalid.
func readBlock(rs io.ReadSeeker, p []byte, index, bs int64) (int64, error) {
payloadLen := int(bs - crc32Len)
if len(p) > payloadLen {
return 0, ErrPayloadSizeTooLarge
}
b := make([]byte, crc32Len)
_, err := rs.Seek(index*bs, os.SEEK_SET)
if err != nil {
return 0, err
}
n, err := rs.Read(b)
// Cannot read full crc
if n > 0 && n < 4 {
return 0, ErrBadCRC
}
if err != nil {
return 0, err
}
crc := binary.BigEndian.Uint32(b)
if len(p) == payloadLen {
// fast path for reading into a buffer of payload size
n, err = rs.Read(p)
if err != nil {
return int64(n), err
}
// Invalid crc
if crc != crc32.Checksum(p[:n], crc32cTable) {
return 0, ErrBadCRC
}
return int64(n), nil
} else {
// p is smaller than payload size
// read into another buffer (so we can compare full block CRC) and copy into p
buf := make([]byte, payloadLen)
n, err = rs.Read(buf)
// If there is an eof returned, two cases may happen
// 1. n < len(p) -- we don't have enough data to
// fill into p, we should return the eof
// 2. n >= len(p) -- we have enough data to fill
// into p, shoudln't return the eof
if err == io.EOF && n >= len(p) {
err = nil
}
if err != nil {
return int64(copy(p, buf[:n])), err
}
if crc != crc32.Checksum(buf[:n], crc32cTable) {
return 0, ErrBadCRC
}
return int64(copy(p, buf[:n])), nil
}
}
func (p fakePeers) PickPeer(key string) (peer ProtoGetter, ok bool) {
if len(p) == 0 {
return
}
n := crc32.Checksum([]byte(key), crc32.IEEETable) % uint32(len(p))
return p[n], p[n] != nil
}
//一个packet包括包头和包体，保证在接收到包头后两秒钟内接收到包体，否则线程会一直阻塞
//因此，引入了超时机制
func RecvPackage(conn net.Conn) (packet *GetPacket, e error, isClose bool) {
var data []byte
header := make([]byte, 12)
if conn == nil {
// e = errors.New("")
fmt.Println("连接已经关闭")
isClose = true
return
}
n, err := io.ReadFull(conn, header)
if n == 0 && err == io.EOF {
// fmt.Println("客户端断开连接")
isClose = true
e = err
return
} else if err != nil {
// fmt.Println("接收数据出错:", err)
isClose = true
e = err
return
}
//数据包长度
size := binary.BigEndian.Uint32(header)
//crc值
crc1 := binary.BigEndian.Uint32(header[4:8])
msgID := binary.BigEndian.Uint32(header[8:12])
data = make([]byte, size)
timeout := NewTimeOut(func() {
n, err = io.ReadFull(conn, data)
if uint32(n) != size {
log.Println("数据包长度不正确", n, "!=", size)
e = errors.New(fmt.Sprint("数据包长度不正确:%d!=%d", n, size))
return
}
if err != nil {
log.Println("读取数据出错:", err)
e = err
return
}
crc2 := crc32.Checksum(data, crc32.IEEETable)
if crc1 != crc2 {
log.Println("crc 数据验证不正确: ", crc1, " != ", crc2)
e = errors.New(fmt.Sprint("crc 数据验证不正确:%d!=%d", crc1, crc2))
return
}
packet = new(GetPacket)
packet.Date = data
packet.MsgID = int32(msgID)
packet.Size = uint32(len(data))
})
isTimeOut := timeout.Do(time.Second * 5)
if isTimeOut {
e = errors.New("数据包头和数据包体不完整")
return
}
return
}
// writeBlock writes a full or partial block into ws. len(p) must be smaller than (bs - crc32Len).
// Each block contains a crc32Len bytes crc32c of the payload and a (bs-crc32Len)
// bytes payload. Any error encountered is returned.
// It is caller's responsibility to ensure that only the last block in the file has
// len(p) < bs - crc32Len
func writeBlock(ws io.WriteSeeker, index, bs int64, p []byte) error {
if int64(len(p)) > bs-crc32Len {
return ErrPayloadSizeTooLarge
}
// seek to the beginning of the block
_, err := ws.Seek(index*bs, os.SEEK_SET)
if err != nil {
return err
}
// write crc32c
// TODO: reuse buffer
b := make([]byte, crc32Len)
binary.BigEndian.PutUint32(b, crc32.Checksum(p, crc32cTable))
_, err = ws.Write(b)
if err != nil {
return err
}
// write payload
_, err = ws.Write(p)
if err != nil {
return err
}
return nil
}
func get(key int64, circle map[uint32]string) string {
keyStr := strconv.Itoa(int(key))
if len(circle) == 0 {
return ""
} else {
hash := crc32.Checksum([]byte(keyStr), crc32q)
var keys []int
for k := range circle {
keys = append(keys, int(k))
}
sort.Ints(keys)
var serverHash uint32
foundHash := false
for i := 0; i < len(keys); i++ {
if keys[i] >= int(hash) {
serverHash = uint32(keys[i])
foundHash = true
break
}
}
if foundHash == true {
return circle[serverHash]
} else {
return circle[uint32(keys[0])]
}
}
}
func (s *blockWriteStream) makePacket() outboundPacket {
packetLength := outboundPacketSize
if s.buf.Len() < outboundPacketSize {
packetLength = s.buf.Len()
}
numChunks := int(math.Ceil(float64(packetLength) / float64(outboundChunkSize)))
packet := outboundPacket{
seqno: s.seqno,
offset: s.offset,
last: false,
checksums: make([]byte, numChunks*4),
data: make([]byte, packetLength),
}
// TODO: we shouldn't actually need this extra copy. We should also be able
// to "reuse" packets.
io.ReadFull(&s.buf, packet.data)
// Fill in the checksum for each chunk of data.
for i := 0; i < numChunks; i++ {
chunkOff := i * outboundChunkSize
chunkEnd := chunkOff + outboundChunkSize
if chunkEnd >= len(packet.data) {
chunkEnd = len(packet.data)
}
checksum := crc32.Checksum(packet.data[chunkOff:chunkEnd], crc32.IEEETable)
binary.BigEndian.PutUint32(packet.checksums[i*4:], checksum)
}
return packet
}
func PacketData(msgID uint32, data *[]byte) *[]byte {
buf := bytes.NewBuffer([]byte{})
binary.Write(buf, binary.BigEndian, uint32(uint32(len(*data))))
crc32 := crc32.Checksum(*data, crc32.IEEETable)
binary.Write(buf, binary.BigEndian, crc32)
// binary.Write(msgID, binary.BigEndian, )
buf.Write(*data)
bs := buf.Bytes()
return &bs
// writer := packet.Writer()
// //size uint32
// writer.WriteU32(uint32(len(*data)))
// //crc32 uint32
// crc32 := crc32.Checksum(*data, crc32.IEEETable)
// writer.WriteU32(crc32)
// //msgID
// writer.WriteU32(msgID)
// //Data
// writer.WriteRawBytes(*data)
// bs := writer.Data()
// return &bs
}
func LockPartition(pg *sql.DB, ns string, max uint64) (uint64, error) {
tab := crc32.MakeTable(crc32.IEEE)
for {
var p uint64
for p = 0; p < max; p++ {
pId := fmt.Sprintf("%s.%d", ns, p)
check := crc32.Checksum([]byte(pId), tab)
rows, err := pg.Query("select pg_try_advisory_lock($1)", check)
if err != nil {
continue
}
for rows.Next() {
var result sql.NullBool
rows.Scan(&result)
if result.Valid && result.Bool {
fmt.Printf("at=%q partition=%d max=%d\n",
"acquired-lock", p, max)
rows.Close()
return p, nil
}
}
rows.Close()
}
fmt.Printf("at=%q\n", "waiting-for-partition-lock")
time.Sleep(time.Second * 10)
}
return 0, errors.New("Unable to lock partition.")
}
func calcNewCRC32(path string, newHeader *BinaryHeader) (uint32, error) {
buf := &bytes.Buffer{}
err := binary.Write(buf, binary.BigEndian, newHeader)
if err != nil {
return 0, err
}
header := buf.Bytes()
crc := crc32.Checksum(header[:28], crc32.IEEETable) // before crc32
crc = crc32.Update(crc, crc32.IEEETable, header[32:])
f, err := os.Open(path)
if err != nil {
return 0, err
}
defer f.Close()
if _, err := f.Seek(int64(len(header)), os.SEEK_SET); err != nil {
return 0, err
}
data := make([]byte, 4*1024*1024)
for {
n, err := f.Read(data)
crc = crc32.Update(crc, crc32.IEEETable, data[:n])
if err != nil {
if err != io.EOF {
return 0, err
}
break
}
}
return crc, nil
}
func TestReadRecord(t *testing.T) {
badInfoRecord := make([]byte, len(infoRecord))
copy(badInfoRecord, infoRecord)
badInfoRecord[len(badInfoRecord)-1] = 'a'
tests := []struct {
data []byte
wr *walpb.Record
we error
}{
{infoRecord, &walpb.Record{Type: 1, Crc: crc32.Checksum(infoData, crcTable), Data: infoData}, nil},
{[]byte(""), &walpb.Record{}, io.EOF},
{infoRecord[:8], &walpb.Record{}, io.ErrUnexpectedEOF},
{infoRecord[:len(infoRecord)-len(infoData)-8], &walpb.Record{}, io.ErrUnexpectedEOF},
{infoRecord[:len(infoRecord)-len(infoData)], &walpb.Record{}, io.ErrUnexpectedEOF},
{infoRecord[:len(infoRecord)-8], &walpb.Record{}, io.ErrUnexpectedEOF},
{badInfoRecord, &walpb.Record{}, walpb.ErrCRCMismatch},
}
rec := &walpb.Record{}
for i, tt := range tests {
buf := bytes.NewBuffer(tt.data)
decoder := newDecoder(ioutil.NopCloser(buf))
e := decoder.decode(rec)
if !reflect.DeepEqual(rec, tt.wr) {
t.Errorf("#%d: block = %v, want %v", i, rec, tt.wr)
}
if !reflect.DeepEqual(e, tt.we) {
t.Errorf("#%d: err = %v, want %v", i, e, tt.we)
}
rec = &walpb.Record{}
}
}
func (c *Connection) parcelValidity(parcel Parcel) uint8 {
verbose(c.peer.PeerIdent(), "Connection.isValidParcel(%s)", parcel.MessageType())
crc := crc32.Checksum(parcel.Payload, CRCKoopmanTable)
switch {
case parcel.Header.NodeID == NodeID: // We are talking to ourselves!
parcel.Trace("Connection.isValidParcel()-loopback", "H")
significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to loopback!: %+v", parcel.Header)
c.peer.QualityScore = MinumumQualityScore - 50 // Ban ourselves for a week
return InvalidDisconnectPeer
case parcel.Header.Network != CurrentNetwork:
parcel.Trace("Connection.isValidParcel()-network", "H")
significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong network. Remote: %0x Us: %0x", parcel.Header.Network, CurrentNetwork)
return InvalidDisconnectPeer
case parcel.Header.Version < ProtocolVersionMinimum:
parcel.Trace("Connection.isValidParcel()-version", "H")
significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong version: %+v", parcel.Header)
return InvalidDisconnectPeer
case parcel.Header.Length != uint32(len(parcel.Payload)):
parcel.Trace("Connection.isValidParcel()-length", "H")
significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to wrong length: %+v", parcel.Header)
return InvalidPeerDemerit
case parcel.Header.Crc32 != crc:
parcel.Trace("Connection.isValidParcel()-checksum", "H")
significant(c.peer.PeerIdent(), "Connection.isValidParcel(), failed due to bad checksum: %+v", parcel.Header)
return InvalidPeerDemerit
default:
parcel.Trace("Connection.isValidParcel()-ParcelValid", "H")
return ParcelValid
}
return ParcelValid
}
// Create an object struct for the given attributes and contents.
//
// LOCKS_REQUIRED(b.mu)
func (b *bucket) mintObject(
req *gcs.CreateObjectRequest,
contents []byte) (o fakeObject) {
md5Sum := md5.Sum(contents)
// Set up basic info.
b.prevGeneration++
o.metadata = gcs.Object{
Name: req.Name,
ContentType: req.ContentType,
ContentLanguage: req.ContentLanguage,
CacheControl: req.CacheControl,
Owner: "user-fake",
Size: uint64(len(contents)),
ContentEncoding: req.ContentEncoding,
ComponentCount: 1,
MD5: &md5Sum,
CRC32C: crc32.Checksum(contents, crc32cTable),
MediaLink: "http://localhost/download/storage/fake/" + req.Name,
Metadata: copyMetadata(req.Metadata),
Generation: b.prevGeneration,
MetaGeneration: 1,
StorageClass: "STANDARD",
Updated: b.clock.Now(),
}
// Set up data.
o.data = contents
return
}
// sendSerializedData sets the leading CRC on buf and writes the data to conn. Any errors on the
// write will be logged but otherwise ignored.
func sendSerializedData(buf []byte, conn io.Writer) {
// Prefix buf with a crc of everything we've appended to it.
AppendUint32(buf[0:0], crc32.Checksum(buf[4:], crcTable))
_, err := conn.Write(buf)
if err != nil {
log.Printf("Failed to write %d bytes in BatchAndSend: %v", err)
}
}
// serializeWriteRow serialize the current block file and offset where new
// will be written into a format suitable for storage into the metadata.
func serializeWriteRow(curBlockFileNum, curFileOffset uint32) []byte {
var serializedRow [12]byte
byteOrder.PutUint32(serializedRow[0:4], curBlockFileNum)
byteOrder.PutUint32(serializedRow[4:8], curFileOffset)
checksum := crc32.Checksum(serializedRow[:8], castagnoli)
byteOrder.PutUint32(serializedRow[8:12], checksum)
return serializedRow[:]
}
func Benchmark_crc32_6(b *testing.B) {
buf, n := initSample(6)
tab := crc32.MakeTable(crc32.Castagnoli)
b.SetBytes(n)
b.ResetTimer()
for i := 0; i < b.N; i++ {
crc32.Checksum(buf, tab)
}
}
//接收客户端消息协程
func (this *ServerConn) recv() {
//处理客户端主动断开连接的情况
defer func(clientConn *ServerConn) {
log.Println("客户端主动关闭连接")
}(this)
for !this.isClose {
var data []byte
header := make([]byte, 12)
n, err := io.ReadFull(this.conn, header)
if n == 0 && err == io.EOF {
// temp := new(GetPacket)
// temp.ConnId = this.session
// temp.MsgID = int32(Close)
// this.inPack <- temp
fmt.Println("客户端断开连接")
return
} else if err != nil {
fmt.Println("接收数据出错:", err)
return
}
//数据包长度
size := binary.BigEndian.Uint32(header)
//crc值
crc1 := binary.BigEndian.Uint32(header[4:8])
// msgID := binary.BigEndian.Uint32(header[8:12])
data = make([]byte, size)
n, err = io.ReadFull(this.conn, data)
if uint32(n) != size {
log.Println("数据包长度不正确", n, "!=", size)
return
}
if err != nil {
log.Println("读取数据出错:", err)
return
}
crc2 := crc32.Checksum(data, crc32.IEEETable)
if crc1 != crc2 {
log.Println("crc 数据验证不正确: ", crc1, " != ", crc2)
return
}
// temp := new(GetPacket)
// temp.ConnId = this.session
// temp.Date = data
// temp.MsgID = int32(msgID)
// temp.Size = uint32(len(data))
// this.inPack <- temp
}
//最后一个包接收了之后关闭chan
//如果有超时包需要等超时了才关闭，目前未做处理
close(this.outData)
}
func (c *crc32Validator) Checksum(data []byte) (rawCRC []byte) {
// Calculate the CRC checksum of data.
crc := crc32.Checksum(data, c.table)
// Transform to a byte slice.
rawCRC = make([]byte, 4)
binary.LittleEndian.PutUint32(rawCRC, crc)
return rawCRC
}
func (c *Context) Login(email string, admin bool) {
c.req.Header.Add("X-AppEngine-Internal-User-Email", email)
c.req.Header.Add("X-AppEngine-Internal-User-Id", strconv.Itoa(int(crc32.Checksum([]byte(email), crc32.IEEETable))))
c.req.Header.Add("X-AppEngine-Internal-User-Federated-Identity", email)
if admin {
c.req.Header.Add("X-AppEngine-Internal-User-Is-Admin", "1")
} else {
c.req.Header.Add("X-AppEngine-Internal-User-Is-Admin", "0")
}
}
func (c *crc32Validator) Validate(data []byte, rawCRC []byte) bool {
// Convert the raw CRC byte slice.
origCRC := binary.LittleEndian.Uint32(rawCRC)
// Calculate the CRC checksum of data.
crc := crc32.Checksum(data, c.table)
// Compare the checksums.
return crc == origCRC
}
func (this *Client) recv() {
for !this.isClose {
header := make([]byte, 12)
n, err := io.ReadFull(this.conn, header)
if n == 0 && err == io.EOF {
fmt.Println("客户端断开连接")
go this.reConnect()
return
} else if err != nil {
fmt.Println("接受数据出错:", err)
go this.reConnect()
return
}
//数据包长度
size := binary.BigEndian.Uint32(header)
//crc值
crc1 := binary.BigEndian.Uint32(header[4:8])
// msgID := binary.BigEndian.Uint32(header[8:12])
data := make([]byte, size)
n, err = io.ReadFull(this.conn, data)
if err != nil {
log.Println("读取数据出错:", err)
go this.reConnect()
return
}
if uint32(n) != size {
log.Println("数据包长度不正确", n, "!=", size)
continue
}
crc2 := crc32.Checksum(data, crc32.IEEETable)
if crc1 != crc2 {
log.Println("crc 数据验证不正确: ", crc1, " != ", crc2)
continue
}
// temp := new(GetPacket)
// temp.ConnId = this.Session
// temp.Date = data
// temp.MsgID = int32(msgID)
// temp.Size = uint32(len(data))
// this.inPack <- temp
}
//最后一个包接收了之后关闭chan
//如果有超时包需要等超时了才关闭，目前未做处理
close(this.outData)
}
// Generates the byte representation of the document, including the header.
// Empty v interpreted as tombstone.
func newDocument(k, v []byte) []byte {
outsize := len(k) + len(v) + 12
out := make([]byte, 12, outsize)
uint32ToBytes(out, 4, uint32(len(k)))
uint32ToBytes(out, 8, uint32(len(v)))
out = append(out, k...)
out = append(out, v...)
hash := crc32.Checksum(out[4:], crcTable)
uint32ToBytes(out, 0, hash)
return out
}
// Set if key not exists, returns whether already exists.
func (sm *SegmentMap) Set(key []byte, value interface{}, force bool) bool {
idx := int64(crc32.Checksum(key, sm.crcTable)) % sm.size
k := string(key)
_, exist := sm.maps[idx][k]
if exist && !force {
return exist
}
sm.maps[idx][k] = value
return exist
}
func (m Message) WriteTo(w io.Writer) (n int64, err error) {
compression := CompressionTypeNone
if compression != CompressionTypeNone {
return -1, fmt.Errorf("Only support none compression for now")
}
totalLen := m.Len() - 4 // Subtract the size of the length
checksum := uint32(crc32.Checksum(m, crc32.IEEETable))
return binwrite(w, totalLen, MagicTypeWithCompression, compression, checksum, m)
}
func MarshalPacket(msgID uint32, data *[]byte) *[]byte {
buf := bytes.NewBuffer([]byte{})
binary.Write(buf, binary.BigEndian, uint32(uint32(len(*data))))
crc32 := crc32.Checksum(*data, crc32.IEEETable)
binary.Write(buf, binary.BigEndian, crc32)
binary.Write(buf, binary.BigEndian, msgID)
buf.Write(*data)
bs := buf.Bytes()
return &bs
}
func (ea *EtcdAero) SetTTL(timerTTL time.Duration) {
ea.timerTTL = timerTTL
ea.etcdLockTTL = timerTTL * 3 / 2
ea.AeroTTL = timerTTL * 5
// randomize sleep ttl form 0.8 to 1.2 of original timerTTL / 2
crc32q := crc32.MakeTable(0xD5828281)
d := 1.0 + float64(int64(crc32.Checksum([]byte(ea.value), crc32q))%1000-500)/2500.0
ea.sleepTTL = time.Duration(float64(timerTTL) * d / 2)
}
func conformNodeID(ip net.IP, nodeID NodeID) NodeID {
var b [20]byte
copy(b[:], nodeID)
// r is supposed to be random but let's make this deterministic and get it
// from the last byte of the original NodeID.
r := byte(b[19]) & 0x07
ip4 := ip.To4()
var y uint32
if ip4 != nil {
var x [4]byte
copy(x[:], ip4)
x[0] &= 0x03
x[1] &= 0x0F
x[2] &= 0x3F
x[3] &= 0xFF
x[0] |= (r << 5)
y = crc32.Checksum(x[:], crc32c)
} else {
var x [16]byte
copy(x[:], ip)
x[0] &= 0x01
x[1] &= 0x03
x[2] &= 0x07
x[3] &= 0x0f
x[4] &= 0x1f
x[5] &= 0x3f
x[6] &= 0x7f
x[7] &= 0xff
x[0] |= (r << 5)
y = crc32.Checksum(x[:], crc32c)
}
b[0] = byte((y >> 24) & 0xFF)
b[1] = byte((y >> 16) & 0xFF)
b[2] = byte((y>>8)&0xF8) | (b[2] & 0x07)
b[19] = (b[19] & 0xF8) | r
return NodeID(string(b[:]))
}
func (s *blockReadStream) validateChecksum(b []byte) error {
checksumOffset := 4 * s.chunkIndex
checksumBytes := s.checksums.Bytes()[checksumOffset : checksumOffset+4]
checksum := binary.BigEndian.Uint32(checksumBytes)
crc := crc32.Checksum(b, s.checksumTab)
if crc != checksum {
return errInvalidChecksum
}
return nil
}
// Generate the CRC used to make or validate secure node ID.
func crcIP(ip net.IP, rand uint8) uint32 {
if ip4 := ip.To4(); ip4 != nil {
ip = ip4
}
// Copy IP so we can make changes. Go sux at this.
ip = append(make(net.IP, 0, len(ip)), ip...)
mask := maskForIP(ip)
for i := range mask {
ip[i] &= mask[i]
}
r := rand & 7
ip[0] |= r << 5
return crc32.Checksum(ip[:len(mask)], crc32.MakeTable(crc32.Castagnoli))
}
func (r *reader) nextFrame() error {
for {
_, err := io.ReadFull(r.reader, r.hdr)
if err != nil {
return err
}
buf, err := r.readBlock()
if err != nil {
return err
}
switch r.hdr[0] {
case 0x00, 0x01:
// compressed or uncompressed bytes
// first 4 bytes are the little endian crc32 checksum
checksum := unmaskChecksum(uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24)
b := buf[4:]
if r.hdr[0] == 0x00 {
// compressed bytes
r.dst, err = snappy.Decode(r.dst, b)
if err != nil {
return err
}
b = r.dst
}
if r.verifyChecksum {
actualChecksum := crc32.Checksum(b, crcTable)
if checksum != actualChecksum {
return errors.New(fmt.Sprintf("invalid checksum %x != %x", checksum, actualChecksum))
}
}
_, err = r.buf.Write(b)
return err
case 0xff:
// stream identifier
if !bytes.Equal(buf, []byte{0x73, 0x4e, 0x61, 0x50, 0x70, 0x59}) {
return errors.New("invalid stream ID")
}
// continue...
default:
return errors.New("invalid frame identifier")
}
}
panic("should never happen")
}
// hashOrder - hashes input key to return returns consistent
// hashed integer slice. Returned integer order is salted
// with an input key. This results in consistent order.
// NOTE: collisions are fine, we are not looking for uniqueness
// in the slices returned.
func hashOrder(key string, cardinality int) []int {
if cardinality < 0 {
// Returns an empty int slice for negative cardinality.
return nil
}
nums := make([]int, cardinality)
keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable)
start := int(uint32(keyCrc)%uint32(cardinality)) | 1
for i := 1; i <= cardinality; i++ {
nums[i-1] = 1 + ((start + i) % cardinality)
}
return nums
}