func (c *Consistent) hashKey(key string) uint32 {
if len(key) < 64 {
copy(c.scratch[:], key)
return crc32.ChecksumIEEE(c.scratch[:len(key)])
}
return crc32.ChecksumIEEE([]byte(key))
}
func compareFiles(srcPath, destPath string) (bool, error) {
src, err := os.Open(srcPath)
if err != nil {
return false, fmt.Errorf("open source failed: %s", err)
}
defer src.Close()
dest, err := os.Open(destPath)
if err != nil {
return false, fmt.Errorf("open dest failed: %s", err)
}
defer dest.Close()
srcContents, err := ioutil.ReadAll(src)
if err != nil {
return false, fmt.Errorf("read source failed: %s", err)
}
destContents, err := ioutil.ReadAll(dest)
if err != nil {
return false, fmt.Errorf("read dest failed: %s", err)
}
contentsHash := fmt.Sprintf("%x", crc32.ChecksumIEEE(srcContents))
srcHash := fmt.Sprintf("%x", crc32.ChecksumIEEE(destContents))
return srcHash == contentsHash, nil
}
// Apply applies the patch data to the given base.
func Apply(base []byte, patch *common.PatchData, skipCRC bool) ([]byte, error) {
if uint64(len(base)) != patch.InputFileSize {
return nil, errors.New("Base file did not have expected size.")
}
if !skipCRC && crc32.ChecksumIEEE(base) != patch.InputChecksum {
return nil, errors.New("Base file did not have expected checksum")
}
output := make([]byte, patch.OutputFileSize)
copy(output, base)
pointer := 0
for _, block := range patch.PatchBlocks {
pointer += int(block.RelativeOffset)
for _, b := range block.Data {
if pointer >= len(base) {
output[pointer] = b
} else {
output[pointer] = base[pointer] ^ b
}
pointer++
}
pointer++
}
if !skipCRC && crc32.ChecksumIEEE(output) != patch.OutputChecksum {
return nil, errors.New("Patch result did not have expected checksum")
}
return output, nil
}
func getCrc(key string) uint32 {
if len(key) < 64 {
var scratch [64]byte
copy(scratch[:], key)
return crc32.ChecksumIEEE(scratch[:len(key)])
}
return crc32.ChecksumIEEE([]byte(key))
}
// Hash function
func (c *Consistent) hashStr(key string) uint32 {
if len(key) < 64 {
var scratch [64]byte
copy(scratch[:], key)
return crc32.ChecksumIEEE(scratch[:len(key)])
}
return crc32.ChecksumIEEE([]byte(key))
}
func main() {
cs := crc32.ChecksumIEEE([]byte("1234667"))
fmt.Println(cs)
cs = crc32.ChecksumIEEE([]byte("dfdfdfdfdf"))
fmt.Println(cs)
}
func (c *Consistent) hashKey(key string) uint32 {
//
//log.Println("key string:", key)
if len(key) < 64 {
var scratch [64]byte
copy(scratch[:], key)
//log.Fatal(len(key), scratch)
return crc32.ChecksumIEEE(scratch[:len(key)])
}
return crc32.ChecksumIEEE([]byte(key))
}
func TestNew_resources(t *testing.T) {
assert := assert.New(t)
r, err := os.Open("testdata/wikipedia.webarchive")
require.NoError(t, err)
a, err := New(r)
assert.Equal(uint32(0x7a24addb), crc32.ChecksumIEEE(a.Content.Data))
assert.Equal("text/html", a.Content.MIMEType)
assert.Equal("UTF-8", a.Content.Encoding)
assert.Equal("https://en.wikipedia.org/wiki/Webarchive", a.Content.URL)
assert.Empty(a.Content.FrameName)
assert.Len(a.Resources, 13)
assert.NoError(err)
resources := []struct {
data uint32
mime string
enc string
url uint32
resp uint32
}{
{0xe0352bc6, "text/css", "utf-8", 0xa88c8657, 0x3e495c5f},
{0xd5fee4ca, "text/css", "utf-8", 0x348faf99, 0xe1b3e4d0},
{0xab5b0999, "text/javascript", "utf-8", 0xe9d253b4, 0x58d61084},
{0xf3c27e95, "text/javascript", "utf-8", 0xfeaf026a, 0x96e53929},
{0x59637a, "image/png", "", 0xbd73e04a, 0x6dd3aad5},
{0xf79b5fc7, "image/png", "", 0x8d210858, 0xfd3b31c4},
{0xfbea27a5, "image/jpeg", "", 0x482f361f, 0xdf0d81fd},
{0xdae1d80a, "image/png", "", 0x2ab8618b, 0x570a3987},
{0x285bcd96, "image/png", "", 0x5903bafa, 0xad214977},
{0xc296a07a, "image/png", "", 0x23103ba7, 0xd845f71},
{0x1af78e35, "image/png", "", 0x9af068df, 0x19fc7a6},
{0x7c4edfa, "image/png", "", 0x7de2a6e8, 0x36084aec},
{0x9885885, "text/css", "utf-8", 0x8c2feaff, 0xb01023b6},
}
for i := 0; i < 13; i++ {
sr := a.Resources[i]
r := resources[i]
s := fmt.Sprintf("subresource %d", i)
assert.Equal(r.data, crc32.ChecksumIEEE(sr.Data), s)
assert.Equal(r.mime, sr.MIMEType, s)
assert.Equal(r.enc, sr.Encoding, s)
assert.Equal(r.url, crc32.ChecksumIEEE([]byte(sr.URL)), s)
assert.Equal(r.resp, crc32.ChecksumIEEE(sr.Response), s)
}
}
func HashValue(value interface{}) uint64 {
switch val := value.(type) {
case int:
return uint64(val)
case uint64:
return uint64(val)
case int64:
return uint64(val)
case string:
return uint64(crc32.ChecksumIEEE(hack.Slice(val)))
case []byte:
return uint64(crc32.ChecksumIEEE(val))
}
panic(NewKeyError("Unexpected key variable type %T", value))
}
func TestEncodeDecodeEntry(t *testing.T) {
/**
crc32 : tStamp : ksz : valueSz : key : value
4 : 4 : 4 : 4 : xxxx : xxxx
**/
//func encodeEntry(tStamp, keySize, valueSize uint32, key, value []byte) []byte {
// EncodeEntry
tStamp := uint32(time.Now().Unix())
key := []byte("Foo")
value := []byte("Bar")
ksz := uint32(len(key))
valuesz := uint32(len(value))
buf := make([]byte, HeaderSize+ksz+valuesz)
binary.LittleEndian.PutUint32(buf[4:8], tStamp)
binary.LittleEndian.PutUint32(buf[8:12], ksz)
binary.LittleEndian.PutUint32(buf[12:16], valuesz)
copy(buf[16:(16+ksz)], key)
copy(buf[(16+ksz):(16+ksz+valuesz)], value)
c32 := crc32.ChecksumIEEE(buf[4:])
binary.LittleEndian.PutUint32(buf[0:4], uint32(c32))
// Test decode
ksz = binary.LittleEndian.Uint32(buf[8:12])
valuesz = binary.LittleEndian.Uint32(buf[12:16])
tStamp = binary.LittleEndian.Uint32(buf[4:8])
c32 = binary.LittleEndian.Uint32(buf[:4])
assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32)
assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp)
assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz)
assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz)
assert.Equal(t, buf[HeaderSize:(HeaderSize+ksz)], key)
assert.Equal(t, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)], value)
// EncodeEntry , ksz = 0, valueSz = 0
ksz = uint32(0)
valuesz = uint32(0)
buf = make([]byte, HeaderSize+ksz+valuesz, HeaderSize+ksz+valuesz)
binary.LittleEndian.PutUint32(buf[4:8], tStamp)
binary.LittleEndian.PutUint32(buf[8:12], ksz)
binary.LittleEndian.PutUint32(buf[12:16], valuesz)
c32 = crc32.ChecksumIEEE(buf[4:])
binary.LittleEndian.PutUint32(buf[0:4], c32)
// decodeEntry, ksz =0, valueSz = 0
assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32)
assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp)
assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz)
assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz)
}
func Read(r io.Reader) (*Message, error) {
var incomingHeader header
err := binary.Read(r, binary.LittleEndian, &incomingHeader)
if err != nil {
return nil, err
}
if incomingHeader.ContentLength > MaximumMessageSize {
return nil, fmt.Errorf("message too large: requested %d bytes but max is %d", incomingHeader.ContentLength, MaximumMessageSize)
}
msg := new(Message)
data := make([]byte, incomingHeader.ContentLength)
// Keep reading so we are resilient to IP fragmentation along the path.
for n := uint32(0); n < incomingHeader.ContentLength; {
nRead, err := r.Read(data[n:])
if err != nil {
return nil, err
}
n += uint32(nRead)
}
// Checksum the incoming data so that transmission errors can be dealt with.
check := crc32.ChecksumIEEE(data)
if check != incomingHeader.Checksum {
return nil, ErrCorruptedMessage
}
err = proto.Unmarshal(data[0:incomingHeader.ContentLength], msg)
return msg, err
}
// Write marshals a Message into the proper on-wire format and sends it
// to the remote system.
func Write(w io.Writer, msg *Message) error {
buf, err := proto.Marshal(msg)
if err != nil {
return err
}
// For now we just compute the length as we don't have tests.
bufHeader := &header{
ContentLength: uint32(len(buf)),
Checksum: crc32.ChecksumIEEE(buf),
Reserved: 0,
}
err = binary.Write(w, binary.LittleEndian, bufHeader)
if err != nil {
return err
}
for totalWritten := 0; totalWritten < len(buf); {
nWritten, err := w.Write(buf)
if err != nil {
return err
}
totalWritten += nWritten
}
return nil
}
func (t *CRCMessage) Marshal(w *wipro.Writer) {
offset := len(w.B)
w.WriteUint32(t.CRC)
start := len(w.B)
t.Message.Marshal(w)
w.SetUint32(offset, crc32.ChecksumIEEE(w.B[start:]))
}
func MakeHashes(word string, methods []string) map[string]string {
hashes := make(map[string]string)
for _, method := range methods {
var hash string
// TODO: Replace this with reflection
// for easier extensibility
switch method {
case "crc32":
hashInt := uint64(crc32.ChecksumIEEE([]byte(word)))
hash = strconv.FormatUint(hashInt, 16)
case "md5":
hash = fmt.Sprintf("%x", md5.Sum([]byte(word)))
case "sha1":
hash = fmt.Sprintf("%x", sha1.Sum([]byte(word)))
case "sha256":
hash = fmt.Sprintf("%x", sha256.Sum256([]byte(word)))
}
hashes[method] = hash
}
return hashes
}
// save writes the snapshot to file.
func (ss *Snapshot) save() error {
// Open the file for writing.
file, err := os.OpenFile(ss.Path, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer file.Close()
// Serialize to JSON.
b, err := json.Marshal(ss)
if err != nil {
return err
}
// Generate checksum and write it to disk.
checksum := crc32.ChecksumIEEE(b)
if _, err = fmt.Fprintf(file, "%08x\n", checksum); err != nil {
return err
}
// Write the snapshot to disk.
if _, err = file.Write(b); err != nil {
return err
}
// Ensure that the snapshot has been flushed to disk before continuing.
if err := file.Sync(); err != nil {
return err
}
return nil
}
func readRequestBody(r io.Reader, header *wire.RequestHeader, request proto.Message) error {
// recv body (end)
compressedPbRequest, err := recvFrame(r)
if err != nil {
return err
}
// checksum
if crc32.ChecksumIEEE(compressedPbRequest) != header.GetChecksum() {
return fmt.Errorf("protorpc.readRequestBody: unexpected checksum.")
}
// decode the compressed data
pbRequest, err := snappy.Decode(nil, compressedPbRequest)
if err != nil {
return err
}
// check wire header: rawMsgLen
if uint32(len(pbRequest)) != header.GetRawRequestLen() {
return fmt.Errorf("protorpc.readRequestBody: Unexcpeted header.RawRequestLen.")
}
// Unmarshal to proto message
if request != nil {
err = proto.Unmarshal(pbRequest, request)
if err != nil {
return err
}
}
return nil
}
func (this Table) calcPartitionsCRC() uint32 {
buf := &bytes.Buffer{}
for _, part := range this.Partitions {
part.write(buf, this.Header.PartitionEntrySize)
}
return crc32.ChecksumIEEE(buf.Bytes())
}
func DecryptAESCFB(src, key []byte) *AuthToken {
encrData := EncrData{}
err := json.Unmarshal(src, &encrData)
if err != nil {
return nil
}
if len(encrData.IV) != len(key) {
return nil
}
rawBytes := make([]byte, len(encrData.Data))
cipher.NewCFBDecrypter(aesCipher(key), encrData.IV).XORKeyStream(rawBytes, encrData.Data)
rawData := RawData{}
err = json.Unmarshal(rawBytes, &rawData)
if err != nil {
return nil
}
cSum := crc32.ChecksumIEEE(rawData.Data)
if cSum != rawData.CSum {
return nil
}
res := AuthToken{}
err = json.Unmarshal(rawData.Data, &res)
if err != nil {
return nil
}
return &res
}
// Send sends the message to the broker, and blocks until an acknowledgement is
// received. If the max number of retries is exceeded, returns the last error.
func (p *Producer) Send(topic string, payload []byte) error {
seqnum := atomic.AddInt64(&p.seqnum, 1)
message := protocol.Message{seqnum, payload, crc32.ChecksumIEEE(payload)}
request := &protocol.ProduceRequest{p.id, topic, message}
log.Debug("Sending %v", request)
p.lock.Lock()
defer p.lock.Unlock()
for {
if _, err := p.socket.Send(request, MAX_RETRIES, origin()); nil != err {
p.socket.Reset(p.register)
continue
}
log.Debug("Acknowledgement received.")
return nil
}
return nil
}
func storage_upload_file(header *Header, conn net.Conn) bool {
buff := make([]byte, header.pkg_len)
n, err := io.ReadFull(conn, buff)
if err != nil || n != int(header.pkg_len) {
return false
}
req := &UploadFileRequest{}
err = req.Unmarshal(buff)
if err != nil {
return false
}
file_ext := storage_format_ext_name(req.file_ext)
crc32 := crc32.ChecksumIEEE(req.file_data)
now := time.Now()
// log.Println(req.store_path_index, req.file_length, len(req.file_data), file_ext, crc32)
addr, _ := conn.RemoteAddr().(*net.TCPAddr)
file_name, full_name := storage_get_filename(addr.IP.String(), int(req.store_path_index), int(now.Unix()), req.file_length, int(crc32), file_ext)
file, err := os.OpenFile(full_name, os.O_WRONLY|os.O_EXCL|os.O_CREATE, 0644)
if err != nil {
log.Print(err)
return send_result(errno(err), conn)
}
n, err = file.Write(req.file_data)
if err != nil || n != len(req.file_data) {
panic("write")
}
file_name = fmt.Sprintf("M00/%s", file_name)
storage_binlog_write(int(now.Unix()), STORAGE_OP_TYPE_SOURCE_CREATE_FILE, file_name)
resp := &UploadFileResponse{"group1", file_name}
return send_response(resp, conn)
}
func DecodeSnapshot4(f *os.File) (*Snapshot4, error) {
// Verify checksum
var checksum uint32
n, err := fmt.Fscanf(f, "%08x\n", &checksum)
if err != nil {
return nil, err
} else if n != 1 {
return nil, errors.New("miss heading checksum")
}
// Load remaining snapshot contents.
b, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
// Generate checksum.
byteChecksum := crc32.ChecksumIEEE(b)
if uint32(checksum) != byteChecksum {
return nil, errors.New("bad checksum")
}
// Decode snapshot.
snapshot := new(Snapshot4)
if err = json.Unmarshal(b, snapshot); err != nil {
return nil, err
}
return snapshot, nil
}
// Get implements the balancing algorithm interface.
func (r *Rendezvous) Get(path string) (*types.UpstreamAddress, error) {
r.RLock()
defer r.RUnlock()
if len(r.buckets) == 0 {
return nil, errors.New("No upstream addresses set!")
}
found := false
maxIdx := 0
var maxScore float64
//!TODO: implement O(log n) version: https://en.wikipedia.org/wiki/Rendezvous_hashing#Implementation
for i := range r.buckets {
key := []byte(r.buckets[i].Host + path)
//!TODO: use faster and better-distributed algorithm than crc32? xxhash? murmur?
score := r.buckets[i].weightPercent * float64(crc32.ChecksumIEEE(key))
if score > maxScore {
found = true
maxIdx = i
maxScore = score
}
}
if !found {
return nil, fmt.Errorf("No upstream addresses found for path %s", path)
}
return &r.buckets[maxIdx].UpstreamAddress, nil
}
func (i *index) Bytes() []byte {
b := make([]byte, headerSize)
w := bytes.NewWriter(b[4:])
binary.Write(w, binary.LittleEndian, i)
binary.LittleEndian.PutUint32(b, crc32.ChecksumIEEE(b[4:]))
return b
}
// validate returns nil if the entry's checksum is valid.
func (entry *LogEntry) validate() error {
expected := crc32.ChecksumIEEE(entry.Payload)
if entry.Checksum == expected {
return nil
}
return errors.New(fmt.Sprintf("Invalid checksum. Expected %d, was %d. Data was %v.", expected, entry.Checksum, entry.Payload))
}
func (t *tsmWriter) WriteBlock(key string, minTime, maxTime time.Time, block []byte) error {
// Nothing to write
if len(block) == 0 {
return nil
}
// Write header only after we have some data to write.
if t.n == 0 {
if err := t.writeHeader(); err != nil {
return err
}
}
checksum := crc32.ChecksumIEEE(block)
n, err := t.w.Write(append(u32tob(checksum), block...))
if err != nil {
return err
}
blockType, err := BlockType(block)
if err != nil {
return err
}
// Record this block in index
t.index.Add(key, blockType, minTime, maxTime, t.n, uint32(n))
// Increment file position pointer
t.n += int64(n)
return nil
}
func BenchmarkCRC32IEEE(b *testing.B) {
var bv uint32
for i := 0; i < b.N; i++ {
bv = crc32.ChecksumIEEE(in)
}
benchVal32 = bv
}
func (task *FlushTask) getRedisConn(key string) redis.Conn {
b := bytes.NewBufferString(key)
b.WriteString("turing")
idx := crc32.ChecksumIEEE(b.Bytes()) % uint32(len(task.redis_conn))
return task.redis_conn[idx]
}
// readHeader reads header part and checks it.
func (r *Reader) readHeader() error {
var err error
r.headerPos, err = r.rs.Seek(0, 1) // get offset of header
if err != nil {
r.curPos = posUnskippableError
return err
}
n, err := r.rs.Read(r.header[:])
if err != nil {
r.curPos = posUnskippableError
return err
}
if n != 16 {
r.curPos = posUnskippableError
return ERR_READ_HEADER_FAILED
}
checksum := binary.LittleEndian.Uint32(r.header[12:])
if checksum != crc32.ChecksumIEEE(r.header[0:12]) {
r.curPos = posUnskippableError
return ERR_CHECK_HEADER_FAILED
}
r.blockType = int8(r.header[HEADER_OFF_BLOCK_TYPE])
r.compressType = int8(r.header[HEADER_OFF_COMPRESS])
r.recNum = binary.LittleEndian.Uint16(r.header[HEADER_OFF_REC_NUM:])
r.bodySize = binary.LittleEndian.Uint32(r.header[HEADER_OFF_BODY_SIZE:])
return nil
}
// encode serializes the log entry to the passed io.Writer.
//
// Entries are serialized in a simple binary format:
//
// ---------------------------------------------
// | uint32 | uint64 | uint64 | uint32 | []byte |
// ---------------------------------------------
// | CRC | TERM | INDEX | SIZE | COMMAND |
// ---------------------------------------------
//
func (e *logEntry) encode(w io.Writer) error {
if len(e.Command) <= 0 {
return errNoCommand
}
if e.Index <= 0 {
return errBadIndex
}
if e.Term <= 0 {
return errBadTerm
}
commandSize := len(e.Command)
buf := make([]byte, 24+commandSize)
binary.LittleEndian.PutUint64(buf[4:12], e.Term)
binary.LittleEndian.PutUint64(buf[12:20], e.Index)
binary.LittleEndian.PutUint32(buf[20:24], uint32(commandSize))
copy(buf[24:], e.Command)
binary.LittleEndian.PutUint32(
buf[0:4],
crc32.ChecksumIEEE(buf[4:]),
)
_, err := w.Write(buf)
return err
}
func EncryptAESCFB(src *AuthToken, key []byte) []byte {
var err error
rawData := RawData{}
rawData.Data, err = json.Marshal(src)
if err != nil {
panic(err)
}
rawData.CSum = crc32.ChecksumIEEE(rawData.Data)
rawBytes, err := json.Marshal(rawData)
encrData := EncrData{
IV: randStr(len(key)),
Data: make([]byte, len(rawBytes)),
}
cipher.NewCFBEncrypter(aesCipher(key), encrData.IV).XORKeyStream(encrData.Data, rawBytes)
encrBytes, err := json.Marshal(encrData)
if err != nil {
panic(err)
}
return encrBytes
}