func (c *Consistent) hashKey(key string) uint32 { if len(key) < 64 { copy(c.scratch[:], key) return crc32.ChecksumIEEE(c.scratch[:len(key)]) } return crc32.ChecksumIEEE([]byte(key)) }
func compareFiles(srcPath, destPath string) (bool, error) { src, err := os.Open(srcPath) if err != nil { return false, fmt.Errorf("open source failed: %s", err) } defer src.Close() dest, err := os.Open(destPath) if err != nil { return false, fmt.Errorf("open dest failed: %s", err) } defer dest.Close() srcContents, err := ioutil.ReadAll(src) if err != nil { return false, fmt.Errorf("read source failed: %s", err) } destContents, err := ioutil.ReadAll(dest) if err != nil { return false, fmt.Errorf("read dest failed: %s", err) } contentsHash := fmt.Sprintf("%x", crc32.ChecksumIEEE(srcContents)) srcHash := fmt.Sprintf("%x", crc32.ChecksumIEEE(destContents)) return srcHash == contentsHash, nil }
// Apply applies the patch data to the given base. func Apply(base []byte, patch *common.PatchData, skipCRC bool) ([]byte, error) { if uint64(len(base)) != patch.InputFileSize { return nil, errors.New("Base file did not have expected size.") } if !skipCRC && crc32.ChecksumIEEE(base) != patch.InputChecksum { return nil, errors.New("Base file did not have expected checksum") } output := make([]byte, patch.OutputFileSize) copy(output, base) pointer := 0 for _, block := range patch.PatchBlocks { pointer += int(block.RelativeOffset) for _, b := range block.Data { if pointer >= len(base) { output[pointer] = b } else { output[pointer] = base[pointer] ^ b } pointer++ } pointer++ } if !skipCRC && crc32.ChecksumIEEE(output) != patch.OutputChecksum { return nil, errors.New("Patch result did not have expected checksum") } return output, nil }
func getCrc(key string) uint32 { if len(key) < 64 { var scratch [64]byte copy(scratch[:], key) return crc32.ChecksumIEEE(scratch[:len(key)]) } return crc32.ChecksumIEEE([]byte(key)) }
// Hash function func (c *Consistent) hashStr(key string) uint32 { if len(key) < 64 { var scratch [64]byte copy(scratch[:], key) return crc32.ChecksumIEEE(scratch[:len(key)]) } return crc32.ChecksumIEEE([]byte(key)) }
func main() { cs := crc32.ChecksumIEEE([]byte("1234667")) fmt.Println(cs) cs = crc32.ChecksumIEEE([]byte("dfdfdfdfdf")) fmt.Println(cs) }
func (c *Consistent) hashKey(key string) uint32 { // //log.Println("key string:", key) if len(key) < 64 { var scratch [64]byte copy(scratch[:], key) //log.Fatal(len(key), scratch) return crc32.ChecksumIEEE(scratch[:len(key)]) } return crc32.ChecksumIEEE([]byte(key)) }
func TestNew_resources(t *testing.T) { assert := assert.New(t) r, err := os.Open("testdata/wikipedia.webarchive") require.NoError(t, err) a, err := New(r) assert.Equal(uint32(0x7a24addb), crc32.ChecksumIEEE(a.Content.Data)) assert.Equal("text/html", a.Content.MIMEType) assert.Equal("UTF-8", a.Content.Encoding) assert.Equal("https://en.wikipedia.org/wiki/Webarchive", a.Content.URL) assert.Empty(a.Content.FrameName) assert.Len(a.Resources, 13) assert.NoError(err) resources := []struct { data uint32 mime string enc string url uint32 resp uint32 }{ {0xe0352bc6, "text/css", "utf-8", 0xa88c8657, 0x3e495c5f}, {0xd5fee4ca, "text/css", "utf-8", 0x348faf99, 0xe1b3e4d0}, {0xab5b0999, "text/javascript", "utf-8", 0xe9d253b4, 0x58d61084}, {0xf3c27e95, "text/javascript", "utf-8", 0xfeaf026a, 0x96e53929}, {0x59637a, "image/png", "", 0xbd73e04a, 0x6dd3aad5}, {0xf79b5fc7, "image/png", "", 0x8d210858, 0xfd3b31c4}, {0xfbea27a5, "image/jpeg", "", 0x482f361f, 0xdf0d81fd}, {0xdae1d80a, "image/png", "", 0x2ab8618b, 0x570a3987}, {0x285bcd96, "image/png", "", 0x5903bafa, 0xad214977}, {0xc296a07a, "image/png", "", 0x23103ba7, 0xd845f71}, {0x1af78e35, "image/png", "", 0x9af068df, 0x19fc7a6}, {0x7c4edfa, "image/png", "", 0x7de2a6e8, 0x36084aec}, {0x9885885, "text/css", "utf-8", 0x8c2feaff, 0xb01023b6}, } for i := 0; i < 13; i++ { sr := a.Resources[i] r := resources[i] s := fmt.Sprintf("subresource %d", i) assert.Equal(r.data, crc32.ChecksumIEEE(sr.Data), s) assert.Equal(r.mime, sr.MIMEType, s) assert.Equal(r.enc, sr.Encoding, s) assert.Equal(r.url, crc32.ChecksumIEEE([]byte(sr.URL)), s) assert.Equal(r.resp, crc32.ChecksumIEEE(sr.Response), s) } }
func HashValue(value interface{}) uint64 { switch val := value.(type) { case int: return uint64(val) case uint64: return uint64(val) case int64: return uint64(val) case string: return uint64(crc32.ChecksumIEEE(hack.Slice(val))) case []byte: return uint64(crc32.ChecksumIEEE(val)) } panic(NewKeyError("Unexpected key variable type %T", value)) }
func TestEncodeDecodeEntry(t *testing.T) { /** crc32 : tStamp : ksz : valueSz : key : value 4 : 4 : 4 : 4 : xxxx : xxxx **/ //func encodeEntry(tStamp, keySize, valueSize uint32, key, value []byte) []byte { // EncodeEntry tStamp := uint32(time.Now().Unix()) key := []byte("Foo") value := []byte("Bar") ksz := uint32(len(key)) valuesz := uint32(len(value)) buf := make([]byte, HeaderSize+ksz+valuesz) binary.LittleEndian.PutUint32(buf[4:8], tStamp) binary.LittleEndian.PutUint32(buf[8:12], ksz) binary.LittleEndian.PutUint32(buf[12:16], valuesz) copy(buf[16:(16+ksz)], key) copy(buf[(16+ksz):(16+ksz+valuesz)], value) c32 := crc32.ChecksumIEEE(buf[4:]) binary.LittleEndian.PutUint32(buf[0:4], uint32(c32)) // Test decode ksz = binary.LittleEndian.Uint32(buf[8:12]) valuesz = binary.LittleEndian.Uint32(buf[12:16]) tStamp = binary.LittleEndian.Uint32(buf[4:8]) c32 = binary.LittleEndian.Uint32(buf[:4]) assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32) assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp) assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz) assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz) assert.Equal(t, buf[HeaderSize:(HeaderSize+ksz)], key) assert.Equal(t, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)], value) // EncodeEntry , ksz = 0, valueSz = 0 ksz = uint32(0) valuesz = uint32(0) buf = make([]byte, HeaderSize+ksz+valuesz, HeaderSize+ksz+valuesz) binary.LittleEndian.PutUint32(buf[4:8], tStamp) binary.LittleEndian.PutUint32(buf[8:12], ksz) binary.LittleEndian.PutUint32(buf[12:16], valuesz) c32 = crc32.ChecksumIEEE(buf[4:]) binary.LittleEndian.PutUint32(buf[0:4], c32) // decodeEntry, ksz =0, valueSz = 0 assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32) assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp) assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz) assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz) }
func Read(r io.Reader) (*Message, error) { var incomingHeader header err := binary.Read(r, binary.LittleEndian, &incomingHeader) if err != nil { return nil, err } if incomingHeader.ContentLength > MaximumMessageSize { return nil, fmt.Errorf("message too large: requested %d bytes but max is %d", incomingHeader.ContentLength, MaximumMessageSize) } msg := new(Message) data := make([]byte, incomingHeader.ContentLength) // Keep reading so we are resilient to IP fragmentation along the path. for n := uint32(0); n < incomingHeader.ContentLength; { nRead, err := r.Read(data[n:]) if err != nil { return nil, err } n += uint32(nRead) } // Checksum the incoming data so that transmission errors can be dealt with. check := crc32.ChecksumIEEE(data) if check != incomingHeader.Checksum { return nil, ErrCorruptedMessage } err = proto.Unmarshal(data[0:incomingHeader.ContentLength], msg) return msg, err }
// Write marshals a Message into the proper on-wire format and sends it // to the remote system. func Write(w io.Writer, msg *Message) error { buf, err := proto.Marshal(msg) if err != nil { return err } // For now we just compute the length as we don't have tests. bufHeader := &header{ ContentLength: uint32(len(buf)), Checksum: crc32.ChecksumIEEE(buf), Reserved: 0, } err = binary.Write(w, binary.LittleEndian, bufHeader) if err != nil { return err } for totalWritten := 0; totalWritten < len(buf); { nWritten, err := w.Write(buf) if err != nil { return err } totalWritten += nWritten } return nil }
func (t *CRCMessage) Marshal(w *wipro.Writer) { offset := len(w.B) w.WriteUint32(t.CRC) start := len(w.B) t.Message.Marshal(w) w.SetUint32(offset, crc32.ChecksumIEEE(w.B[start:])) }
func MakeHashes(word string, methods []string) map[string]string { hashes := make(map[string]string) for _, method := range methods { var hash string // TODO: Replace this with reflection // for easier extensibility switch method { case "crc32": hashInt := uint64(crc32.ChecksumIEEE([]byte(word))) hash = strconv.FormatUint(hashInt, 16) case "md5": hash = fmt.Sprintf("%x", md5.Sum([]byte(word))) case "sha1": hash = fmt.Sprintf("%x", sha1.Sum([]byte(word))) case "sha256": hash = fmt.Sprintf("%x", sha256.Sum256([]byte(word))) } hashes[method] = hash } return hashes }
// save writes the snapshot to file. func (ss *Snapshot) save() error { // Open the file for writing. file, err := os.OpenFile(ss.Path, os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return err } defer file.Close() // Serialize to JSON. b, err := json.Marshal(ss) if err != nil { return err } // Generate checksum and write it to disk. checksum := crc32.ChecksumIEEE(b) if _, err = fmt.Fprintf(file, "%08x\n", checksum); err != nil { return err } // Write the snapshot to disk. if _, err = file.Write(b); err != nil { return err } // Ensure that the snapshot has been flushed to disk before continuing. if err := file.Sync(); err != nil { return err } return nil }
func readRequestBody(r io.Reader, header *wire.RequestHeader, request proto.Message) error { // recv body (end) compressedPbRequest, err := recvFrame(r) if err != nil { return err } // checksum if crc32.ChecksumIEEE(compressedPbRequest) != header.GetChecksum() { return fmt.Errorf("protorpc.readRequestBody: unexpected checksum.") } // decode the compressed data pbRequest, err := snappy.Decode(nil, compressedPbRequest) if err != nil { return err } // check wire header: rawMsgLen if uint32(len(pbRequest)) != header.GetRawRequestLen() { return fmt.Errorf("protorpc.readRequestBody: Unexcpeted header.RawRequestLen.") } // Unmarshal to proto message if request != nil { err = proto.Unmarshal(pbRequest, request) if err != nil { return err } } return nil }
func (this Table) calcPartitionsCRC() uint32 { buf := &bytes.Buffer{} for _, part := range this.Partitions { part.write(buf, this.Header.PartitionEntrySize) } return crc32.ChecksumIEEE(buf.Bytes()) }
func DecryptAESCFB(src, key []byte) *AuthToken { encrData := EncrData{} err := json.Unmarshal(src, &encrData) if err != nil { return nil } if len(encrData.IV) != len(key) { return nil } rawBytes := make([]byte, len(encrData.Data)) cipher.NewCFBDecrypter(aesCipher(key), encrData.IV).XORKeyStream(rawBytes, encrData.Data) rawData := RawData{} err = json.Unmarshal(rawBytes, &rawData) if err != nil { return nil } cSum := crc32.ChecksumIEEE(rawData.Data) if cSum != rawData.CSum { return nil } res := AuthToken{} err = json.Unmarshal(rawData.Data, &res) if err != nil { return nil } return &res }
// Send sends the message to the broker, and blocks until an acknowledgement is // received. If the max number of retries is exceeded, returns the last error. func (p *Producer) Send(topic string, payload []byte) error { seqnum := atomic.AddInt64(&p.seqnum, 1) message := protocol.Message{seqnum, payload, crc32.ChecksumIEEE(payload)} request := &protocol.ProduceRequest{p.id, topic, message} log.Debug("Sending %v", request) p.lock.Lock() defer p.lock.Unlock() for { if _, err := p.socket.Send(request, MAX_RETRIES, origin()); nil != err { p.socket.Reset(p.register) continue } log.Debug("Acknowledgement received.") return nil } return nil }
func storage_upload_file(header *Header, conn net.Conn) bool { buff := make([]byte, header.pkg_len) n, err := io.ReadFull(conn, buff) if err != nil || n != int(header.pkg_len) { return false } req := &UploadFileRequest{} err = req.Unmarshal(buff) if err != nil { return false } file_ext := storage_format_ext_name(req.file_ext) crc32 := crc32.ChecksumIEEE(req.file_data) now := time.Now() // log.Println(req.store_path_index, req.file_length, len(req.file_data), file_ext, crc32) addr, _ := conn.RemoteAddr().(*net.TCPAddr) file_name, full_name := storage_get_filename(addr.IP.String(), int(req.store_path_index), int(now.Unix()), req.file_length, int(crc32), file_ext) file, err := os.OpenFile(full_name, os.O_WRONLY|os.O_EXCL|os.O_CREATE, 0644) if err != nil { log.Print(err) return send_result(errno(err), conn) } n, err = file.Write(req.file_data) if err != nil || n != len(req.file_data) { panic("write") } file_name = fmt.Sprintf("M00/%s", file_name) storage_binlog_write(int(now.Unix()), STORAGE_OP_TYPE_SOURCE_CREATE_FILE, file_name) resp := &UploadFileResponse{"group1", file_name} return send_response(resp, conn) }
func DecodeSnapshot4(f *os.File) (*Snapshot4, error) { // Verify checksum var checksum uint32 n, err := fmt.Fscanf(f, "%08x\n", &checksum) if err != nil { return nil, err } else if n != 1 { return nil, errors.New("miss heading checksum") } // Load remaining snapshot contents. b, err := ioutil.ReadAll(f) if err != nil { return nil, err } // Generate checksum. byteChecksum := crc32.ChecksumIEEE(b) if uint32(checksum) != byteChecksum { return nil, errors.New("bad checksum") } // Decode snapshot. snapshot := new(Snapshot4) if err = json.Unmarshal(b, snapshot); err != nil { return nil, err } return snapshot, nil }
// Get implements the balancing algorithm interface. func (r *Rendezvous) Get(path string) (*types.UpstreamAddress, error) { r.RLock() defer r.RUnlock() if len(r.buckets) == 0 { return nil, errors.New("No upstream addresses set!") } found := false maxIdx := 0 var maxScore float64 //!TODO: implement O(log n) version: https://en.wikipedia.org/wiki/Rendezvous_hashing#Implementation for i := range r.buckets { key := []byte(r.buckets[i].Host + path) //!TODO: use faster and better-distributed algorithm than crc32? xxhash? murmur? score := r.buckets[i].weightPercent * float64(crc32.ChecksumIEEE(key)) if score > maxScore { found = true maxIdx = i maxScore = score } } if !found { return nil, fmt.Errorf("No upstream addresses found for path %s", path) } return &r.buckets[maxIdx].UpstreamAddress, nil }
func (i *index) Bytes() []byte { b := make([]byte, headerSize) w := bytes.NewWriter(b[4:]) binary.Write(w, binary.LittleEndian, i) binary.LittleEndian.PutUint32(b, crc32.ChecksumIEEE(b[4:])) return b }
// validate returns nil if the entry's checksum is valid. func (entry *LogEntry) validate() error { expected := crc32.ChecksumIEEE(entry.Payload) if entry.Checksum == expected { return nil } return errors.New(fmt.Sprintf("Invalid checksum. Expected %d, was %d. Data was %v.", expected, entry.Checksum, entry.Payload)) }
func (t *tsmWriter) WriteBlock(key string, minTime, maxTime time.Time, block []byte) error { // Nothing to write if len(block) == 0 { return nil } // Write header only after we have some data to write. if t.n == 0 { if err := t.writeHeader(); err != nil { return err } } checksum := crc32.ChecksumIEEE(block) n, err := t.w.Write(append(u32tob(checksum), block...)) if err != nil { return err } blockType, err := BlockType(block) if err != nil { return err } // Record this block in index t.index.Add(key, blockType, minTime, maxTime, t.n, uint32(n)) // Increment file position pointer t.n += int64(n) return nil }
func BenchmarkCRC32IEEE(b *testing.B) { var bv uint32 for i := 0; i < b.N; i++ { bv = crc32.ChecksumIEEE(in) } benchVal32 = bv }
func (task *FlushTask) getRedisConn(key string) redis.Conn { b := bytes.NewBufferString(key) b.WriteString("turing") idx := crc32.ChecksumIEEE(b.Bytes()) % uint32(len(task.redis_conn)) return task.redis_conn[idx] }
// readHeader reads header part and checks it. func (r *Reader) readHeader() error { var err error r.headerPos, err = r.rs.Seek(0, 1) // get offset of header if err != nil { r.curPos = posUnskippableError return err } n, err := r.rs.Read(r.header[:]) if err != nil { r.curPos = posUnskippableError return err } if n != 16 { r.curPos = posUnskippableError return ERR_READ_HEADER_FAILED } checksum := binary.LittleEndian.Uint32(r.header[12:]) if checksum != crc32.ChecksumIEEE(r.header[0:12]) { r.curPos = posUnskippableError return ERR_CHECK_HEADER_FAILED } r.blockType = int8(r.header[HEADER_OFF_BLOCK_TYPE]) r.compressType = int8(r.header[HEADER_OFF_COMPRESS]) r.recNum = binary.LittleEndian.Uint16(r.header[HEADER_OFF_REC_NUM:]) r.bodySize = binary.LittleEndian.Uint32(r.header[HEADER_OFF_BODY_SIZE:]) return nil }
// encode serializes the log entry to the passed io.Writer. // // Entries are serialized in a simple binary format: // // --------------------------------------------- // | uint32 | uint64 | uint64 | uint32 | []byte | // --------------------------------------------- // | CRC | TERM | INDEX | SIZE | COMMAND | // --------------------------------------------- // func (e *logEntry) encode(w io.Writer) error { if len(e.Command) <= 0 { return errNoCommand } if e.Index <= 0 { return errBadIndex } if e.Term <= 0 { return errBadTerm } commandSize := len(e.Command) buf := make([]byte, 24+commandSize) binary.LittleEndian.PutUint64(buf[4:12], e.Term) binary.LittleEndian.PutUint64(buf[12:20], e.Index) binary.LittleEndian.PutUint32(buf[20:24], uint32(commandSize)) copy(buf[24:], e.Command) binary.LittleEndian.PutUint32( buf[0:4], crc32.ChecksumIEEE(buf[4:]), ) _, err := w.Write(buf) return err }
func EncryptAESCFB(src *AuthToken, key []byte) []byte { var err error rawData := RawData{} rawData.Data, err = json.Marshal(src) if err != nil { panic(err) } rawData.CSum = crc32.ChecksumIEEE(rawData.Data) rawBytes, err := json.Marshal(rawData) encrData := EncrData{ IV: randStr(len(key)), Data: make([]byte, len(rawBytes)), } cipher.NewCFBEncrypter(aesCipher(key), encrData.IV).XORKeyStream(encrData.Data, rawBytes) encrBytes, err := json.Marshal(encrData) if err != nil { panic(err) } return encrBytes }