Exemple #1
0
func (c *Consistent) hashKey(key string) uint32 {
	if len(key) < 64 {
		copy(c.scratch[:], key)
		return crc32.ChecksumIEEE(c.scratch[:len(key)])
	}
	return crc32.ChecksumIEEE([]byte(key))
}
Exemple #2
0
func compareFiles(srcPath, destPath string) (bool, error) {
	src, err := os.Open(srcPath)
	if err != nil {
		return false, fmt.Errorf("open source failed: %s", err)
	}
	defer src.Close()

	dest, err := os.Open(destPath)
	if err != nil {
		return false, fmt.Errorf("open dest failed: %s", err)
	}
	defer dest.Close()

	srcContents, err := ioutil.ReadAll(src)
	if err != nil {
		return false, fmt.Errorf("read source failed: %s", err)
	}
	destContents, err := ioutil.ReadAll(dest)
	if err != nil {
		return false, fmt.Errorf("read dest failed: %s", err)
	}

	contentsHash := fmt.Sprintf("%x", crc32.ChecksumIEEE(srcContents))
	srcHash := fmt.Sprintf("%x", crc32.ChecksumIEEE(destContents))

	return srcHash == contentsHash, nil
}
Exemple #3
0
// Apply applies the patch data to the given base.
func Apply(base []byte, patch *common.PatchData, skipCRC bool) ([]byte, error) {
	if uint64(len(base)) != patch.InputFileSize {
		return nil, errors.New("Base file did not have expected size.")
	}
	if !skipCRC && crc32.ChecksumIEEE(base) != patch.InputChecksum {
		return nil, errors.New("Base file did not have expected checksum")
	}

	output := make([]byte, patch.OutputFileSize)
	copy(output, base)

	pointer := 0
	for _, block := range patch.PatchBlocks {
		pointer += int(block.RelativeOffset)

		for _, b := range block.Data {
			if pointer >= len(base) {
				output[pointer] = b
			} else {
				output[pointer] = base[pointer] ^ b
			}
			pointer++
		}

		pointer++
	}

	if !skipCRC && crc32.ChecksumIEEE(output) != patch.OutputChecksum {
		return nil, errors.New("Patch result did not have expected checksum")
	}

	return output, nil
}
Exemple #4
0
func getCrc(key string) uint32 {
	if len(key) < 64 {
		var scratch [64]byte
		copy(scratch[:], key)
		return crc32.ChecksumIEEE(scratch[:len(key)])
	}
	return crc32.ChecksumIEEE([]byte(key))
}
Exemple #5
0
// Hash function
func (c *Consistent) hashStr(key string) uint32 {
	if len(key) < 64 {
		var scratch [64]byte
		copy(scratch[:], key)
		return crc32.ChecksumIEEE(scratch[:len(key)])
	}
	return crc32.ChecksumIEEE([]byte(key))
}
func main() {

	cs := crc32.ChecksumIEEE([]byte("1234667"))
	fmt.Println(cs)

	cs = crc32.ChecksumIEEE([]byte("dfdfdfdfdf"))
	fmt.Println(cs)
}
Exemple #7
0
func (c *Consistent) hashKey(key string) uint32 {
	//
	//log.Println("key string:", key)
	if len(key) < 64 {
		var scratch [64]byte
		copy(scratch[:], key)
		//log.Fatal(len(key), scratch)
		return crc32.ChecksumIEEE(scratch[:len(key)])
	}
	return crc32.ChecksumIEEE([]byte(key))
}
func TestNew_resources(t *testing.T) {
	assert := assert.New(t)

	r, err := os.Open("testdata/wikipedia.webarchive")
	require.NoError(t, err)

	a, err := New(r)
	assert.Equal(uint32(0x7a24addb), crc32.ChecksumIEEE(a.Content.Data))
	assert.Equal("text/html", a.Content.MIMEType)
	assert.Equal("UTF-8", a.Content.Encoding)
	assert.Equal("https://en.wikipedia.org/wiki/Webarchive", a.Content.URL)
	assert.Empty(a.Content.FrameName)
	assert.Len(a.Resources, 13)
	assert.NoError(err)

	resources := []struct {
		data uint32
		mime string
		enc  string
		url  uint32
		resp uint32
	}{
		{0xe0352bc6, "text/css", "utf-8", 0xa88c8657, 0x3e495c5f},
		{0xd5fee4ca, "text/css", "utf-8", 0x348faf99, 0xe1b3e4d0},
		{0xab5b0999, "text/javascript", "utf-8", 0xe9d253b4, 0x58d61084},
		{0xf3c27e95, "text/javascript", "utf-8", 0xfeaf026a, 0x96e53929},
		{0x59637a, "image/png", "", 0xbd73e04a, 0x6dd3aad5},
		{0xf79b5fc7, "image/png", "", 0x8d210858, 0xfd3b31c4},
		{0xfbea27a5, "image/jpeg", "", 0x482f361f, 0xdf0d81fd},
		{0xdae1d80a, "image/png", "", 0x2ab8618b, 0x570a3987},
		{0x285bcd96, "image/png", "", 0x5903bafa, 0xad214977},
		{0xc296a07a, "image/png", "", 0x23103ba7, 0xd845f71},
		{0x1af78e35, "image/png", "", 0x9af068df, 0x19fc7a6},
		{0x7c4edfa, "image/png", "", 0x7de2a6e8, 0x36084aec},
		{0x9885885, "text/css", "utf-8", 0x8c2feaff, 0xb01023b6},
	}

	for i := 0; i < 13; i++ {
		sr := a.Resources[i]
		r := resources[i]
		s := fmt.Sprintf("subresource %d", i)

		assert.Equal(r.data, crc32.ChecksumIEEE(sr.Data), s)
		assert.Equal(r.mime, sr.MIMEType, s)
		assert.Equal(r.enc, sr.Encoding, s)
		assert.Equal(r.url, crc32.ChecksumIEEE([]byte(sr.URL)), s)
		assert.Equal(r.resp, crc32.ChecksumIEEE(sr.Response), s)
	}
}
Exemple #9
0
func HashValue(value interface{}) uint64 {
	switch val := value.(type) {
	case int:
		return uint64(val)
	case uint64:
		return uint64(val)
	case int64:
		return uint64(val)
	case string:
		return uint64(crc32.ChecksumIEEE(hack.Slice(val)))
	case []byte:
		return uint64(crc32.ChecksumIEEE(val))
	}
	panic(NewKeyError("Unexpected key variable type %T", value))
}
Exemple #10
0
func TestEncodeDecodeEntry(t *testing.T) {
	/**
		crc32	:	tStamp	:	ksz	:	valueSz	:	key	:	value
		4 		:	4 		: 	4 	: 		4	:	xxxx	: xxxx
	**/
	//func encodeEntry(tStamp, keySize, valueSize uint32, key, value []byte) []byte {
	// EncodeEntry
	tStamp := uint32(time.Now().Unix())
	key := []byte("Foo")
	value := []byte("Bar")
	ksz := uint32(len(key))
	valuesz := uint32(len(value))
	buf := make([]byte, HeaderSize+ksz+valuesz)
	binary.LittleEndian.PutUint32(buf[4:8], tStamp)
	binary.LittleEndian.PutUint32(buf[8:12], ksz)
	binary.LittleEndian.PutUint32(buf[12:16], valuesz)
	copy(buf[16:(16+ksz)], key)
	copy(buf[(16+ksz):(16+ksz+valuesz)], value)
	c32 := crc32.ChecksumIEEE(buf[4:])
	binary.LittleEndian.PutUint32(buf[0:4], uint32(c32))
	// Test decode

	ksz = binary.LittleEndian.Uint32(buf[8:12])
	valuesz = binary.LittleEndian.Uint32(buf[12:16])
	tStamp = binary.LittleEndian.Uint32(buf[4:8])
	c32 = binary.LittleEndian.Uint32(buf[:4])
	assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32)
	assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp)
	assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz)
	assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz)
	assert.Equal(t, buf[HeaderSize:(HeaderSize+ksz)], key)
	assert.Equal(t, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)], value)

	// EncodeEntry , ksz = 0, valueSz = 0
	ksz = uint32(0)
	valuesz = uint32(0)
	buf = make([]byte, HeaderSize+ksz+valuesz, HeaderSize+ksz+valuesz)
	binary.LittleEndian.PutUint32(buf[4:8], tStamp)
	binary.LittleEndian.PutUint32(buf[8:12], ksz)
	binary.LittleEndian.PutUint32(buf[12:16], valuesz)
	c32 = crc32.ChecksumIEEE(buf[4:])
	binary.LittleEndian.PutUint32(buf[0:4], c32)
	// decodeEntry, ksz =0, valueSz = 0
	assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32)
	assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp)
	assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz)
	assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz)
}
Exemple #11
0
func Read(r io.Reader) (*Message, error) {
	var incomingHeader header

	err := binary.Read(r, binary.LittleEndian, &incomingHeader)
	if err != nil {
		return nil, err
	}

	if incomingHeader.ContentLength > MaximumMessageSize {
		return nil, fmt.Errorf("message too large: requested %d bytes but max is %d", incomingHeader.ContentLength, MaximumMessageSize)
	}

	msg := new(Message)
	data := make([]byte, incomingHeader.ContentLength)

	// Keep reading so we are resilient to IP fragmentation along the path.
	for n := uint32(0); n < incomingHeader.ContentLength; {
		nRead, err := r.Read(data[n:])
		if err != nil {
			return nil, err
		}
		n += uint32(nRead)
	}

	// Checksum the incoming data so that transmission errors can be dealt with.
	check := crc32.ChecksumIEEE(data)
	if check != incomingHeader.Checksum {
		return nil, ErrCorruptedMessage
	}

	err = proto.Unmarshal(data[0:incomingHeader.ContentLength], msg)
	return msg, err
}
Exemple #12
0
// Write marshals a Message into the proper on-wire format and sends it
// to the remote system.
func Write(w io.Writer, msg *Message) error {
	buf, err := proto.Marshal(msg)
	if err != nil {
		return err
	}

	// For now we just compute the length as we don't have tests.
	bufHeader := &header{
		ContentLength: uint32(len(buf)),
		Checksum:      crc32.ChecksumIEEE(buf),
		Reserved:      0,
	}

	err = binary.Write(w, binary.LittleEndian, bufHeader)
	if err != nil {
		return err
	}

	for totalWritten := 0; totalWritten < len(buf); {
		nWritten, err := w.Write(buf)
		if err != nil {
			return err
		}
		totalWritten += nWritten
	}
	return nil
}
func (t *CRCMessage) Marshal(w *wipro.Writer) {
	offset := len(w.B)
	w.WriteUint32(t.CRC)
	start := len(w.B)
	t.Message.Marshal(w)
	w.SetUint32(offset, crc32.ChecksumIEEE(w.B[start:]))
}
func MakeHashes(word string, methods []string) map[string]string {
	hashes := make(map[string]string)
	for _, method := range methods {
		var hash string

		// TODO: Replace this with reflection
		// 		 for easier extensibility
		switch method {
		case "crc32":
			hashInt := uint64(crc32.ChecksumIEEE([]byte(word)))

			hash = strconv.FormatUint(hashInt, 16)
		case "md5":
			hash = fmt.Sprintf("%x", md5.Sum([]byte(word)))

		case "sha1":
			hash = fmt.Sprintf("%x", sha1.Sum([]byte(word)))

		case "sha256":
			hash = fmt.Sprintf("%x", sha256.Sum256([]byte(word)))

		}

		hashes[method] = hash
	}

	return hashes
}
Exemple #15
0
// save writes the snapshot to file.
func (ss *Snapshot) save() error {
	// Open the file for writing.
	file, err := os.OpenFile(ss.Path, os.O_CREATE|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer file.Close()

	// Serialize to JSON.
	b, err := json.Marshal(ss)
	if err != nil {
		return err
	}

	// Generate checksum and write it to disk.
	checksum := crc32.ChecksumIEEE(b)
	if _, err = fmt.Fprintf(file, "%08x\n", checksum); err != nil {
		return err
	}

	// Write the snapshot to disk.
	if _, err = file.Write(b); err != nil {
		return err
	}

	// Ensure that the snapshot has been flushed to disk before continuing.
	if err := file.Sync(); err != nil {
		return err
	}

	return nil
}
Exemple #16
0
func readRequestBody(r io.Reader, header *wire.RequestHeader, request proto.Message) error {
	// recv body (end)
	compressedPbRequest, err := recvFrame(r)
	if err != nil {
		return err
	}

	// checksum
	if crc32.ChecksumIEEE(compressedPbRequest) != header.GetChecksum() {
		return fmt.Errorf("protorpc.readRequestBody: unexpected checksum.")
	}

	// decode the compressed data
	pbRequest, err := snappy.Decode(nil, compressedPbRequest)
	if err != nil {
		return err
	}
	// check wire header: rawMsgLen
	if uint32(len(pbRequest)) != header.GetRawRequestLen() {
		return fmt.Errorf("protorpc.readRequestBody: Unexcpeted header.RawRequestLen.")
	}

	// Unmarshal to proto message
	if request != nil {
		err = proto.Unmarshal(pbRequest, request)
		if err != nil {
			return err
		}
	}

	return nil
}
Exemple #17
0
Fichier : gpt.go Projet : rekby/gpt
func (this Table) calcPartitionsCRC() uint32 {
	buf := &bytes.Buffer{}
	for _, part := range this.Partitions {
		part.write(buf, this.Header.PartitionEntrySize)
	}
	return crc32.ChecksumIEEE(buf.Bytes())
}
func DecryptAESCFB(src, key []byte) *AuthToken {
	encrData := EncrData{}
	err := json.Unmarshal(src, &encrData)
	if err != nil {
		return nil
	}

	if len(encrData.IV) != len(key) {
		return nil
	}

	rawBytes := make([]byte, len(encrData.Data))
	cipher.NewCFBDecrypter(aesCipher(key), encrData.IV).XORKeyStream(rawBytes, encrData.Data)

	rawData := RawData{}
	err = json.Unmarshal(rawBytes, &rawData)
	if err != nil {
		return nil
	}

	cSum := crc32.ChecksumIEEE(rawData.Data)
	if cSum != rawData.CSum {
		return nil
	}

	res := AuthToken{}
	err = json.Unmarshal(rawData.Data, &res)
	if err != nil {
		return nil
	}

	return &res
}
Exemple #19
0
// Send sends the message to the broker, and blocks until an acknowledgement is
// received. If the max number of retries is exceeded, returns the last error.
func (p *Producer) Send(topic string, payload []byte) error {

	seqnum := atomic.AddInt64(&p.seqnum, 1)
	message := protocol.Message{seqnum, payload, crc32.ChecksumIEEE(payload)}
	request := &protocol.ProduceRequest{p.id, topic, message}

	log.Debug("Sending %v", request)

	p.lock.Lock()
	defer p.lock.Unlock()

	for {

		if _, err := p.socket.Send(request, MAX_RETRIES, origin()); nil != err {
			p.socket.Reset(p.register)
			continue
		}

		log.Debug("Acknowledgement received.")
		return nil

	}

	return nil

}
Exemple #20
0
func storage_upload_file(header *Header, conn net.Conn) bool {
	buff := make([]byte, header.pkg_len)
	n, err := io.ReadFull(conn, buff)
	if err != nil || n != int(header.pkg_len) {
		return false
	}
	req := &UploadFileRequest{}
	err = req.Unmarshal(buff)
	if err != nil {
		return false
	}

	file_ext := storage_format_ext_name(req.file_ext)
	crc32 := crc32.ChecksumIEEE(req.file_data)
	now := time.Now()
	//	log.Println(req.store_path_index, req.file_length,  len(req.file_data), file_ext, crc32)

	addr, _ := conn.RemoteAddr().(*net.TCPAddr)
	file_name, full_name := storage_get_filename(addr.IP.String(), int(req.store_path_index), int(now.Unix()), req.file_length, int(crc32), file_ext)
	file, err := os.OpenFile(full_name, os.O_WRONLY|os.O_EXCL|os.O_CREATE, 0644)
	if err != nil {
		log.Print(err)
		return send_result(errno(err), conn)
	}
	n, err = file.Write(req.file_data)
	if err != nil || n != len(req.file_data) {
		panic("write")
	}

	file_name = fmt.Sprintf("M00/%s", file_name)
	storage_binlog_write(int(now.Unix()), STORAGE_OP_TYPE_SOURCE_CREATE_FILE, file_name)

	resp := &UploadFileResponse{"group1", file_name}
	return send_response(resp, conn)
}
func DecodeSnapshot4(f *os.File) (*Snapshot4, error) {
	// Verify checksum
	var checksum uint32
	n, err := fmt.Fscanf(f, "%08x\n", &checksum)
	if err != nil {
		return nil, err
	} else if n != 1 {
		return nil, errors.New("miss heading checksum")
	}

	// Load remaining snapshot contents.
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	// Generate checksum.
	byteChecksum := crc32.ChecksumIEEE(b)
	if uint32(checksum) != byteChecksum {
		return nil, errors.New("bad checksum")
	}

	// Decode snapshot.
	snapshot := new(Snapshot4)
	if err = json.Unmarshal(b, snapshot); err != nil {
		return nil, err
	}
	return snapshot, nil
}
Exemple #22
0
// Get implements the balancing algorithm interface.
func (r *Rendezvous) Get(path string) (*types.UpstreamAddress, error) {
	r.RLock()
	defer r.RUnlock()
	if len(r.buckets) == 0 {
		return nil, errors.New("No upstream addresses set!")
	}

	found := false
	maxIdx := 0
	var maxScore float64

	//!TODO: implement O(log n) version: https://en.wikipedia.org/wiki/Rendezvous_hashing#Implementation
	for i := range r.buckets {
		key := []byte(r.buckets[i].Host + path)
		//!TODO: use faster and better-distributed algorithm than crc32? xxhash? murmur?
		score := r.buckets[i].weightPercent * float64(crc32.ChecksumIEEE(key))
		if score > maxScore {
			found = true
			maxIdx = i
			maxScore = score
		}
	}

	if !found {
		return nil, fmt.Errorf("No upstream addresses found for path %s", path)
	}

	return &r.buckets[maxIdx].UpstreamAddress, nil
}
Exemple #23
0
func (i *index) Bytes() []byte {
	b := make([]byte, headerSize)
	w := bytes.NewWriter(b[4:])
	binary.Write(w, binary.LittleEndian, i)
	binary.LittleEndian.PutUint32(b, crc32.ChecksumIEEE(b[4:]))
	return b
}
Exemple #24
0
// validate returns nil if the entry's checksum is valid.
func (entry *LogEntry) validate() error {
	expected := crc32.ChecksumIEEE(entry.Payload)
	if entry.Checksum == expected {
		return nil
	}
	return errors.New(fmt.Sprintf("Invalid checksum. Expected %d, was %d. Data was %v.", expected, entry.Checksum, entry.Payload))
}
Exemple #25
0
func (t *tsmWriter) WriteBlock(key string, minTime, maxTime time.Time, block []byte) error {
	// Nothing to write
	if len(block) == 0 {
		return nil
	}

	// Write header only after we have some data to write.
	if t.n == 0 {
		if err := t.writeHeader(); err != nil {
			return err
		}
	}

	checksum := crc32.ChecksumIEEE(block)

	n, err := t.w.Write(append(u32tob(checksum), block...))
	if err != nil {
		return err
	}

	blockType, err := BlockType(block)
	if err != nil {
		return err
	}
	// Record this block in index
	t.index.Add(key, blockType, minTime, maxTime, t.n, uint32(n))

	// Increment file position pointer
	t.n += int64(n)

	return nil
}
Exemple #26
0
func BenchmarkCRC32IEEE(b *testing.B) {
	var bv uint32
	for i := 0; i < b.N; i++ {
		bv = crc32.ChecksumIEEE(in)
	}
	benchVal32 = bv
}
Exemple #27
0
func (task *FlushTask) getRedisConn(key string) redis.Conn {
	b := bytes.NewBufferString(key)
	b.WriteString("turing")

	idx := crc32.ChecksumIEEE(b.Bytes()) % uint32(len(task.redis_conn))
	return task.redis_conn[idx]
}
Exemple #28
0
// readHeader reads header part and checks it.
func (r *Reader) readHeader() error {
	var err error
	r.headerPos, err = r.rs.Seek(0, 1) // get offset of header
	if err != nil {
		r.curPos = posUnskippableError
		return err
	}
	n, err := r.rs.Read(r.header[:])
	if err != nil {
		r.curPos = posUnskippableError
		return err
	}
	if n != 16 {
		r.curPos = posUnskippableError
		return ERR_READ_HEADER_FAILED
	}

	checksum := binary.LittleEndian.Uint32(r.header[12:])
	if checksum != crc32.ChecksumIEEE(r.header[0:12]) {
		r.curPos = posUnskippableError
		return ERR_CHECK_HEADER_FAILED
	}

	r.blockType = int8(r.header[HEADER_OFF_BLOCK_TYPE])
	r.compressType = int8(r.header[HEADER_OFF_COMPRESS])
	r.recNum = binary.LittleEndian.Uint16(r.header[HEADER_OFF_REC_NUM:])
	r.bodySize = binary.LittleEndian.Uint32(r.header[HEADER_OFF_BODY_SIZE:])
	return nil
}
Exemple #29
0
// encode serializes the log entry to the passed io.Writer.
//
// Entries are serialized in a simple binary format:
//
//		 ---------------------------------------------
//		| uint32 | uint64 | uint64 | uint32 | []byte  |
//		 ---------------------------------------------
//		| CRC    | TERM   | INDEX  | SIZE   | COMMAND |
//		 ---------------------------------------------
//
func (e *logEntry) encode(w io.Writer) error {
	if len(e.Command) <= 0 {
		return errNoCommand
	}
	if e.Index <= 0 {
		return errBadIndex
	}
	if e.Term <= 0 {
		return errBadTerm
	}

	commandSize := len(e.Command)
	buf := make([]byte, 24+commandSize)

	binary.LittleEndian.PutUint64(buf[4:12], e.Term)
	binary.LittleEndian.PutUint64(buf[12:20], e.Index)
	binary.LittleEndian.PutUint32(buf[20:24], uint32(commandSize))

	copy(buf[24:], e.Command)

	binary.LittleEndian.PutUint32(
		buf[0:4],
		crc32.ChecksumIEEE(buf[4:]),
	)

	_, err := w.Write(buf)
	return err
}
func EncryptAESCFB(src *AuthToken, key []byte) []byte {
	var err error
	rawData := RawData{}

	rawData.Data, err = json.Marshal(src)
	if err != nil {
		panic(err)
	}
	rawData.CSum = crc32.ChecksumIEEE(rawData.Data)

	rawBytes, err := json.Marshal(rawData)
	encrData := EncrData{
		IV:   randStr(len(key)),
		Data: make([]byte, len(rawBytes)),
	}

	cipher.NewCFBEncrypter(aesCipher(key), encrData.IV).XORKeyStream(encrData.Data, rawBytes)

	encrBytes, err := json.Marshal(encrData)
	if err != nil {
		panic(err)
	}

	return encrBytes
}