Exemplo n.º 1
2
func GetHash(a string) (hash.Hash, error) {
	var h hash.Hash
	switch a {
	case "adler32":
		h = adler32.New()
	case "crc32", "crc32ieee":
		h = crc32.New(crc32.MakeTable(crc32.IEEE))
	case "crc32castagnoli":
		h = crc32.New(crc32.MakeTable(crc32.Castagnoli))
	case "crc32koopman":
		h = crc32.New(crc32.MakeTable(crc32.Koopman))
	case "crc64", "crc64iso":
		h = crc64.New(crc64.MakeTable(crc64.ISO))
	case "crc64ecma":
		h = crc64.New(crc64.MakeTable(crc64.ECMA))
	case "fnv", "fnv32":
		h = fnv.New32()
	case "fnv32a":
		h = fnv.New32a()
	case "fnv64":
		h = fnv.New64()
	case "fnv64a":
		h = fnv.New64a()
	case "hmac", "hmacsha256":
		h = hmac.New(sha256.New, []byte(key))
	case "hmacmd5":
		h = hmac.New(md5.New, []byte(key))
	case "hmacsha1":
		h = hmac.New(sha1.New, []byte(key))
	case "hmacsha512":
		h = hmac.New(sha512.New, []byte(key))
	case "md4":
		h = md4.New()
	case "md5":
		h = md5.New()
	case "ripemd160":
		h = ripemd160.New()
	case "sha1":
		h = sha1.New()
	case "sha224":
		h = sha256.New224()
	case "sha256":
		h = sha256.New()
	case "sha384":
		h = sha512.New384()
	case "sha512":
		h = sha512.New()
	default:
		return nil, errors.New("Invalid algorithm")
	}
	return h, nil
}
Exemplo n.º 2
0
func LockPartition(pg *sql.DB, ns string, max uint64) (uint64, error) {
	tab := crc32.MakeTable(crc32.IEEE)
	for {
		var p uint64
		for p = 0; p < max; p++ {
			pId := fmt.Sprintf("%s.%d", ns, p)
			check := crc32.Checksum([]byte(pId), tab)
			rows, err := pg.Query("select pg_try_advisory_lock($1)", check)
			if err != nil {
				continue
			}
			for rows.Next() {
				var result sql.NullBool
				rows.Scan(&result)
				if result.Valid && result.Bool {
					fmt.Printf("at=%q partition=%d max=%d\n",
						"acquired-lock", p, max)
					rows.Close()
					return p, nil
				}
			}
			rows.Close()
		}
		fmt.Printf("at=%q\n", "waiting-for-partition-lock")
		time.Sleep(time.Second * 10)
	}
	return 0, errors.New("Unable to lock partition.")
}
Exemplo n.º 3
0
func (b *backend) Hash(ignores map[IgnoreKey]struct{}) (uint32, error) {
	h := crc32.New(crc32.MakeTable(crc32.Castagnoli))

	b.mu.RLock()
	defer b.mu.RUnlock()
	err := b.db.View(func(tx *bolt.Tx) error {
		c := tx.Cursor()
		for next, _ := c.First(); next != nil; next, _ = c.Next() {
			b := tx.Bucket(next)
			if b == nil {
				return fmt.Errorf("cannot get hash of bucket %s", string(next))
			}
			h.Write(next)
			b.ForEach(func(k, v []byte) error {
				bk := IgnoreKey{Bucket: string(next), Key: string(k)}
				if _, ok := ignores[bk]; !ok {
					h.Write(k)
					h.Write(v)
				}
				return nil
			})
		}
		return nil
	})

	if err != nil {
		return 0, err
	}

	return h.Sum32(), nil
}
Exemplo n.º 4
0
func (e *Engine) crc32_castagnoli() error {
	data, err := computeHash(crc32.New(crc32.MakeTable(crc32.Castagnoli)), e.stack.Pop())
	if err == nil {
		e.stack.Push(data)
	}
	return err
}
Exemplo n.º 5
0
func (e *Engine) crc32_koopman() error {
	data, err := computeHash(crc32.New(crc32.MakeTable(crc32.Koopman)), e.stack.Pop())
	if err == nil {
		e.stack.Push(data)
	}
	return err
}
Exemplo n.º 6
0
func computeOffsets(index *nodeIndex, n *trieNode) uint16 {
	if n.leaf {
		return n.value
	}
	hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
	// We only index continuation bytes.
	for i := 0; i < 64; i++ {
		var v uint16 = 0
		if nn := n.table[0x80+i]; nn != nil {
			v = computeOffsets(index, nn)
		}
		hasher.Write([]byte{uint8(v >> 8), uint8(v)})
	}
	h := hasher.Sum32()
	if n.isInternal() {
		v, ok := index.lookupBlockIdx[h]
		if !ok {
			v = uint16(len(index.lookupBlocks))
			index.lookupBlocks = append(index.lookupBlocks, n)
			index.lookupBlockIdx[h] = v
		}
		n.value = v
	} else {
		v, ok := index.valueBlockIdx[h]
		if !ok {
			v = uint16(len(index.valueBlocks))
			index.valueBlocks = append(index.valueBlocks, n)
			index.valueBlockIdx[h] = v
		}
		n.value = v
	}
	return n.value
}
Exemplo n.º 7
0
func GetFileChecksum(file *os.File) uint32 {
	fileInfo, err := file.Stat()
	if err != nil {
		log.Println(err)
		return 0
	}
	if fileInfo.Size() > CheckSumMaxSize && CheckSumMaxSize != -1 {
		return 0
	}
	hasher := crc32.New(crc32.MakeTable(crc32.Castagnoli))
	byteBuf := make([]byte, ChunkSize)
	byteChan := make(chan []byte, ChunkSize)
	go func() {
		for val := range byteChan {
			hasher.Write(val)
		}
	}()
	for done := false; !done; {
		numRead, err := file.Read(byteBuf)
		if err != nil && err != io.EOF {
			log.Println(err)
		}
		if numRead < ChunkSize {
			byteBuf = byteBuf[:numRead]
			done = true
		}
		byteChan <- byteBuf
	}
	close(byteChan)
	return hasher.Sum32()
}
Exemplo n.º 8
0
func (b *backend) Hash() (uint32, error) {
	h := crc32.New(crc32.MakeTable(crc32.Castagnoli))

	err := b.db.View(func(tx *bolt.Tx) error {
		c := tx.Cursor()
		for next, _ := c.First(); next != nil; next, _ = c.Next() {
			b := tx.Bucket(next)
			if b == nil {
				return fmt.Errorf("cannot get hash of bucket %s", string(next))
			}
			h.Write(next)
			b.ForEach(func(k, v []byte) error {
				h.Write(k)
				h.Write(v)
				return nil
			})
		}
		return nil
	})

	if err != nil {
		return 0, err
	}

	return h.Sum32(), nil
}
Exemplo n.º 9
0
func (s *store) Hash() (uint32, error) {
	h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
	_, err := s.Snapshot(h)
	if err != nil {
		return 0, err
	}
	return h.Sum32(), nil
}
Exemplo n.º 10
0
func emptyHashes() []HashSum {
	return []HashSum{
		{Name: "md5", hash: md5.New()},
		{Name: "sha1", hash: sha1.New()},
		{Name: "sha256", hash: sha256.New()},
		{Name: "sha512", hash: sha512.New()},
		{Name: "adler32", hash: adler32.New()},
		{Name: "crc32 (IEEE)", hash: crc32.New(crc32.MakeTable(crc32.IEEE))},
		{Name: "crc32 (Castagnoli)", hash: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
		{Name: "crc32 (Koopman)", hash: crc32.New(crc32.MakeTable(crc32.Koopman))},
		{Name: "crc64 (ISO)", hash: crc64.New(crc64.MakeTable(crc64.ISO))},
		{Name: "crc64 (ECMA)", hash: crc64.New(crc64.MakeTable(crc64.ECMA))},
		{Name: "fnv32-1", hash: fnv.New32()},
		{Name: "fnv32-1a", hash: fnv.New32a()},
		{Name: "fnv64-1", hash: fnv.New64()},
		{Name: "fnv64-1a", hash: fnv.New64a()},
	}
}
Exemplo n.º 11
0
func Benchmark_crc32_6(b *testing.B) {
	buf, n := initSample(6)
	tab := crc32.MakeTable(crc32.Castagnoli)
	b.SetBytes(n)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		crc32.Checksum(buf, tab)
	}
}
Exemplo n.º 12
0
// connectNext pops a datanode from the list based on previous failures, and
// connects to it.
func (br *BlockReader) connectNext() error {
	address := br.datanodes.next()

	conn, err := net.DialTimeout("tcp", address, connectTimeout)
	if err != nil {
		return err
	}

	err = br.writeBlockReadRequest(conn)
	if err != nil {
		return err
	}

	resp, err := readBlockOpResponse(conn)
	if err != nil {
		return err
	} else if resp.GetStatus() != hdfs.Status_SUCCESS {
		return fmt.Errorf("Error from datanode: %s (%s)", resp.GetStatus().String(), resp.GetMessage())
	}

	readInfo := resp.GetReadOpChecksumInfo()
	checksumInfo := readInfo.GetChecksum()

	var checksumTab *crc32.Table
	checksumType := checksumInfo.GetType()
	switch checksumType {
	case hdfs.ChecksumTypeProto_CHECKSUM_CRC32:
		checksumTab = crc32.IEEETable
	case hdfs.ChecksumTypeProto_CHECKSUM_CRC32C:
		checksumTab = crc32.MakeTable(crc32.Castagnoli)
	default:
		return fmt.Errorf("Unsupported checksum type: %d", checksumType)
	}

	chunkSize := int(checksumInfo.GetBytesPerChecksum())
	stream := newBlockReadStream(conn, chunkSize, checksumTab)

	// The read will start aligned to a chunk boundary, so we need to seek forward
	// to the requested offset.
	amountToDiscard := br.offset - int64(readInfo.GetChunkOffset())
	if amountToDiscard > 0 {
		_, err := io.CopyN(ioutil.Discard, stream, amountToDiscard)
		if err != nil {
			if err == io.EOF {
				err = io.ErrUnexpectedEOF
			}

			conn.Close()
			return err
		}
	}

	br.stream = stream
	br.conn = conn
	return nil
}
Exemplo n.º 13
0
func main() {
	flag.Parse()

	if flag.NArg() < 1 || flag.Arg(0) == "" {
		fmt.Printf("usage: crc32 <file>\n")
		os.Exit(1)
	}
	filename := flag.Arg(0)

	var poly uint32
	switch strings.ToLower(*polynomial) {
	case "ieee":
		poly = crc32.IEEE
	case "castagnoli":
		poly = crc32.Castagnoli
	case "koopman":
		poly = crc32.Koopman
	default:
		fmt.Printf("unknown -polynomial %s\n", *polynomial)
		os.Exit(1)
	}

	var format string
	switch strings.ToLower(*output) {
	case "hex":
		format = "%x\n"
	case "dec":
		format = "%d\n"
	case "oct":
		format = "%o\n"
	default:
		fmt.Printf("unknown -output %s\n", *output)
		os.Exit(1)
	}

	f, err := os.Open(filename)
	if err != nil {
		fmt.Printf("%s: %s\n", filename, err)
		os.Exit(1)
	}
	defer f.Close()

	// http://blog.vzv.ca/2012/06/crc64-file-hash-in-gogolang.html
	h := crc32.New(crc32.MakeTable(poly))
	buf := make([]byte, 8192)
	read, err := f.Read(buf)
	for read > -1 && err == nil {
		h.Write(buf)
		read, err = f.Read(buf)
	}

	s := h.Sum32()
	fmt.Printf(format, s)
}
Exemplo n.º 14
0
func (ea *EtcdAero) SetTTL(timerTTL time.Duration) {

	ea.timerTTL = timerTTL
	ea.etcdLockTTL = timerTTL * 3 / 2
	ea.AeroTTL = timerTTL * 5

	// randomize sleep ttl form 0.8 to 1.2 of original timerTTL / 2
	crc32q := crc32.MakeTable(0xD5828281)
	d := 1.0 + float64(int64(crc32.Checksum([]byte(ea.value), crc32q))%1000-500)/2500.0
	ea.sleepTTL = time.Duration(float64(timerTTL) * d / 2)
}
Exemplo n.º 15
0
func TestBadCRC(t *testing.T) {
	dir := path.Join(os.TempDir(), "snapshot")
	err := os.Mkdir(dir, 0700)
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(dir)
	ss := New(dir)
	err = ss.save(testSnap)
	if err != nil {
		t.Fatal(err)
	}
	defer func() { crcTable = crc32.MakeTable(crc32.Castagnoli) }()
	// switch to use another crc table
	// fake a crc mismatch
	crcTable = crc32.MakeTable(crc32.Koopman)

	_, err = ss.Load()
	if err == nil || err != ErrCRCMismatch {
		t.Errorf("err = %v, want %v", err, ErrCRCMismatch)
	}
}
Exemplo n.º 16
0
// Generate the CRC used to make or validate secure node ID.
func crcIP(ip net.IP, rand uint8) uint32 {
	if ip4 := ip.To4(); ip4 != nil {
		ip = ip4
	}
	// Copy IP so we can make changes. Go sux at this.
	ip = append(make(net.IP, 0, len(ip)), ip...)
	mask := maskForIP(ip)
	for i := range mask {
		ip[i] &= mask[i]
	}
	r := rand & 7
	ip[0] |= r << 5
	return crc32.Checksum(ip[:len(mask)], crc32.MakeTable(crc32.Castagnoli))
}
Exemplo n.º 17
0
// NewSegmentMap create a new SegmentMap.
func NewSegmentMap(size int64) (*SegmentMap, error) {
	if size <= 0 {
		return nil, errors.Errorf("Invalid size: %d", size)
	}

	sm := &SegmentMap{
		maps: make([]map[string]interface{}, size),
		size: size,
	}
	for i := int64(0); i < size; i++ {
		sm.maps[i] = make(map[string]interface{})
	}

	sm.crcTable = crc32.MakeTable(crc32.Castagnoli)
	return sm, nil
}
Exemplo n.º 18
0
// SerializeTo is for gopacket.SerializableLayer.
func (s SCTP) SerializeTo(b gopacket.SerializeBuffer, opts gopacket.SerializeOptions) error {
	bytes, err := b.PrependBytes(12)
	if err != nil {
		return err
	}
	binary.BigEndian.PutUint16(bytes[0:2], uint16(s.SrcPort))
	binary.BigEndian.PutUint16(bytes[2:4], uint16(s.DstPort))
	binary.BigEndian.PutUint32(bytes[4:8], s.VerificationTag)
	if opts.ComputeChecksums {
		// Note:  MakeTable(Castagnoli) actually only creates the table once, then
		// passes back a singleton on every other call, so this shouldn't cause
		// excessive memory allocation.
		binary.LittleEndian.PutUint32(bytes[8:12], crc32.Checksum(b.Bytes(), crc32.MakeTable(crc32.Castagnoli)))
	}
	return nil
}
Exemplo n.º 19
0
func setEntity(entities map[uint32][]byte, data []byte) uint32 {
	table := crc32.MakeTable(crc32.Castagnoli)
	crc := crc32.Checksum(data, table)
	for {
		other, ok := entities[crc]
		if !ok {
			entities[crc] = data
			return crc
		}
		if bytes.Equal(other, data) {
			return crc
		}
		crc++
	}

	panic("unreachable")
}
Exemplo n.º 20
0
func ExampleMakeTable() {
	// In this package, the CRC polynomial is represented in reversed notation,
	// or LSB-first representation.
	//
	// LSB-first representation is a hexadecimal number with n bits, in which the
	// most significant bit represents the coefficient of x⁰ and the least significant
	// bit represents the coefficient of xⁿ⁻¹ (the coefficient for xⁿ is implicit).
	//
	// For example, CRC32-Q, as defined by the following polynomial,
	//	x³²+ x³¹+ x²⁴+ x²²+ x¹⁶+ x¹⁴+ x⁸+ x⁷+ x⁵+ x³+ x¹+ x⁰
	// has the reversed notation 0b11010101100000101000001010000001, so the value
	// that should be passed to MakeTable is 0xD5828281.
	crc32q := crc32.MakeTable(0xD5828281)
	fmt.Printf("%08x\n", crc32.Checksum([]byte("Hello world"), crc32q))
	// Output:
	// 2964d064
}
Exemplo n.º 21
0
func dbStatus(p string) dbstatus {
	if _, err := os.Stat(p); err != nil {
		ExitWithError(ExitError, err)
	}

	ds := dbstatus{}

	db, err := bolt.Open(p, 0400, nil)
	if err != nil {
		ExitWithError(ExitError, err)
	}
	defer db.Close()

	h := crc32.New(crc32.MakeTable(crc32.Castagnoli))

	err = db.View(func(tx *bolt.Tx) error {
		ds.TotalSize = tx.Size()
		c := tx.Cursor()
		for next, _ := c.First(); next != nil; next, _ = c.Next() {
			b := tx.Bucket(next)
			if b == nil {
				return fmt.Errorf("cannot get hash of bucket %s", string(next))
			}
			h.Write(next)
			iskeyb := (string(next) == "key")
			b.ForEach(func(k, v []byte) error {
				h.Write(k)
				h.Write(v)
				if iskeyb {
					rev := bytesToRev(k)
					ds.Revision = rev.main
				}
				ds.TotalKey++
				return nil
			})
		}
		return nil
	})

	if err != nil {
		ExitWithError(ExitError, err)
	}

	ds.Hash = h.Sum32()
	return ds
}
Exemplo n.º 22
0
func getCRC32Validator() *crc32Validator {
	// Lock the mutex.
	crcMutex.Lock()
	defer crcMutex.Unlock()

	// If already created, return it.
	if _crc32Validator != nil {
		return _crc32Validator
	}

	// Create a new validator.
	_crc32Validator = &crc32Validator{
		table: crc32.MakeTable(crc32Polynomial),
	}

	return _crc32Validator
}
Exemplo n.º 23
0
func init() {
	crc32CastagnoliTable := crc32.MakeTable(crc32.Castagnoli)

	ChecksumTypeNone.pool().New = func() interface{} {
		return nullChecksum{}
	}
	ChecksumTypeCrc32.pool().New = func() interface{} {
		return newHashChecksum(ChecksumTypeCrc32, crc32.NewIEEE())
	}
	ChecksumTypeCrc32C.pool().New = func() interface{} {
		return newHashChecksum(ChecksumTypeCrc32C, crc32.New(crc32CastagnoliTable))
	}

	// TODO: Implement farm hash.
	ChecksumTypeFarmhash.pool().New = func() interface{} {
		return nullChecksum{}
	}
}
Exemplo n.º 24
0
func computeOffsets(index *nodeIndex, n *trieNode) int {
	if n.leaf {
		return n.value
	}
	hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
	// We only index continuation bytes.
	for i := 0; i < blockSize; i++ {
		v := 0
		if nn := n.table[0x80+i]; nn != nil {
			v = computeOffsets(index, nn)
		}
		hasher.Write([]byte{uint8(v >> 8), uint8(v)})
	}
	h := hasher.Sum32()
	if n.isInternal() {
		v, ok := index.lookupBlockIdx[h]
		if !ok {
			v = len(index.lookupBlocks)
			index.lookupBlocks = append(index.lookupBlocks, n)
			index.lookupBlockIdx[h] = v
		}
		n.value = v
	} else {
		v, ok := index.valueBlockIdx[h]
		if !ok {
			if c := n.countSparseEntries(); c > maxSparseEntries {
				v = len(index.valueBlocks)
				index.valueBlocks = append(index.valueBlocks, n)
				index.valueBlockIdx[h] = v
			} else {
				v = -len(index.sparseOffset)
				index.sparseBlocks = append(index.sparseBlocks, n)
				index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount))
				index.sparseCount += c + 1
				index.valueBlockIdx[h] = v
			}
		}
		n.value = v
	}
	return n.value
}
Exemplo n.º 25
0
func main() {

	servers = []string{
		"http://localhost:3000/",
		"http://localhost:3001/",
		"http://localhost:3002/"}
	crc32q = crc32.MakeTable(0xD5828281)

	circle = make(map[uint32]string)

	for i := 0; i < len(servers); i++ {
		val := crc32.Checksum([]byte(servers[i]), crc32q)
		circle[val] = servers[i]
	}

	cmdArgs := os.Args[1:]
	//	Start Client

	StartClient(cmdArgs)

}
Exemplo n.º 26
0
func main() {
	ConsistencyCircle = make(map[uint32]string)
	//All servers
	serversList = []string{
		"http://localhost:3000/",
		"http://localhost:3001/",
		"http://localhost:3002/"}

	crc32q = crc32.MakeTable(0xD5828281)

	//Server selection based
	for i := 0; i < len(serversList); i++ {
		value := crc32.Checksum([]byte(serversList[i]), crc32q)
		ConsistencyCircle[value] = serversList[i]
	}

	cmdArgs := os.Args[1:]

	//Client function
	Client(cmdArgs)

}
Exemplo n.º 27
0
Arquivo: needle.go Projeto: wtmmac/bfs
	PaddingSize  = 8
	paddingAlign = PaddingSize - 1
	paddingByte  = byte(0)

	// flags
	FlagOK     = byte(0)
	FlagDel    = byte(1)
	FlagOffset = magicSize + cookieSize + keySize
	// display
	displayData = 16
)

var (
	padding = [][]byte{nil}
	// crc32 checksum table, goroutine safe
	crc32Table = crc32.MakeTable(crc32.Koopman)
	// magic number
	headerMagic = []byte{0x12, 0x34, 0x56, 0x78}
	footerMagic = []byte{0x87, 0x65, 0x43, 0x21}
	// flag
	FlagDelBytes = []byte{FlagDel}
	// needle min size, which data is one byte
	MinSize = align(HeaderSize + FooterSize + 1)
)

// init the padding table
func init() {
	var i int
	for i = 1; i < PaddingSize; i++ {
		padding = append(padding, bytes.Repeat([]byte{paddingByte}, i))
	}
Exemplo n.º 28
0
const (
	infoType int64 = iota + 1
	entryType
	stateType
	crcType

	// the owner can make/remove files inside the directory
	privateDirMode = 0700
)

var (
	ErrIDMismatch  = errors.New("wal: unmatch id")
	ErrNotFound    = errors.New("wal: file is not found")
	ErrCRCMismatch = errors.New("wal: crc mismatch")
	crcTable       = crc32.MakeTable(crc32.Castagnoli)
)

// WAL is a logical repersentation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
	dir string // the living directory of the underlay files

	ri      int64    // index of entry to start reading
	decoder *decoder // decoder to decode records

	f       *os.File // underlay file opened for appending, sync
	seq     int64    // sequence of the wal file currently used for writes
Exemplo n.º 29
0
package storage

import "hash/crc32"

var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)

func newCheckSum(b []byte) uint32 {
	return crc32.Checksum(b, castagnoliTable)
}
Exemplo n.º 30
0
//string to hash
func MakeHash(s string) string {
	const IEEE = 0xedb88320
	var IEEETable = crc32.MakeTable(IEEE)
	hash := fmt.Sprintf("%x", crc32.Checksum([]byte(s), IEEETable))
	return hash
}