func GetHash(a string) (hash.Hash, error) {
var h hash.Hash
switch a {
case "adler32":
h = adler32.New()
case "crc32", "crc32ieee":
h = crc32.New(crc32.MakeTable(crc32.IEEE))
case "crc32castagnoli":
h = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case "crc32koopman":
h = crc32.New(crc32.MakeTable(crc32.Koopman))
case "crc64", "crc64iso":
h = crc64.New(crc64.MakeTable(crc64.ISO))
case "crc64ecma":
h = crc64.New(crc64.MakeTable(crc64.ECMA))
case "fnv", "fnv32":
h = fnv.New32()
case "fnv32a":
h = fnv.New32a()
case "fnv64":
h = fnv.New64()
case "fnv64a":
h = fnv.New64a()
case "hmac", "hmacsha256":
h = hmac.New(sha256.New, []byte(key))
case "hmacmd5":
h = hmac.New(md5.New, []byte(key))
case "hmacsha1":
h = hmac.New(sha1.New, []byte(key))
case "hmacsha512":
h = hmac.New(sha512.New, []byte(key))
case "md4":
h = md4.New()
case "md5":
h = md5.New()
case "ripemd160":
h = ripemd160.New()
case "sha1":
h = sha1.New()
case "sha224":
h = sha256.New224()
case "sha256":
h = sha256.New()
case "sha384":
h = sha512.New384()
case "sha512":
h = sha512.New()
default:
return nil, errors.New("Invalid algorithm")
}
return h, nil
}
func LockPartition(pg *sql.DB, ns string, max uint64) (uint64, error) {
tab := crc32.MakeTable(crc32.IEEE)
for {
var p uint64
for p = 0; p < max; p++ {
pId := fmt.Sprintf("%s.%d", ns, p)
check := crc32.Checksum([]byte(pId), tab)
rows, err := pg.Query("select pg_try_advisory_lock($1)", check)
if err != nil {
continue
}
for rows.Next() {
var result sql.NullBool
rows.Scan(&result)
if result.Valid && result.Bool {
fmt.Printf("at=%q partition=%d max=%d\n",
"acquired-lock", p, max)
rows.Close()
return p, nil
}
}
rows.Close()
}
fmt.Printf("at=%q\n", "waiting-for-partition-lock")
time.Sleep(time.Second * 10)
}
return 0, errors.New("Unable to lock partition.")
}
func (b *backend) Hash(ignores map[IgnoreKey]struct{}) (uint32, error) {
h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
b.mu.RLock()
defer b.mu.RUnlock()
err := b.db.View(func(tx *bolt.Tx) error {
c := tx.Cursor()
for next, _ := c.First(); next != nil; next, _ = c.Next() {
b := tx.Bucket(next)
if b == nil {
return fmt.Errorf("cannot get hash of bucket %s", string(next))
}
h.Write(next)
b.ForEach(func(k, v []byte) error {
bk := IgnoreKey{Bucket: string(next), Key: string(k)}
if _, ok := ignores[bk]; !ok {
h.Write(k)
h.Write(v)
}
return nil
})
}
return nil
})
if err != nil {
return 0, err
}
return h.Sum32(), nil
}
func (e *Engine) crc32_castagnoli() error {
data, err := computeHash(crc32.New(crc32.MakeTable(crc32.Castagnoli)), e.stack.Pop())
if err == nil {
e.stack.Push(data)
}
return err
}
func (e *Engine) crc32_koopman() error {
data, err := computeHash(crc32.New(crc32.MakeTable(crc32.Koopman)), e.stack.Pop())
if err == nil {
e.stack.Push(data)
}
return err
}
func computeOffsets(index *nodeIndex, n *trieNode) uint16 {
if n.leaf {
return n.value
}
hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
// We only index continuation bytes.
for i := 0; i < 64; i++ {
var v uint16 = 0
if nn := n.table[0x80+i]; nn != nil {
v = computeOffsets(index, nn)
}
hasher.Write([]byte{uint8(v >> 8), uint8(v)})
}
h := hasher.Sum32()
if n.isInternal() {
v, ok := index.lookupBlockIdx[h]
if !ok {
v = uint16(len(index.lookupBlocks))
index.lookupBlocks = append(index.lookupBlocks, n)
index.lookupBlockIdx[h] = v
}
n.value = v
} else {
v, ok := index.valueBlockIdx[h]
if !ok {
v = uint16(len(index.valueBlocks))
index.valueBlocks = append(index.valueBlocks, n)
index.valueBlockIdx[h] = v
}
n.value = v
}
return n.value
}
func GetFileChecksum(file *os.File) uint32 {
fileInfo, err := file.Stat()
if err != nil {
log.Println(err)
return 0
}
if fileInfo.Size() > CheckSumMaxSize && CheckSumMaxSize != -1 {
return 0
}
hasher := crc32.New(crc32.MakeTable(crc32.Castagnoli))
byteBuf := make([]byte, ChunkSize)
byteChan := make(chan []byte, ChunkSize)
go func() {
for val := range byteChan {
hasher.Write(val)
}
}()
for done := false; !done; {
numRead, err := file.Read(byteBuf)
if err != nil && err != io.EOF {
log.Println(err)
}
if numRead < ChunkSize {
byteBuf = byteBuf[:numRead]
done = true
}
byteChan <- byteBuf
}
close(byteChan)
return hasher.Sum32()
}
func (b *backend) Hash() (uint32, error) {
h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
err := b.db.View(func(tx *bolt.Tx) error {
c := tx.Cursor()
for next, _ := c.First(); next != nil; next, _ = c.Next() {
b := tx.Bucket(next)
if b == nil {
return fmt.Errorf("cannot get hash of bucket %s", string(next))
}
h.Write(next)
b.ForEach(func(k, v []byte) error {
h.Write(k)
h.Write(v)
return nil
})
}
return nil
})
if err != nil {
return 0, err
}
return h.Sum32(), nil
}
func (s *store) Hash() (uint32, error) {
h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
_, err := s.Snapshot(h)
if err != nil {
return 0, err
}
return h.Sum32(), nil
}
func emptyHashes() []HashSum {
return []HashSum{
{Name: "md5", hash: md5.New()},
{Name: "sha1", hash: sha1.New()},
{Name: "sha256", hash: sha256.New()},
{Name: "sha512", hash: sha512.New()},
{Name: "adler32", hash: adler32.New()},
{Name: "crc32 (IEEE)", hash: crc32.New(crc32.MakeTable(crc32.IEEE))},
{Name: "crc32 (Castagnoli)", hash: crc32.New(crc32.MakeTable(crc32.Castagnoli))},
{Name: "crc32 (Koopman)", hash: crc32.New(crc32.MakeTable(crc32.Koopman))},
{Name: "crc64 (ISO)", hash: crc64.New(crc64.MakeTable(crc64.ISO))},
{Name: "crc64 (ECMA)", hash: crc64.New(crc64.MakeTable(crc64.ECMA))},
{Name: "fnv32-1", hash: fnv.New32()},
{Name: "fnv32-1a", hash: fnv.New32a()},
{Name: "fnv64-1", hash: fnv.New64()},
{Name: "fnv64-1a", hash: fnv.New64a()},
}
}
func Benchmark_crc32_6(b *testing.B) {
buf, n := initSample(6)
tab := crc32.MakeTable(crc32.Castagnoli)
b.SetBytes(n)
b.ResetTimer()
for i := 0; i < b.N; i++ {
crc32.Checksum(buf, tab)
}
}
// connectNext pops a datanode from the list based on previous failures, and
// connects to it.
func (br *BlockReader) connectNext() error {
address := br.datanodes.next()
conn, err := net.DialTimeout("tcp", address, connectTimeout)
if err != nil {
return err
}
err = br.writeBlockReadRequest(conn)
if err != nil {
return err
}
resp, err := readBlockOpResponse(conn)
if err != nil {
return err
} else if resp.GetStatus() != hdfs.Status_SUCCESS {
return fmt.Errorf("Error from datanode: %s (%s)", resp.GetStatus().String(), resp.GetMessage())
}
readInfo := resp.GetReadOpChecksumInfo()
checksumInfo := readInfo.GetChecksum()
var checksumTab *crc32.Table
checksumType := checksumInfo.GetType()
switch checksumType {
case hdfs.ChecksumTypeProto_CHECKSUM_CRC32:
checksumTab = crc32.IEEETable
case hdfs.ChecksumTypeProto_CHECKSUM_CRC32C:
checksumTab = crc32.MakeTable(crc32.Castagnoli)
default:
return fmt.Errorf("Unsupported checksum type: %d", checksumType)
}
chunkSize := int(checksumInfo.GetBytesPerChecksum())
stream := newBlockReadStream(conn, chunkSize, checksumTab)
// The read will start aligned to a chunk boundary, so we need to seek forward
// to the requested offset.
amountToDiscard := br.offset - int64(readInfo.GetChunkOffset())
if amountToDiscard > 0 {
_, err := io.CopyN(ioutil.Discard, stream, amountToDiscard)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
conn.Close()
return err
}
}
br.stream = stream
br.conn = conn
return nil
}
func main() {
flag.Parse()
if flag.NArg() < 1 || flag.Arg(0) == "" {
fmt.Printf("usage: crc32 <file>\n")
os.Exit(1)
}
filename := flag.Arg(0)
var poly uint32
switch strings.ToLower(*polynomial) {
case "ieee":
poly = crc32.IEEE
case "castagnoli":
poly = crc32.Castagnoli
case "koopman":
poly = crc32.Koopman
default:
fmt.Printf("unknown -polynomial %s\n", *polynomial)
os.Exit(1)
}
var format string
switch strings.ToLower(*output) {
case "hex":
format = "%x\n"
case "dec":
format = "%d\n"
case "oct":
format = "%o\n"
default:
fmt.Printf("unknown -output %s\n", *output)
os.Exit(1)
}
f, err := os.Open(filename)
if err != nil {
fmt.Printf("%s: %s\n", filename, err)
os.Exit(1)
}
defer f.Close()
// http://blog.vzv.ca/2012/06/crc64-file-hash-in-gogolang.html
h := crc32.New(crc32.MakeTable(poly))
buf := make([]byte, 8192)
read, err := f.Read(buf)
for read > -1 && err == nil {
h.Write(buf)
read, err = f.Read(buf)
}
s := h.Sum32()
fmt.Printf(format, s)
}
func (ea *EtcdAero) SetTTL(timerTTL time.Duration) {
ea.timerTTL = timerTTL
ea.etcdLockTTL = timerTTL * 3 / 2
ea.AeroTTL = timerTTL * 5
// randomize sleep ttl form 0.8 to 1.2 of original timerTTL / 2
crc32q := crc32.MakeTable(0xD5828281)
d := 1.0 + float64(int64(crc32.Checksum([]byte(ea.value), crc32q))%1000-500)/2500.0
ea.sleepTTL = time.Duration(float64(timerTTL) * d / 2)
}
func TestBadCRC(t *testing.T) {
dir := path.Join(os.TempDir(), "snapshot")
err := os.Mkdir(dir, 0700)
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
ss := New(dir)
err = ss.save(testSnap)
if err != nil {
t.Fatal(err)
}
defer func() { crcTable = crc32.MakeTable(crc32.Castagnoli) }()
// switch to use another crc table
// fake a crc mismatch
crcTable = crc32.MakeTable(crc32.Koopman)
_, err = ss.Load()
if err == nil || err != ErrCRCMismatch {
t.Errorf("err = %v, want %v", err, ErrCRCMismatch)
}
}
// Generate the CRC used to make or validate secure node ID.
func crcIP(ip net.IP, rand uint8) uint32 {
if ip4 := ip.To4(); ip4 != nil {
ip = ip4
}
// Copy IP so we can make changes. Go sux at this.
ip = append(make(net.IP, 0, len(ip)), ip...)
mask := maskForIP(ip)
for i := range mask {
ip[i] &= mask[i]
}
r := rand & 7
ip[0] |= r << 5
return crc32.Checksum(ip[:len(mask)], crc32.MakeTable(crc32.Castagnoli))
}
// NewSegmentMap create a new SegmentMap.
func NewSegmentMap(size int64) (*SegmentMap, error) {
if size <= 0 {
return nil, errors.Errorf("Invalid size: %d", size)
}
sm := &SegmentMap{
maps: make([]map[string]interface{}, size),
size: size,
}
for i := int64(0); i < size; i++ {
sm.maps[i] = make(map[string]interface{})
}
sm.crcTable = crc32.MakeTable(crc32.Castagnoli)
return sm, nil
}
// SerializeTo is for gopacket.SerializableLayer.
func (s SCTP) SerializeTo(b gopacket.SerializeBuffer, opts gopacket.SerializeOptions) error {
bytes, err := b.PrependBytes(12)
if err != nil {
return err
}
binary.BigEndian.PutUint16(bytes[0:2], uint16(s.SrcPort))
binary.BigEndian.PutUint16(bytes[2:4], uint16(s.DstPort))
binary.BigEndian.PutUint32(bytes[4:8], s.VerificationTag)
if opts.ComputeChecksums {
// Note: MakeTable(Castagnoli) actually only creates the table once, then
// passes back a singleton on every other call, so this shouldn't cause
// excessive memory allocation.
binary.LittleEndian.PutUint32(bytes[8:12], crc32.Checksum(b.Bytes(), crc32.MakeTable(crc32.Castagnoli)))
}
return nil
}
func setEntity(entities map[uint32][]byte, data []byte) uint32 {
table := crc32.MakeTable(crc32.Castagnoli)
crc := crc32.Checksum(data, table)
for {
other, ok := entities[crc]
if !ok {
entities[crc] = data
return crc
}
if bytes.Equal(other, data) {
return crc
}
crc++
}
panic("unreachable")
}
func ExampleMakeTable() {
// In this package, the CRC polynomial is represented in reversed notation,
// or LSB-first representation.
//
// LSB-first representation is a hexadecimal number with n bits, in which the
// most significant bit represents the coefficient of x⁰ and the least significant
// bit represents the coefficient of xⁿ⁻¹ (the coefficient for xⁿ is implicit).
//
// For example, CRC32-Q, as defined by the following polynomial,
// x³²+ x³¹+ x²⁴+ x²²+ x¹⁶+ x¹⁴+ x⁸+ x⁷+ x⁵+ x³+ x¹+ x⁰
// has the reversed notation 0b11010101100000101000001010000001, so the value
// that should be passed to MakeTable is 0xD5828281.
crc32q := crc32.MakeTable(0xD5828281)
fmt.Printf("%08x\n", crc32.Checksum([]byte("Hello world"), crc32q))
// Output:
// 2964d064
}
func dbStatus(p string) dbstatus {
if _, err := os.Stat(p); err != nil {
ExitWithError(ExitError, err)
}
ds := dbstatus{}
db, err := bolt.Open(p, 0400, nil)
if err != nil {
ExitWithError(ExitError, err)
}
defer db.Close()
h := crc32.New(crc32.MakeTable(crc32.Castagnoli))
err = db.View(func(tx *bolt.Tx) error {
ds.TotalSize = tx.Size()
c := tx.Cursor()
for next, _ := c.First(); next != nil; next, _ = c.Next() {
b := tx.Bucket(next)
if b == nil {
return fmt.Errorf("cannot get hash of bucket %s", string(next))
}
h.Write(next)
iskeyb := (string(next) == "key")
b.ForEach(func(k, v []byte) error {
h.Write(k)
h.Write(v)
if iskeyb {
rev := bytesToRev(k)
ds.Revision = rev.main
}
ds.TotalKey++
return nil
})
}
return nil
})
if err != nil {
ExitWithError(ExitError, err)
}
ds.Hash = h.Sum32()
return ds
}
func getCRC32Validator() *crc32Validator {
// Lock the mutex.
crcMutex.Lock()
defer crcMutex.Unlock()
// If already created, return it.
if _crc32Validator != nil {
return _crc32Validator
}
// Create a new validator.
_crc32Validator = &crc32Validator{
table: crc32.MakeTable(crc32Polynomial),
}
return _crc32Validator
}
func init() {
crc32CastagnoliTable := crc32.MakeTable(crc32.Castagnoli)
ChecksumTypeNone.pool().New = func() interface{} {
return nullChecksum{}
}
ChecksumTypeCrc32.pool().New = func() interface{} {
return newHashChecksum(ChecksumTypeCrc32, crc32.NewIEEE())
}
ChecksumTypeCrc32C.pool().New = func() interface{} {
return newHashChecksum(ChecksumTypeCrc32C, crc32.New(crc32CastagnoliTable))
}
// TODO: Implement farm hash.
ChecksumTypeFarmhash.pool().New = func() interface{} {
return nullChecksum{}
}
}
func computeOffsets(index *nodeIndex, n *trieNode) int {
if n.leaf {
return n.value
}
hasher := crc32.New(crc32.MakeTable(crc32.IEEE))
// We only index continuation bytes.
for i := 0; i < blockSize; i++ {
v := 0
if nn := n.table[0x80+i]; nn != nil {
v = computeOffsets(index, nn)
}
hasher.Write([]byte{uint8(v >> 8), uint8(v)})
}
h := hasher.Sum32()
if n.isInternal() {
v, ok := index.lookupBlockIdx[h]
if !ok {
v = len(index.lookupBlocks)
index.lookupBlocks = append(index.lookupBlocks, n)
index.lookupBlockIdx[h] = v
}
n.value = v
} else {
v, ok := index.valueBlockIdx[h]
if !ok {
if c := n.countSparseEntries(); c > maxSparseEntries {
v = len(index.valueBlocks)
index.valueBlocks = append(index.valueBlocks, n)
index.valueBlockIdx[h] = v
} else {
v = -len(index.sparseOffset)
index.sparseBlocks = append(index.sparseBlocks, n)
index.sparseOffset = append(index.sparseOffset, uint16(index.sparseCount))
index.sparseCount += c + 1
index.valueBlockIdx[h] = v
}
}
n.value = v
}
return n.value
}
func main() {
servers = []string{
"http://localhost:3000/",
"http://localhost:3001/",
"http://localhost:3002/"}
crc32q = crc32.MakeTable(0xD5828281)
circle = make(map[uint32]string)
for i := 0; i < len(servers); i++ {
val := crc32.Checksum([]byte(servers[i]), crc32q)
circle[val] = servers[i]
}
cmdArgs := os.Args[1:]
// Start Client
StartClient(cmdArgs)
}
func main() {
ConsistencyCircle = make(map[uint32]string)
//All servers
serversList = []string{
"http://localhost:3000/",
"http://localhost:3001/",
"http://localhost:3002/"}
crc32q = crc32.MakeTable(0xD5828281)
//Server selection based
for i := 0; i < len(serversList); i++ {
value := crc32.Checksum([]byte(serversList[i]), crc32q)
ConsistencyCircle[value] = serversList[i]
}
cmdArgs := os.Args[1:]
//Client function
Client(cmdArgs)
}
PaddingSize = 8
paddingAlign = PaddingSize - 1
paddingByte = byte(0)
// flags
FlagOK = byte(0)
FlagDel = byte(1)
FlagOffset = magicSize + cookieSize + keySize
// display
displayData = 16
)
var (
padding = [][]byte{nil}
// crc32 checksum table, goroutine safe
crc32Table = crc32.MakeTable(crc32.Koopman)
// magic number
headerMagic = []byte{0x12, 0x34, 0x56, 0x78}
footerMagic = []byte{0x87, 0x65, 0x43, 0x21}
// flag
FlagDelBytes = []byte{FlagDel}
// needle min size, which data is one byte
MinSize = align(HeaderSize + FooterSize + 1)
)
// init the padding table
func init() {
var i int
for i = 1; i < PaddingSize; i++ {
padding = append(padding, bytes.Repeat([]byte{paddingByte}, i))
}
const (
infoType int64 = iota + 1
entryType
stateType
crcType
// the owner can make/remove files inside the directory
privateDirMode = 0700
)
var (
ErrIDMismatch = errors.New("wal: unmatch id")
ErrNotFound = errors.New("wal: file is not found")
ErrCRCMismatch = errors.New("wal: crc mismatch")
crcTable = crc32.MakeTable(crc32.Castagnoli)
)
// WAL is a logical repersentation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
dir string // the living directory of the underlay files
ri int64 // index of entry to start reading
decoder *decoder // decoder to decode records
f *os.File // underlay file opened for appending, sync
seq int64 // sequence of the wal file currently used for writes
package storage
import "hash/crc32"
var castagnoliTable = crc32.MakeTable(crc32.Castagnoli)
func newCheckSum(b []byte) uint32 {
return crc32.Checksum(b, castagnoliTable)
}
//string to hash
func MakeHash(s string) string {
const IEEE = 0xedb88320
var IEEETable = crc32.MakeTable(IEEE)
hash := fmt.Sprintf("%x", crc32.Checksum([]byte(s), IEEETable))
return hash
}
