func (g *FieldGrouping) ModHash(tuple Values) uint32 { h := fnv.New32() values := g.SourceFields.Select(g.Selector, tuple) for _, value := range values { h.Write(value.Encode()) } h = fnv.New32() return h.Sum32() % uint32(len(g.Dests)) }
// Get bucket (or sharded map) by key in 1 cache instance with built in hash function of Golang func (cache *Cache) getBucketWithBuiltInHasher(key string) (*ConcurrentMap, uint) { hasher := fnv.New32() hasher.Write([]byte(key)) bucketsCount := len(cache.buckets) bucketIndex := uint(hasher.Sum32()) % uint(bucketsCount) return cache.buckets[bucketIndex], bucketIndex }
func BenchmarkFnv32(b *testing.B) { h := fnv.New32() for i := 0; i < b.N; i++ { h.Write(in) h.Sum(nil) } }
func (self matchHash) Key(target matchHash) uint32 { hasher := fnv.New32() for k, s := range self { var value, mask []byte if bKey, ok := k.(OxmKeyBasic); !ok || oxm.Header(bKey).Class() != ofp4.OFPXMC_OPENFLOW_BASIC { continue } else { length, _ := ofp4.OxmOfDefs(uint32(bKey)) value = make([]byte, length) mask = make([]byte, length) } if t, ok := target[k]; ok { a := s.(OxmValueMask) b := t.(OxmValueMask) for i, _ := range mask { mask[i] = 0xFF } if a.Mask != nil { for i, _ := range mask { mask[i] &= a.Mask[i] } } if b.Mask != nil { for i, _ := range mask { mask[i] &= b.Mask[i] } } for i, _ := range value { value[i] = b.Value[i] & mask[i] } } hasher.Write(value) } return hasher.Sum32() }
// Computes the hash of the cursor by computing the hash of every enpdoint supplied func computeHash(endpoints []loadbalance.Endpoint) uint32 { h := fnv.New32() for _, endpoint := range endpoints { h.Write([]byte(endpoint.Id())) } return h.Sum32() }
// hashKey hashes a key into a uint32 func hashKey(key []byte) uint32 { // need to convert to a []byte h := fnv.New32() h.Write(key) return h.Sum32() }
// init initializes the matcher. func (m *matcher) init(prog *syntax.Prog, n int) error { m.prog = prog m.dstate = make(map[uint32]*dstate) m.numMatch = n m.maxState = 10 m.allTail = &m.all m.numByte = 256 for i := range m.undo { m.undo[i] = byte(i) } m.h = fnv.New32() m.z1.q.Init(uint32(len(prog.Inst))) m.z2.q.Init(uint32(len(prog.Inst))) m.z3.q.Init(uint32(len(prog.Inst))) m.ids = make([]int, 0, len(prog.Inst)) m.addq(&m.z1.q, uint32(prog.Start), syntax.EmptyBeginLine|syntax.EmptyBeginText) m.z1.flag = flagBOL | flagBOT m.start = m.cache(&m.z1, nil, 0) m.z1.q.Reset() m.addq(&m.z1.q, uint32(prog.Start), syntax.EmptyBeginLine) m.z1.flag = flagBOL m.startLine = m.cache(&m.z1, nil, 0) m.crunchProg() return nil }
// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the // specified capacity. func NewInverseBloomFilter(capacity uint) *InverseBloomFilter { return &InverseBloomFilter{ array: make([]*[]byte, capacity), hash: fnv.New32(), capacity: capacity, } }
func (e *Engine) fnv32() error { data, err := computeHash(fnv.New32(), e.stack.Pop()) if err == nil { e.stack.Push(data) } return err }
func TestHyperLogLogIntersectLarge(t *testing.T) { rand.Seed(time.Now().UnixNano()) runs := uint64(10000000) registerSize := uint(2048) a, _ := New(registerSize) b, _ := New(registerSize) hash := fnv.New32() for i := uint64(0); i < runs; i++ { hash.Write([]byte(randStringBytesMaskImprSrc(10))) s := hash.Sum32() a.Add(s) if i%2 == 0 { b.Add(s) } hash.Reset() } intersected, _ := a.Intersect(b) maxIntersect := (float64(runs) / 2) * 1.022 minIntersect := (float64(runs) / 2) * 0.988 if float64(intersected) > maxIntersect || float64(intersected) < minIntersect { log.Printf("Intersect exceeded deviation bounderies (min: %f max: %f result: %d)", minIntersect, maxIntersect, intersected) t.Fail() } }
func TestHyperLogLogIntersectNone(t *testing.T) { a, _ := New(2048) b, _ := New(2048) hash := fnv.New32() // Apple in both hash.Write([]byte("apple")) s := hash.Sum32() a.Add(s) hash.Reset() // Beer in both hash.Write([]byte("beer")) s = hash.Sum32() a.Add(s) hash.Reset() // Banana in a hash.Write([]byte("banana")) s = hash.Sum32() a.Add(s) hash.Reset() intersected, _ := a.Intersect(b) log.Printf("none %d", intersected) }
func computeOffsets(index *nodeIndex, n *trieNode) int64 { if n.leaf { return n.value } hasher := fnv.New32() // We only index continuation bytes. for i := 0; i < blockSize; i++ { v := int64(0) if nn := n.table[0x80+i]; nn != nil { v = computeOffsets(index, nn) } hasher.Write([]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}) } h := hasher.Sum32() if n.isInternal() { v, ok := index.lookupBlockIdx[h] if !ok { v = int64(len(index.lookupBlocks)) index.lookupBlocks = append(index.lookupBlocks, n) index.lookupBlockIdx[h] = v } n.value = v } else { v, ok := index.valueBlockIdx[h] if !ok { v = int64(len(index.valueBlocks)) index.valueBlocks = append(index.valueBlocks, n) index.valueBlockIdx[h] = v } n.value = v } return n.value }
// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the // specified capacity. func NewInverseBloomFilter(capacity uint) *InverseBloomFilter { return &InverseBloomFilter{ array: make([]*[]byte, capacity), hashPool: &sync.Pool{New: func() interface{} { return fnv.New32() }}, capacity: capacity, } }
// getShard gets the shard responsible for the given key func (pm *ParMap) getShard(key string) *shard { hasher := fnv.New32() hasher.Write([]byte(key)) shard := pm.shards[hasher.Sum32()%pm.count] return shard }
func testHyperLogLog(t *testing.T, n, low_b, high_b int) { words := dictionary(n) bad := 0 n_words := uint64(len(words)) for i := low_b; i < high_b; i++ { m := uint(math.Pow(2, float64(i))) h, err := New(m) if err != nil { t.Fatalf("can't make New(%d): %v", m, err) } hash := fnv.New32() for _, word := range words { hash.Write([]byte(word)) h.Add(hash.Sum32()) hash.Reset() } expected_error := 1.04 / math.Sqrt(float64(m)) actual_error := math.Abs(geterror(n_words, h.Count())) if actual_error > expected_error { bad++ t.Logf("m=%d: error=%.5f, expected <%.5f; actual=%d, estimated=%d\n", m, actual_error, expected_error, n_words, h.Count()) } } t.Logf("%d of %d tests exceeded estimated error", bad, high_b-low_b) }
// Returns shard under given key func (m ConcurrentMap) GetShard(key int64) *ConcurrentMapShared { hasher := fnv.New32() b := make([]byte, 8) binary.LittleEndian.PutUint64(b, uint64(key)) hasher.Write(b) // hasher.Write([]byte(key)) return m[int(hasher.Sum32())%SHARD_COUNT] }
func calcFileHash(filename string) string { bytes, _ := ioutil.ReadFile(filename) h := fnv.New32() h.Write(bytes) return fmt.Sprint(h.Sum32()) }
func shardMatch(name string) bool { if *shards == 0 { return true } h := fnv.New32() io.WriteString(h, name) return int(h.Sum32()%uint32(*shards)) == *shard }
func styleHash(mapType string, mml string, mss []string) uint32 { f := fnv.New32() f.Write([]byte(mapType)) f.Write([]byte(mml)) for i := range mss { f.Write([]byte(mss[i])) } return f.Sum32() }
func (i *IndexString) Hash(n int) int { hash := fnv.New32() _, err := hash.Write(i.Bytes()) if err != nil { Errorf("Could not write to fnv hash in IndexString.Hash()") return 0 } return int(hash.Sum32()) % n }
func (i *IndexBytes) Hash(n int) int { hash := fnv.New32() _, err := hash.Write([]byte(*i)) if err != nil { Errorf("Could not write to fnv hash in IndexBytes.Hash()") return 0 } return int(hash.Sum32()) % n }
func newconn(w http.ResponseWriter, r *http.Request) (*conn, error) { wconn, err := websocket.Upgrade(w, r.Header, nil, 1024, 1024) if err != nil { return nil, err } hash := fnv.New32() hash.Write([]byte(r.RemoteAddr)) return &conn{Id(hash.Sum32()), make(chan Msg, 64), wconn, time.NewTicker(pingPeriod)}, nil }
func hashStr(s dvid.IZYXString, n int) int { hash := fnv.New32() _, err := hash.Write([]byte(s)) if err != nil { dvid.Criticalf("Could not write to fnv hash in labelblk.hashStr()") return 0 } return int(hash.Sum32()) % n }
func hashNGram(ngram []string) uint32 { h := fnv.New32() h.Write([]byte(ngram[0])) for _, w := range ngram[1:] { h.Write([]byte("\x00")) h.Write([]byte(w)) } return h.Sum32() }
func TestFnv32(t *testing.T) { key := []byte("ABC") hasher := fnv.New32() hasher.Write(key) if fnv32(string(key)) != hasher.Sum32() { t.Errorf("Bundled fnv32 produced %d, expected result from hash/fnv32 is %d", fnv32(string(key)), hasher.Sum32()) } }
func generateIndex(str string) uint { h := fnv.New32() b := []byte(str) h.Write(b) rand.Seed(int64(h.Sum32())) return uint(rand.Intn(m)) }
func BenchmarkFnv32(b *testing.B) { var bv []byte h := fnv.New32() for i := 0; i < b.N; i++ { h.Write(in) bv = h.Sum(nil) h.Reset() } benchValByte = bv }
func (bloom Bloom) hashKey(index int, elem uint64) uint32 { initialValue := bloom.hashes[index] buffer := make([]byte, 24, 24) binary.PutUvarint(buffer, initialValue) binary.PutUvarint(buffer[8:], elem) hash := fnv.New32() hash.Write(buffer) hashValue := hash.Sum32() return hashValue % uint32(bloom.size) }
func (h *consistentHash) bucketIdx(key []byte, i int) (keyUint uint32, idx int) { fnvH := fnv.New32() fnvH.Write(key) if err := binary.Write(fnvH, binary.LittleEndian, uint32(i)); err != nil { log.Panicf("binary.Write() failed: [%s]", err) } keyUint = fnvH.Sum32() idx = int(keyUint % uint32(h.BucketsCount)) return }
func main() { println("=====================[ emdr-relay-go ]=====================") println("Starting emdr-relay-go 1.1...") cache := cache.NewLRUCache(cache_size_limit) receiver, _ := zmq.NewSocket(zmq.SUB) receiver.Connect("tcp://master.eve-emdr.com:8050") receiver.Connect("tcp://secondary.eve-emdr.com:8050") receiver.SetSubscribe("") defer receiver.Close() sender, _ := zmq.NewSocket(zmq.PUB) sender.Bind("tcp://*:8050") defer sender.Close() println("Listening on port 8050.") println("===========================================================") // Ensure subscriber connection has time to complete time.Sleep(time.Second) // We auto-exit every number of hours so we can recover from some // weird edge case conditions that disrupt the network. They're not common, // but we'll do this to be absolutely sure. go periodic_exiter() for { msg, zmq_err := receiver.Recv(0) if zmq_err != nil { println("RECV ERROR:", zmq_err.Error()) } var h hash.Hash = fnv.New32() h.Write([]byte(msg)) checksum := h.Sum([]byte{}) cache_key := fmt.Sprintf("%x", checksum) cache_item, cache_hit := cache.Get(cache_key) if cache_hit { // We've already seen this before, ignore it. continue } // At this point, we know we've encountered a message we haven't // seen in the recent past. cache_item = &CacheValue{found: true} // Insert the cache entry to prevent future re-sends of this message. cache.Set(cache_key, cache_item) // A cache miss means that the incoming message is not a dupe. // Send the message to subscribers. sender.Send(msg, 0) } }