func (g *FieldGrouping) ModHash(tuple Values) uint32 {
h := fnv.New32()
values := g.SourceFields.Select(g.Selector, tuple)
for _, value := range values {
h.Write(value.Encode())
}
h = fnv.New32()
return h.Sum32() % uint32(len(g.Dests))
}
// Get bucket (or sharded map) by key in 1 cache instance with built in hash function of Golang
func (cache *Cache) getBucketWithBuiltInHasher(key string) (*ConcurrentMap, uint) {
hasher := fnv.New32()
hasher.Write([]byte(key))
bucketsCount := len(cache.buckets)
bucketIndex := uint(hasher.Sum32()) % uint(bucketsCount)
return cache.buckets[bucketIndex], bucketIndex
}
func BenchmarkFnv32(b *testing.B) {
h := fnv.New32()
for i := 0; i < b.N; i++ {
h.Write(in)
h.Sum(nil)
}
}
func (self matchHash) Key(target matchHash) uint32 {
hasher := fnv.New32()
for k, s := range self {
var value, mask []byte
if bKey, ok := k.(OxmKeyBasic); !ok || oxm.Header(bKey).Class() != ofp4.OFPXMC_OPENFLOW_BASIC {
continue
} else {
length, _ := ofp4.OxmOfDefs(uint32(bKey))
value = make([]byte, length)
mask = make([]byte, length)
}
if t, ok := target[k]; ok {
a := s.(OxmValueMask)
b := t.(OxmValueMask)
for i, _ := range mask {
mask[i] = 0xFF
}
if a.Mask != nil {
for i, _ := range mask {
mask[i] &= a.Mask[i]
}
}
if b.Mask != nil {
for i, _ := range mask {
mask[i] &= b.Mask[i]
}
}
for i, _ := range value {
value[i] = b.Value[i] & mask[i]
}
}
hasher.Write(value)
}
return hasher.Sum32()
}
// Computes the hash of the cursor by computing the hash of every enpdoint supplied
func computeHash(endpoints []loadbalance.Endpoint) uint32 {
h := fnv.New32()
for _, endpoint := range endpoints {
h.Write([]byte(endpoint.Id()))
}
return h.Sum32()
}
// hashKey hashes a key into a uint32
func hashKey(key []byte) uint32 {
// need to convert to a []byte
h := fnv.New32()
h.Write(key)
return h.Sum32()
}
// init initializes the matcher.
func (m *matcher) init(prog *syntax.Prog, n int) error {
m.prog = prog
m.dstate = make(map[uint32]*dstate)
m.numMatch = n
m.maxState = 10
m.allTail = &m.all
m.numByte = 256
for i := range m.undo {
m.undo[i] = byte(i)
}
m.h = fnv.New32()
m.z1.q.Init(uint32(len(prog.Inst)))
m.z2.q.Init(uint32(len(prog.Inst)))
m.z3.q.Init(uint32(len(prog.Inst)))
m.ids = make([]int, 0, len(prog.Inst))
m.addq(&m.z1.q, uint32(prog.Start), syntax.EmptyBeginLine|syntax.EmptyBeginText)
m.z1.flag = flagBOL | flagBOT
m.start = m.cache(&m.z1, nil, 0)
m.z1.q.Reset()
m.addq(&m.z1.q, uint32(prog.Start), syntax.EmptyBeginLine)
m.z1.flag = flagBOL
m.startLine = m.cache(&m.z1, nil, 0)
m.crunchProg()
return nil
}
// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the
// specified capacity.
func NewInverseBloomFilter(capacity uint) *InverseBloomFilter {
return &InverseBloomFilter{
array: make([]*[]byte, capacity),
hash: fnv.New32(),
capacity: capacity,
}
}
func (e *Engine) fnv32() error {
data, err := computeHash(fnv.New32(), e.stack.Pop())
if err == nil {
e.stack.Push(data)
}
return err
}
func TestHyperLogLogIntersectLarge(t *testing.T) {
rand.Seed(time.Now().UnixNano())
runs := uint64(10000000)
registerSize := uint(2048)
a, _ := New(registerSize)
b, _ := New(registerSize)
hash := fnv.New32()
for i := uint64(0); i < runs; i++ {
hash.Write([]byte(randStringBytesMaskImprSrc(10)))
s := hash.Sum32()
a.Add(s)
if i%2 == 0 {
b.Add(s)
}
hash.Reset()
}
intersected, _ := a.Intersect(b)
maxIntersect := (float64(runs) / 2) * 1.022
minIntersect := (float64(runs) / 2) * 0.988
if float64(intersected) > maxIntersect || float64(intersected) < minIntersect {
log.Printf("Intersect exceeded deviation bounderies (min: %f max: %f result: %d)", minIntersect, maxIntersect, intersected)
t.Fail()
}
}
func TestHyperLogLogIntersectNone(t *testing.T) {
a, _ := New(2048)
b, _ := New(2048)
hash := fnv.New32()
// Apple in both
hash.Write([]byte("apple"))
s := hash.Sum32()
a.Add(s)
hash.Reset()
// Beer in both
hash.Write([]byte("beer"))
s = hash.Sum32()
a.Add(s)
hash.Reset()
// Banana in a
hash.Write([]byte("banana"))
s = hash.Sum32()
a.Add(s)
hash.Reset()
intersected, _ := a.Intersect(b)
log.Printf("none %d", intersected)
}
func computeOffsets(index *nodeIndex, n *trieNode) int64 {
if n.leaf {
return n.value
}
hasher := fnv.New32()
// We only index continuation bytes.
for i := 0; i < blockSize; i++ {
v := int64(0)
if nn := n.table[0x80+i]; nn != nil {
v = computeOffsets(index, nn)
}
hasher.Write([]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
}
h := hasher.Sum32()
if n.isInternal() {
v, ok := index.lookupBlockIdx[h]
if !ok {
v = int64(len(index.lookupBlocks))
index.lookupBlocks = append(index.lookupBlocks, n)
index.lookupBlockIdx[h] = v
}
n.value = v
} else {
v, ok := index.valueBlockIdx[h]
if !ok {
v = int64(len(index.valueBlocks))
index.valueBlocks = append(index.valueBlocks, n)
index.valueBlockIdx[h] = v
}
n.value = v
}
return n.value
}
// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the
// specified capacity.
func NewInverseBloomFilter(capacity uint) *InverseBloomFilter {
return &InverseBloomFilter{
array: make([]*[]byte, capacity),
hashPool: &sync.Pool{New: func() interface{} { return fnv.New32() }},
capacity: capacity,
}
}
// getShard gets the shard responsible for the given key
func (pm *ParMap) getShard(key string) *shard {
hasher := fnv.New32()
hasher.Write([]byte(key))
shard := pm.shards[hasher.Sum32()%pm.count]
return shard
}
func testHyperLogLog(t *testing.T, n, low_b, high_b int) {
words := dictionary(n)
bad := 0
n_words := uint64(len(words))
for i := low_b; i < high_b; i++ {
m := uint(math.Pow(2, float64(i)))
h, err := New(m)
if err != nil {
t.Fatalf("can't make New(%d): %v", m, err)
}
hash := fnv.New32()
for _, word := range words {
hash.Write([]byte(word))
h.Add(hash.Sum32())
hash.Reset()
}
expected_error := 1.04 / math.Sqrt(float64(m))
actual_error := math.Abs(geterror(n_words, h.Count()))
if actual_error > expected_error {
bad++
t.Logf("m=%d: error=%.5f, expected <%.5f; actual=%d, estimated=%d\n",
m, actual_error, expected_error, n_words, h.Count())
}
}
t.Logf("%d of %d tests exceeded estimated error", bad, high_b-low_b)
}
// Returns shard under given key
func (m ConcurrentMap) GetShard(key int64) *ConcurrentMapShared {
hasher := fnv.New32()
b := make([]byte, 8)
binary.LittleEndian.PutUint64(b, uint64(key))
hasher.Write(b)
// hasher.Write([]byte(key))
return m[int(hasher.Sum32())%SHARD_COUNT]
}
func calcFileHash(filename string) string {
bytes, _ := ioutil.ReadFile(filename)
h := fnv.New32()
h.Write(bytes)
return fmt.Sprint(h.Sum32())
}
func shardMatch(name string) bool {
if *shards == 0 {
return true
}
h := fnv.New32()
io.WriteString(h, name)
return int(h.Sum32()%uint32(*shards)) == *shard
}
func styleHash(mapType string, mml string, mss []string) uint32 {
f := fnv.New32()
f.Write([]byte(mapType))
f.Write([]byte(mml))
for i := range mss {
f.Write([]byte(mss[i]))
}
return f.Sum32()
}
func (i *IndexString) Hash(n int) int {
hash := fnv.New32()
_, err := hash.Write(i.Bytes())
if err != nil {
Errorf("Could not write to fnv hash in IndexString.Hash()")
return 0
}
return int(hash.Sum32()) % n
}
func (i *IndexBytes) Hash(n int) int {
hash := fnv.New32()
_, err := hash.Write([]byte(*i))
if err != nil {
Errorf("Could not write to fnv hash in IndexBytes.Hash()")
return 0
}
return int(hash.Sum32()) % n
}
func newconn(w http.ResponseWriter, r *http.Request) (*conn, error) {
wconn, err := websocket.Upgrade(w, r.Header, nil, 1024, 1024)
if err != nil {
return nil, err
}
hash := fnv.New32()
hash.Write([]byte(r.RemoteAddr))
return &conn{Id(hash.Sum32()), make(chan Msg, 64), wconn, time.NewTicker(pingPeriod)}, nil
}
func hashStr(s dvid.IZYXString, n int) int {
hash := fnv.New32()
_, err := hash.Write([]byte(s))
if err != nil {
dvid.Criticalf("Could not write to fnv hash in labelblk.hashStr()")
return 0
}
return int(hash.Sum32()) % n
}
func hashNGram(ngram []string) uint32 {
h := fnv.New32()
h.Write([]byte(ngram[0]))
for _, w := range ngram[1:] {
h.Write([]byte("\x00"))
h.Write([]byte(w))
}
return h.Sum32()
}
func TestFnv32(t *testing.T) {
key := []byte("ABC")
hasher := fnv.New32()
hasher.Write(key)
if fnv32(string(key)) != hasher.Sum32() {
t.Errorf("Bundled fnv32 produced %d, expected result from hash/fnv32 is %d", fnv32(string(key)), hasher.Sum32())
}
}
func generateIndex(str string) uint {
h := fnv.New32()
b := []byte(str)
h.Write(b)
rand.Seed(int64(h.Sum32()))
return uint(rand.Intn(m))
}
func BenchmarkFnv32(b *testing.B) {
var bv []byte
h := fnv.New32()
for i := 0; i < b.N; i++ {
h.Write(in)
bv = h.Sum(nil)
h.Reset()
}
benchValByte = bv
}
func (bloom Bloom) hashKey(index int, elem uint64) uint32 {
initialValue := bloom.hashes[index]
buffer := make([]byte, 24, 24)
binary.PutUvarint(buffer, initialValue)
binary.PutUvarint(buffer[8:], elem)
hash := fnv.New32()
hash.Write(buffer)
hashValue := hash.Sum32()
return hashValue % uint32(bloom.size)
}
func (h *consistentHash) bucketIdx(key []byte, i int) (keyUint uint32, idx int) {
fnvH := fnv.New32()
fnvH.Write(key)
if err := binary.Write(fnvH, binary.LittleEndian, uint32(i)); err != nil {
log.Panicf("binary.Write() failed: [%s]", err)
}
keyUint = fnvH.Sum32()
idx = int(keyUint % uint32(h.BucketsCount))
return
}
func main() {
println("=====================[ emdr-relay-go ]=====================")
println("Starting emdr-relay-go 1.1...")
cache := cache.NewLRUCache(cache_size_limit)
receiver, _ := zmq.NewSocket(zmq.SUB)
receiver.Connect("tcp://master.eve-emdr.com:8050")
receiver.Connect("tcp://secondary.eve-emdr.com:8050")
receiver.SetSubscribe("")
defer receiver.Close()
sender, _ := zmq.NewSocket(zmq.PUB)
sender.Bind("tcp://*:8050")
defer sender.Close()
println("Listening on port 8050.")
println("===========================================================")
// Ensure subscriber connection has time to complete
time.Sleep(time.Second)
// We auto-exit every number of hours so we can recover from some
// weird edge case conditions that disrupt the network. They're not common,
// but we'll do this to be absolutely sure.
go periodic_exiter()
for {
msg, zmq_err := receiver.Recv(0)
if zmq_err != nil {
println("RECV ERROR:", zmq_err.Error())
}
var h hash.Hash = fnv.New32()
h.Write([]byte(msg))
checksum := h.Sum([]byte{})
cache_key := fmt.Sprintf("%x", checksum)
cache_item, cache_hit := cache.Get(cache_key)
if cache_hit {
// We've already seen this before, ignore it.
continue
}
// At this point, we know we've encountered a message we haven't
// seen in the recent past.
cache_item = &CacheValue{found: true}
// Insert the cache entry to prevent future re-sends of this message.
cache.Set(cache_key, cache_item)
// A cache miss means that the incoming message is not a dupe.
// Send the message to subscribers.
sender.Send(msg, 0)
}
}
