예제 #1
0
func (g *FieldGrouping) ModHash(tuple Values) uint32 {
	h := fnv.New32()
	values := g.SourceFields.Select(g.Selector, tuple)
	for _, value := range values {
		h.Write(value.Encode())
	}
	h = fnv.New32()
	return h.Sum32() % uint32(len(g.Dests))
}
예제 #2
0
// Get bucket (or sharded map) by key in 1 cache instance with built in hash function of Golang
func (cache *Cache) getBucketWithBuiltInHasher(key string) (*ConcurrentMap, uint) {
	hasher := fnv.New32()
	hasher.Write([]byte(key))
	bucketsCount := len(cache.buckets)
	bucketIndex := uint(hasher.Sum32()) % uint(bucketsCount)
	return cache.buckets[bucketIndex], bucketIndex
}
예제 #3
0
func BenchmarkFnv32(b *testing.B) {
	h := fnv.New32()
	for i := 0; i < b.N; i++ {
		h.Write(in)
		h.Sum(nil)
	}
}
예제 #4
0
파일: match.go 프로젝트: hkwi/gopenflow
func (self matchHash) Key(target matchHash) uint32 {
	hasher := fnv.New32()
	for k, s := range self {
		var value, mask []byte
		if bKey, ok := k.(OxmKeyBasic); !ok || oxm.Header(bKey).Class() != ofp4.OFPXMC_OPENFLOW_BASIC {
			continue
		} else {
			length, _ := ofp4.OxmOfDefs(uint32(bKey))
			value = make([]byte, length)
			mask = make([]byte, length)
		}
		if t, ok := target[k]; ok {
			a := s.(OxmValueMask)
			b := t.(OxmValueMask)
			for i, _ := range mask {
				mask[i] = 0xFF
			}
			if a.Mask != nil {
				for i, _ := range mask {
					mask[i] &= a.Mask[i]
				}
			}
			if b.Mask != nil {
				for i, _ := range mask {
					mask[i] &= b.Mask[i]
				}
			}
			for i, _ := range value {
				value[i] = b.Value[i] & mask[i]
			}
		}
		hasher.Write(value)
	}
	return hasher.Sum32()
}
예제 #5
0
// Computes the hash of the cursor by computing the hash of every enpdoint supplied
func computeHash(endpoints []loadbalance.Endpoint) uint32 {
	h := fnv.New32()
	for _, endpoint := range endpoints {
		h.Write([]byte(endpoint.Id()))
	}
	return h.Sum32()
}
예제 #6
0
파일: trie.go 프로젝트: eliothedeman/immut
// hashKey hashes a key into a uint32
func hashKey(key []byte) uint32 {

	// need to convert to a []byte
	h := fnv.New32()
	h.Write(key)
	return h.Sum32()
}
예제 #7
0
파일: match.go 프로젝트: 0x7cc/rsc
// init initializes the matcher.
func (m *matcher) init(prog *syntax.Prog, n int) error {
	m.prog = prog
	m.dstate = make(map[uint32]*dstate)
	m.numMatch = n
	m.maxState = 10
	m.allTail = &m.all
	m.numByte = 256
	for i := range m.undo {
		m.undo[i] = byte(i)
	}
	m.h = fnv.New32()

	m.z1.q.Init(uint32(len(prog.Inst)))
	m.z2.q.Init(uint32(len(prog.Inst)))
	m.z3.q.Init(uint32(len(prog.Inst)))
	m.ids = make([]int, 0, len(prog.Inst))

	m.addq(&m.z1.q, uint32(prog.Start), syntax.EmptyBeginLine|syntax.EmptyBeginText)
	m.z1.flag = flagBOL | flagBOT
	m.start = m.cache(&m.z1, nil, 0)

	m.z1.q.Reset()
	m.addq(&m.z1.q, uint32(prog.Start), syntax.EmptyBeginLine)
	m.z1.flag = flagBOL
	m.startLine = m.cache(&m.z1, nil, 0)

	m.crunchProg()

	return nil
}
예제 #8
0
// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the
// specified capacity.
func NewInverseBloomFilter(capacity uint) *InverseBloomFilter {
	return &InverseBloomFilter{
		array:    make([]*[]byte, capacity),
		hash:     fnv.New32(),
		capacity: capacity,
	}
}
예제 #9
0
파일: hash.go 프로젝트: ancientlore/hashsrv
func (e *Engine) fnv32() error {
	data, err := computeHash(fnv.New32(), e.stack.Pop())
	if err == nil {
		e.stack.Push(data)
	}
	return err
}
예제 #10
0
func TestHyperLogLogIntersectLarge(t *testing.T) {
	rand.Seed(time.Now().UnixNano())

	runs := uint64(10000000)
	registerSize := uint(2048)

	a, _ := New(registerSize)
	b, _ := New(registerSize)
	hash := fnv.New32()

	for i := uint64(0); i < runs; i++ {
		hash.Write([]byte(randStringBytesMaskImprSrc(10)))
		s := hash.Sum32()
		a.Add(s)
		if i%2 == 0 {
			b.Add(s)
		}
		hash.Reset()
	}

	intersected, _ := a.Intersect(b)
	maxIntersect := (float64(runs) / 2) * 1.022
	minIntersect := (float64(runs) / 2) * 0.988

	if float64(intersected) > maxIntersect || float64(intersected) < minIntersect {
		log.Printf("Intersect exceeded deviation bounderies (min: %f max: %f result: %d)", minIntersect, maxIntersect, intersected)
		t.Fail()
	}
}
예제 #11
0
func TestHyperLogLogIntersectNone(t *testing.T) {
	a, _ := New(2048)
	b, _ := New(2048)

	hash := fnv.New32()

	// Apple in both
	hash.Write([]byte("apple"))
	s := hash.Sum32()
	a.Add(s)
	hash.Reset()

	// Beer in both
	hash.Write([]byte("beer"))
	s = hash.Sum32()
	a.Add(s)
	hash.Reset()

	// Banana in a
	hash.Write([]byte("banana"))
	s = hash.Sum32()
	a.Add(s)
	hash.Reset()

	intersected, _ := a.Intersect(b)
	log.Printf("none %d", intersected)
}
예제 #12
0
func computeOffsets(index *nodeIndex, n *trieNode) int64 {
	if n.leaf {
		return n.value
	}
	hasher := fnv.New32()
	// We only index continuation bytes.
	for i := 0; i < blockSize; i++ {
		v := int64(0)
		if nn := n.table[0x80+i]; nn != nil {
			v = computeOffsets(index, nn)
		}
		hasher.Write([]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
	}
	h := hasher.Sum32()
	if n.isInternal() {
		v, ok := index.lookupBlockIdx[h]
		if !ok {
			v = int64(len(index.lookupBlocks))
			index.lookupBlocks = append(index.lookupBlocks, n)
			index.lookupBlockIdx[h] = v
		}
		n.value = v
	} else {
		v, ok := index.valueBlockIdx[h]
		if !ok {
			v = int64(len(index.valueBlocks))
			index.valueBlocks = append(index.valueBlocks, n)
			index.valueBlockIdx[h] = v
		}
		n.value = v
	}
	return n.value
}
예제 #13
0
// NewInverseBloomFilter creates and returns a new InverseBloomFilter with the
// specified capacity.
func NewInverseBloomFilter(capacity uint) *InverseBloomFilter {
	return &InverseBloomFilter{
		array:    make([]*[]byte, capacity),
		hashPool: &sync.Pool{New: func() interface{} { return fnv.New32() }},
		capacity: capacity,
	}
}
예제 #14
0
파일: parmap.go 프로젝트: pombredanne/pbtc
// getShard gets the shard responsible for the given key
func (pm *ParMap) getShard(key string) *shard {
	hasher := fnv.New32()
	hasher.Write([]byte(key))
	shard := pm.shards[hasher.Sum32()%pm.count]

	return shard
}
예제 #15
0
func testHyperLogLog(t *testing.T, n, low_b, high_b int) {
	words := dictionary(n)
	bad := 0
	n_words := uint64(len(words))
	for i := low_b; i < high_b; i++ {
		m := uint(math.Pow(2, float64(i)))

		h, err := New(m)
		if err != nil {
			t.Fatalf("can't make New(%d): %v", m, err)
		}

		hash := fnv.New32()
		for _, word := range words {
			hash.Write([]byte(word))
			h.Add(hash.Sum32())
			hash.Reset()
		}

		expected_error := 1.04 / math.Sqrt(float64(m))
		actual_error := math.Abs(geterror(n_words, h.Count()))

		if actual_error > expected_error {
			bad++
			t.Logf("m=%d: error=%.5f, expected <%.5f; actual=%d, estimated=%d\n",
				m, actual_error, expected_error, n_words, h.Count())
		}

	}
	t.Logf("%d of %d tests exceeded estimated error", bad, high_b-low_b)
}
예제 #16
0
// Returns shard under given key
func (m ConcurrentMap) GetShard(key int64) *ConcurrentMapShared {
	hasher := fnv.New32()
	b := make([]byte, 8)
	binary.LittleEndian.PutUint64(b, uint64(key))
	hasher.Write(b)
	//	hasher.Write([]byte(key))
	return m[int(hasher.Sum32())%SHARD_COUNT]
}
예제 #17
0
func calcFileHash(filename string) string {
	bytes, _ := ioutil.ReadFile(filename)

	h := fnv.New32()
	h.Write(bytes)

	return fmt.Sprint(h.Sum32())
}
예제 #18
0
파일: run.go 프로젝트: hurkgu/go
func shardMatch(name string) bool {
	if *shards == 0 {
		return true
	}
	h := fnv.New32()
	io.WriteString(h, name)
	return int(h.Sum32()%uint32(*shards)) == *shard
}
예제 #19
0
func styleHash(mapType string, mml string, mss []string) uint32 {
	f := fnv.New32()
	f.Write([]byte(mapType))
	f.Write([]byte(mml))
	for i := range mss {
		f.Write([]byte(mss[i]))
	}
	return f.Sum32()
}
예제 #20
0
파일: index.go 프로젝트: tartavull/dvid
func (i *IndexString) Hash(n int) int {
	hash := fnv.New32()
	_, err := hash.Write(i.Bytes())
	if err != nil {
		Errorf("Could not write to fnv hash in IndexString.Hash()")
		return 0
	}
	return int(hash.Sum32()) % n
}
예제 #21
0
파일: index.go 프로젝트: tartavull/dvid
func (i *IndexBytes) Hash(n int) int {
	hash := fnv.New32()
	_, err := hash.Write([]byte(*i))
	if err != nil {
		Errorf("Could not write to fnv hash in IndexBytes.Hash()")
		return 0
	}
	return int(hash.Sum32()) % n
}
예제 #22
0
파일: conn.go 프로젝트: napsy/lab
func newconn(w http.ResponseWriter, r *http.Request) (*conn, error) {
	wconn, err := websocket.Upgrade(w, r.Header, nil, 1024, 1024)
	if err != nil {
		return nil, err
	}
	hash := fnv.New32()
	hash.Write([]byte(r.RemoteAddr))
	return &conn{Id(hash.Sum32()), make(chan Msg, 64), wconn, time.NewTicker(pingPeriod)}, nil
}
예제 #23
0
파일: sync.go 프로젝트: hanslovsky/dvid
func hashStr(s dvid.IZYXString, n int) int {
	hash := fnv.New32()
	_, err := hash.Write([]byte(s))
	if err != nil {
		dvid.Criticalf("Could not write to fnv hash in labelblk.hashStr()")
		return 0
	}
	return int(hash.Sum32()) % n
}
예제 #24
0
func hashNGram(ngram []string) uint32 {
	h := fnv.New32()
	h.Write([]byte(ngram[0]))
	for _, w := range ngram[1:] {
		h.Write([]byte("\x00"))
		h.Write([]byte(w))
	}
	return h.Sum32()
}
func TestFnv32(t *testing.T) {
	key := []byte("ABC")

	hasher := fnv.New32()
	hasher.Write(key)
	if fnv32(string(key)) != hasher.Sum32() {
		t.Errorf("Bundled fnv32 produced %d, expected result from hash/fnv32 is %d", fnv32(string(key)), hasher.Sum32())
	}
}
예제 #26
0
func generateIndex(str string) uint {
	h := fnv.New32()
	b := []byte(str)

	h.Write(b)

	rand.Seed(int64(h.Sum32()))
	return uint(rand.Intn(m))
}
예제 #27
0
func BenchmarkFnv32(b *testing.B) {
	var bv []byte
	h := fnv.New32()
	for i := 0; i < b.N; i++ {
		h.Write(in)
		bv = h.Sum(nil)
		h.Reset()
	}
	benchValByte = bv
}
예제 #28
0
파일: bloom.go 프로젝트: wchau/exercise
func (bloom Bloom) hashKey(index int, elem uint64) uint32 {
	initialValue := bloom.hashes[index]
	buffer := make([]byte, 24, 24)
	binary.PutUvarint(buffer, initialValue)
	binary.PutUvarint(buffer[8:], elem)
	hash := fnv.New32()
	hash.Write(buffer)
	hashValue := hash.Sum32()
	return hashValue % uint32(bloom.size)
}
예제 #29
0
func (h *consistentHash) bucketIdx(key []byte, i int) (keyUint uint32, idx int) {
	fnvH := fnv.New32()
	fnvH.Write(key)
	if err := binary.Write(fnvH, binary.LittleEndian, uint32(i)); err != nil {
		log.Panicf("binary.Write() failed: [%s]", err)
	}
	keyUint = fnvH.Sum32()
	idx = int(keyUint % uint32(h.BucketsCount))
	return
}
예제 #30
0
func main() {
	println("=====================[ emdr-relay-go ]=====================")
	println("Starting emdr-relay-go 1.1...")
	cache := cache.NewLRUCache(cache_size_limit)

	receiver, _ := zmq.NewSocket(zmq.SUB)
	receiver.Connect("tcp://master.eve-emdr.com:8050")
	receiver.Connect("tcp://secondary.eve-emdr.com:8050")
	receiver.SetSubscribe("")
	defer receiver.Close()

	sender, _ := zmq.NewSocket(zmq.PUB)
	sender.Bind("tcp://*:8050")
	defer sender.Close()

	println("Listening on port 8050.")
	println("===========================================================")
	//  Ensure subscriber connection has time to complete
	time.Sleep(time.Second)

	// We auto-exit every number of hours so we can recover from some
	// weird edge case conditions that disrupt the network. They're not common,
	// but we'll do this to be absolutely sure.
	go periodic_exiter()

	for {
		msg, zmq_err := receiver.Recv(0)
		if zmq_err != nil {
			println("RECV ERROR:", zmq_err.Error())
		}

		var h hash.Hash = fnv.New32()
		h.Write([]byte(msg))

		checksum := h.Sum([]byte{})
		cache_key := fmt.Sprintf("%x", checksum)

		cache_item, cache_hit := cache.Get(cache_key)
		if cache_hit {
			// We've already seen this before, ignore it.
			continue
		}

		// At this point, we know we've encountered a message we haven't
		// seen in the recent past.
		cache_item = &CacheValue{found: true}
		// Insert the cache entry to prevent future re-sends of this message.
		cache.Set(cache_key, cache_item)

		// A cache miss means that the incoming message is not a dupe.
		// Send the message to subscribers.
		sender.Send(msg, 0)
	}
}