Exemple #1
0
func NewBloomWrap(capSize uint) BloomWrap {
	wrap := []bloom.Bloom{}
	newFilter := func(h hash.Hash) {
		filter := standard.New(capSize)
		filter.SetHasher(h)
		wrap = append(wrap, filter)
	}
	newFilter(cityhash.New64())
	newFilter(fnv.New64())
	newFilter(md5.New())
	log.Println("bloomLen: ", len(wrap))
	return wrap
}
Exemple #2
0
func BenchmarkBloomCityHash(b *testing.B) {
	var lines []string
	lines = append(lines, web2...)
	for len(lines) < b.N {
		lines = append(lines, web2...)
	}

	bf := New(uint(b.N))
	bf.SetHasher(cityhash.New64())
	fn := 0

	b.ResetTimer()

	for l := 0; l < b.N; l++ {
		if !(bf.Add([]byte(lines[l])).Check([]byte(lines[l]))) {
			fn++
		}
	}

	b.StopTimer()
}
Exemple #3
0
func generateGarbage() {
	var capSize uint = 1000000000
	filter := standard.New(capSize)
	filter.SetHasher(cityhash.New64())
	v := []byte("Love")
	b := filter.Add(v).Check(v)
	log.Println("check @v:", b)
	bad := 0
	for i := 0; i < 1000000000; i++ {
		if i%1000000 == 0 {
			log.Println(i)
			debug.FreeOSMemory()
		}
		d := []byte(fmt.Sprint("data", i))
		if filter.Check(d) == true {
			bad++
			//			panic(fmt.Sprint("should not exist @d:",string(d)))
		}
		if flag := filter.Add(d).Check(d); flag == false {
			panic(d)
		}
	}
	log.Println("====>>That is all @bad:", bad)
}
Exemple #4
0
func TestBloomFilter(t *testing.T) {
	l := []uint{uint(len(web2)), 200000, 100000, 50000}
	h := []hash.Hash{fnv.New64(), crc64.New(crc64.MakeTable(crc64.ECMA)), murmur3.New64(), cityhash.New64(), md5.New(), sha1.New()}
	n := []string{"fnv.New64()", "crc64.New()", "murmur3.New64()", "cityhash.New64()", "md5.New()", "sha1.New()"}

	for i := range l {
		for j := range h {
			fmt.Printf("\n\nTesting %s with size %d\n", n[j], l[i])
			bf := New(l[i])
			bf.SetHasher(h[j])
			testBloomFilter(t, bf)
		}
	}
}
Exemple #5
0
func TestBloomFilter(t *testing.T) {
	l := []uint{uint(len(web2)), 200000, 100000, 50000}
	h := []hash.Hash{fnv.New64(), crc64.New(crc64.MakeTable(crc64.ECMA)), murmur3.New64(), cityhash.New64(), md5.New(), sha1.New()}
	n := []string{"fnv.New64()", "crc64.New()", "murmur3.New64()", "cityhash.New64()", "md5.New()", "sha1.New()"}
	b := []func(uint) bloom.Bloom{standard.New, partitioned.New}
	bn := []string{"standard", "partitioned"}

	for i := range l {
		for j := range h {
			for k := range b {
				fmt.Printf("\n\nTesting %s with size %d using %s\n", n[j], l[i], bn[k])
				bf := New(l[i])
				bf.SetHasher(h[j])
				bf.(*ScalableBloom).SetBloomFilter(b[k])
				bf.Reset()
				testBloomFilter(t, bf)
			}
		}
	}
}