func hash(ids ...string) uint64 {
id := murmur3.New64()
for _, i := range ids {
id.Write([]byte(i))
}
return id.Sum64()
}
func smartHash(file string) (uint64, error) {
h := murmur3.New64()
f, err := os.Open(file)
defer f.Close()
fi, err := f.Stat()
if err != nil {
return 0, err
}
if fi.Size() <= FULL_HASH_LIMIT {
buffer := make([]byte, fi.Size())
f.Read(buffer)
h.Write(buffer)
} else {
buffer := make([]byte, SAMPLE_SIZE)
f.Read(buffer)
h.Write(buffer)
f.Seek(fi.Size()/2, 0)
f.Read(buffer)
h.Write(buffer)
f.Seek(-SAMPLE_SIZE, 2)
f.Read(buffer)
h.Write(buffer)
}
return h.Sum64(), nil
}
// AuthorID generates a unique ID based on the murmur hash of the public key.
func (key *PrivateKey) AuthorID() (string, error) {
hasher := murmur3.New64()
buf, err := x509.MarshalPKIXPublicKey(&(*ecdsa.PrivateKey)(key).PublicKey)
if err != nil {
return "", err
}
hasher.Write(buf)
return "degdb:author_" + strconv.Itoa(int(hasher.Sum64())), nil
}
func (dumbMerger) Merge(reports []report.Report) report.Report {
rpt := report.MakeReport()
id := murmur3.New64()
for _, r := range reports {
rpt = rpt.Merge(r)
id.Write([]byte(r.ID))
}
rpt.ID = fmt.Sprintf("%x", id.Sum64())
return rpt
}
func hashFiles(files <-chan *File, catalog *Catalog) {
buffer := make([]byte, SAMPLE_SIZE)
h := murmur3.New64()
for file := range files {
h.Reset()
f, _ := os.Open(file.Path)
f.Read(buffer)
h.Write(buffer)
f.Close()
file.Hash = h.Sum64()
catalog.AddFile(file)
}
}
func TestBloomFilter(t *testing.T) {
l := []uint{uint(len(web2)), 200000, 100000, 50000}
h := []hash.Hash{fnv.New64(), crc64.New(crc64.MakeTable(crc64.ECMA)), murmur3.New64(), cityhash.New64(), md5.New(), sha1.New()}
n := []string{"fnv.New64()", "crc64.New()", "murmur3.New64()", "cityhash.New64()", "md5.New()", "sha1.New()"}
for i := range l {
for j := range h {
fmt.Printf("\n\nTesting %s with size %d\n", n[j], l[i])
bf := New(l[i])
bf.SetHasher(h[j])
testBloomFilter(t, bf)
}
}
}
func TestBloomFilter(t *testing.T) {
l := []uint{uint(len(web2)), 200000, 100000, 50000}
h := []hash.Hash{fnv.New64(), crc64.New(crc64.MakeTable(crc64.ECMA)), murmur3.New64(), cityhash.New64(), md5.New(), sha1.New()}
n := []string{"fnv.New64()", "crc64.New()", "murmur3.New64()", "cityhash.New64()", "md5.New()", "sha1.New()"}
b := []func(uint) bloom.Bloom{standard.New, partitioned.New}
bn := []string{"standard", "partitioned"}
for i := range l {
for j := range h {
for k := range b {
fmt.Printf("\n\nTesting %s with size %d using %s\n", n[j], l[i], bn[k])
bf := New(l[i])
bf.SetHasher(h[j])
bf.(*ScalableBloom).SetBloomFilter(b[k])
bf.Reset()
testBloomFilter(t, bf)
}
}
}
}
func BenchmarkBloomMurmur3(b *testing.B) {
var lines []string
lines = append(lines, web2...)
for len(lines) < b.N {
lines = append(lines, web2...)
}
bf := New(uint(b.N))
bf.SetHasher(murmur3.New64())
fn := 0
b.ResetTimer()
for l := 0; l < b.N; l++ {
if !(bf.Add([]byte(lines[l])).Check([]byte(lines[l]))) {
fn++
}
}
b.StopTimer()
}
// Report returns a merged report over all added reports. It implements
// Reporter.
func (c *collector) Report() report.Report {
c.mtx.Lock()
defer c.mtx.Unlock()
// If the oldest report is still within range,
// and there is a cached report, return that.
if c.cached != nil && len(c.reports) > 0 {
oldest := now().Add(-c.window)
if c.reports[0].timestamp.Before(oldest) {
return *c.cached
}
}
c.reports = clean(c.reports, c.window)
rpt := report.MakeReport()
id := murmur3.New64()
for _, tr := range c.reports {
rpt = rpt.Merge(tr.report)
id.Write([]byte(tr.report.ID))
}
rpt.ID = fmt.Sprintf("%x", id.Sum64())
c.cached = &rpt
return rpt
}
// newHasher allocates and return a new Hasher.
func newHasher() *hasher {
return &hasher{mmh3: murmur3.New64()}
}
// RandomUint64 creates a random uint64 number
func RandomUint64(data string) uint64 {
hasher := murmur3.New64()
hasher.Write([]byte(uuid.NewUUID()))
return hasher.Sum64()
}
func BenchmarkUseMurmur(b *testing.B) {
h := murmur3.New64()
benchmarkHash(b, h)
}
func TestUseMurmur_Collissions(t *testing.T) {
h := murmur3.New64()
testCollissions(t, h)
}
func versionFromBytes(b []byte) string {
hasher := murmur3.New64()
hasher.Write(b)
return fmt.Sprintf("%d", hasher.Sum64())
}
