func makeId(dv driver.Value) uint64 {
switch vt := dv.(type) {
case int:
return uint64(vt)
case int64:
return uint64(vt)
case []byte:
return siphash.Hash(0, 1, vt)
// iv, err := strconv.ParseUint(string(vt), 10, 64)
// if err != nil {
// u.Warnf("could not create id: %v for %v", err, dv)
// }
// return iv
case string:
return siphash.Hash(0, 1, []byte(vt))
// iv, err := strconv.ParseUint(vt, 10, 64)
// if err != nil {
// u.Warnf("could not create id: %v for %v", err, dv)
// }
// return iv
case *Key:
//u.Infof("got %#v", vt)
return vt.Id
case datasource.KeyCol:
//u.Infof("got %#v", vt)
return makeId(vt.Val)
default:
//u.LogTracef(u.WARN, "wat")
u.Warnf("not implemented conversion: %T", dv)
}
return 0
}
// New Replica Factor 1 with random
func New(randomized bool) *RF1R {
res := RF1R{}
if randomized {
res.hasher1 = func(b []byte) uint64 { return siphash.Hash(0, 0, b) }
res.hasher2 = func(b []byte) uint64 { return siphash.Hash(2, 3, b) }
} else {
res.hasher1 = func(b []byte) uint64 { return siphash.Hash(0, 0, b) }
res.hasher2 = func(b []byte) uint64 { return siphash.Hash(0, 0, b) }
}
return &res
}
func BenchmarkSipHashFast(b *testing.B) {
var key0, key1 uint64 = 0, 1
b.StartTimer()
for i := 0; i < b.N; i++ {
siphash.Hash(key0, key1, []byte("Hello World"))
}
}
func (t *T) Add(key string, val interface{}) {
newitem := slruItem{0, key, val, siphash.Hash(0, 0, stringToSlice(key))}
oitem, evicted := t.lru.add(newitem)
if !evicted {
return
}
// estimate count of what will be evicted from slru
victim := t.slru.victim()
if victim == nil {
t.slru.add(oitem)
return
}
if !t.bouncer.allow(oitem.keyh) {
return
}
vcount := t.c.estimate(victim.keyh)
ocount := t.c.estimate(oitem.keyh)
if ocount < vcount {
return
}
t.slru.add(oitem)
}
// New Replica Factor 1 with random
func New() *RF2 {
res := RF2{}
res.hasher = func(b []byte) uint64 { return siphash.Hash(0, 0, b) }
return &res
}
func main() {
verbose := flag.Bool("v", false, "verbose")
flag.Parse()
scanner := bufio.NewScanner(os.Stdin)
w := bufio.NewWriter(os.Stdout)
defer w.Flush()
for scanner.Scan() {
fname := scanner.Text()
b, err := ioutil.ReadFile(fname)
if err != nil {
log.Println(err)
continue
}
sip := siphash.Hash(0, 0, b)
data, err := lz4.Encode(nil, b)
if err != nil {
log.Println("error packing: ", err)
continue
}
const headerSize = 4 + 8 + 2 + 4 // + len(fname)
h := struct {
PacketSize uint32
SipHash uint64
Fname []byte `binpack:"lenprefix=uint16"`
UncompressedSize uint32
}{
PacketSize: uint32(headerSize + len(fname) + len(data)),
SipHash: sip,
Fname: []byte(fname),
UncompressedSize: uint32(len(b)),
}
if *verbose {
log.Printf("%s: %d -> %d\n", fname, len(b), len(data))
}
err = binpack.Write(w, binary.LittleEndian, h)
if err != nil {
log.Printf("error writing header %+v\n", err)
break
}
_, err = w.Write(data)
if err != nil {
log.Printf("error writing %+v\n", err)
break
}
}
}
func (c *OneConnection) SendCmpctBlk(hash *btc.Uint256) {
crec := GetchBlockForBIP152(hash)
if crec == nil {
fmt.Println(c.ConnID, "cmpctblock not sent for", hash.String())
return
}
k0 := binary.LittleEndian.Uint64(crec.BIP152[8:16])
k1 := binary.LittleEndian.Uint64(crec.BIP152[16:24])
msg := new(bytes.Buffer)
msg.Write(crec.Data[:80])
msg.Write(crec.BIP152[:8])
btc.WriteVlen(msg, uint64(len(crec.Block.Txs)-1)) // all except coinbase
for i := 1; i < len(crec.Block.Txs); i++ {
var lsb [8]byte
var hasz *btc.Uint256
if c.Node.SendCmpctVer == 2 {
hasz = crec.Block.Txs[i].WTxID()
} else {
hasz = crec.Block.Txs[i].Hash
}
binary.LittleEndian.PutUint64(lsb[:], siphash.Hash(k0, k1, hasz.Hash[:]))
msg.Write(lsb[:6])
}
msg.Write([]byte{1}) // one preffiled tx
msg.Write([]byte{0}) // coinbase - index 0
if c.Node.SendCmpctVer == 2 {
msg.Write(crec.Block.Txs[0].Raw) // coinbase - index 0
} else {
crec.Block.Txs[0].WriteSerialized(msg) // coinbase - index 0
}
c.SendRawMsg("cmpctblock", msg.Bytes())
}
// Hash returns a simhash value for the document returned by the scanner
func Hash(scanner FeatureScanner) uint64 {
var signs [64]int64
for scanner.Scan() {
b := scanner.Bytes()
h := siphash.Hash(0, 0, b)
for i := 0; i < 64; i++ {
negate := int(h) & 1
// if negate is 1, we will negate '-1', below
r := (-1 ^ -negate) + negate
signs[i] += int64(r)
h >>= 1
}
}
var shash uint64
// TODO: can probably be done with SSE?
for i := 63; i >= 0; i-- {
shash <<= 1
shash |= uint64(signs[i]>>63) & 1
}
return shash
}
func (t *T) Get(key string) (interface{}, bool) {
t.w++
if t.w == t.samples {
t.c.reset()
t.bouncer.reset()
t.w = 0
}
val, ok := t.data[key]
if !ok {
keyh := siphash.Hash(0, 0, stringToSlice(key))
t.c.add(keyh)
return nil, false
}
item := val.Value.(*slruItem)
t.c.add(item.keyh)
v := item.value
if item.listid == 0 {
t.lru.get(val)
} else {
t.slru.get(val)
}
return v, true
}
func main() {
f := flag.String("f", "/dev/stdin", "input file")
n := flag.Int("n", 100e6, "cardinality estimate of set")
fprate := flag.Float64("fp", 0.00000001, "false positive rate")
flag.Parse()
file, err := os.Open(*f)
if err != nil {
log.Fatal(err)
}
var lines int
b := bloomf.New(*n, *fprate, func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines++
if !b.Lookup(scanner.Bytes()) {
os.Stdout.Write(scanner.Bytes())
os.Stdout.Write([]byte("\n"))
}
b.Insert(scanner.Bytes())
if lines%(1<<20) == 0 {
log.Println(lines)
}
}
}
func (d *Dict) Set(key []byte, value interface{}) {
p := siphash.Hash(d.k0, d.k1, key) % dictSize
node := d.data[p]
if node == nil {
d.data[p] = &dictNode{
k: key,
v: value,
next: nil,
}
d.size += 1
return
}
for {
if bytes.Equal(node.k, key) {
node.v = value
return
}
if node.next != nil {
node = node.next
continue
}
node.next = &dictNode{
k: key,
v: value,
next: nil,
}
d.size += 1
return
}
}
func (d *Dict) Delete(key []byte) {
p := siphash.Hash(d.k0, d.k1, key) % dictSize
node := d.data[p]
if node == nil {
return
}
if bytes.Equal(node.k, key) {
d.data[p] = nil
d.size -= 1
return
}
next := node.next
for {
if next == nil {
return
}
if bytes.Equal(next.k, key) {
node.next = next.next
d.size -= 1
return
}
next = node.next
}
panic("unreachable code")
}
func main() {
scan := bufio.NewScanner(os.Stdin)
for scan.Scan() {
u := siphash.Hash(0, 0, scan.Bytes())
fmt.Printf("%016x\n", u)
}
}
func hashOne(k0, k1 uint64, r io.Reader, name string) {
buf, err := ioutil.ReadAll(r)
if err != nil {
log.Println(name, err)
return
}
h := siphash.Hash(k0, k1, buf)
fmt.Printf("%016x %s\n", h, name)
}
func sipuintptr(s []uintptr) uint64 {
b := make([]byte, len(s)*8)
for i, v := range s {
binary.LittleEndian.PutUint64(b[i*8:], uint64(v))
}
return siphash.Hash(0, 0, b)
}
func Benchmark_Siphash(b *testing.B) {
size := 1024
k1, k2 := uint64(rand.Int63()), uint64(rand.Int63())
msg := randArray(size)
b.SetBytes(int64(size))
b.ResetTimer()
for i := 0; i < b.N; i++ {
siphash.Hash(k1, k2, msg)
}
}
func TestSiphash(t *testing.T) {
k1, k2 := uint64(rand.Int63()), uint64(rand.Int63())
msg := randArray(1 << 9) // 512B
var loop int = 2e6
t1 := time.Now()
for i := 0; i < loop; i++ {
siphash.Hash(k1, k2, msg)
}
tu := float64(time.Since(t1).Nanoseconds()) / 1e9
t.Logf("tu=%.2fms speed=%.2fm", tu*1e3, float64(loop*len(msg))/float64(1<<20)/tu)
}
func timeErrorUnit(sp []byte, errors int) bool {
_, _, hk := extractKeys(sp)
// generate
head := makeDbcHello(1, sp)
// make error
tc := uint64(time.Now().Unix()/TIME_STEP + int64(errors))
errSum := siphash.Hash(hk, tc, head[:DPH_LEN1])
binary.BigEndian.PutUint64(head[DPH_LEN1:], errSum)
// verify
tcArr := calculateTimeCounter(true)
ok, _, _ := verifyDbcHello(head, sp, tcArr)
return ok
}
func makeDbcHello(data byte, secret []byte) []byte {
randLen := rand.Int() % DP_LEN1 // 8bit
buf := randArray(randLen + DP_P2I)
pos, sKey, hKey := extractKeys(secret)
// actually f is uint16
f := (randLen << 8) | int(data)
f = (f + sKey) % DP_MOD
binary.BigEndian.PutUint16(buf[pos:pos+2], uint16(f))
sum := siphash.Hash(hKey, calculateTimeCounter(false)[0], buf[:DP_LEN1])
binary.BigEndian.PutUint64(buf[DP_LEN1:DP_P2I], sum)
return buf
}
// Lookup queries the cuckoo filter for an item
func (cf *CF) Lookup(x []byte) bool {
h := siphash.Hash(0, 0, x)
i1 := uint32(h) % uint32(len(cf.t))
f := byte(h >> 32)
if cf.hasFP(i1, f) {
return true
}
i2 := (i1 ^ hashfp(f)) % uint32(len(cf.t))
return cf.hasFP(i2, f)
}
func pickUpstream(req *dns.Msg) (*string, error) {
name := strings.ToLower(req.Question[0].Name)
h := siphash.Hash(sipHashKey.k1, sipHashKey.k2, []byte(name))
upstreamServers.lock.RLock()
defer upstreamServers.lock.RUnlock()
liveCount := uint64(len(upstreamServers.live))
if liveCount <= 0 {
return nil, errors.New("All upstream servers are down")
}
i := h / (math.MaxUint64 / liveCount)
if i >= liveCount {
i = liveCount - 1
}
res := upstreamServers.live[i]
return &res, nil
}
func (d *Dict) Get(key []byte) (interface{}, bool) {
node := d.data[siphash.Hash(d.k0, d.k1, key)%dictSize]
if node == nil {
return nil, false
}
for {
if !bytes.Equal(node.k, key) {
node = node.next
if node == nil {
return nil, false
}
continue
}
return node.v, true
}
panic("unreachable code")
}
func main() {
verbose := flag.Bool("v", false, "verbose")
flag.Parse()
r := bufio.NewReader(os.Stdin)
for {
var header struct {
PacketSize uint32
SipHash uint64
Fname []byte `binpack:"lenprefix=uint16"`
UncompressedSize uint32
}
err := binpack.Read(r, binary.LittleEndian, &header)
if err == io.EOF {
break
}
compressedSize := header.PacketSize - 16 - 2 - uint32(len(header.Fname))
if *verbose {
log.Printf("%s: %d -> %d\n", string(header.Fname), compressedSize, header.UncompressedSize)
}
data := make([]byte, compressedSize)
n, err := io.ReadFull(r, data)
if n != len(data) || err != nil {
log.Fatalf("bad read: n=%d len(data)=%d err=%s\n", n, len(data), err)
}
data, err = lz4.Decode(nil, data)
h := siphash.Hash(0, 0, data)
if err != nil || h != header.SipHash {
log.Println("FAIL: err=", err)
}
}
}
func verifyDbcHello(buf []byte, secret []byte, tc []uint64) (trusted bool, data, len2 byte) {
pos, sKey, hKey := extractKeys(secret)
p1 := buf[:DP_LEN1]
var sum, cltSum uint64
cltSum = binary.BigEndian.Uint64(buf[DP_LEN1:DP_P2I])
for i := 0; !trusted && i < len(tc); i++ {
sum = siphash.Hash(hKey, tc[i], p1)
trusted = cltSum == sum
}
if !trusted {
return
}
z := int(binary.BigEndian.Uint16(buf[pos : pos+2]))
z = (z - sKey + DP_MOD) % DP_MOD
len2, data = byte(z>>8), byte(z&0xff)
return
}
// TestAndSet queries the filter for a given byte sequence, inserts the
// sequence, and returns if it was present before the insertion operation.
func (f *ReplayFilter) TestAndSet(now time.Time, buf []byte) bool {
digest := siphash.Hash(f.key[0], f.key[1], buf)
f.Lock()
defer f.Unlock()
f.compactFilter(now)
if e := f.filter[digest]; e != nil {
// Hit. Just return.
return true
}
// Miss. Add a new entry.
e := new(entry)
e.digest = digest
e.firstSeen = now
e.element = f.fifo.PushBack(e)
f.filter[digest] = e
return false
}
// Insert adds an element to the filter and returns if the insertion was successful.
func (cf *CF) Insert(x []byte) bool {
h := siphash.Hash(0, 0, x)
i1 := uint32(h) % uint32(len(cf.t))
f := byte(h >> 32)
i2 := (i1 ^ hashfp(f)) % uint32(len(cf.t))
if idx, ok := cf.hasSpace(i1); ok {
cf.setOccupied(i1, idx, f)
return true
}
if idx, ok := cf.hasSpace(i2); ok {
cf.setOccupied(i2, idx, f)
return true
}
i := i1
cf.rnd = rnd(cf.rnd)
if cf.rnd&1 == 1 {
i = i2
}
for n := 0; n < 500; n++ {
f = cf.evict(i, f)
i = (i ^ hashfp(f)) % uint32(len(cf.t))
if idx, ok := cf.hasSpace(i); ok {
cf.setOccupied(i, idx, f)
return true
}
}
return false
}
func main() {
chooserType := flag.String("chooser", "jump", "shardedkv chooser")
expectedShard := flag.String("shard", "", "expected shard name")
numShards := flag.Int("n", 16, "number of shards")
fname := flag.String("f", "", "input file name")
flag.Parse()
var buckets []string
for i := 1; i <= *numShards; i++ {
buckets = append(buckets, fmt.Sprintf("shard%d", i))
}
var chooser shardedkv.Chooser
switch *chooserType {
case "jump":
chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
case "ketama":
chooser = ketama.New()
case "chash":
chooser = chash.New()
default:
log.Fatalf("unknown chooser: %s; known jump,ketama,chash", *chooserType)
}
chooser.SetBuckets(buckets)
results := make(map[string]int)
var f io.Reader
f, err := os.Open(*fname)
if err != nil {
log.Fatal("error loading input file", fname, ":", err)
}
if strings.HasSuffix(*fname, ".gz") {
f, _ = gzip.NewReader(f)
}
scan := bufio.NewScanner(f)
for scan.Scan() {
shard := chooser.Choose(scan.Text())
if *expectedShard != "" && shard != *expectedShard {
fmt.Printf("wrong shard %s %+v\n", shard, scan.Text())
}
results[shard]++
}
stats := onlinestats.NewRunning()
fmt.Println("chooser", *chooserType)
for _, b := range buckets {
v := results[b]
fmt.Printf("%-10s %d\n", b, v)
stats.Push(float64(v))
}
if *expectedShard == "" {
fmt.Println("mean", stats.Mean())
fmt.Println("stdev", stats.Stddev(), stats.Stddev()/stats.Mean())
}
}
// Hash returns the hash code for the Value.
func (v Var) Hash() int {
h := siphash.Hash(hashSeed0, hashSeed1, *(*[]byte)(unsafe.Pointer(&v)))
return int(h)
}
func main() {
numBuckets := flag.Int("b", 8, "buckets")
chooserType := flag.String("chooser", "jump", "shardedkv chooser")
verbose := flag.Bool("v", false, "verbose")
replicas := flag.Int("r", 0, "replicas")
flag.Parse()
var buckets []string
for i := 0; i < *numBuckets; i++ {
// buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0x0ddc0ffeebadf00d, 0xcafebabedeadbeef, []byte(fmt.Sprintf("192.168.0.%d", 10+i)))))
// buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0, 0, []byte(fmt.Sprintf("192.168.0.%d", 10+i)))))
buckets = append(buckets, fmt.Sprintf("192.168.0.%d", 10+i))
}
var chooser shardedkv.Chooser
switch *chooserType {
case "jump":
chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
case "ketama":
chooser = ketama.New()
case "mpc":
chooser = mpc.New(func(b []byte, seed uint64) uint64 { return siphash.Hash(seed, 0, b) }, [2]uint64{0x0ddc0ffeebadf00d, 0xcafebabedeadbeef}, 21)
case "chash":
// 5120*8 + 5120*(8+16)*2 bytes
chooser = chash.New()
default:
log.Fatalf("unknown chooser: %s; known jump,ketama,chash,mpc", *chooserType)
}
chooser.SetBuckets(buckets)
results := make(map[string]int)
scan := bufio.NewScanner(os.Stdin)
for scan.Scan() {
key := scan.Text()
shard := chooser.Choose(key)
if *verbose {
fmt.Printf("%s %s\n", shard, key)
}
results[shard]++
}
stats := onlinestats.NewRunning()
min := int(^uint(0) / 2)
max := 0
fmt.Println("chooser", *chooserType)
for _, b := range buckets {
v := results[b]
if v < min {
min = v
}
if v > max {
max = v
}
fmt.Printf("%-10s %d\n", b, v)
stats.Push(float64(v))
}
fmt.Printf("min=%d max=%d\n", min, max)
fmt.Printf("mean %.2f\n", stats.Mean())
fmt.Printf("stdev %.2f %.2f%%\n", stats.Stddev(), 100*stats.Stddev()/stats.Mean())
fmt.Printf("peak/mean %.2f\n", float64(max)/stats.Mean())
}
func getSlot(v []byte) uint64 {
return siphash.Hash(hashKey0, hashKey1, v) & routeMask
}
