func main() {
chooserType := flag.String("chooser", "jump", "shardedkv chooser")
expectedShard := flag.String("shard", "", "expected shard name")
numShards := flag.Int("n", 16, "number of shards")
fname := flag.String("f", "", "input file name")
flag.Parse()
var buckets []string
for i := 1; i <= *numShards; i++ {
buckets = append(buckets, fmt.Sprintf("shard%d", i))
}
var chooser shardedkv.Chooser
switch *chooserType {
case "jump":
chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
case "ketama":
chooser = ketama.New()
case "chash":
chooser = chash.New()
default:
log.Fatalf("unknown chooser: %s; known jump,ketama,chash", *chooserType)
}
chooser.SetBuckets(buckets)
results := make(map[string]int)
var f io.Reader
f, err := os.Open(*fname)
if err != nil {
log.Fatal("error loading input file", fname, ":", err)
}
if strings.HasSuffix(*fname, ".gz") {
f, _ = gzip.NewReader(f)
}
scan := bufio.NewScanner(f)
for scan.Scan() {
shard := chooser.Choose(scan.Text())
if *expectedShard != "" && shard != *expectedShard {
fmt.Printf("wrong shard %s %+v\n", shard, scan.Text())
}
results[shard]++
}
stats := onlinestats.NewRunning()
fmt.Println("chooser", *chooserType)
for _, b := range buckets {
v := results[b]
fmt.Printf("%-10s %d\n", b, v)
stats.Push(float64(v))
}
if *expectedShard == "" {
fmt.Println("mean", stats.Mean())
fmt.Println("stdev", stats.Stddev(), stats.Stddev()/stats.Mean())
}
}
func main() {
chooser := flag.String("chooser", "jump", "chooser to use")
numBuckets := flag.Int("b", 8, "buckets")
replicas := flag.Int("r", 1, "replicas")
cpuprofile := flag.String("cpuprofile", "", "cpu profile")
flag.Parse()
if *cpuprofile != "" {
f, err := os.Create(*cpuprofile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
var buckets []string
for i := 0; i < *numBuckets; i++ {
buckets = append(buckets, fmt.Sprintf("192.168.%d.%d", i, 10+2*i))
}
results := make(map[string]int)
scan := bufio.NewScanner(os.Stdin)
hasher := func(b []byte) uint64 { h := fnv.New64a(); h.Write(b); return h.Sum64() }
var j replichooser
switch *chooser {
case "ketama":
j = ketama.New()
case "chash":
j = chash.New()
case "jump":
j = jump.New(hasher)
default:
log.Fatal("unknown chooser", *chooser)
}
j.SetBuckets(buckets)
var totalMetrics int
t0 := time.Now()
for scan.Scan() {
key := scan.Text()
totalMetrics++
r := j.ChooseReplicas(key, *replicas)
for _, rr := range r {
results[rr]++
}
}
t1 := time.Since(t0)
stats := onlinestats.NewRunning()
fmt.Printf("total metrics processed: %d, time spent ~%ds (%d/s)\n", totalMetrics, int(t1.Seconds()), int(float64(totalMetrics)/float64(t1.Seconds())))
fmt.Printf("replication count: %d, server count: %d\n", *replicas, *numBuckets)
fmt.Println("server distribution:")
min := int(^uint(0) / 2)
max := 0
for _, b := range buckets {
v := results[b]
if v < min {
min = v
}
if v > max {
max = v
}
fmt.Printf("- server %15s: %6d (%.2f%%)\n", b, v, 100*float64(v)/float64(totalMetrics))
stats.Push(float64(v))
}
fmt.Printf("band: %d - %d (diff %d, %.1f%%), mean: %.2f, stddev: %.2f\n", min, max, (max - min), 100*float64(max-min)/float64(totalMetrics), stats.Mean(), stats.Stddev())
}
func main() {
numBuckets := flag.Int("b", 8, "buckets")
chooserType := flag.String("chooser", "jump", "shardedkv chooser")
verbose := flag.Bool("v", false, "verbose")
replicas := flag.Int("r", 0, "replicas")
flag.Parse()
var buckets []string
for i := 0; i < *numBuckets; i++ {
// buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0x0ddc0ffeebadf00d, 0xcafebabedeadbeef, []byte(fmt.Sprintf("192.168.0.%d", 10+i)))))
// buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0, 0, []byte(fmt.Sprintf("192.168.0.%d", 10+i)))))
buckets = append(buckets, fmt.Sprintf("192.168.0.%d", 10+i))
}
var chooser shardedkv.Chooser
switch *chooserType {
case "jump":
chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
case "ketama":
chooser = ketama.New()
case "mpc":
chooser = mpc.New(func(b []byte, seed uint64) uint64 { return siphash.Hash(seed, 0, b) }, [2]uint64{0x0ddc0ffeebadf00d, 0xcafebabedeadbeef}, 21)
case "chash":
// 5120*8 + 5120*(8+16)*2 bytes
chooser = chash.New()
default:
log.Fatalf("unknown chooser: %s; known jump,ketama,chash,mpc", *chooserType)
}
chooser.SetBuckets(buckets)
results := make(map[string]int)
scan := bufio.NewScanner(os.Stdin)
for scan.Scan() {
key := scan.Text()
shard := chooser.Choose(key)
if *verbose {
fmt.Printf("%s %s\n", shard, key)
}
results[shard]++
}
stats := onlinestats.NewRunning()
min := int(^uint(0) / 2)
max := 0
fmt.Println("chooser", *chooserType)
for _, b := range buckets {
v := results[b]
if v < min {
min = v
}
if v > max {
max = v
}
fmt.Printf("%-10s %d\n", b, v)
stats.Push(float64(v))
}
fmt.Printf("min=%d max=%d\n", min, max)
fmt.Printf("mean %.2f\n", stats.Mean())
fmt.Printf("stdev %.2f %.2f%%\n", stats.Stddev(), 100*stats.Stddev()/stats.Mean())
fmt.Printf("peak/mean %.2f\n", float64(max)/stats.Mean())
}
func BenchmarkKetama2048(b *testing.B) { benchmarkChooser(b, 2048, ketama.New()) }
func BenchmarkKetama8192(b *testing.B) { benchmarkChooser(b, 8192, ketama.New()) }
func BenchmarkKetama128(b *testing.B) { benchmarkChooser(b, 128, ketama.New()) }
func BenchmarkKetama512(b *testing.B) { benchmarkChooser(b, 512, ketama.New()) }
func main() {
workers := flag.Int("w", 4, "workers")
chooserType := flag.String("chooser", "jump", "shardedkv chooser")
verbose := flag.Bool("v", false, "verbose")
flag.Parse()
filech := make(chan string)
donech := make(chan map[string]int)
var buckets []string
for i := 1; i <= 32; i++ {
buckets = append(buckets, fmt.Sprintf("shard%d", i))
}
var chooser shardedkv.Chooser
switch *chooserType {
case "jump":
chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
case "ketama":
chooser = ketama.New()
case "chash":
// 5120*8 + 5120*(8+16)*2 bytes
chooser = chash.New()
default:
log.Fatalf("unknown chooser: %s; known jump,ketama,chash", *chooserType)
}
chooser.SetBuckets(buckets)
for i := 0; i < *workers; i++ {
go worker(chooser, filech, donech, *verbose)
}
files, _ := filepath.Glob("data/*")
for _, f := range files {
filech <- f
}
close(filech)
results := make(map[string]int)
for i := 0; i < *workers; i++ {
shards := <-donech
for k, v := range shards {
results[k] += v
}
}
stats := onlinestats.NewRunning()
fmt.Println("chooser", *chooserType)
for _, b := range buckets {
v := results[b]
fmt.Printf("%-10s %d\n", b, v)
stats.Push(float64(v))
}
fmt.Println("mean", stats.Mean())
fmt.Println("stdev", stats.Stddev(), stats.Stddev()/stats.Mean())
}
func TestDistributionKetama8192(t *testing.T) { testDistribution(t, 8192, ketama.New()) }
func TestDistributionKetama2048(t *testing.T) { testDistribution(t, 2048, ketama.New()) }
func TestDistributionKetama512(t *testing.T) { testDistribution(t, 512, ketama.New()) }
func TestDistributionKetama128(t *testing.T) { testDistribution(t, 128, ketama.New()) }