Exemplo n.º 1
0
func TestShardedkv(t *testing.T) {
	var shards []Shard
	nElements := 1000
	nShards := 10

	for i := 0; i < nShards; i++ {
		label := "test_shard" + strconv.Itoa(i)
		shards = append(shards, Shard{Name: label, Backend: st.New()})
	}

	chooser := ch.New()

	kv := New(chooser, shards)

	for i := 0; i < nElements; i++ {
		kv.Set("test"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i)))
	}

	for i := 0; i < nElements; i++ {
		k := "test" + strconv.Itoa(i)

		v, ok, err := kv.Get(k)
		if ok != true {
			t.Errorf("failed  to get key: %s\n", err)
		}

		if string(v) != "value"+strconv.Itoa(i) {
			t.Errorf("failed to get a valid value: %s != \"value%d\"\n", v, i)
		}
	}

	var migrationBuckets []string

	for i := nShards; i < nShards*2; i++ {
		label := "test_shard" + strconv.Itoa(i)
		migrationBuckets = append(migrationBuckets, label)
		backend := st.New()
		shards = append(shards, Shard{Name: label, Backend: backend})
		kv.AddShard(label, backend)
	}

	migration := ch.New()
	migration.SetBuckets(migrationBuckets)

	kv.BeginMigration(migration)

	// make sure requesting still works
	for i := 0; i < nElements; i++ {
		k := "test" + strconv.Itoa(i)

		v, ok, err := kv.Get(k)
		if ok != true {
			t.Errorf("failed  to get key: %s\n", err)
		}

		if string(v) != "value"+strconv.Itoa(i) {
			t.Errorf("failed to get a valid value: %s != \"value%d\"\n", v, i)
		}
	}

	// make sure setting still works
	for i := 0; i < nElements; i++ {
		kv.Set("test"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i)))
	}

	for i := 0; i < nElements; i++ {
		k := "test" + strconv.Itoa(i)

		v, ok, err := kv.Get(k)
		if ok != true {
			t.Errorf("failed  to get key: %s\n", err)
		}

		if string(v) != "value"+strconv.Itoa(i) {
			t.Errorf("failed to get a valid value: %s != \"value%d\"\n", v, i)
		}
	}

	// and that deleting removes from both during a migration
	for i := 0; i < nElements; i++ {
		kv.Delete("test" + strconv.Itoa(i))
	}

	for i := 0; i < nElements; i++ {
		k := "test" + strconv.Itoa(i)

		_, ok, _ := kv.Get(k)
		if ok {
			t.Errorf("got a key that shouldn't have been there")
		}
	}

	// set the keys again
	for i := 0; i < nElements; i++ {
		kv.Set("test"+strconv.Itoa(i), []byte("value"+strconv.Itoa(i)))
	}

	// end the migration
	kv.EndMigration()

	// delete the old shards
	for i := 0; i < nShards; i++ {
		label := "test_shard" + strconv.Itoa(i)
		kv.DeleteShard(label)
	}

	// and make sure we can still get to the keys
	for i := 0; i < nElements; i++ {
		k := "test" + strconv.Itoa(i)

		v, ok, err := kv.Get(k)
		if ok != true {
			t.Errorf("failed  to get key: %s\n", err)
		}

		if string(v) != "value"+strconv.Itoa(i) {
			t.Errorf("failed to get a valid value: %s != \"value%d\"\n", v, i)
		}
	}
}
Exemplo n.º 2
0
func main() {

	chooserType := flag.String("chooser", "jump", "shardedkv chooser")
	expectedShard := flag.String("shard", "", "expected shard name")
	numShards := flag.Int("n", 16, "number of shards")
	fname := flag.String("f", "", "input file name")

	flag.Parse()

	var buckets []string
	for i := 1; i <= *numShards; i++ {
		buckets = append(buckets, fmt.Sprintf("shard%d", i))
	}

	var chooser shardedkv.Chooser

	switch *chooserType {
	case "jump":
		chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
	case "ketama":
		chooser = ketama.New()
	case "chash":
		chooser = chash.New()
	default:
		log.Fatalf("unknown chooser: %s; known jump,ketama,chash", *chooserType)
	}

	chooser.SetBuckets(buckets)

	results := make(map[string]int)

	var f io.Reader

	f, err := os.Open(*fname)
	if err != nil {
		log.Fatal("error loading input file", fname, ":", err)
	}

	if strings.HasSuffix(*fname, ".gz") {
		f, _ = gzip.NewReader(f)
	}

	scan := bufio.NewScanner(f)
	for scan.Scan() {
		shard := chooser.Choose(scan.Text())
		if *expectedShard != "" && shard != *expectedShard {
			fmt.Printf("wrong shard %s %+v\n", shard, scan.Text())
		}
		results[shard]++
	}

	stats := onlinestats.NewRunning()

	fmt.Println("chooser", *chooserType)
	for _, b := range buckets {
		v := results[b]
		fmt.Printf("%-10s %d\n", b, v)
		stats.Push(float64(v))
	}

	if *expectedShard == "" {
		fmt.Println("mean", stats.Mean())
		fmt.Println("stdev", stats.Stddev(), stats.Stddev()/stats.Mean())
	}
}
Exemplo n.º 3
0
func main() {

	chooser := flag.String("chooser", "jump", "chooser to use")
	numBuckets := flag.Int("b", 8, "buckets")
	replicas := flag.Int("r", 1, "replicas")
	cpuprofile := flag.String("cpuprofile", "", "cpu profile")

	flag.Parse()

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	var buckets []string
	for i := 0; i < *numBuckets; i++ {
		buckets = append(buckets, fmt.Sprintf("192.168.%d.%d", i, 10+2*i))
	}
	results := make(map[string]int)

	scan := bufio.NewScanner(os.Stdin)

	hasher := func(b []byte) uint64 { h := fnv.New64a(); h.Write(b); return h.Sum64() }

	var j replichooser

	switch *chooser {
	case "ketama":
		j = ketama.New()
	case "chash":
		j = chash.New()
	case "jump":
		j = jump.New(hasher)
	default:
		log.Fatal("unknown chooser", *chooser)
	}

	j.SetBuckets(buckets)

	var totalMetrics int
	t0 := time.Now()
	for scan.Scan() {
		key := scan.Text()
		totalMetrics++

		r := j.ChooseReplicas(key, *replicas)

		for _, rr := range r {
			results[rr]++
		}
	}
	t1 := time.Since(t0)

	stats := onlinestats.NewRunning()

	fmt.Printf("total metrics processed: %d, time spent ~%ds (%d/s)\n", totalMetrics, int(t1.Seconds()), int(float64(totalMetrics)/float64(t1.Seconds())))
	fmt.Printf("replication count: %d, server count: %d\n", *replicas, *numBuckets)
	fmt.Println("server distribution:")

	min := int(^uint(0) / 2)
	max := 0
	for _, b := range buckets {
		v := results[b]
		if v < min {
			min = v
		}
		if v > max {
			max = v
		}
		fmt.Printf("- server %15s: %6d (%.2f%%)\n", b, v, 100*float64(v)/float64(totalMetrics))
		stats.Push(float64(v))
	}

	fmt.Printf("band: %d - %d (diff %d, %.1f%%), mean: %.2f, stddev: %.2f\n", min, max, (max - min), 100*float64(max-min)/float64(totalMetrics), stats.Mean(), stats.Stddev())
}
Exemplo n.º 4
0
func main() {

	numBuckets := flag.Int("b", 8, "buckets")
	chooserType := flag.String("chooser", "jump", "shardedkv chooser")
	verbose := flag.Bool("v", false, "verbose")
	replicas := flag.Int("r", 0, "replicas")

	flag.Parse()

	var buckets []string
	for i := 0; i < *numBuckets; i++ {
		//  	buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0x0ddc0ffeebadf00d, 0xcafebabedeadbeef, []byte(fmt.Sprintf("192.168.0.%d", 10+i)))))
		// buckets = append(buckets, fmt.Sprintf("%016x", siphash.Hash(0, 0, []byte(fmt.Sprintf("192.168.0.%d", 10+i)))))
		buckets = append(buckets, fmt.Sprintf("192.168.0.%d", 10+i))
	}

	var chooser shardedkv.Chooser

	switch *chooserType {
	case "jump":
		chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
	case "ketama":
		chooser = ketama.New()
	case "mpc":
		chooser = mpc.New(func(b []byte, seed uint64) uint64 { return siphash.Hash(seed, 0, b) }, [2]uint64{0x0ddc0ffeebadf00d, 0xcafebabedeadbeef}, 21)
	case "chash":
		// 5120*8 + 5120*(8+16)*2 bytes
		chooser = chash.New()
	default:
		log.Fatalf("unknown chooser: %s; known jump,ketama,chash,mpc", *chooserType)
	}

	chooser.SetBuckets(buckets)

	results := make(map[string]int)

	scan := bufio.NewScanner(os.Stdin)
	for scan.Scan() {
		key := scan.Text()
		shard := chooser.Choose(key)
		if *verbose {
			fmt.Printf("%s %s\n", shard, key)
		}
		results[shard]++
	}

	stats := onlinestats.NewRunning()

	min := int(^uint(0) / 2)
	max := 0
	fmt.Println("chooser", *chooserType)
	for _, b := range buckets {
		v := results[b]
		if v < min {
			min = v
		}
		if v > max {
			max = v
		}
		fmt.Printf("%-10s %d\n", b, v)
		stats.Push(float64(v))
	}

	fmt.Printf("min=%d max=%d\n", min, max)
	fmt.Printf("mean %.2f\n", stats.Mean())
	fmt.Printf("stdev %.2f %.2f%%\n", stats.Stddev(), 100*stats.Stddev()/stats.Mean())
	fmt.Printf("peak/mean %.2f\n", float64(max)/stats.Mean())
}
Exemplo n.º 5
0
func BenchmarkChash2048(b *testing.B) { benchmarkChooser(b, 2048, chash.New()) }
Exemplo n.º 6
0
func BenchmarkChash8192(b *testing.B) { benchmarkChooser(b, 8192, chash.New()) }
Exemplo n.º 7
0
func BenchmarkChash128(b *testing.B)  { benchmarkChooser(b, 128, chash.New()) }
Exemplo n.º 8
0
func BenchmarkChash512(b *testing.B)  { benchmarkChooser(b, 512, chash.New()) }
Exemplo n.º 9
0
func main() {

	workers := flag.Int("w", 4, "workers")
	chooserType := flag.String("chooser", "jump", "shardedkv chooser")
	verbose := flag.Bool("v", false, "verbose")

	flag.Parse()

	filech := make(chan string)
	donech := make(chan map[string]int)

	var buckets []string
	for i := 1; i <= 32; i++ {
		buckets = append(buckets, fmt.Sprintf("shard%d", i))
	}

	var chooser shardedkv.Chooser

	switch *chooserType {
	case "jump":
		chooser = jump.New(func(b []byte) uint64 { return siphash.Hash(0, 0, b) })
	case "ketama":
		chooser = ketama.New()
	case "chash":
		// 5120*8 + 5120*(8+16)*2 bytes
		chooser = chash.New()
	default:
		log.Fatalf("unknown chooser: %s; known jump,ketama,chash", *chooserType)
	}

	chooser.SetBuckets(buckets)

	for i := 0; i < *workers; i++ {
		go worker(chooser, filech, donech, *verbose)
	}

	files, _ := filepath.Glob("data/*")

	for _, f := range files {
		filech <- f
	}

	close(filech)

	results := make(map[string]int)

	for i := 0; i < *workers; i++ {
		shards := <-donech
		for k, v := range shards {
			results[k] += v
		}
	}

	stats := onlinestats.NewRunning()

	fmt.Println("chooser", *chooserType)
	for _, b := range buckets {
		v := results[b]
		fmt.Printf("%-10s %d\n", b, v)
		stats.Push(float64(v))
	}

	fmt.Println("mean", stats.Mean())
	fmt.Println("stdev", stats.Stddev(), stats.Stddev()/stats.Mean())
}
Exemplo n.º 10
0
func BenchmarkChash32(b *testing.B)   { benchmarkChooser(b, 32, chash.New()) }
Exemplo n.º 11
0
func TestDistributionChash8192(t *testing.T) { testDistribution(t, 8192, chash.New()) }
Exemplo n.º 12
0
func TestDistributionChash2048(t *testing.T) { testDistribution(t, 2048, chash.New()) }
Exemplo n.º 13
0
func TestDistributionChash512(t *testing.T)  { testDistribution(t, 512, chash.New()) }
Exemplo n.º 14
0
func TestDistributionChash128(t *testing.T)  { testDistribution(t, 128, chash.New()) }
Exemplo n.º 15
0
func TestDistributionChash32(t *testing.T)   { testDistribution(t, 32, chash.New()) }