// deletePrefix performs a RangeRequest to get keys on a given prefix func deletePrefix(kv pb.KVClient, prefix string) (*pb.DeleteRangeResponse, error) { return kv.DeleteRange( context.TODO(), &pb.DeleteRangeRequest{ Key: []byte(prefix), RangeEnd: []byte(prefixEnd(prefix))}) }
func newStressRange(kvc pb.KVClient, keySuffixRange int) stressFunc { return func(ctx context.Context) error { _, err := kvc.Range(ctx, &pb.RangeRequest{ Key: []byte(fmt.Sprintf("foo%d", rand.Intn(keySuffixRange))), }, grpc.FailFast(false)) return err } }
func newStressDelete(kvc pb.KVClient, keySuffixRange int) stressFunc { return func(ctx context.Context) (error, int64) { _, err := kvc.DeleteRange(ctx, &pb.DeleteRangeRequest{ Key: []byte(fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange))), }, grpc.FailFast(false)) return err, 1 } }
func newStressPut(kvc pb.KVClient, keySuffixRange, keySize int) stressFunc { return func(ctx context.Context) (error, int64) { _, err := kvc.Put(ctx, &pb.PutRequest{ Key: []byte(fmt.Sprintf("foo%016x", rand.Intn(keySuffixRange))), Value: randBytes(keySize), }, grpc.FailFast(false)) return err, 1 } }
func doPutForWatch(ctx context.Context, client etcdserverpb.KVClient, requests <-chan etcdserverpb.PutRequest) { for r := range requests { _, err := client.Put(ctx, &r) if err != nil { fmt.Fprintln(os.Stderr, "failed to Put for watch benchmark: %s", err) os.Exit(1) } } }
func newStressRangeInterval(kvc pb.KVClient, keySuffixRange int) stressFunc { return func(ctx context.Context) (error, int64) { start := rand.Intn(keySuffixRange) end := start + 500 _, err := kvc.Range(ctx, &pb.RangeRequest{ Key: []byte(fmt.Sprintf("foo%016x", start)), RangeEnd: []byte(fmt.Sprintf("foo%016x", end)), }, grpc.FailFast(false)) return err, 0 } }
func doRange(client etcdserverpb.KVClient, requests <-chan etcdserverpb.RangeRequest) { defer wg.Done() for req := range requests { st := time.Now() _, err := client.Range(context.Background(), &req) var errStr string if err != nil { errStr = err.Error() } results <- result{errStr: errStr, duration: time.Since(st)} bar.Increment() } }
func doPut(ctx context.Context, client etcdserverpb.KVClient, requests <-chan etcdserverpb.PutRequest) { defer wg.Done() for r := range requests { st := time.Now() _, err := client.Put(ctx, &r) var errStr string if err != nil { errStr = err.Error() } results <- result{errStr: errStr, duration: time.Since(st)} bar.Increment() } }
func put(client etcdserverpb.KVClient, requests <-chan *etcdserverpb.PutRequest) { defer wg.Done() for r := range requests { st := time.Now() _, err := client.Put(context.Background(), r) var errStr string if err != nil { errStr = err.Error() } results <- &result{ errStr: errStr, duration: time.Now().Sub(st), } bar.Increment() } }
func get(client etcdserverpb.KVClient, key, end []byte, requests <-chan struct{}) { defer wg.Done() req := &etcdserverpb.RangeRequest{Key: key, RangeEnd: end} for _ = range requests { st := time.Now() _, err := client.Range(context.Background(), req) var errStr string if err != nil { errStr = err.Error() } results <- &result{ errStr: errStr, duration: time.Now().Sub(st), } bar.Increment() } }
// deleteRevKey deletes a key by revision, returning false if key is missing func deleteRevKey(kvc pb.KVClient, key string, rev int64) (bool, error) { cmp := &pb.Compare{ Result: pb.Compare_EQUAL, Target: pb.Compare_MOD, Key: []byte(key), TargetUnion: &pb.Compare_ModRevision{ModRevision: rev}, } req := &pb.RequestUnion{Request: &pb.RequestUnion_RequestDeleteRange{ RequestDeleteRange: &pb.DeleteRangeRequest{Key: []byte(key)}}} txnresp, err := kvc.Txn( context.TODO(), &pb.TxnRequest{ Compare: []*pb.Compare{cmp}, Success: []*pb.RequestUnion{req}, Failure: nil, }) if err != nil { return false, err } else if txnresp.Succeeded == false { return false, nil } return true, nil }
func putEmptyKey(kv pb.KVClient, key string) (*pb.PutResponse, error) { return kv.Put(context.TODO(), &pb.PutRequest{Key: []byte(key), Value: []byte{}}) }
// waitForRestart tries a range request until the client's server responds. // This is mainly a stop-gap function until grpcproxy's KVClient adapter // (and by extension, clientv3) supports grpc.CallOption pass-through so // FailFast=false works with Put. func waitForRestart(t *testing.T, kvc pb.KVClient) { req := &pb.RangeRequest{Key: []byte("_"), Serializable: true} if _, err := kvc.Range(context.TODO(), req, grpc.FailFast(false)); err != nil { t.Fatal(err) } }
func (s *stresser) run(ctx context.Context, kvc pb.KVClient) { defer s.wg.Done() for { if err := s.rateLimiter.Wait(ctx); err == context.Canceled { return } // TODO: 10-second is enough timeout to cover leader failure // and immediate leader election. Find out what other cases this // could be timed out. putctx, putcancel := context.WithTimeout(ctx, 10*time.Second) _, err := kvc.Put(putctx, &pb.PutRequest{ Key: []byte(fmt.Sprintf("foo%d", rand.Intn(s.KeySuffixRange))), Value: []byte(randStr(s.KeySize)), }, grpc.FailFast(false)) putcancel() if err != nil { shouldContinue := false switch grpc.ErrorDesc(err) { case context.DeadlineExceeded.Error(): // This retries when request is triggered at the same time as // leader failure. When we terminate the leader, the request to // that leader cannot be processed, and times out. Also requests // to followers cannot be forwarded to the old leader, so timing out // as well. We want to keep stressing until the cluster elects a // new leader and start processing requests again. shouldContinue = true case etcdserver.ErrTimeoutDueToLeaderFail.Error(), etcdserver.ErrTimeout.Error(): // This retries when request is triggered at the same time as // leader failure and follower nodes receive time out errors // from losing their leader. Followers should retry to connect // to the new leader. shouldContinue = true case etcdserver.ErrStopped.Error(): // one of the etcd nodes stopped from failure injection shouldContinue = true case transport.ErrConnClosing.Desc: // server closed the transport (failure injected node) shouldContinue = true case rpctypes.ErrNotCapable.Error(): // capability check has not been done (in the beginning) shouldContinue = true // default: // errors from stresser.Cancel method: // rpc error: code = 1 desc = context canceled (type grpc.rpcError) // rpc error: code = 2 desc = grpc: the client connection is closing (type grpc.rpcError) } if shouldContinue { continue } return } s.mu.Lock() s.success++ s.mu.Unlock() } }