// TestV3LeaseExists creates a lease on a random client, then sends a keepalive on another // client to confirm it's visible to the whole cluster. func TestV3LeaseExists(t *testing.T) { defer testutil.AfterTest(t) clus := NewClusterV3(t, &ClusterConfig{Size: 3}) defer clus.Terminate(t) // create lease ctx0, cancel0 := context.WithCancel(context.Background()) defer cancel0() lresp, err := clus.RandClient().Lease.LeaseCreate( ctx0, &pb.LeaseCreateRequest{TTL: 30}) if err != nil { t.Fatal(err) } if lresp.Error != "" { t.Fatal(lresp.Error) } // confirm keepalive ctx1, cancel1 := context.WithCancel(context.Background()) defer cancel1() lac, err := clus.RandClient().Lease.LeaseKeepAlive(ctx1) if err != nil { t.Fatal(err) } defer lac.CloseSend() if err = lac.Send(&pb.LeaseKeepAliveRequest{ID: lresp.ID}); err != nil { t.Fatal(err) } if _, err = lac.Recv(); err != nil { t.Fatal(err) } }
func newWatcher(c *EtcdClient, key string, rev int64, isPrefix bool) (*Watcher, error) { ctx, cancel := context.WithCancel(context.Background()) w, err := c.Watch.Watch(ctx) if err != nil { return nil, err } req := &pb.WatchCreateRequest{StartRevision: rev} if isPrefix { req.Prefix = []byte(key) } else { req.Key = []byte(key) } if err := w.Send(&pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{CreateRequest: req}}); err != nil { return nil, err } wresp, err := w.Recv() if err != nil { return nil, err } if len(wresp.Events) != 0 || wresp.Created != true { return nil, ErrWaitMismatch } ret := &Watcher{ wstream: w, cancel: cancel, donec: make(chan struct{}), id: storage.WatchID(wresp.WatchId), recvc: make(chan *storagepb.Event), } go ret.recvLoop() return ret, nil }
func TestNodeAdvance(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() storage := NewMemoryStorage() c := &Config{ ID: 1, ElectionTick: 10, HeartbeatTick: 1, Storage: storage, MaxSizePerMsg: noLimit, MaxInflightMsgs: 256, } n := StartNode(c, []Peer{{ID: 1}}) n.Campaign(ctx) <-n.Ready() n.Propose(ctx, []byte("foo")) var rd Ready select { case rd = <-n.Ready(): t.Fatalf("unexpected Ready before Advance: %+v", rd) case <-time.After(time.Millisecond): } storage.Append(rd.Entries) n.Advance() select { case <-n.Ready(): case <-time.After(100 * time.Millisecond): t.Errorf("expect Ready after Advance, but there is no Ready available") } }
func testWatchCancelRunning(t *testing.T, wctx *watchctx) { ctx, cancel := context.WithCancel(context.Background()) if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil { t.Fatalf("expected non-nil watcher channel") } if _, err := wctx.kv.Put(ctx, "a", "a"); err != nil { t.Fatal(err) } cancel() select { case <-time.After(time.Second): t.Fatalf("took too long to cancel") case v, ok := <-wctx.ch: if !ok { // closed before getting put; OK break } // got the PUT; should close next select { case <-time.After(time.Second): t.Fatalf("took too long to close") case v, ok = <-wctx.ch: if ok { t.Fatalf("expected watcher channel to close, got %v", v) } } } }
func TestSimpleHTTPClientDoCancelContextWaitForRoundTrip(t *testing.T) { tr := newFakeTransport() c := &simpleHTTPClient{transport: tr} donechan := make(chan struct{}) ctx, cancel := context.WithCancel(context.Background()) go func() { c.Do(ctx, &fakeAction{}) close(donechan) }() // This should call CancelRequest and begin the cancellation process cancel() select { case <-donechan: t.Fatalf("simpleHTTPClient.Do should not have exited yet") default: } tr.finishCancel <- struct{}{} select { case <-donechan: //expected behavior return case <-time.After(time.Second): t.Fatalf("simpleHTTPClient.Do did not exit within 1s") } }
func TestSimpleHTTPClientDoCancelContextResponseBodyClosed(t *testing.T) { tr := newFakeTransport() c := &simpleHTTPClient{transport: tr} // create an already-cancelled context ctx, cancel := context.WithCancel(context.Background()) cancel() body := &checkableReadCloser{ReadCloser: ioutil.NopCloser(strings.NewReader("foo"))} go func() { // wait that simpleHTTPClient knows the context is already timed out, // and calls CancelRequest testutil.WaitSchedule() // response is returned before cancel effects tr.respchan <- &http.Response{Body: body} }() _, _, err := c.Do(ctx, &fakeAction{}) if err == nil { t.Fatalf("expected non-nil error, got nil") } if !body.closed { t.Fatalf("expected closed body") } }
func newClient(cfg *Config) (*Client, error) { if cfg == nil { cfg = &Config{RetryDialer: dialEndpointList} } var creds *credentials.TransportAuthenticator if cfg.TLS != nil { c := credentials.NewTLS(cfg.TLS) creds = &c } // use a temporary skeleton client to bootstrap first connection ctx, cancel := context.WithCancel(context.TODO()) conn, err := cfg.RetryDialer(&Client{cfg: *cfg, creds: creds, ctx: ctx}) if err != nil { return nil, err } client := &Client{ conn: conn, cfg: *cfg, creds: creds, ctx: ctx, cancel: cancel, } client.Cluster = NewCluster(client) client.KV = NewKV(client) client.Lease = NewLease(client) client.Watcher = NewWatcher(client) client.Auth = NewAuth(client) client.Maintenance = &maintenance{c: client} return client, nil }
// DoCancelAfterFirstResponse cancels the RPC after receiving the first message from the server. func DoCancelAfterFirstResponse(tc testpb.TestServiceClient) { ctx, cancel := context.WithCancel(context.Background()) stream, err := tc.FullDuplexCall(ctx) if err != nil { grpclog.Fatalf("%v.FullDuplexCall(_) = _, %v", tc, err) } respParam := []*testpb.ResponseParameters{ { Size: proto.Int32(31415), }, } pl := clientNewPayload(testpb.PayloadType_COMPRESSABLE, 27182) req := &testpb.StreamingOutputCallRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), ResponseParameters: respParam, Payload: pl, } if err := stream.Send(req); err != nil { grpclog.Fatalf("%v.Send(%v) = %v", stream, req, err) } if _, err := stream.Recv(); err != nil { grpclog.Fatalf("%v.Recv() = %v", stream, err) } cancel() if _, err := stream.Recv(); grpc.Code(err) != codes.Canceled { grpclog.Fatalf("%v compleled with error code %d, want %d", stream, grpc.Code(err), codes.Canceled) } grpclog.Println("CancelAfterFirstResponse done") }
// Register register service // to configured etcd instance. // Once the Register is called, the client // also periodically // calls the refresh goroutine. func (e *EtcdReigistryClient) Register() { e.etcdKey = buildKey(e.ServiceName, e.InstanceName) value := registerDTO{ e.BaseURL, } val, _ := json.Marshal(value) e.keepAliveTicker = time.NewTicker(KeepAlivePeriod) ctx, c := context.WithCancel(context.TODO()) e.cancel = c insertFunc := func() { e.etcdKApi.Set(context.Background(), e.etcdKey, string(val), &client.SetOptions{ TTL: TTL, }) } insertFunc() // Exec the keep alive goroutine go func() { for { select { case <-e.keepAliveTicker.C: insertFunc() log.Printf("Keep alive routine for %s", e.ServiceName) case <-ctx.Done(): log.Printf("Shutdown keep alive routine for %s", e.ServiceName) return } } }() }
// TestV3LeaseKeepAlive ensures keepalive keeps the lease alive. func TestV3LeaseKeepAlive(t *testing.T) { defer testutil.AfterTest(t) testLeaseRemoveLeasedKey(t, func(clus *ClusterV3, leaseID int64) error { lc := toGRPC(clus.RandClient()).Lease lreq := &pb.LeaseKeepAliveRequest{ID: leaseID} ctx, cancel := context.WithCancel(context.Background()) defer cancel() lac, err := lc.LeaseKeepAlive(ctx) if err != nil { return err } defer lac.CloseSend() // renew long enough so lease would've expired otherwise for i := 0; i < 3; i++ { if err = lac.Send(lreq); err != nil { return err } lresp, rxerr := lac.Recv() if rxerr != nil { return rxerr } if lresp.ID != leaseID { return fmt.Errorf("expected lease ID %v, got %v", leaseID, lresp.ID) } time.Sleep(time.Duration(lresp.TTL/2) * time.Second) } _, err = lc.LeaseRevoke(context.TODO(), &pb.LeaseRevokeRequest{ID: leaseID}) return err }) }
func BenchmarkOneNode(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() n := newNode() s := NewMemoryStorage() r := newTestRaft(1, []uint64{1}, 10, 1, s) go n.run(r) defer n.Stop() n.Campaign(ctx) go func() { for i := 0; i < b.N; i++ { n.Propose(ctx, []byte("foo")) } }() for { rd := <-n.Ready() s.Append(rd.Entries) // a reasonable disk sync latency time.Sleep(1 * time.Millisecond) n.Advance() if rd.HardState.Commit == uint64(b.N+1) { return } } }
func observe(c *clientv3.Client, election string) error { e := concurrency.NewElection(context.TODO(), c, election) ctx, cancel := context.WithCancel(context.TODO()) donec := make(chan struct{}) sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, os.Kill) go func() { <-sigc cancel() }() go func() { for resp := range e.Observe(ctx) { display.Get(resp) } close(donec) }() <-donec select { case <-ctx.Done(): default: return errors.New("elect: observer lost") } return nil }
func testCancel(t *testing.T, e env) { s, addr := serverSetUp(t, nil, math.MaxUint32, nil, nil, e) cc := clientSetUp(t, addr, nil, nil, "", e) tc := testpb.NewTestServiceClient(cc) defer tearDown(s, cc) argSize := 2718 respSize := 314 payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(argSize)) if err != nil { t.Fatal(err) } req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), ResponseSize: proto.Int32(int32(respSize)), Payload: payload, } ctx, cancel := context.WithCancel(context.Background()) time.AfterFunc(1*time.Millisecond, cancel) reply, err := tc.UnaryCall(ctx, req) if grpc.Code(err) != codes.Canceled { t.Fatalf(`TestService/UnaryCall(_, _) = %v, %v; want <nil>, error code: %d`, reply, err, codes.Canceled) } }
func (t *http2Client) newStream(ctx context.Context, callHdr *CallHdr) *Stream { fc := &inFlow{ limit: initialWindowSize, conn: t.fc, } // TODO(zhaoq): Handle uint32 overflow of Stream.id. s := &Stream{ id: t.nextID, method: callHdr.Method, buf: newRecvBuffer(), fc: fc, sendQuotaPool: newQuotaPool(int(t.streamSendQuota)), headerChan: make(chan struct{}), } t.nextID += 2 s.windowHandler = func(n int) { t.updateWindow(s, uint32(n)) } // Make a stream be able to cancel the pending operations by itself. s.ctx, s.cancel = context.WithCancel(ctx) s.dec = &recvBufferReader{ ctx: s.ctx, recv: s.buf, } return s }
// TestV3LeaseSwitch tests a key can be switched from one lease to another. func TestV3LeaseSwitch(t *testing.T) { defer testutil.AfterTest(t) clus := NewClusterV3(t, &ClusterConfig{Size: 3}) defer clus.Terminate(t) key := "foo" // create lease ctx, cancel := context.WithCancel(context.Background()) defer cancel() lresp1, err1 := toGRPC(clus.RandClient()).Lease.LeaseCreate(ctx, &pb.LeaseCreateRequest{TTL: 30}) if err1 != nil { t.Fatal(err1) } lresp2, err2 := toGRPC(clus.RandClient()).Lease.LeaseCreate(ctx, &pb.LeaseCreateRequest{TTL: 30}) if err2 != nil { t.Fatal(err2) } // attach key on lease1 then switch it to lease2 put1 := &pb.PutRequest{Key: []byte(key), Lease: lresp1.ID} _, err := toGRPC(clus.RandClient()).KV.Put(ctx, put1) if err != nil { t.Fatal(err) } put2 := &pb.PutRequest{Key: []byte(key), Lease: lresp2.ID} _, err = toGRPC(clus.RandClient()).KV.Put(ctx, put2) if err != nil { t.Fatal(err) } // revoke lease1 should not remove key _, err = toGRPC(clus.RandClient()).Lease.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: lresp1.ID}) if err != nil { t.Fatal(err) } rreq := &pb.RangeRequest{Key: []byte("foo")} rresp, err := toGRPC(clus.RandClient()).KV.Range(context.TODO(), rreq) if err != nil { t.Fatal(err) } if len(rresp.Kvs) != 1 { t.Fatalf("unexpect removal of key") } // revoke lease2 should remove key _, err = toGRPC(clus.RandClient()).Lease.LeaseRevoke(ctx, &pb.LeaseRevokeRequest{ID: lresp2.ID}) if err != nil { t.Fatal(err) } rresp, err = toGRPC(clus.RandClient()).KV.Range(context.TODO(), rreq) if err != nil { t.Fatal(err) } if len(rresp.Kvs) != 0 { t.Fatalf("lease removed but key remains") } }
func (e *Election) observe(ctx context.Context, ch chan<- v3.GetResponse) { defer close(ch) for { resp, err := e.client.Get(ctx, e.keyPrefix, v3.WithFirstCreate()...) if err != nil { return } var kv *storagepb.KeyValue cctx, cancel := context.WithCancel(ctx) if len(resp.Kvs) == 0 { // wait for first key put on prefix opts := []v3.OpOption{v3.WithRev(resp.Header.Revision), v3.WithPrefix()} wch := e.client.Watch(cctx, e.keyPrefix, opts...) for kv == nil { wr, ok := <-wch if !ok || wr.Err() != nil { cancel() return } // only accept PUTs; a DELETE will make observe() spin for _, ev := range wr.Events { if ev.Type == storagepb.PUT { kv = ev.Kv break } } } } else { kv = resp.Kvs[0] } wch := e.client.Watch(cctx, string(kv.Key), v3.WithRev(kv.ModRevision)) keyDeleted := false for !keyDeleted { wr, ok := <-wch if !ok { return } for _, ev := range wr.Events { if ev.Type == storagepb.DELETE { keyDeleted = true break } resp.Header = &wr.Header resp.Kvs = []*storagepb.KeyValue{ev.Kv} select { case ch <- *resp: case <-cctx.Done(): return } } } cancel() } }
func (le *LeaderElector) Start() <-chan ElectionResponse { var waitIndex uint64 = 0 out := make(chan ElectionResponse, 2) await := make(chan bool, 2) renew := make(chan bool, 2) le.ctx, le.cancel = context.WithCancel(context.Background()) if le.ctx == nil || le.cancel == nil { out <- ElectionResponse{ err: errors.New("Couldn't instantiate context/cancel objects"), } close(out) return out } go func() { for { select { case <-le.ctx.Done(): le.obsvr.Stop() close(out) return case <-await: resp, err := le.waitForLeadership(waitIndex) if err != nil { time.Sleep(LE_SLEEP_INTERVAL) } else if resp != nil { waitIndex = resp.Node.ModifiedIndex } case <-renew: errChan := le.ec.RenewTTL(le.ctx, le.keyPath, le.value, TTL_VAL, TTL_REFRESH_TIMEOUT) for e := range errChan { out <- ElectionResponse{err: e} time.Sleep(LE_SLEEP_INTERVAL) break } default: _, err := le.acquireLeadership() if err != nil { errObj, ok := err.(client.Error) if ok == true && errObj.Code == client.ErrorCodeNodeExist { await <- true } else { time.Sleep(LE_SLEEP_INTERVAL) } } else { renew <- true out <- ElectionResponse{err: nil} } } } }() return out }
// TestElectionFailover tests that an election will func TestElectionFailover(t *testing.T) { clus := NewClusterV3(t, &ClusterConfig{Size: 3}) defer clus.Terminate(t) defer dropSessionLease(clus) cctx, cancel := context.WithCancel(context.TODO()) defer cancel() // first leader (elected) e := concurrency.NewElection(clus.clients[0], "test-election") if err := e.Campaign(context.TODO(), "foo"); err != nil { t.Fatalf("failed volunteer (%v)", err) } // check first leader resp, ok := <-e.Observe(cctx) if !ok { t.Fatalf("could not wait for first election; channel closed") } s := string(resp.Kvs[0].Value) if s != "foo" { t.Fatalf("wrong election result. got %s, wanted foo", s) } // next leader electedc := make(chan struct{}) go func() { ee := concurrency.NewElection(clus.clients[1], "test-election") if eer := ee.Campaign(context.TODO(), "bar"); eer != nil { t.Fatal(eer) } electedc <- struct{}{} }() // invoke leader failover session, serr := concurrency.NewSession(clus.clients[0]) if serr != nil { t.Fatal(serr) } if err := session.Close(); err != nil { t.Fatal(err) } // check new leader e = concurrency.NewElection(clus.clients[2], "test-election") resp, ok = <-e.Observe(cctx) if !ok { t.Fatalf("could not wait for second election; channel closed") } s = string(resp.Kvs[0].Value) if s != "bar" { t.Fatalf("wrong election result. got %s, wanted bar", s) } // leader must ack election (otherwise, Campaign may see closed conn) <-electedc }
func waitUpdate(ctx context.Context, client *v3.Client, key string, opts ...v3.OpOption) error { cctx, cancel := context.WithCancel(ctx) defer cancel() wresp, ok := <-client.Watch(cctx, key, opts...) if !ok { return ctx.Err() } return wresp.Err() }
// NewFIFOScheduler returns a Scheduler that schedules jobs in FIFO // order sequentially func NewFIFOScheduler() Scheduler { f := &fifo{ resume: make(chan struct{}, 1), donec: make(chan struct{}, 1), } f.finishCond = sync.NewCond(&f.mu) f.ctx, f.cancel = context.WithCancel(context.Background()) go f.run() return f }
// TestMultiNodeStart ensures that a node can be started correctly. The node should // start with correct configuration change entries, and can accept and commit // proposals. func TestMultiNodeStart(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() cc := raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1} ccdata, err := cc.Marshal() if err != nil { t.Fatalf("unexpected marshal error: %v", err) } wants := []Ready{ { SoftState: &SoftState{Lead: 1, RaftState: StateLeader}, HardState: raftpb.HardState{Term: 2, Commit: 2, Vote: 1}, Entries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, {Term: 2, Index: 2}, }, CommittedEntries: []raftpb.Entry{ {Type: raftpb.EntryConfChange, Term: 1, Index: 1, Data: ccdata}, {Term: 2, Index: 2}, }, }, { HardState: raftpb.HardState{Term: 2, Commit: 3, Vote: 1}, Entries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, CommittedEntries: []raftpb.Entry{{Term: 2, Index: 3, Data: []byte("foo")}}, }, } mn := StartMultiNode(1) storage := NewMemoryStorage() mn.CreateGroup(1, newTestConfig(1, nil, 10, 1, storage), []Peer{{ID: 1}}) mn.Campaign(ctx, 1) gs := <-mn.Ready() g := gs[1] if !reflect.DeepEqual(g, wants[0]) { t.Fatalf("#%d: g = %+v,\n w %+v", 1, g, wants[0]) } else { storage.Append(g.Entries) mn.Advance(gs) } mn.Propose(ctx, 1, []byte("foo")) if gs2 := <-mn.Ready(); !reflect.DeepEqual(gs2[1], wants[1]) { t.Errorf("#%d: g = %+v,\n w %+v", 2, gs2[1], wants[1]) } else { storage.Append(gs2[1].Entries) mn.Advance(gs2) } select { case rd := <-mn.Ready(): t.Errorf("unexpected Ready: %+v", rd) case <-time.After(time.Millisecond): } }
// TestV3LeaseFailover ensures the old leader drops lease keepalive requests within // election timeout after it loses its quorum. And the new leader extends the TTL of // the lease to at least TTL + election timeout. func TestV3LeaseFailover(t *testing.T) { clus := NewClusterV3(t, &ClusterConfig{Size: 3}) defer clus.Terminate(t) toIsolate := clus.waitLeader(t, clus.Members) lc := toGRPC(clus.Client(toIsolate)).Lease // create lease lresp, err := lc.LeaseCreate(context.TODO(), &pb.LeaseCreateRequest{TTL: 5}) if err != nil { t.Fatal(err) } if lresp.Error != "" { t.Fatal(lresp.Error) } // isolate the current leader with its followers. clus.Members[toIsolate].Pause() lreq := &pb.LeaseKeepAliveRequest{ID: lresp.ID} ctx, cancel := context.WithCancel(context.Background()) defer cancel() lac, err := lc.LeaseKeepAlive(ctx) if err != nil { t.Fatal(err) } defer lac.CloseSend() // send keep alive to old leader until the old leader starts // to drop lease request. var expectedExp time.Time for { if err = lac.Send(lreq); err != nil { break } lkresp, rxerr := lac.Recv() if rxerr != nil { break } expectedExp = time.Now().Add(time.Duration(lkresp.TTL) * time.Second) time.Sleep(time.Duration(lkresp.TTL/2) * time.Second) } clus.Members[toIsolate].Resume() clus.waitLeader(t, clus.Members) // lease should not expire at the last received expire deadline. time.Sleep(expectedExp.Sub(time.Now()) - 500*time.Millisecond) if !leaseExist(t, clus, lresp.ID) { t.Error("unexpected lease not exists") } }
func testV3WatchCancel(t *testing.T, startRev int64) { clus := newClusterGRPC(t, &clusterConfig{size: 3}) wAPI := pb.NewWatchClient(clus.RandConn()) ctx, cancel := context.WithCancel(context.Background()) defer cancel() wStream, errW := wAPI.Watch(ctx) if errW != nil { t.Fatalf("wAPI.Watch error: %v", errW) } wreq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CreateRequest{ CreateRequest: &pb.WatchCreateRequest{ Key: []byte("foo"), StartRevision: startRev}}} if err := wStream.Send(wreq); err != nil { t.Fatalf("wStream.Send error: %v", err) } wresp, errR := wStream.Recv() if errR != nil { t.Errorf("wStream.Recv error: %v", errR) } if !wresp.Created { t.Errorf("wresp.Created got = %v, want = true", wresp.Created) } creq := &pb.WatchRequest{RequestUnion: &pb.WatchRequest_CancelRequest{ CancelRequest: &pb.WatchCancelRequest{ WatchId: wresp.WatchId}}} if err := wStream.Send(creq); err != nil { t.Fatalf("wStream.Send error: %v", err) } cresp, err := wStream.Recv() if err != nil { t.Errorf("wStream.Recv error: %v", err) } if !cresp.Canceled { t.Errorf("cresp.Canceled got = %v, want = true", cresp.Canceled) } kvc := pb.NewKVClient(clus.RandConn()) if _, err := kvc.Put(context.TODO(), &pb.PutRequest{Key: []byte("foo"), Value: []byte("bar")}); err != nil { t.Errorf("couldn't put key (%v)", err) } // watch got canceled, so this should block rok, nr := WaitResponse(wStream, 1*time.Second) if !rok { t.Errorf("unexpected pb.WatchResponse is received %+v", nr) } clus.Terminate(t) }
func (s *stresser) Stress() error { conn, err := grpc.Dial(s.Endpoint, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return fmt.Errorf("%v (%s)", err, s.Endpoint) } defer conn.Close() ctx, cancel := context.WithCancel(context.Background()) wg := &sync.WaitGroup{} wg.Add(s.N) s.mu.Lock() s.conn = conn s.cancel = cancel s.wg = wg s.mu.Unlock() kvc := pb.NewKVClient(conn) for i := 0; i < s.N; i++ { go func(i int) { defer wg.Done() for { // TODO: 10-second is enough timeout to cover leader failure // and immediate leader election. Find out what other cases this // could be timed out. putctx, putcancel := context.WithTimeout(ctx, 10*time.Second) _, err := kvc.Put(putctx, &pb.PutRequest{ Key: []byte(fmt.Sprintf("foo%d", rand.Intn(s.KeySuffixRange))), Value: []byte(randStr(s.KeySize)), }) putcancel() if err != nil { if grpc.ErrorDesc(err) == context.DeadlineExceeded.Error() { // This retries when request is triggered at the same time as // leader failure. When we terminate the leader, the request to // that leader cannot be processed, and times out. Also requests // to followers cannot be forwarded to the old leader, so timing out // as well. We want to keep stressing until the cluster elects a // new leader and start processing requests again. continue } return } s.mu.Lock() s.success++ s.mu.Unlock() } }(i) } <-ctx.Done() return nil }
func (clm *clientLeaseMgr) sessionLease(client *clientv3.Client, ttl int64) (lease.LeaseID, error) { clm.mu.Lock() defer clm.mu.Unlock() if lka, ok := clm.leases[client]; ok { return lka.id, nil } resp, err := client.Lease.LeaseCreate(context.TODO(), &pb.LeaseCreateRequest{TTL: ttl}) if err != nil { return lease.NoLease, err } id := lease.LeaseID(resp.ID) ctx, cancel := context.WithCancel(context.Background()) keepAlive, err := client.Lease.LeaseKeepAlive(ctx) if err != nil || keepAlive == nil { return lease.NoLease, err } lka := &leaseKeepAlive{id: id, donec: make(chan struct{})} clm.leases[client] = lka // keep the lease alive until clientection error go func() { defer func() { keepAlive.CloseSend() clm.mu.Lock() delete(clm.leases, client) clm.mu.Unlock() cancel() close(lka.donec) }() ttl := resp.TTL for { lreq := &pb.LeaseKeepAliveRequest{ID: int64(id)} select { case <-lka.donec: return case <-time.After(time.Duration(ttl/2) * time.Second): } if err := keepAlive.Send(lreq); err != nil { break } resp, err := keepAlive.Recv() if err != nil { break } ttl = resp.TTL } }() return id, nil }
func (e *etcd) WatchRouteChanges(filter string) (<-chan Event, <-chan error, context.CancelFunc) { events := make(chan Event) errors := make(chan error) cxt, cancel := context.WithCancel(e.ctx) go e.dispatchWatchEvents(cxt, filter, events, errors) time.Sleep(100 * time.Millisecond) //give the watcher a chance to connect return events, errors, cancel }
func testWatchCancelImmediate(t *testing.T, wctx *watchctx) { ctx, cancel := context.WithCancel(context.Background()) cancel() wch := wctx.w.Watch(ctx, "a") select { case wresp, ok := <-wch: if ok { t.Fatalf("read wch got %v; expected closed channel", wresp) } default: t.Fatalf("closed watcher channel should not block") } }
func setupFakeEtcd(keys client.KeysAPI) db.DB { nodeURLs := []string{"127.0.0.1:5000"} cfg := client.Config{ Endpoints: nodeURLs, Transport: client.DefaultTransport, } client, err := client.New(cfg) Expect(err).NotTo(HaveOccurred()) ctx, cancel := context.WithCancel(context.Background()) return db.New(client, keys, ctx, cancel) }
// DoCancelAfterBegin cancels the RPC after metadata has been sent but before payloads are sent. func DoCancelAfterBegin(tc testpb.TestServiceClient) { ctx, cancel := context.WithCancel(metadata.NewContext(context.Background(), testMetadata)) stream, err := tc.StreamingInputCall(ctx) if err != nil { grpclog.Fatalf("%v.StreamingInputCall(_) = _, %v", tc, err) } cancel() _, err = stream.CloseAndRecv() if grpc.Code(err) != codes.Canceled { grpclog.Fatalf("%v.CloseAndRecv() got error code %d, want %d", stream, grpc.Code(err), codes.Canceled) } grpclog.Println("CancelAfterBegin done") }
func TestIsHalted(t *testing.T) { if !isHalted(nil, fmt.Errorf("etcdserver: some etcdserver error")) { t.Errorf(`error prefixed with "etcdserver: " should be Halted`) } ctx, cancel := context.WithCancel(context.TODO()) if isHalted(ctx, nil) { t.Errorf("no error and active context should not be Halted") } cancel() if !isHalted(ctx, nil) { t.Errorf("cancel on context should be Halted") } }