func TestEvictCacheOnError(t *testing.T) { defer leaktest.AfterTest(t) // if rpcError is true, the first attempt gets an RPC error, otherwise // the RPC call succeeds but there is an error in the RequestHeader. // Currently leader and cached range descriptor are treated equally. testCases := []struct{ rpcError, retryable, shouldClearLeader, shouldClearReplica bool }{ {false, false, false, false}, // non-retryable replica error {false, true, false, false}, // retryable replica error {true, false, true, true}, // RPC error aka all nodes dead {true, true, false, false}, // retryable RPC error } for i, tc := range testCases { g, s := makeTestGossip(t) defer s() leader := proto.Replica{ NodeID: 99, StoreID: 999, } first := true var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, _ func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { if !first { return []gogoproto.Message{getReply()}, nil } first = false err := rpc.NewSendError("boom", tc.retryable) if tc.rpcError { return nil, err } reply := getReply() reply.(proto.Response).Header().SetGoError(err) return []gogoproto.Message{reply}, nil } ctx := &DistSenderContext{ RPCSend: testFn, RangeDescriptorDB: mockRangeDescriptorDB(func(_ proto.Key, _ lookupOptions) ([]proto.RangeDescriptor, error) { return []proto.RangeDescriptor{testRangeDescriptor}, nil }), } ds := NewDistSender(ctx, g) ds.updateLeaderCache(1, leader) call := proto.PutCall(proto.Key("a"), proto.Value{Bytes: []byte("value")}) reply := call.Reply.(*proto.PutResponse) client.SendCallConverted(ds, context.Background(), call) if err := reply.GoError(); err != nil && !testutils.IsError(err, "boom") { t.Errorf("put encountered unexpected error: %s", err) } if cur := ds.leaderCache.Lookup(1); reflect.DeepEqual(cur, &proto.Replica{}) && !tc.shouldClearLeader { t.Errorf("%d: leader cache eviction: shouldClearLeader=%t, but value is %v", i, tc.shouldClearLeader, cur) } _, cachedDesc := ds.rangeCache.getCachedRangeDescriptor(call.Args.Header().Key, false /* !inclusive */) if cachedDesc == nil != tc.shouldClearReplica { t.Errorf("%d: unexpected second replica lookup behaviour: wanted=%t", i, tc.shouldClearReplica) } } }
func TestEvictCacheOnError(t *testing.T) { defer leaktest.AfterTest(t) // if rpcError is true, the first attempt gets an RPC error, otherwise // the RPC call succeeds but there is an error in the RequestHeader. // Currently leader and cached range descriptor are treated equally. testCases := []struct{ rpcError, retryable, shouldClearLeader, shouldClearReplica bool }{ {false, false, false, false}, // non-retryable replica error {false, true, false, false}, // retryable replica error {true, false, true, true}, // RPC error aka all nodes dead {true, true, false, false}, // retryable RPC error } for i, tc := range testCases { g, s := makeTestGossip(t) defer s() leader := roachpb.ReplicaDescriptor{ NodeID: 99, StoreID: 999, } first := true var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) { if !first { return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil } first = false if tc.rpcError { return nil, rpc.NewSendError("boom", tc.retryable) } var err error if tc.retryable { err = &roachpb.RangeKeyMismatchError{} } else { err = errors.New("boom") } reply := getReply() reply.(*roachpb.BatchResponse).SetGoError(err) return []proto.Message{reply}, nil } ctx := &DistSenderContext{ RPCSend: testFn, RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { return []roachpb.RangeDescriptor{testRangeDescriptor}, nil }), } ds := NewDistSender(ctx, g) ds.updateLeaderCache(1, leader) put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value")).(*roachpb.PutRequest) if _, pErr := client.SendWrapped(ds, nil, put); pErr != nil && !testutils.IsError(pErr.GoError(), "boom") { t.Errorf("put encountered unexpected error: %s", pErr) } if cur := ds.leaderCache.Lookup(1); reflect.DeepEqual(cur, &roachpb.ReplicaDescriptor{}) && !tc.shouldClearLeader { t.Errorf("%d: leader cache eviction: shouldClearLeader=%t, but value is %v", i, tc.shouldClearLeader, cur) } _, cachedDesc := ds.rangeCache.getCachedRangeDescriptor(roachpb.RKey(put.Key), false /* !inclusive */) if cachedDesc == nil != tc.shouldClearReplica { t.Errorf("%d: unexpected second replica lookup behaviour: wanted=%t", i, tc.shouldClearReplica) } } }