// TestUnretryableError verifies that Send returns an unretryable
// error when it hits a critical error.
func TestUnretryableError(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
nodeContext := newNodeTestContext(nil, stopper)
_, ln := newTestServer(t, nodeContext)
sp := tracing.NewTracer().StartSpan("node test")
defer sp.Finish()
opts := SendOptions{
Ordering: orderStable,
SendNextTimeout: 1 * time.Second,
Timeout: 10 * time.Second,
Trace: sp,
}
sendOneFn = func(client *batchClient, timeout time.Duration,
context *rpc.Context, trace opentracing.Span, done chan *netrpc.Call) {
call := netrpc.Call{
Reply: &roachpb.BatchResponse{},
}
call.Error = errors.New("unretryable")
done <- &call
}
defer func() { sendOneFn = sendOne }()
_, err := sendBatch(opts, []net.Addr{ln.Addr()}, nodeContext)
if err == nil {
t.Fatalf("Unexpected success")
}
retryErr, ok := err.(retry.Retryable)
if !ok {
t.Fatalf("Unexpected error type: %v", err)
}
if retryErr.CanRetry() {
t.Errorf("Unexpected retryable error: %v", retryErr)
}
}
func (master *Master) Go(cookie string, userServiceMethod string, args interface{}, reply interface{}, done chan *rpc.Call) *rpc.Call {
serviceMethod := "User." + userServiceMethod
if slave, err := master.getStartedSlave(cookie); err != nil {
call := new(rpc.Call)
call.ServiceMethod = serviceMethod
call.Args = args
call.Reply = reply
if done == nil {
done = make(chan *rpc.Call, 10) // buffered.
} else {
// If caller passes done != nil, it must arrange that
// done has enough buffer for the number of simultaneous
// RPCs that will be using that channel. If the channel
// is totally unbuffered, it's best not to run at all.
if cap(done) == 0 {
log.Panic("rpc: done channel is unbuffered")
}
}
call.Done = done
call.Error = err
call.Done <- call
return call
} else {
return slave.conn.Go(serviceMethod, args, reply, done)
}
}
func (s *state) requestVoteRequest(req *RequestVoteRequest, resp *RequestVoteResponse,
call *rpc.Call) {
g, ok := s.groups[req.GroupID]
if !ok {
call.Error = util.Errorf("unknown group %v", req.GroupID)
call.Done <- call
return
}
if g.metadata.VotedFor.isSet() && g.metadata.VotedFor != req.CandidateID {
resp.VoteGranted = false
} else {
// TODO: check log positions
g.metadata.CurrentTerm = req.Term
resp.VoteGranted = true
}
resp.Term = g.metadata.CurrentTerm
g.pendingCalls.PushBack(&pendingCall{call, g.metadata.CurrentTerm, -1})
s.updateDirtyStatus(g)
}
func (s *state) requestVoteRequest(req *RequestVoteRequest, resp *RequestVoteResponse,
call *rpc.Call) {
g, ok := s.groups[req.GroupID]
if !ok {
call.Error = util.Errorf("unknown group %v", req.GroupID)
call.Done <- call
return
}
if g.electionState.VotedFor.isSet() && g.electionState.VotedFor != req.CandidateID {
resp.VoteGranted = false
} else {
// TODO: check log positions
g.electionState.CurrentTerm = req.Term
resp.VoteGranted = true
}
log.V(1).Infof("node %v responding %v to vote request from node %v in term %v", s.nodeID,
resp.VoteGranted, req.CandidateID, req.Term)
resp.Term = g.electionState.CurrentTerm
s.addPendingCall(g, &pendingCall{call, g.electionState.CurrentTerm, -1})
s.updateDirtyStatus(g)
}
// TestComplexScenarios verifies various complex success/failure scenarios by
// mocking sendOne.
func TestComplexScenarios(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
nodeContext := NewNodeTestContext(nil, stopper)
testCases := []struct {
numServers int
numRequests int
numErrors int
numRetryableErrors int
success bool
isRetryableErrorExpected bool
}{
// --- Success scenarios ---
{1, 1, 0, 0, true, false},
{5, 1, 0, 0, true, false},
{5, 5, 0, 0, true, false},
// There are some errors, but enough RPCs succeed.
{5, 1, 1, 0, true, false},
{5, 1, 4, 0, true, false},
{5, 3, 2, 0, true, false},
// --- Failure scenarios ---
// Too many requests.
{1, 5, 0, 0, false, false},
// All RPCs fail.
{5, 1, 5, 0, false, false},
// Some RPCs fail and we do not have enough remaining clients.
{5, 3, 3, 0, false, false},
// All RPCs fail, but some of the errors are retryable.
{5, 1, 5, 1, false, true},
{5, 3, 5, 3, false, true},
// Some RPCs fail, but we do have enough remaining clients and recoverable errors.
{5, 3, 3, 1, false, true},
{5, 3, 4, 2, false, true},
}
for i, test := range testCases {
// Copy the values to avoid data race. sendOneFn might
// be called after this test case finishes.
numErrors := test.numErrors
numRetryableErrors := test.numRetryableErrors
var serverAddrs []net.Addr
for j := 0; j < test.numServers; j++ {
s := createAndStartNewServer(t, nodeContext)
serverAddrs = append(serverAddrs, s.Addr())
}
opts := Options{
N: test.numRequests,
Ordering: OrderStable,
SendNextTimeout: 1 * time.Second,
Timeout: 1 * time.Second,
}
getArgs := func(addr net.Addr) gogoproto.Message {
return &proto.PingRequest{}
}
getReply := func() gogoproto.Message {
return &proto.PingResponse{}
}
// Mock sendOne.
sendOneFn = func(client *Client, timeout time.Duration, method string, args, reply gogoproto.Message, done chan *rpc.Call) {
addr := client.addr
addrID := -1
for serverAddrID, serverAddr := range serverAddrs {
if serverAddr.String() == addr.String() {
addrID = serverAddrID
break
}
}
if addrID == -1 {
t.Fatalf("%d: %v is not found in serverAddrs: %v", i, addr, serverAddrs)
}
call := rpc.Call{
Reply: reply,
}
if addrID < numErrors {
call.Error = SendError{
errMsg: "test",
canRetry: addrID < numRetryableErrors,
}
}
done <- &call
}
defer func() { sendOneFn = sendOne }()
replies, err := Send(opts, "Heartbeat.Ping", serverAddrs, getArgs, getReply, nodeContext)
if test.success {
if len(replies) != test.numRequests {
t.Errorf("%d: %v replies are expected, but got %v", i, test.numRequests, len(replies))
}
continue
}
retryErr, ok := err.(retry.Retryable)
if !ok {
t.Fatalf("%d: Unexpected error type: %v", i, err)
}
if retryErr.CanRetry() != test.isRetryableErrorExpected {
t.Errorf("%d: Unexpected error: %v", i, retryErr)
}
}
}
// TestComplexScenarios verifies various complex success/failure scenarios by
// mocking sendOne.
func TestComplexScenarios(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
nodeContext := newNodeTestContext(nil, stopper)
testCases := []struct {
numServers int
numErrors int
numRetryableErrors int
success bool
isRetryableErrorExpected bool
}{
// --- Success scenarios ---
{1, 0, 0, true, false},
{5, 0, 0, true, false},
// There are some errors, but enough RPCs succeed.
{5, 1, 0, true, false},
{5, 4, 0, true, false},
{5, 2, 0, true, false},
// --- Failure scenarios ---
// All RPCs fail.
{5, 5, 0, false, false},
// All RPCs fail, but some of the errors are retryable.
{5, 5, 1, false, true},
{5, 5, 3, false, true},
// Some RPCs fail, but we do have enough remaining clients and recoverable errors.
{5, 5, 2, false, true},
}
for i, test := range testCases {
// Copy the values to avoid data race. sendOneFn might
// be called after this test case finishes.
numErrors := test.numErrors
numRetryableErrors := test.numRetryableErrors
var serverAddrs []net.Addr
for j := 0; j < test.numServers; j++ {
_, ln := newTestServer(t, nodeContext)
serverAddrs = append(serverAddrs, ln.Addr())
}
sp := tracing.NewTracer().StartSpan("node test")
defer sp.Finish()
opts := SendOptions{
Ordering: orderStable,
SendNextTimeout: 1 * time.Second,
Timeout: 10 * time.Second,
Trace: sp,
}
// Mock sendOne.
sendOneFn = func(client *batchClient, timeout time.Duration,
context *rpc.Context, trace opentracing.Span, done chan *netrpc.Call) {
addr := client.RemoteAddr()
addrID := -1
for serverAddrID, serverAddr := range serverAddrs {
if serverAddr.String() == addr.String() {
addrID = serverAddrID
break
}
}
if addrID == -1 {
t.Fatalf("%d: %v is not found in serverAddrs: %v", i, addr, serverAddrs)
}
call := netrpc.Call{
Reply: &roachpb.BatchResponse{},
}
if addrID < numErrors {
call.Error = roachpb.NewSendError("test", addrID < numRetryableErrors)
}
done <- &call
}
defer func() { sendOneFn = sendOne }()
reply, err := sendBatch(opts, serverAddrs, nodeContext)
if test.success {
if reply == nil {
t.Errorf("%d: expected reply", i)
}
continue
}
retryErr, ok := err.(retry.Retryable)
if !ok {
t.Fatalf("%d: Unexpected error type: %v", i, err)
}
if retryErr.CanRetry() != test.isRetryableErrorExpected {
t.Errorf("%d: Unexpected error: %v", i, retryErr)
}
}
}