Example #1
0
// TestUnretryableError verifies that Send returns an unretryable
// error when it hits a critical error.
func TestUnretryableError(t *testing.T) {
	defer leaktest.AfterTest(t)()

	stopper := stop.NewStopper()
	defer stopper.Stop()

	nodeContext := newNodeTestContext(nil, stopper)
	_, ln := newTestServer(t, nodeContext)

	sp := tracing.NewTracer().StartSpan("node test")
	defer sp.Finish()

	opts := SendOptions{
		Ordering:        orderStable,
		SendNextTimeout: 1 * time.Second,
		Timeout:         10 * time.Second,
		Trace:           sp,
	}

	sendOneFn = func(client *batchClient, timeout time.Duration,
		context *rpc.Context, trace opentracing.Span, done chan *netrpc.Call) {
		call := netrpc.Call{
			Reply: &roachpb.BatchResponse{},
		}
		call.Error = errors.New("unretryable")
		done <- &call
	}
	defer func() { sendOneFn = sendOne }()

	_, err := sendBatch(opts, []net.Addr{ln.Addr()}, nodeContext)
	if err == nil {
		t.Fatalf("Unexpected success")
	}
	retryErr, ok := err.(retry.Retryable)
	if !ok {
		t.Fatalf("Unexpected error type: %v", err)
	}
	if retryErr.CanRetry() {
		t.Errorf("Unexpected retryable error: %v", retryErr)
	}
}
Example #2
0
func (master *Master) Go(cookie string, userServiceMethod string, args interface{}, reply interface{}, done chan *rpc.Call) *rpc.Call {
	serviceMethod := "User." + userServiceMethod
	if slave, err := master.getStartedSlave(cookie); err != nil {
		call := new(rpc.Call)
		call.ServiceMethod = serviceMethod
		call.Args = args
		call.Reply = reply
		if done == nil {
			done = make(chan *rpc.Call, 10) // buffered.
		} else {
			// If caller passes done != nil, it must arrange that
			// done has enough buffer for the number of simultaneous
			// RPCs that will be using that channel.  If the channel
			// is totally unbuffered, it's best not to run at all.
			if cap(done) == 0 {
				log.Panic("rpc: done channel is unbuffered")
			}
		}
		call.Done = done
		call.Error = err
		call.Done <- call
		return call
	} else {
		return slave.conn.Go(serviceMethod, args, reply, done)
	}
}
Example #3
0
func (s *state) requestVoteRequest(req *RequestVoteRequest, resp *RequestVoteResponse,
	call *rpc.Call) {
	g, ok := s.groups[req.GroupID]
	if !ok {
		call.Error = util.Errorf("unknown group %v", req.GroupID)
		call.Done <- call
		return
	}
	if g.metadata.VotedFor.isSet() && g.metadata.VotedFor != req.CandidateID {
		resp.VoteGranted = false
	} else {
		// TODO: check log positions
		g.metadata.CurrentTerm = req.Term
		resp.VoteGranted = true
	}
	resp.Term = g.metadata.CurrentTerm
	g.pendingCalls.PushBack(&pendingCall{call, g.metadata.CurrentTerm, -1})
	s.updateDirtyStatus(g)
}
Example #4
0
func (s *state) requestVoteRequest(req *RequestVoteRequest, resp *RequestVoteResponse,
	call *rpc.Call) {
	g, ok := s.groups[req.GroupID]
	if !ok {
		call.Error = util.Errorf("unknown group %v", req.GroupID)
		call.Done <- call
		return
	}
	if g.electionState.VotedFor.isSet() && g.electionState.VotedFor != req.CandidateID {
		resp.VoteGranted = false
	} else {
		// TODO: check log positions
		g.electionState.CurrentTerm = req.Term
		resp.VoteGranted = true
	}
	log.V(1).Infof("node %v responding %v to vote request from node %v in term %v", s.nodeID,
		resp.VoteGranted, req.CandidateID, req.Term)
	resp.Term = g.electionState.CurrentTerm
	s.addPendingCall(g, &pendingCall{call, g.electionState.CurrentTerm, -1})
	s.updateDirtyStatus(g)
}
Example #5
0
// TestComplexScenarios verifies various complex success/failure scenarios by
// mocking sendOne.
func TestComplexScenarios(t *testing.T) {
	defer leaktest.AfterTest(t)

	stopper := stop.NewStopper()
	defer stopper.Stop()

	nodeContext := NewNodeTestContext(nil, stopper)

	testCases := []struct {
		numServers               int
		numRequests              int
		numErrors                int
		numRetryableErrors       int
		success                  bool
		isRetryableErrorExpected bool
	}{
		// --- Success scenarios ---
		{1, 1, 0, 0, true, false},
		{5, 1, 0, 0, true, false},
		{5, 5, 0, 0, true, false},
		// There are some errors, but enough RPCs succeed.
		{5, 1, 1, 0, true, false},
		{5, 1, 4, 0, true, false},
		{5, 3, 2, 0, true, false},

		// --- Failure scenarios ---
		// Too many requests.
		{1, 5, 0, 0, false, false},
		// All RPCs fail.
		{5, 1, 5, 0, false, false},
		// Some RPCs fail and we do not have enough remaining clients.
		{5, 3, 3, 0, false, false},
		// All RPCs fail, but some of the errors are retryable.
		{5, 1, 5, 1, false, true},
		{5, 3, 5, 3, false, true},
		// Some RPCs fail, but we do have enough remaining clients and recoverable errors.
		{5, 3, 3, 1, false, true},
		{5, 3, 4, 2, false, true},
	}
	for i, test := range testCases {
		// Copy the values to avoid data race. sendOneFn might
		// be called after this test case finishes.
		numErrors := test.numErrors
		numRetryableErrors := test.numRetryableErrors

		var serverAddrs []net.Addr
		for j := 0; j < test.numServers; j++ {
			s := createAndStartNewServer(t, nodeContext)
			serverAddrs = append(serverAddrs, s.Addr())
		}

		opts := Options{
			N:               test.numRequests,
			Ordering:        OrderStable,
			SendNextTimeout: 1 * time.Second,
			Timeout:         1 * time.Second,
		}
		getArgs := func(addr net.Addr) gogoproto.Message {
			return &proto.PingRequest{}
		}
		getReply := func() gogoproto.Message {
			return &proto.PingResponse{}
		}

		// Mock sendOne.
		sendOneFn = func(client *Client, timeout time.Duration, method string, args, reply gogoproto.Message, done chan *rpc.Call) {
			addr := client.addr
			addrID := -1
			for serverAddrID, serverAddr := range serverAddrs {
				if serverAddr.String() == addr.String() {
					addrID = serverAddrID
					break
				}
			}
			if addrID == -1 {
				t.Fatalf("%d: %v is not found in serverAddrs: %v", i, addr, serverAddrs)
			}
			call := rpc.Call{
				Reply: reply,
			}
			if addrID < numErrors {
				call.Error = SendError{
					errMsg:   "test",
					canRetry: addrID < numRetryableErrors,
				}
			}
			done <- &call
		}
		defer func() { sendOneFn = sendOne }()

		replies, err := Send(opts, "Heartbeat.Ping", serverAddrs, getArgs, getReply, nodeContext)
		if test.success {
			if len(replies) != test.numRequests {
				t.Errorf("%d: %v replies are expected, but got %v", i, test.numRequests, len(replies))
			}
			continue
		}

		retryErr, ok := err.(retry.Retryable)
		if !ok {
			t.Fatalf("%d: Unexpected error type: %v", i, err)
		}
		if retryErr.CanRetry() != test.isRetryableErrorExpected {
			t.Errorf("%d: Unexpected error: %v", i, retryErr)
		}
	}
}
Example #6
0
// TestComplexScenarios verifies various complex success/failure scenarios by
// mocking sendOne.
func TestComplexScenarios(t *testing.T) {
	defer leaktest.AfterTest(t)()

	stopper := stop.NewStopper()
	defer stopper.Stop()

	nodeContext := newNodeTestContext(nil, stopper)

	testCases := []struct {
		numServers               int
		numErrors                int
		numRetryableErrors       int
		success                  bool
		isRetryableErrorExpected bool
	}{
		// --- Success scenarios ---
		{1, 0, 0, true, false},
		{5, 0, 0, true, false},
		// There are some errors, but enough RPCs succeed.
		{5, 1, 0, true, false},
		{5, 4, 0, true, false},
		{5, 2, 0, true, false},

		// --- Failure scenarios ---
		// All RPCs fail.
		{5, 5, 0, false, false},
		// All RPCs fail, but some of the errors are retryable.
		{5, 5, 1, false, true},
		{5, 5, 3, false, true},
		// Some RPCs fail, but we do have enough remaining clients and recoverable errors.
		{5, 5, 2, false, true},
	}
	for i, test := range testCases {
		// Copy the values to avoid data race. sendOneFn might
		// be called after this test case finishes.
		numErrors := test.numErrors
		numRetryableErrors := test.numRetryableErrors

		var serverAddrs []net.Addr
		for j := 0; j < test.numServers; j++ {
			_, ln := newTestServer(t, nodeContext)
			serverAddrs = append(serverAddrs, ln.Addr())
		}

		sp := tracing.NewTracer().StartSpan("node test")
		defer sp.Finish()

		opts := SendOptions{
			Ordering:        orderStable,
			SendNextTimeout: 1 * time.Second,
			Timeout:         10 * time.Second,
			Trace:           sp,
		}

		// Mock sendOne.
		sendOneFn = func(client *batchClient, timeout time.Duration,
			context *rpc.Context, trace opentracing.Span, done chan *netrpc.Call) {
			addr := client.RemoteAddr()
			addrID := -1
			for serverAddrID, serverAddr := range serverAddrs {
				if serverAddr.String() == addr.String() {
					addrID = serverAddrID
					break
				}
			}
			if addrID == -1 {
				t.Fatalf("%d: %v is not found in serverAddrs: %v", i, addr, serverAddrs)
			}
			call := netrpc.Call{
				Reply: &roachpb.BatchResponse{},
			}
			if addrID < numErrors {
				call.Error = roachpb.NewSendError("test", addrID < numRetryableErrors)
			}
			done <- &call
		}
		defer func() { sendOneFn = sendOne }()

		reply, err := sendBatch(opts, serverAddrs, nodeContext)
		if test.success {
			if reply == nil {
				t.Errorf("%d: expected reply", i)
			}
			continue
		}

		retryErr, ok := err.(retry.Retryable)
		if !ok {
			t.Fatalf("%d: Unexpected error type: %v", i, err)
		}
		if retryErr.CanRetry() != test.isRetryableErrorExpected {
			t.Errorf("%d: Unexpected error: %v", i, retryErr)
		}
	}
}