func TestSendNext_AllRetryableApplicationErrors(t *testing.T) {
defer leaktest.AfterTest(t)()
doneChans, sendChan, stopper := setupSendNextTest(t)
defer stopper.Stop()
// All replicas finish with a retryable error.
for _, ch := range doneChans {
ch <- BatchCall{
Reply: &roachpb.BatchResponse{
BatchResponse_Header: roachpb.BatchResponse_Header{
Error: roachpb.NewError(roachpb.NewRangeNotFoundError(1)),
},
},
}
}
// The client send finishes with one of the errors, wrapped in a SendError.
bc := <-sendChan
if bc.Err == nil {
t.Fatalf("expected SendError, got err=nil and reply=%s", bc.Reply)
} else if _, ok := bc.Err.(*roachpb.SendError); !ok {
t.Fatalf("expected SendError, got err=%s", bc.Err)
} else if exp := "range 1 was not found"; !testutils.IsError(bc.Err, exp) {
t.Errorf("expected SendError to contain %q, but got %v", exp, bc.Err)
}
}
func TestSendNext_RetryableApplicationErrorThenSuccess(t *testing.T) {
defer leaktest.AfterTest(t)()
doneChans, sendChan, stopper := setupSendNextTest(t)
defer stopper.Stop()
// One replica finishes with a retryable error.
doneChans[1] <- BatchCall{
Reply: &roachpb.BatchResponse{
BatchResponse_Header: roachpb.BatchResponse_Header{
Error: roachpb.NewError(roachpb.NewRangeNotFoundError(1)),
},
},
}
// A second replica finishes successfully.
doneChans[2] <- BatchCall{
Reply: &roachpb.BatchResponse{},
}
// The client send finishes with the second response.
bc := <-sendChan
if bc.Err != nil {
t.Fatalf("unexpected RPC error: %s", bc.Err)
}
if bc.Reply.Error != nil {
t.Errorf("expected successful reply, got %s", bc.Reply.Error)
}
}
// LookupReplica looks up replica by key [range]. Lookups are done
// by consulting each store in turn via Store.LookupReplica(key).
// Returns RangeID and replica on success; RangeKeyMismatch error
// if not found.
// If end is nil, a replica containing start is looked up.
// This is only for testing usage; performance doesn't matter.
func (ls *Stores) LookupReplica(
start, end roachpb.RKey,
) (roachpb.RangeID, roachpb.ReplicaDescriptor, error) {
ls.mu.RLock()
defer ls.mu.RUnlock()
var rangeID roachpb.RangeID
var repDesc roachpb.ReplicaDescriptor
var repDescFound bool
for _, store := range ls.storeMap {
replica := store.LookupReplica(start, nil)
if replica == nil {
continue
}
// Verify that the descriptor contains the entire range.
if desc := replica.Desc(); !desc.ContainsKeyRange(start, end) {
ctx := ls.AnnotateCtx(context.TODO())
log.Warningf(ctx, "range not contained in one range: [%s,%s), but have [%s,%s)",
start, end, desc.StartKey, desc.EndKey)
err := roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), desc)
return 0, roachpb.ReplicaDescriptor{}, err
}
rangeID = replica.RangeID
var err error
repDesc, err = replica.GetReplicaDescriptor()
if err != nil {
if _, ok := err.(*roachpb.RangeNotFoundError); ok {
// We are not holding a lock across this block; the replica could have
// been removed from the range (via down-replication) between the
// LookupReplica and the GetReplicaDescriptor calls. In this case just
// ignore this replica.
continue
}
return 0, roachpb.ReplicaDescriptor{}, err
}
if repDescFound {
// We already found the range; this should never happen outside of tests.
err := errors.Errorf("range %+v exists on additional store: %+v", replica, store)
return 0, roachpb.ReplicaDescriptor{}, err
}
repDescFound = true
}
if !repDescFound {
return 0, roachpb.ReplicaDescriptor{}, roachpb.NewRangeNotFoundError(0)
}
return rangeID, repDesc, nil
}