// sendRPC sends one or more RPCs to replicas from the supplied // roachpb.Replica slice. Returns an RPC error if the request could // not be sent. Note that the reply may contain a higher level error // and must be checked in addition to the RPC error. // // The replicas are assumed to be ordered by preference, with closer // ones (i.e. expected lowest latency) first. func (ds *DistSender) sendRPC( ctx context.Context, rangeID roachpb.RangeID, replicas ReplicaSlice, ba roachpb.BatchRequest, ) (*roachpb.BatchResponse, error) { if len(replicas) == 0 { return nil, roachpb.NewSendError( fmt.Sprintf("no replica node addresses available via gossip for range %d", rangeID)) } // TODO(pmattis): This needs to be tested. If it isn't set we'll // still route the request appropriately by key, but won't receive // RangeNotFoundErrors. ba.RangeID = rangeID // Set RPC opts with stipulation that one of N RPCs must succeed. rpcOpts := SendOptions{ ctx: ctx, SendNextTimeout: ds.sendNextTimeout, transportFactory: ds.transportFactory, } tracing.AnnotateTrace() defer tracing.AnnotateTrace() reply, err := ds.sendToReplicas(rpcOpts, rangeID, replicas, ba, ds.rpcContext) if err != nil { return nil, err } return reply, nil }
// Test the behavior of SendNextTimeout when some servers return // RPC errors but one succeeds. func TestSendNext_RPCErrorThenSuccess(t *testing.T) { defer leaktest.AfterTest(t)() doneChans, sendChan, stopper := setupSendNextTest(t) defer stopper.Stop() // Now that all replicas have been contacted, let two finish with // retryable errors. for i := 1; i <= 2; i++ { doneChans[i] <- BatchCall{ Reply: nil, Err: roachpb.NewSendError("boom"), } } // The client is still waiting for the third slow RPC to complete. select { case bc := <-sendChan: t.Fatalf("got unexpected response %v", bc) default: } // Now let the final server complete the RPC successfully. doneChans[0] <- BatchCall{ Reply: &roachpb.BatchResponse{}, Err: nil, } // The client side now completes successfully. bc := <-sendChan if bc.Err != nil { t.Fatal(bc.Err) } }
// sendRPC sends one or more RPCs to replicas from the supplied // roachpb.Replica slice. Returns an RPC error if the request could // not be sent. Note that the reply may contain a higher level error // and must be checked in addition to the RPC error. // // The replicas are assumed to be ordered by preference, with closer // ones (i.e. expected lowest latency) first. func (ds *DistSender) sendRPC( ctx context.Context, rangeID roachpb.RangeID, replicas ReplicaSlice, ba roachpb.BatchRequest, ) (*roachpb.BatchResponse, error) { if len(replicas) == 0 { return nil, roachpb.NewSendError( fmt.Sprintf("no replica node addresses available via gossip for range %d", rangeID)) } // TODO(pmattis): This needs to be tested. If it isn't set we'll // still route the request appropriately by key, but won't receive // RangeNotFoundErrors. ba.RangeID = rangeID // A given RPC may generate retries to multiple replicas, but as soon as we // get a response from one we want to cancel those other RPCs. ctx, cancel := context.WithCancel(ctx) defer cancel() // Set RPC opts with stipulation that one of N RPCs must succeed. rpcOpts := SendOptions{ SendNextTimeout: ds.sendNextTimeout, transportFactory: ds.transportFactory, metrics: &ds.metrics, } tracing.AnnotateTrace() defer tracing.AnnotateTrace() reply, err := ds.sendToReplicas(ctx, rpcOpts, rangeID, replicas, ba, ds.rpcContext) if err != nil { return nil, err } return reply, nil }
func (f *firstNErrorTransport) SendNext(done chan<- BatchCall) { call := BatchCall{ Reply: &roachpb.BatchResponse{}, } if f.numSent < f.numErrors { call.Err = roachpb.NewSendError("test") } f.numSent++ done <- call }
// Send implements the Sender interface. func (s sender) Send( ctx context.Context, ba roachpb.BatchRequest, ) (*roachpb.BatchResponse, *roachpb.Error) { br, err := s.Batch(ctx, &ba) if err != nil { return nil, roachpb.NewError(roachpb.NewSendError(err.Error())) } pErr := br.Error br.Error = nil return br, pErr }
// sendToReplicas sends one or more RPCs to clients specified by the // slice of replicas. On success, Send returns the first successful // reply. If an error occurs which is not specific to a single // replica, it's returned immediately. Otherwise, when all replicas // have been tried and failed, returns a send error. func (ds *DistSender) sendToReplicas( opts SendOptions, rangeID roachpb.RangeID, replicas ReplicaSlice, args roachpb.BatchRequest, rpcContext *rpc.Context, ) (*roachpb.BatchResponse, error) { if len(replicas) < 1 { return nil, roachpb.NewSendError( fmt.Sprintf("insufficient replicas (%d) to satisfy send request of %d", len(replicas), 1)) } var ambiguousResult bool var haveCommit bool // We only check for committed txns, not aborts because aborts may // be retried without any risk of inconsistencies. if etArg, ok := args.GetArg(roachpb.EndTransaction); ok && etArg.(*roachpb.EndTransactionRequest).Commit { haveCommit = true } done := make(chan BatchCall, len(replicas)) transportFactory := opts.transportFactory if transportFactory == nil { transportFactory = grpcTransportFactory } transport, err := transportFactory(opts, rpcContext, replicas, args) if err != nil { return nil, err } defer transport.Close() if transport.IsExhausted() { return nil, roachpb.NewSendError( fmt.Sprintf("sending to all %d replicas failed", len(replicas))) } // Send the first request. pending := 1 log.VEventf(opts.ctx, 2, "sending RPC for batch: %s", args.Summary()) transport.SendNext(done) // Wait for completions. This loop will retry operations that fail // with errors that reflect per-replica state and may succeed on // other replicas. var sendNextTimer timeutil.Timer defer sendNextTimer.Stop() for { sendNextTimer.Reset(opts.SendNextTimeout) select { case <-sendNextTimer.C: sendNextTimer.Read = true // On successive RPC timeouts, send to additional replicas if available. if !transport.IsExhausted() { log.VEventf(opts.ctx, 2, "timeout, trying next peer") pending++ transport.SendNext(done) } case call := <-done: pending-- err := call.Err if err == nil { if log.V(2) { log.Infof(opts.ctx, "RPC reply: %s", call.Reply) } else if log.V(1) && call.Reply.Error != nil { log.Infof(opts.ctx, "application error: %s", call.Reply.Error) } if call.Reply.Error == nil { return call.Reply, nil } else if !ds.handlePerReplicaError(opts.ctx, transport, rangeID, call.Reply.Error) { // The error received is not specific to this replica, so we // should return it instead of trying other replicas. However, // if we're trying to commit a transaction and there are // still other RPCs outstanding or an ambiguous RPC error // was already received, we must return an ambiguous commit // error instead of returned error. if haveCommit && (pending > 0 || ambiguousResult) { return nil, roachpb.NewAmbiguousResultError() } return call.Reply, nil } // Extract the detail so it can be included in the error // message if this is our last replica. // // TODO(bdarnell): The last error is not necessarily the best // one to return; we may want to remember the "best" error // we've seen (for example, a NotLeaseHolderError conveys more // information than a RangeNotFound). err = call.Reply.Error.GoError() } else { if log.V(1) { log.Warningf(opts.ctx, "RPC error: %s", err) } // All connection errors except for an unavailable node (this // is GRPC's fail-fast error), may mean that the request // succeeded on the remote server, but we were unable to // receive the reply. Set the ambiguous commit flag. // // We retry ambiguous commit batches to avoid returning the // unrecoverable AmbiguousResultError. This is safe because // repeating an already-successfully applied batch is // guaranteed to return either a TransactionReplayError (in // case the replay happens at the original leader), or a // TransactionRetryError (in case the replay happens at a new // leader). If the original attempt merely timed out or was // lost, then the batch will succeed and we can be assured the // commit was applied just once. // // The Unavailable code is used by GRPC to indicate that a // request fails fast and is not sent, so we can be sure there // is no ambiguity on these errors. Note that these are common // if a node is down. // See https://github.com/grpc/grpc-go/blob/52f6504dc290bd928a8139ba94e3ab32ed9a6273/call.go#L182 // See https://github.com/grpc/grpc-go/blob/52f6504dc290bd928a8139ba94e3ab32ed9a6273/stream.go#L158 if haveCommit && grpc.Code(err) != codes.Unavailable { ambiguousResult = true } } // Send to additional replicas if available. if !transport.IsExhausted() { log.VEventf(opts.ctx, 2, "error, trying next peer: %s", err) pending++ transport.SendNext(done) } if pending == 0 { if ambiguousResult { err = roachpb.NewAmbiguousResultError() } else { err = roachpb.NewSendError( fmt.Sprintf("sending to all %d replicas failed; last error: %v", len(replicas), err), ) } if log.V(2) { log.ErrEvent(opts.ctx, err.Error()) } return nil, err } } } }