// MaybeWrap wraps the given argument in a batch, unless it is already one.
func maybeWrap(args proto.Request) (*proto.BatchRequest, func(*proto.BatchResponse) proto.Response) {
if ba, ok := args.(*proto.BatchRequest); ok {
return ba, func(br *proto.BatchResponse) proto.Response { return br }
}
ba := &proto.BatchRequest{}
ba.RequestHeader = *(gogoproto.Clone(args.Header()).(*proto.RequestHeader))
ba.Add(args)
return ba, func(br *proto.BatchResponse) proto.Response {
var unwrappedReply proto.Response
if len(br.Responses) == 0 {
unwrappedReply = args.CreateReply()
} else {
unwrappedReply = br.Responses[0].GetInner()
}
// The ReplyTxn is propagated from one response to the next request,
// and we adopt the mechanism that whenever the Txn changes, it needs
// to be set in the reply, for example to ratched up the transaction
// timestamp on writes when necessary.
// This is internally necessary to sequentially execute the batch,
// so it makes some sense to take the burden of updating the Txn
// from TxnCoordSender - it will only need to act on retries/aborts
// in the future.
unwrappedReply.Header().Txn = br.Txn
if unwrappedReply.Header().Error == nil {
unwrappedReply.Header().Error = br.Error
}
return unwrappedReply
}
}
// sendRPC sends one or more RPCs to replicas from the supplied proto.Replica
// slice. First, replicas which have gossiped addresses are corralled (and
// rearranged depending on proximity and whether the request needs to go to a
// leader) and then sent via rpc.Send, with requirement that one RPC to a
// server must succeed. Returns an RPC error if the request could not be sent.
// Note that the reply may contain a higher level error and must be checked in
// addition to the RPC error.
func (ds *DistSender) sendRPC(raftID proto.RaftID, replicas replicaSlice, order rpc.OrderingPolicy,
args proto.Request, reply proto.Response) error {
if len(replicas) == 0 {
return util.Errorf("%s: replicas set is empty", args.Method())
}
// Build a slice of replica addresses (if gossiped).
var addrs []net.Addr
replicaMap := map[string]*proto.Replica{}
for i := range replicas {
nd := &replicas[i].NodeDesc
addr := util.MakeUnresolvedAddr(nd.Address.Network, nd.Address.Address)
addrs = append(addrs, addr)
replicaMap[addr.String()] = &replicas[i].Replica
}
if len(addrs) == 0 {
return noNodeAddrsAvailError{}
}
// TODO(pmattis): This needs to be tested. If it isn't set we'll
// still route the request appropriately by key, but won't receive
// RangeNotFoundErrors.
args.Header().RaftID = raftID
// Set RPC opts with stipulation that one of N RPCs must succeed.
rpcOpts := rpc.Options{
N: 1,
Ordering: order,
SendNextTimeout: defaultSendNextTimeout,
Timeout: defaultRPCTimeout,
}
// getArgs clones the arguments on demand for all but the first replica.
firstArgs := true
getArgs := func(addr net.Addr) interface{} {
var a proto.Request
// Use the supplied args proto if this is our first address.
if firstArgs {
firstArgs = false
a = args
} else {
// Otherwise, copy the args value and set the replica in the header.
a = gogoproto.Clone(args).(proto.Request)
}
a.Header().Replica = *replicaMap[addr.String()]
return a
}
// RPCs are sent asynchronously and there is no synchronized access to
// the reply object, so we don't pass itself to rpcSend.
// Otherwise there maybe a race case:
// If the RPC call times out using our original reply object,
// we must not use it any more; the rpc call might still return
// and just write to it at any time.
// args.CreateReply() should be cheaper than gogoproto.Clone which use reflect.
getReply := func() interface{} {
return args.CreateReply()
}
replies, err := ds.rpcSend(rpcOpts, "Node."+args.Method().String(),
addrs, getArgs, getReply, ds.gossip.RPCContext)
if err == nil {
// Set content of replies[0] back to reply
dst := reflect.ValueOf(reply).Elem()
dst.Set(reflect.ValueOf(replies[0]).Elem())
}
return err
}
// applyRaftCommandInBatch executes the command in a batch engine and
// returns the batch containing the results. The caller is responsible
// for committing the batch, even on error.
func (r *Range) applyRaftCommandInBatch(ctx context.Context, index uint64, originNode proto.RaftNodeID,
args proto.Request, ms *engine.MVCCStats) (engine.Engine, proto.Response, error) {
// Create a new batch for the command to ensure all or nothing semantics.
batch := r.rm.Engine().NewBatch()
if lease := r.getLease(); args.Method() != proto.InternalLeaderLease &&
(!lease.OwnedBy(originNode) || !lease.Covers(args.Header().Timestamp)) {
// Verify the leader lease is held, unless this command is trying to
// obtain it. Any other Raft command has had the leader lease held
// by the replica at proposal time, but this may no longer be the case.
// Corruption aside, the most likely reason is a leadership change (the
// most recent leader assumes responsibility for all past timestamps as
// well). In that case, it's not valid to go ahead with the execution:
// Writes must be aware of the last time the mutated key was read, and
// since reads are served locally by the lease holder without going
// through Raft, a read which was not taken into account may have been
// served. Hence, we must retry at the current leader.
//
// It's crucial that we don't update the response cache for the error
// returned below since the request is going to be retried with the
// same ClientCmdID and would get the distributed sender stuck in an
// infinite loop, retrieving a stale NotLeaderError over and over
// again, even when proposing at the correct replica.
return batch, nil, r.newNotLeaderError(lease, originNode)
}
// Check the response cache to ensure idempotency.
if proto.IsWrite(args) {
if reply, err := r.respCache.GetResponse(batch, args.Header().CmdID); err != nil {
// Any error encountered while fetching the response cache entry means corruption.
return batch, reply, newReplicaCorruptionError(util.Errorf("could not read from response cache"), err)
} else if reply != nil {
if log.V(1) {
log.Infoc(ctx, "found response cache entry for %+v", args.Header().CmdID)
}
// TODO(tamird): move this into the response cache itself
defer func() { reply.Header().Error = nil }()
// We successfully read from the response cache, so return whatever error
// was present in the cached entry (if any).
return batch, reply, reply.Header().GoError()
}
}
// Execute the command.
reply, intents, rErr := r.executeCmd(batch, ms, args)
// Regardless of error, add result to the response cache if this is
// a write method. This must be done as part of the execution of
// raft commands so that every replica maintains the same responses
// to continue request idempotence, even if leadership changes.
if proto.IsWrite(args) {
if rErr == nil {
// If command was successful, flush the MVCC stats to the batch.
if err := r.stats.MergeMVCCStats(batch, ms, args.Header().Timestamp.WallTime); err != nil {
log.Fatalc(ctx, "setting mvcc stats in a batch should never fail: %s", err)
}
} else {
// Otherwise, reset the batch to clear out partial execution and
// prepare for the failed response cache entry.
batch.Close()
batch = r.rm.Engine().NewBatch()
}
// TODO(tamird): move this into the response cache itself
if reply == nil {
reply = args.CreateReply()
}
if reply.Header().Error != nil {
panic("the world is on fire")
}
reply.Header().SetGoError(rErr)
if err := r.respCache.PutResponse(batch, args.Header().CmdID, reply); err != nil {
log.Fatalc(ctx, "putting a response cache entry in a batch should never fail: %s", err)
}
reply.Header().Error = nil
}
// If the execution of the command wasn't successful, stop here.
if rErr != nil {
return batch, reply, rErr
}
// On success and only on the replica on which this command originated,
// resolve skipped intents asynchronously.
if originNode == r.rm.RaftNodeID() {
r.handleSkippedIntents(args, intents)
}
return batch, reply, nil
}
// sendRPC sends one or more RPCs to replicas from the supplied proto.Replica
// slice. First, replicas which have gossiped addresses are corralled (and
// rearranged depending on proximity and whether the request needs to go to a
// leader) and then sent via rpc.Send, with requirement that one RPC to a
// server must succeed. Returns an RPC error if the request could not be sent.
// Note that the reply may contain a higher level error and must be checked in
// addition to the RPC error.
func (ds *DistSender) sendRPC(trace *tracer.Trace, rangeID proto.RangeID, replicas replicaSlice, order rpc.OrderingPolicy,
args proto.Request) (proto.Response, error) {
if len(replicas) == 0 {
// TODO(tschottdorf): this gets in the way of some tests. Consider
// refactoring so that gossip is mocked out more easily. Provisional
// code. return nil, util.Errorf("%s: replicas set is empty",
// args.Method())
}
// Build a slice of replica addresses (if gossiped).
var addrs []net.Addr
replicaMap := map[string]*proto.Replica{}
for i := range replicas {
addr := replicas[i].NodeDesc.Address
addrs = append(addrs, addr)
replicaMap[addr.String()] = &replicas[i].Replica
}
if len(addrs) == 0 {
// TODO(tschottdorf): see len(replicas) above.
// return nil, noNodeAddrsAvailError{}
}
// TODO(pmattis): This needs to be tested. If it isn't set we'll
// still route the request appropriately by key, but won't receive
// RangeNotFoundErrors.
args.Header().RangeID = rangeID
// Set RPC opts with stipulation that one of N RPCs must succeed.
rpcOpts := rpc.Options{
N: 1,
Ordering: order,
SendNextTimeout: defaultSendNextTimeout,
Timeout: defaultRPCTimeout,
Trace: trace,
}
// getArgs clones the arguments on demand for all but the first replica.
firstArgs := true
getArgs := func(addr net.Addr) gogoproto.Message {
var a proto.Request
// Use the supplied args proto if this is our first address.
if firstArgs {
firstArgs = false
a = args
} else {
// Otherwise, copy the args value and set the replica in the header.
a = gogoproto.Clone(args).(proto.Request)
}
if addr != nil {
// TODO(tschottdorf): see len(replicas) above.
a.Header().Replica = *replicaMap[addr.String()]
}
return a
}
// RPCs are sent asynchronously and there is no synchronized access to
// the reply object, so we don't pass itself to RPCSend.
// Otherwise there maybe a race case:
// If the RPC call times out using our original reply object,
// we must not use it any more; the rpc call might still return
// and just write to it at any time.
// args.CreateReply() should be cheaper than gogoproto.Clone which use reflect.
getReply := func() gogoproto.Message {
return args.CreateReply()
}
replies, err := ds.rpcSend(rpcOpts, "Node."+args.Method().String(),
addrs, getArgs, getReply, ds.gossip.RPCContext)
if err != nil {
return nil, err
}
return replies[0].(proto.Response), nil
}
