// Send implements the client.Sender interface. If the call is part // of a transaction, the coordinator will initialize the transaction // if it's not nil but has an empty ID. func (tc *TxnCoordSender) Send(ctx context.Context, call proto.Call) { header := call.Args.Header() tc.maybeBeginTxn(header) header.CmdID = header.GetOrCreateCmdID(tc.clock.PhysicalNow()) // This is the earliest point at which the request has a ClientCmdID and/or // TxnID (if applicable). Begin a Trace which follows this request. trace := tc.tracer.NewTrace(call.Args.Header()) defer trace.Finalize() defer trace.Epoch(fmt.Sprintf("sending %s", call.Method()))() defer func() { if err := call.Reply.Header().GoError(); err != nil { trace.Event(fmt.Sprintf("reply error: %T", err)) } }() ctx = tracer.ToCtx(ctx, trace) // Process batch specially; otherwise, send via wrapped sender. switch args := call.Args.(type) { case *proto.BatchRequest: trace.Event("batch processing") tc.sendBatch(ctx, args, call.Reply.(*proto.BatchResponse)) default: // TODO(tschottdorf): should treat all calls as Batch. After all, that // will be almost all calls. tc.sendOne(ctx, call) } }
// executeCmd interprets the given message as a *roachpb.BatchRequest and sends it // via the local sender. func (n *Node) executeCmd(argsI proto.Message) (proto.Message, error) { ba := argsI.(*roachpb.BatchRequest) var br *roachpb.BatchResponse f := func() { // TODO(tschottdorf) get a hold of the client's ID, add it to the // context before dispatching, and create an ID for tracing the request. trace := n.ctx.Tracer.NewTrace(tracer.Node, ba) defer trace.Finalize() defer trace.Epoch("node")() ctx := tracer.ToCtx((*Node)(n).context(), trace) tStart := time.Now() var pErr *roachpb.Error br, pErr = n.stores.Send(ctx, *ba) if pErr != nil { br = &roachpb.BatchResponse{} trace.Event(fmt.Sprintf("error: %T", pErr.GoError())) } if br.Error != nil { panic(roachpb.ErrorUnexpectedlySet(n.stores, br)) } n.feed.CallComplete(*ba, time.Now().Sub(tStart), pErr) br.Error = pErr } if !n.stopper.RunTask(f) { return nil, util.Errorf("node %d stopped", n.Descriptor.NodeID) } return br, nil }
// executeCmd creates a proto.Call struct and sends it via our local sender. func (n *nodeServer) executeCmd(args proto.Request, reply proto.Response) error { // TODO(tschottdorf) get a hold of the client's ID, add it to the // context before dispatching, and create an ID for tracing the request. header := args.Header() header.CmdID = header.GetOrCreateCmdID(n.ctx.Clock.PhysicalNow()) trace := n.ctx.Tracer.NewTrace(header) defer trace.Finalize() defer trace.Epoch("node")() ctx := tracer.ToCtx((*Node)(n).context(), trace) n.lSender.Send(ctx, proto.Call{Args: args, Reply: reply}) n.feed.CallComplete(args, reply) if err := reply.Header().GoError(); err != nil { trace.Event(fmt.Sprintf("error: %T", err)) } return nil }
// heartbeatLoop periodically sends a HeartbeatTxn RPC to an extant // transaction, stopping in the event the transaction is aborted or // committed after attempting to resolve the intents. When the // heartbeat stops, the transaction is unregistered from the // coordinator, func (tc *TxnCoordSender) heartbeatLoop(id string) { var tickChan <-chan time.Time { ticker := time.NewTicker(tc.heartbeatInterval) tickChan = ticker.C defer ticker.Stop() } defer func() { tc.Lock() tc.unregisterTxnLocked(id) tc.Unlock() }() var closer <-chan struct{} var trace *tracer.Trace { tc.Lock() txnMeta := tc.txns[id] // do not leak to outer scope closer = txnMeta.txnEnd trace = tc.tracer.NewTrace(tracer.Coord, &txnMeta.txn) defer trace.Finalize() tc.Unlock() } if closer == nil { // Avoid race in which a Txn is cleaned up before the heartbeat // goroutine gets a chance to start. return } ctx := tracer.ToCtx(context.Background(), trace) // Loop with ticker for periodic heartbeats. for { select { case <-tickChan: if !tc.heartbeat(id, trace, ctx) { return } case <-closer: // Transaction finished normally. return case <-tc.stopper.ShouldDrain(): return } } }
// executeCmd interprets the given message as a *roachpb.BatchRequest and sends it // via the local sender. func (n *Node) executeCmd(argsI proto.Message) (proto.Message, error) { ba := argsI.(*roachpb.BatchRequest) // TODO(tschottdorf) get a hold of the client's ID, add it to the // context before dispatching, and create an ID for tracing the request. trace := n.ctx.Tracer.NewTrace(tracer.Node, ba) defer trace.Finalize() defer trace.Epoch("node")() ctx := tracer.ToCtx((*Node)(n).context(), trace) br, pErr := n.stores.Send(ctx, *ba) if pErr != nil { br = &roachpb.BatchResponse{} trace.Event(fmt.Sprintf("error: %T", pErr.GoError())) } if br.Error != nil { panic(roachpb.ErrorUnexpectedlySet(n.stores, br)) } n.feed.CallComplete(*ba, pErr) br.Error = pErr return br, nil }
// Send implements the client.Sender interface. If the call is part // of a transaction, the coordinator will initialize the transaction // if it's not nil but has an empty ID. func (tc *TxnCoordSender) Send(ctx context.Context, call proto.Call) { header := call.Args.Header() tc.maybeBeginTxn(header) header.CmdID = header.GetOrCreateCmdID(tc.clock.PhysicalNow()) // This is the earliest point at which the request has a ClientCmdID and/or // TxnID (if applicable). Begin a Trace which follows this request. trace := tc.tracer.NewTrace(call.Args.Header()) defer trace.Finalize() defer trace.Epoch(fmt.Sprintf("sending %s", call.Method()))() defer func() { if err := call.Reply.Header().GoError(); err != nil { trace.Event(fmt.Sprintf("reply error: %T", err)) } }() ctx = tracer.ToCtx(ctx, trace) // Process batch specially; otherwise, send via wrapped sender. switch args := call.Args.(type) { case *proto.InternalBatchRequest: trace.Event("batch processing") tc.sendBatch(ctx, args, call.Reply.(*proto.InternalBatchResponse)) case *proto.BatchRequest: // Convert the batch request to internal-batch request. internalArgs := &proto.InternalBatchRequest{RequestHeader: args.RequestHeader} internalReply := &proto.InternalBatchResponse{} for i := range args.Requests { internalArgs.Add(args.Requests[i].GetValue().(proto.Request)) } tc.sendBatch(ctx, internalArgs, internalReply) reply := call.Reply.(*proto.BatchResponse) reply.ResponseHeader = internalReply.ResponseHeader // Convert from internal-batch response to batch response. for i := range internalReply.Responses { reply.Add(internalReply.Responses[i].GetValue().(proto.Response)) } default: tc.sendOne(ctx, call) } }
// executeCmd creates a proto.Call struct and sends it via our local sender. func (n *Node) executeCmd(argsI gogoproto.Message) (gogoproto.Message, error) { args := argsI.(proto.Request) // TODO(tschottdorf) get a hold of the client's ID, add it to the // context before dispatching, and create an ID for tracing the request. header := args.Header() header.CmdID = header.GetOrCreateCmdID(n.ctx.Clock.PhysicalNow()) trace := n.ctx.Tracer.NewTrace(header) defer trace.Finalize() defer trace.Epoch("node")() ctx := tracer.ToCtx((*Node)(n).context(), trace) ba, unwrap := client.MaybeWrap(args) br, pErr := n.lSender.Send(ctx, *ba) if pErr != nil { br = &proto.BatchResponse{} trace.Event(fmt.Sprintf("error: %T", pErr.GoError())) } if br.Error != nil { panic(proto.ErrorUnexpectedlySet(n.lSender, br)) } br.Error = pErr n.feed.CallComplete(ba, br) return unwrap(br), nil }
// Send implements the batch.Sender interface. If the request is part of a // transaction, the TxnCoordSender adds the transaction to a map of active // transactions and begins heartbeating it. Every subsequent request for the // same transaction updates the lastUpdate timestamp to prevent live // transactions from being considered abandoned and garbage collected. // Read/write mutating requests have their key or key range added to the // transaction's interval tree of key ranges for eventual cleanup via resolved // write intents; they're tagged to an outgoing EndTransaction request, with // the receiving replica in charge of resolving them. func (tc *TxnCoordSender) Send(ctx context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { if err := tc.maybeBeginTxn(&ba); err != nil { return nil, roachpb.NewError(err) } ba.CmdID = ba.GetOrCreateCmdID(tc.clock.PhysicalNow()) var startNS int64 // This is the earliest point at which the request has a ClientCmdID and/or // TxnID (if applicable). Begin a Trace which follows this request. trace := tc.tracer.NewTrace(tracer.Coord, &ba) defer trace.Finalize() defer trace.Epoch("sending batch")() ctx = tracer.ToCtx(ctx, trace) var id string // optional transaction ID if ba.Txn != nil { // If this request is part of a transaction... id = string(ba.Txn.ID) // Verify that if this Transaction is not read-only, we have it on // file. If not, refuse writes - the client must have issued a write on // another coordinator previously. if ba.Txn.Writing && ba.IsTransactionWrite() { tc.Lock() _, ok := tc.txns[id] tc.Unlock() if !ok { return nil, roachpb.NewError(util.Errorf("transaction must not write on multiple coordinators")) } } // Set the timestamp to the original timestamp for read-only // commands and to the transaction timestamp for read/write // commands. if ba.IsReadOnly() { ba.Timestamp = ba.Txn.OrigTimestamp } else { ba.Timestamp = ba.Txn.Timestamp } if rArgs, ok := ba.GetArg(roachpb.EndTransaction); ok { et := rArgs.(*roachpb.EndTransactionRequest) if len(et.Key) != 0 { return nil, roachpb.NewError(util.Errorf("EndTransaction must not have a Key set")) } et.Key = ba.Txn.Key // Remember when EndTransaction started in case we want to // be linearizable. startNS = tc.clock.PhysicalNow() if len(et.Intents) > 0 { // TODO(tschottdorf): it may be useful to allow this later. // That would be part of a possible plan to allow txns which // write on multiple coordinators. return nil, roachpb.NewError(util.Errorf("client must not pass intents to EndTransaction")) } tc.Lock() txnMeta, metaOK := tc.txns[id] if id != "" && metaOK { et.Intents = txnMeta.intents() } tc.Unlock() if intents := ba.GetIntents(); len(intents) > 0 { // Writes in Batch, so EndTransaction is fine. Should add // outstanding intents to EndTransaction, though. // TODO(tschottdorf): possible issues when the batch fails, // but the intents have been added anyways. // TODO(tschottdorf): some of these intents may be covered // by others, for example {[a,b), a}). This can lead to // some extra requests when those are non-local to the txn // record. But it doesn't seem worth optimizing now. et.Intents = append(et.Intents, intents...) } else if !metaOK { // If we don't have the transaction, then this must be a retry // by the client. We can no longer reconstruct a correct // request so we must fail. // // TODO(bdarnell): if we had a GetTransactionStatus API then // we could lookup the transaction and return either nil or // TransactionAbortedError instead of this ambivalent error. return nil, roachpb.NewError(util.Errorf("transaction is already committed or aborted")) } if len(et.Intents) == 0 { // If there aren't any intents, then there's factually no // transaction to end. Read-only txns have all of their state in // the client. return nil, roachpb.NewError(util.Errorf("cannot commit a read-only transaction")) } if log.V(1) { for _, intent := range et.Intents { trace.Event(fmt.Sprintf("intent: [%s,%s)", intent.Key, intent.EndKey)) } } } } // Send the command through wrapped sender, taking appropriate measures // on error. var br *roachpb.BatchResponse { var pErr *roachpb.Error br, pErr = tc.wrapped.Send(ctx, ba) if _, ok := pErr.GoError().(*roachpb.OpRequiresTxnError); ok { br, pErr = tc.resendWithTxn(ba) } if pErr := tc.updateState(ctx, ba, br, pErr); pErr != nil { return nil, pErr } } if br.Txn == nil { return br, nil } if _, ok := ba.GetArg(roachpb.EndTransaction); !ok { return br, nil } // If the --linearizable flag is set, we want to make sure that // all the clocks in the system are past the commit timestamp // of the transaction. This is guaranteed if either // - the commit timestamp is MaxOffset behind startNS // - MaxOffset ns were spent in this function // when returning to the client. Below we choose the option // that involves less waiting, which is likely the first one // unless a transaction commits with an odd timestamp. if tsNS := br.Txn.Timestamp.WallTime; startNS > tsNS { startNS = tsNS } sleepNS := tc.clock.MaxOffset() - time.Duration(tc.clock.PhysicalNow()-startNS) if tc.linearizable && sleepNS > 0 { defer func() { if log.V(1) { log.Infof("%v: waiting %s on EndTransaction for linearizability", br.Txn.Short(), util.TruncateDuration(sleepNS, time.Millisecond)) } time.Sleep(sleepNS) }() } if br.Txn.Status != roachpb.PENDING { tc.cleanupTxn(trace, *br.Txn) } return br, nil }
// heartbeat periodically sends a HeartbeatTxn RPC to an extant // transaction, stopping in the event the transaction is aborted or // committed after attempting to resolve the intents. When the // heartbeat stops, the transaction is unregistered from the // coordinator, func (tc *TxnCoordSender) heartbeat(id string) { var tickChan <-chan time.Time { ticker := time.NewTicker(tc.heartbeatInterval) tickChan = ticker.C defer ticker.Stop() } defer tc.unregisterTxn(id) var closer <-chan struct{} var trace *tracer.Trace { tc.Lock() txnMeta := tc.txns[id] // do not leak to outer scope closer = txnMeta.txnEnd trace = tc.tracer.NewTrace(&txnMeta.txn) tc.Unlock() } if closer == nil { // Avoid race in which a Txn is cleaned up before the heartbeat // goroutine gets a chance to start. return } ctx := tracer.ToCtx(context.Background(), trace) defer trace.Finalize() // Loop with ticker for periodic heartbeats. for { select { case <-tickChan: tc.Lock() proceed := true txnMeta := tc.txns[id] // Before we send a heartbeat, determine whether this transaction // should be considered abandoned. If so, exit heartbeat. if txnMeta.hasClientAbandonedCoord(tc.clock.PhysicalNow()) { // TODO(tschottdorf): should we be more proactive here? // The client might be continuing the transaction // through another coordinator, but in the most likely // case it's just gone and the open transaction record // could block concurrent operations. if log.V(1) { log.Infof("transaction %s abandoned; stopping heartbeat", txnMeta.txn) } proceed = false } // txnMeta.txn is possibly replaced concurrently, // so grab a copy before unlocking. txn := txnMeta.txn tc.Unlock() if !proceed { return } request := &proto.HeartbeatTxnRequest{ RequestHeader: proto.RequestHeader{ Key: txn.Key, User: security.RootUser, Txn: &txn, }, } request.Header().Timestamp = tc.clock.Now() reply := &proto.HeartbeatTxnResponse{} call := proto.Call{ Args: request, Reply: reply, } epochEnds := trace.Epoch("heartbeat") tc.wrapped.Send(ctx, call) epochEnds() // If the transaction is not in pending state, then we can stop // the heartbeat. It's either aborted or committed, and we resolve // write intents accordingly. if reply.GoError() != nil { log.Warningf("heartbeat to %s failed: %s", txn, reply.GoError()) } else if reply.Txn != nil && reply.Txn.Status != proto.PENDING { // Signal cleanup. Doesn't do much but stop this goroutine, but // let's be future-proof. tc.cleanupTxn(trace, *reply.Txn) return } case <-closer: // Transaction finished normally. return } } }
// resolveIntents resolves the given intents. For those which are local to the // range, we submit directly to the range-local Raft instance; the call returns // as soon as all resolve commands have been **proposed** (not executed). This // ensures that if a waiting client retries immediately after conflict // resolution, it will not hit the same intents again. All non-local intents // are resolved asynchronously in a batch. // TODO(tschottdorf): once Txn records have a list of possibly open intents, // resolveIntents should send an RPC to update the transaction(s) as well (for // those intents with non-pending Txns). func (r *Replica) resolveIntents(ctx context.Context, intents []proto.Intent) { trace := tracer.FromCtx(ctx) tracer.ToCtx(ctx, nil) // we're doing async stuff below; those need new traces trace.Event("resolving intents [async]") var wg sync.WaitGroup bArgs := &proto.BatchRequest{} bArgs.User = security.RootUser for i := range intents { intent := intents[i] // avoids a race in `i, intent := range ...` var resolveArgs proto.Request var local bool // whether this intent lives on this Range { header := proto.RequestHeader{ // Use the pushee's timestamp, which might be lower than the // pusher's request timestamp. No need to push the intent higher // than the pushee's txn! Timestamp: intent.Txn.Timestamp, Key: intent.Key, EndKey: intent.EndKey, User: security.RootUser, Txn: &intent.Txn, } if len(intent.EndKey) == 0 { resolveArgs = &proto.ResolveIntentRequest{RequestHeader: header} local = r.ContainsKey(intent.Key) } else { resolveArgs = &proto.ResolveIntentRangeRequest{RequestHeader: header} local = r.ContainsKeyRange(intent.Key, intent.EndKey) } } // If the intent isn't (completely) local, we'll need to send an external request. // We'll batch them all up and send at the end. if !local { bArgs.Add(resolveArgs) continue } // If it is local, it goes directly into Raft. // TODO(tschottdorf): this may be premature optimization. Consider just // treating everything as an external request. This means having to // wait for complete execution of the command (whereas now we just wait // for proposition) and some more overhead sending things around. wg.Add(1) action := func() { // Trace this under the ID of the intent owner. ctx := tracer.ToCtx(ctx, r.rm.Tracer().NewTrace(resolveArgs.Header().Txn)) if _, err := r.addWriteCmd(ctx, resolveArgs, &wg); err != nil && log.V(1) { log.Warningc(ctx, "resolve for key %s failed: %s", intent.Key, err) } } if !r.rm.Stopper().RunAsyncTask(action) { // Still run the task. Our caller already has a task and going async // here again is merely for performance, but some intents need to // be resolved because they might block other tasks. See #1684. // Note that handleSkippedIntents has a TODO in case #1684 comes // back. action() } } // Resolve all of the intents which aren't local to the Range. This is a // no-op if all are local. b := &client.Batch{} b.InternalAddCall(proto.Call{Args: bArgs, Reply: &proto.BatchResponse{}}) action := func() { // TODO(tschottdorf): no tracing here yet. Probably useful at some point, // but needs a) the corresponding interface and b) facilities for tracing // multiple tracees at the same time (batch full of possibly individual // txns). if err := r.rm.DB().Run(b); err != nil { if log.V(1) { log.Infoc(ctx, "%s", err) } } } if !r.rm.Stopper().RunAsyncTask(action) { // As with local intents, try async to not keep the caller waiting, but // when draining just go ahead and do it synchronously. See #1684. action() } // Wait until all the local `ResolveIntent`s have been submitted to raft. // No-op if all were external. wg.Wait() }
// heartbeat periodically sends an InternalHeartbeatTxn RPC to an // extant transaction, stopping in the event the transaction is // aborted or committed or if the TxnCoordSender is closed. func (tc *TxnCoordSender) heartbeat(id string) { tc.stopper.RunWorker(func() { ticker := time.NewTicker(tc.heartbeatInterval) defer ticker.Stop() tc.Lock() var closer chan struct{} if txnMeta, ok := tc.txns[id]; ok { closer = txnMeta.txnEnd } tc.Unlock() if closer == nil { return } // Loop with ticker for periodic heartbeats. for { select { case <-ticker.C: tc.Lock() var txn proto.Transaction _, proceed := tc.txns[id] if proceed { txnMeta := tc.txns[id] // assign only here for local scope // Before we send a heartbeat, determine whether this transaction // should be considered abandoned. If so, exit heartbeat. if txnMeta.hasClientAbandonedCoord(tc.clock.PhysicalNow()) { tc.unregisterTxnLocked(txnMeta) // TODO(tschottdorf): should we be more proactive here? // The client might be continuing the transaction // through another coordinator, but in the most likely // case it's just gone and the open transaction record // could block concurrent operations. if log.V(1) { log.Infof("transaction %s abandoned; stopping heartbeat", txnMeta.txn) } proceed = false } // txnMeta.txn is possibly replaced concurrently, // so grab a copy. txn = txnMeta.txn } tc.Unlock() if !proceed { return } request := &proto.InternalHeartbeatTxnRequest{ RequestHeader: proto.RequestHeader{ Key: txn.Key, User: security.RootUser, Txn: &txn, }, } request.Header().Timestamp = tc.clock.Now() reply := &proto.InternalHeartbeatTxnResponse{} call := proto.Call{ Args: request, Reply: reply, } tc.stopper.RunTask(func() { // Each heartbeat gets its own Trace. trace := tc.tracer.NewTrace(&txn) ctx := tracer.ToCtx(context.Background(), trace) epochEnds := trace.Epoch("heartbeat") tc.wrapped.Send(ctx, call) epochEnds() // If the transaction is not in pending state, then we can stop // the heartbeat. It's either aborted or committed, and we resolve // write intents accordingly. if reply.GoError() != nil { log.Warningf("heartbeat to %s failed: %s", txn, reply.GoError()) } else if reply.Txn != nil && reply.Txn.Status != proto.PENDING { tc.cleanupTxn(trace, *reply.Txn, nil) proceed = false } trace.Finalize() }) if !proceed { return } case <-closer: // Transaction finished. return case <-tc.stopper.ShouldStop(): // System shutdown. return } } }) }
// close sends resolve intent commands for all key ranges this // transaction has covered, clears the keys cache and closes the // metadata heartbeat. Any keys listed in the resolved slice have // already been resolved and do not receive resolve intent commands. func (tm *txnMetadata) close(trace *tracer.Trace, txn *proto.Transaction, resolved []proto.Key, sender client.Sender, stopper *stop.Stopper) { close(tm.txnEnd) // stop heartbeat trace.Event("coordinator stops") if tm.keys.Len() > 0 { if log.V(2) { log.Infof("cleaning up %d intent(s) for transaction %s", tm.keys.Len(), txn) } } // TODO(tschottdorf): Should create a Batch here. for _, o := range tm.keys.GetOverlaps(proto.KeyMin, proto.KeyMax) { // If the op was range based, end key != start key: resolve a range. var call proto.Call key := o.Key.Start().(proto.Key) endKey := o.Key.End().(proto.Key) if !key.Next().Equal(endKey) { call.Args = &proto.InternalResolveIntentRangeRequest{ RequestHeader: proto.RequestHeader{ Timestamp: txn.Timestamp, Key: key, EndKey: endKey, User: security.RootUser, Txn: txn, }, } call.Reply = &proto.InternalResolveIntentRangeResponse{} } else { // Check if the key has already been resolved; skip if yes. found := false for _, k := range resolved { if key.Equal(k) { found = true } } if found { continue } call.Args = &proto.InternalResolveIntentRequest{ RequestHeader: proto.RequestHeader{ Timestamp: txn.Timestamp, Key: key, User: security.RootUser, Txn: txn, }, } call.Reply = &proto.InternalResolveIntentResponse{} } // We don't care about the reply channel; these are best // effort. We simply fire and forget, each in its own goroutine. ctx := tracer.ToCtx(context.Background(), trace.Fork()) stopper.RunAsyncTask(func() { if log.V(2) { log.Infof("cleaning up intent %q for txn %s", call.Args.Header().Key, txn) } sender.Send(ctx, call) if call.Reply.Header().Error != nil { log.Warningf("failed to cleanup %q intent: %s", call.Args.Header().Key, call.Reply.Header().GoError()) } }) } tm.keys.Clear() }
// resolve sends resolve intent commands for all key ranges this transaction // has covered. Any keys listed in the resolved slice have already been // resolved and are skipped. func (tm *txnMetadata) resolve(trace *tracer.Trace, resolved []proto.Key, sender client.Sender) { txn := &tm.txn if tm.keys.Len() > 0 { if log.V(2) { log.Infof("cleaning up %d intent(s) for transaction %s", tm.keys.Len(), txn) } } // TODO(tschottdorf): Should create a Batch here. However, we're resolving // intents and if those are on meta records, there may be a certain order // in which they need to be resolved so that they can get routed to the // correct range. Since a batch runs its commands one by one and we don't // know the correct order, we prefer to fire them off in parallel. var wg sync.WaitGroup for _, o := range tm.keys.GetOverlaps(proto.KeyMin, proto.KeyMax) { // If the op was range based, end key != start key: resolve a range. var call proto.Call key := o.Key.Start().(proto.Key) endKey := o.Key.End().(proto.Key) if !key.Next().Equal(endKey) { call.Args = &proto.InternalResolveIntentRangeRequest{ RequestHeader: proto.RequestHeader{ Timestamp: txn.Timestamp, Key: key, EndKey: endKey, User: security.RootUser, Txn: txn, }, } call.Reply = &proto.InternalResolveIntentRangeResponse{} } else { // Check if the key has already been resolved; skip if yes. found := false for _, k := range resolved { if key.Equal(k) { if log.V(2) { log.Warningf("skipping previously resolved intent at %q", k) } found = true } } if found { continue } call.Args = &proto.InternalResolveIntentRequest{ RequestHeader: proto.RequestHeader{ Timestamp: txn.Timestamp, Key: key, User: security.RootUser, Txn: txn, }, } call.Reply = &proto.InternalResolveIntentResponse{} } ctx := tracer.ToCtx(context.Background(), trace.Fork()) if log.V(2) { log.Infof("cleaning up intent %q for txn %s", call.Args.Header().Key, txn) } // Each operation gets their own goroutine. We only want to return to // the caller after the operations have finished. wg.Add(1) go func() { sender.Send(ctx, call) wg.Done() if call.Reply.Header().Error != nil { log.Warningf("failed to cleanup %q intent: %s", call.Args.Header().Key, call.Reply.Header().GoError()) } }() } defer trace.Epoch("waiting for intent resolution")() wg.Wait() tm.keys.Clear() }