// sendOne invokes the specified RPC on the supplied client when the
// client is ready. On success, the reply is sent on the channel;
// otherwise an error is sent.
//
// Do not call directly, but instead use sendOneFn. Tests mock out this method
// via sendOneFn in order to test various error cases.
func sendOne(client *batchClient, timeout time.Duration,
context *rpc.Context, trace opentracing.Span, done chan *netrpc.Call) {
const method = "Node.Batch"
addr := client.RemoteAddr()
args := &client.args
args.Replica = client.replica.ReplicaDescriptor
if log.V(2) {
log.Infof("sending request to %s: %+v", addr, args)
}
trace.LogEvent(fmt.Sprintf("sending to %s", addr))
if enableLocalCalls && context.LocalServer != nil && addr.String() == context.LocalAddr {
if context.LocalServer.LocalCall(method, args, done) {
return
}
}
reply := &roachpb.BatchResponse{}
// Don't bother firing off a goroutine in the common case where a client
// is already healthy.
select {
case <-client.Healthy():
client.Go(method, args, reply, done)
return
default:
}
go func() {
var timeoutChan <-chan time.Time
if timeout != 0 {
timeoutChan = time.After(timeout)
}
select {
case <-client.Healthy():
client.Go(method, args, reply, done)
case <-client.Closed:
done <- &netrpc.Call{Error: newRPCError(
util.Errorf("rpc to %s failed as client connection was closed", method))}
case <-timeoutChan:
done <- &netrpc.Call{Error: newRPCError(
util.Errorf("rpc to %s: client not ready after %s", method, timeout))}
}
}()
}
// heartbeatLoop periodically sends a HeartbeatTxn RPC to an extant
// transaction, stopping in the event the transaction is aborted or
// committed after attempting to resolve the intents. When the
// heartbeat stops, the transaction is unregistered from the
// coordinator,
func (tc *TxnCoordSender) heartbeatLoop(txnID uuid.UUID) {
var tickChan <-chan time.Time
{
ticker := time.NewTicker(tc.heartbeatInterval)
tickChan = ticker.C
defer ticker.Stop()
}
defer func() {
tc.Lock()
duration, restarts, status := tc.unregisterTxnLocked(txnID)
tc.Unlock()
tc.updateStats(duration, int64(restarts), status)
}()
var closer <-chan struct{}
var sp opentracing.Span
{
tc.Lock()
txnMeta := tc.txns[txnID] // do not leak to outer scope
closer = txnMeta.txnEnd
// TODO(tschottdorf): this should join to the trace of the request
// which starts this goroutine.
sp = tc.tracer.StartSpan(opHeartbeatLoop)
defer sp.Finish()
tc.Unlock()
}
if closer == nil {
// Avoid race in which a Txn is cleaned up before the heartbeat
// goroutine gets a chance to start.
return
}
ctx := opentracing.ContextWithSpan(context.Background(), sp)
// Loop with ticker for periodic heartbeats.
for {
select {
case <-tickChan:
if !tc.heartbeat(txnID, sp, ctx) {
return
}
case <-closer:
// Transaction finished normally.
return
case <-tc.stopper.ShouldDrain():
return
}
}
}
// cleanupTxn is called when a transaction ends. The transaction record is
// updated and the heartbeat goroutine signaled to clean up the transaction
// gracefully.
func (tc *TxnCoordSender) cleanupTxn(trace opentracing.Span, txn roachpb.Transaction) {
trace.LogEvent("coordinator stops")
tc.Lock()
defer tc.Unlock()
txnMeta, ok := tc.txns[*txn.ID]
// The heartbeat might've already removed the record.
if !ok {
return
}
// The supplied txn may be newer than the one in txnMeta, which is relevant
// for stats.
txnMeta.txn = txn
// Trigger heartbeat shutdown.
close(txnMeta.txnEnd)
txnMeta.txnEnd = nil
}
// sendSingleRange gathers and rearranges the replicas, and makes an RPC call.
func (ds *DistSender) sendSingleRange(trace opentracing.Span, ba roachpb.BatchRequest, desc *roachpb.RangeDescriptor) (*roachpb.BatchResponse, *roachpb.Error) {
trace.LogEvent(fmt.Sprintf("sending RPC to [%s, %s)", desc.StartKey, desc.EndKey))
leader := ds.leaderCache.Lookup(roachpb.RangeID(desc.RangeID))
// Try to send the call.
replicas := newReplicaSlice(ds.gossip, desc)
// Rearrange the replicas so that those replicas with long common
// prefix of attributes end up first. If there's no prefix, this is a
// no-op.
order := ds.optimizeReplicaOrder(replicas)
// If this request needs to go to a leader and we know who that is, move
// it to the front.
if !(ba.IsReadOnly() && ba.ReadConsistency == roachpb.INCONSISTENT) &&
leader.StoreID > 0 {
if i := replicas.FindReplica(leader.StoreID); i >= 0 {
replicas.MoveToFront(i)
order = orderStable
}
}
// Increase the sequence counter in the per-range loop (not
// outside) since we might hit the same range twice by
// accident. For example, we might send multiple requests to
// the same Replica if (1) the descriptor cache has post-split
// descriptors that are still write intents and (2) the split
// has not yet been completed.
ba.SetNewRequest()
// TODO(tschottdorf): should serialize the trace here, not higher up.
br, pErr := ds.sendRPC(trace, desc.RangeID, replicas, order, ba)
if pErr != nil {
return nil, pErr
}
// Untangle the error from the received response.
pErr = br.Error
br.Error = nil // scrub the response error
return br, pErr
}
func (tc *TxnCoordSender) heartbeat(txnID uuid.UUID, trace opentracing.Span, ctx context.Context) bool {
tc.Lock()
proceed := true
txnMeta := tc.txns[txnID]
var intentSpans []roachpb.Span
// Before we send a heartbeat, determine whether this transaction
// should be considered abandoned. If so, exit heartbeat.
if txnMeta.hasClientAbandonedCoord(tc.clock.PhysicalNow()) {
// TODO(tschottdorf): should we be more proactive here?
// The client might be continuing the transaction
// through another coordinator, but in the most likely
// case it's just gone and the open transaction record
// could block concurrent operations.
if log.V(1) {
log.Infof("transaction %s abandoned; stopping heartbeat",
txnMeta.txn)
}
proceed = false
// Grab the intents here to avoid potential race.
intentSpans = collectIntentSpans(txnMeta.keys)
txnMeta.keys.Clear()
}
// txnMeta.txn is possibly replaced concurrently,
// so grab a copy before unlocking.
txn := txnMeta.txn.Clone()
tc.Unlock()
ba := roachpb.BatchRequest{}
ba.Txn = &txn
if !proceed {
// Actively abort the transaction and its intents since we assume it's abandoned.
et := &roachpb.EndTransactionRequest{
Span: roachpb.Span{
Key: txn.Key,
},
Commit: false,
IntentSpans: intentSpans,
}
ba.Add(et)
tc.stopper.RunAsyncTask(func() {
// Use the wrapped sender since the normal Sender
// does not allow clients to specify intents.
// TODO(tschottdorf): not using the existing context here since that
// leads to use-after-finish of the contained trace. Should fork off
// before the goroutine.
if _, pErr := tc.wrapped.Send(context.Background(), ba); pErr != nil {
if log.V(1) {
log.Warningf("abort due to inactivity failed for %s: %s ", txn, pErr)
}
}
})
return false
}
hb := &roachpb.HeartbeatTxnRequest{
Now: tc.clock.Now(),
}
hb.Key = txn.Key
ba.Add(hb)
trace.LogEvent("heartbeat")
_, err := tc.wrapped.Send(ctx, ba)
// If the transaction is not in pending state, then we can stop
// the heartbeat. It's either aborted or committed, and we resolve
// write intents accordingly.
if err != nil {
log.Warningf("heartbeat to %s failed: %s", txn, err)
}
// TODO(bdarnell): once we have gotten a heartbeat response with
// Status != PENDING, future heartbeats are useless. However, we
// need to continue the heartbeatLoop until the client either
// commits or abandons the transaction. We could save a little
// pointless work by restructuring this loop to stop sending
// heartbeats between the time that the transaction is aborted and
// the client finds out. Furthermore, we could use this information
// to send TransactionAbortedErrors to the client so it can restart
// immediately instead of running until its EndTransaction.
return true
}
// Add adds a bool tag to the `span`
func (tag boolTagName) Set(span opentracing.Span, value bool) {
span.SetTag(string(tag), value)
}
// Set adds a uint16 tag to the `span`
func (tag uint16TagName) Set(span opentracing.Span, value uint16) {
span.SetTag(string(tag), value)
}
// Set adds a string tag to the `span`
func (tag stringTagName) Set(span opentracing.Span, value string) {
span.SetTag(string(tag), value)
}
// Set adds a string tag to the `span`
func (tag spanKindTagName) Set(span opentracing.Span, value SpanKindEnum) {
span.SetTag(string(tag), value)
}
// Add adds a uint32 tag to the `span`
func (tag uint32Tag) Add(span opentracing.Span, value uint32) {
span.SetTag(string(tag), value)
}
// addHeaderTags adds header key:value pairs to a span as a tag with the prefix
// "Request.Header.*"
func addHeaderTags(span opentracing.Span, h http.Header) {
for k, v := range h {
span.SetTag(headerTagPrefix+k, strings.Join(v, ", "))
}
}
// FinishSpan closes the given span (if not nil). It is a convenience wrapper
// for span.Finish() which tolerates nil spans.
func FinishSpan(span opentracing.Span) {
if span != nil {
span.Finish()
}
}
