func makeReplicaKeyRanges(
d *roachpb.RangeDescriptor, metaFunc func(roachpb.RangeID) roachpb.Key,
) []keyRange {
// The first range in the keyspace starts at KeyMin, which includes the
// node-local space. We need the original StartKey to find the range
// metadata, but the actual data starts at LocalMax.
dataStartKey := d.StartKey.AsRawKey()
if d.StartKey.Equal(roachpb.RKeyMin) {
dataStartKey = keys.LocalMax
}
sysRangeIDKey := metaFunc(d.RangeID)
return []keyRange{
{
start: engine.MakeMVCCMetadataKey(sysRangeIDKey),
end: engine.MakeMVCCMetadataKey(sysRangeIDKey.PrefixEnd()),
},
{
start: engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.StartKey)),
end: engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.EndKey)),
},
{
start: engine.MakeMVCCMetadataKey(dataStartKey),
end: engine.MakeMVCCMetadataKey(d.EndKey.AsRawKey()),
},
}
}
// truncate restricts all contained requests to the given key range
// and returns a new BatchRequest.
// All requests contained in that batch are "truncated" to the given
// span, inserting NoopRequest appropriately to replace requests which
// are left without a key range to operate on. The number of non-noop
// requests after truncation is returned.
func truncate(ba roachpb.BatchRequest, rs roachpb.RSpan) (roachpb.BatchRequest, int, error) {
truncateOne := func(args roachpb.Request) (bool, roachpb.Span, error) {
if _, ok := args.(*roachpb.NoopRequest); ok {
return true, emptySpan, nil
}
header := args.Header()
if !roachpb.IsRange(args) {
// This is a point request.
if len(header.EndKey) > 0 {
return false, emptySpan, errors.Errorf("%T is not a range command, but EndKey is set", args)
}
keyAddr, err := keys.Addr(header.Key)
if err != nil {
return false, emptySpan, err
}
if !rs.ContainsKey(keyAddr) {
return false, emptySpan, nil
}
return true, header, nil
}
// We're dealing with a range-spanning request.
local := false
keyAddr, err := keys.Addr(header.Key)
if err != nil {
return false, emptySpan, err
}
endKeyAddr, err := keys.Addr(header.EndKey)
if err != nil {
return false, emptySpan, err
}
if l, r := !keyAddr.Equal(header.Key), !endKeyAddr.Equal(header.EndKey); l || r {
if !l || !r {
return false, emptySpan, errors.Errorf("local key mixed with global key in range")
}
local = true
}
if keyAddr.Less(rs.Key) {
// rs.Key can't be local because it contains range split points, which
// are never local.
if !local {
header.Key = rs.Key.AsRawKey()
} else {
// The local start key should be truncated to the boundary of local keys which
// address to rs.Key.
header.Key = keys.MakeRangeKeyPrefix(rs.Key)
}
}
if !endKeyAddr.Less(rs.EndKey) {
// rs.EndKey can't be local because it contains range split points, which
// are never local.
if !local {
header.EndKey = rs.EndKey.AsRawKey()
} else {
// The local end key should be truncated to the boundary of local keys which
// address to rs.EndKey.
header.EndKey = keys.MakeRangeKeyPrefix(rs.EndKey)
}
}
// Check whether the truncation has left any keys in the range. If not,
// we need to cut it out of the request.
if header.Key.Compare(header.EndKey) >= 0 {
return false, emptySpan, nil
}
return true, header, nil
}
var numNoop int
truncBA := ba
truncBA.Requests = make([]roachpb.RequestUnion, len(ba.Requests))
for pos, arg := range ba.Requests {
hasRequest, newHeader, err := truncateOne(arg.GetInner())
if !hasRequest {
// We omit this one, i.e. replace it with a Noop.
numNoop++
union := roachpb.RequestUnion{}
union.MustSetInner(&noopRequest)
truncBA.Requests[pos] = union
} else {
// Keep the old one. If we must adjust the header, must copy.
if inner := ba.Requests[pos].GetInner(); newHeader.Equal(inner.Header()) {
truncBA.Requests[pos] = ba.Requests[pos]
} else {
shallowCopy := inner.ShallowCopy()
shallowCopy.SetHeader(newHeader)
union := &truncBA.Requests[pos] // avoid operating on copy
union.MustSetInner(shallowCopy)
}
}
if err != nil {
return roachpb.BatchRequest{}, 0, err
}
}
return truncBA, len(ba.Requests) - numNoop, nil
}
func TestTruncate(t *testing.T) {
defer leaktest.AfterTest(t)()
loc := func(s string) string {
return string(keys.RangeDescriptorKey(roachpb.RKey(s)))
}
locPrefix := func(s string) string {
return string(keys.MakeRangeKeyPrefix(roachpb.RKey(s)))
}
testCases := []struct {
keys [][2]string
expKeys [][2]string
from, to string
desc [2]string // optional, defaults to {from,to}
err string
}{
{
// Keys inside of active range.
keys: [][2]string{{"a", "q"}, {"c"}, {"b, e"}, {"q"}},
expKeys: [][2]string{{"a", "q"}, {"c"}, {"b, e"}, {"q"}},
from: "a", to: "q\x00",
},
{
// Keys outside of active range.
keys: [][2]string{{"a"}, {"a", "b"}, {"q"}, {"q", "z"}},
expKeys: [][2]string{{}, {}, {}, {}},
from: "b", to: "q",
},
{
// Range-local keys inside of active range.
keys: [][2]string{{loc("b")}, {loc("c")}},
expKeys: [][2]string{{loc("b")}, {loc("c")}},
from: "b", to: "e",
},
{
// Range-local key outside of active range.
keys: [][2]string{{loc("a")}},
expKeys: [][2]string{{}},
from: "b", to: "e",
},
{
// Range-local range contained in active range.
keys: [][2]string{{loc("b"), loc("e") + "\x00"}},
expKeys: [][2]string{{loc("b"), loc("e") + "\x00"}},
from: "b", to: "e\x00",
},
{
// Range-local range not contained in active range.
keys: [][2]string{{loc("a"), loc("b")}},
expKeys: [][2]string{{}},
from: "c", to: "e",
},
{
// Range-local range not contained in active range.
keys: [][2]string{{loc("a"), locPrefix("b")}, {loc("e"), loc("f")}},
expKeys: [][2]string{{}, {}},
from: "b", to: "e",
},
{
// Range-local range partially contained in active range.
keys: [][2]string{{loc("a"), loc("b")}},
expKeys: [][2]string{{loc("a"), locPrefix("b")}},
from: "a", to: "b",
},
{
// Range-local range partially contained in active range.
keys: [][2]string{{loc("a"), loc("b")}},
expKeys: [][2]string{{locPrefix("b"), loc("b")}},
from: "b", to: "e",
},
{
// Range-local range contained in active range.
keys: [][2]string{{locPrefix("b"), loc("b")}},
expKeys: [][2]string{{locPrefix("b"), loc("b")}},
from: "b", to: "c",
},
{
// Mixed range-local vs global key range.
keys: [][2]string{{loc("c"), "d\x00"}},
from: "b", to: "e",
err: "local key mixed with global key",
},
{
// Key range touching and intersecting active range.
keys: [][2]string{{"a", "b"}, {"a", "c"}, {"p", "q"}, {"p", "r"}, {"a", "z"}},
expKeys: [][2]string{{}, {"b", "c"}, {"p", "q"}, {"p", "q"}, {"b", "q"}},
from: "b", to: "q",
},
// Active key range is intersection of descriptor and [from,to).
{
keys: [][2]string{{"c", "q"}},
expKeys: [][2]string{{"d", "p"}},
from: "a", to: "z",
desc: [2]string{"d", "p"},
},
{
keys: [][2]string{{"c", "q"}},
expKeys: [][2]string{{"d", "p"}},
from: "d", to: "p",
desc: [2]string{"a", "z"},
},
}
for i, test := range testCases {
goldenOriginal := roachpb.BatchRequest{}
for _, ks := range test.keys {
if len(ks[1]) > 0 {
u := uuid.MakeV4()
goldenOriginal.Add(&roachpb.ResolveIntentRangeRequest{
Span: roachpb.Span{Key: roachpb.Key(ks[0]), EndKey: roachpb.Key(ks[1])},
IntentTxn: enginepb.TxnMeta{ID: &u},
})
} else {
goldenOriginal.Add(&roachpb.GetRequest{
Span: roachpb.Span{Key: roachpb.Key(ks[0])},
})
}
}
original := roachpb.BatchRequest{Requests: make([]roachpb.RequestUnion, len(goldenOriginal.Requests))}
for i, request := range goldenOriginal.Requests {
original.Requests[i].SetValue(request.GetInner().ShallowCopy())
}
desc := &roachpb.RangeDescriptor{
StartKey: roachpb.RKey(test.desc[0]), EndKey: roachpb.RKey(test.desc[1]),
}
if len(desc.StartKey) == 0 {
desc.StartKey = roachpb.RKey(test.from)
}
if len(desc.EndKey) == 0 {
desc.EndKey = roachpb.RKey(test.to)
}
rs := roachpb.RSpan{Key: roachpb.RKey(test.from), EndKey: roachpb.RKey(test.to)}
rs, err := rs.Intersect(desc)
if err != nil {
t.Errorf("%d: intersection failure: %v", i, err)
continue
}
ba, num, err := truncate(original, rs)
if err != nil || test.err != "" {
if !testutils.IsError(err, test.err) {
t.Errorf("%d: %v (expected: %q)", i, err, test.err)
}
continue
}
var reqs int
for j, arg := range ba.Requests {
req := arg.GetInner()
if _, ok := req.(*roachpb.NoopRequest); ok {
continue
}
if h := req.Header(); !bytes.Equal(h.Key, roachpb.Key(test.expKeys[j][0])) || !bytes.Equal(h.EndKey, roachpb.Key(test.expKeys[j][1])) {
t.Errorf("%d.%d: range mismatch: actual [%q,%q), wanted [%q,%q)", i, j,
h.Key, h.EndKey, test.expKeys[j][0], test.expKeys[j][1])
} else if _, ok := req.(*roachpb.NoopRequest); ok != (len(h.Key) == 0) {
t.Errorf("%d.%d: expected NoopRequest, got %T", i, j, req)
} else if len(h.Key) != 0 {
reqs++
}
}
if reqs != num {
t.Errorf("%d: counted %d requests, but truncation indicated %d", i, reqs, num)
}
if !reflect.DeepEqual(original, goldenOriginal) {
t.Errorf("%d: truncation mutated original:\nexpected: %s\nactual: %s",
i, goldenOriginal, original)
}
}
}
// processLocalKeyRange scans the local range key entries, consisting of
// transaction records, queue last processed timestamps, and range descriptors.
//
// - Transaction entries: updates txnMap with those transactions which
// are old and either PENDING or with intents registered. In the
// first case we want to push the transaction so that it is aborted,
// and in the second case we may have to resolve the intents
// success- fully before GCing the entry. The transaction records
// which can be gc'ed are returned separately and are not added to
// txnMap nor intentSpanMap.
//
// - Queue last processed times: cleanup any entries which don't match
// this range's start key. This can happen on range merges.
func processLocalKeyRange(
ctx context.Context,
snap engine.Reader,
desc *roachpb.RangeDescriptor,
txnMap map[uuid.UUID]*roachpb.Transaction,
cutoff hlc.Timestamp,
infoMu *lockableGCInfo,
resolveIntents resolveFunc,
) ([]roachpb.GCRequest_GCKey, error) {
infoMu.Lock()
defer infoMu.Unlock()
var gcKeys []roachpb.GCRequest_GCKey
handleOneTransaction := func(kv roachpb.KeyValue) error {
var txn roachpb.Transaction
if err := kv.Value.GetProto(&txn); err != nil {
return err
}
infoMu.TransactionSpanTotal++
if !txn.LastActive().Less(cutoff) {
return nil
}
txnID := *txn.ID
// The transaction record should be considered for removal.
switch txn.Status {
case roachpb.PENDING:
// Marked as running, so we need to push it to abort it but won't
// try to GC it in this cycle (for convenience).
// TODO(tschottdorf): refactor so that we can GC PENDING entries
// in the same cycle, but keeping the calls to pushTxn in a central
// location (keeping it easy to batch them up in the future).
infoMu.TransactionSpanGCPending++
txnMap[txnID] = &txn
return nil
case roachpb.ABORTED:
// If we remove this transaction, it effectively still counts as
// ABORTED (by design). So this can be GC'ed even if we can't
// resolve the intents.
// Note: Most aborted transaction weren't aborted by their client,
// but instead by the coordinator - those will not have any intents
// persisted, though they still might exist in the system.
infoMu.TransactionSpanGCAborted++
func() {
infoMu.Unlock() // intentional
defer infoMu.Lock()
if err := resolveIntents(roachpb.AsIntents(txn.Intents, &txn),
true /* wait */, false /* !poison */); err != nil {
log.Warningf(ctx, "failed to resolve intents of aborted txn on gc: %s", err)
}
}()
case roachpb.COMMITTED:
// It's committed, so it doesn't need a push but we can only
// GC it after its intents are resolved.
if err := func() error {
infoMu.Unlock() // intentional
defer infoMu.Lock()
return resolveIntents(roachpb.AsIntents(txn.Intents, &txn), true /* wait */, false /* !poison */)
}(); err != nil {
log.Warningf(ctx, "unable to resolve intents of committed txn on gc: %s", err)
// Returning the error here would abort the whole GC run, and
// we don't want that. Instead, we simply don't GC this entry.
return nil
}
infoMu.TransactionSpanGCCommitted++
default:
panic(fmt.Sprintf("invalid transaction state: %s", txn))
}
gcKeys = append(gcKeys, roachpb.GCRequest_GCKey{Key: kv.Key}) // zero timestamp
return nil
}
handleOneQueueLastProcessed := func(kv roachpb.KeyValue, rangeKey roachpb.RKey) error {
if !rangeKey.Equal(desc.StartKey) {
// Garbage collect the last processed timestamp if it doesn't match start key.
gcKeys = append(gcKeys, roachpb.GCRequest_GCKey{Key: kv.Key}) // zero timestamp
}
return nil
}
handleOne := func(kv roachpb.KeyValue) error {
rangeKey, suffix, _, err := keys.DecodeRangeKey(kv.Key)
if err != nil {
return err
}
if suffix.Equal(keys.LocalTransactionSuffix.AsRawKey()) {
if err := handleOneTransaction(kv); err != nil {
return err
}
} else if suffix.Equal(keys.LocalQueueLastProcessedSuffix.AsRawKey()) {
if err := handleOneQueueLastProcessed(kv, roachpb.RKey(rangeKey)); err != nil {
return err
}
}
return nil
}
startKey := keys.MakeRangeKeyPrefix(desc.StartKey)
endKey := keys.MakeRangeKeyPrefix(desc.EndKey)
_, err := engine.MVCCIterate(ctx, snap, startKey, endKey,
hlc.ZeroTimestamp, true /* consistent */, nil, /* txn */
false /* !reverse */, func(kv roachpb.KeyValue) (bool, error) {
return false, handleOne(kv)
})
return gcKeys, err
}