func makeReplicaKeyRanges( d *roachpb.RangeDescriptor, metaFunc func(roachpb.RangeID) roachpb.Key, ) []keyRange { // The first range in the keyspace starts at KeyMin, which includes the // node-local space. We need the original StartKey to find the range // metadata, but the actual data starts at LocalMax. dataStartKey := d.StartKey.AsRawKey() if d.StartKey.Equal(roachpb.RKeyMin) { dataStartKey = keys.LocalMax } sysRangeIDKey := metaFunc(d.RangeID) return []keyRange{ { start: engine.MakeMVCCMetadataKey(sysRangeIDKey), end: engine.MakeMVCCMetadataKey(sysRangeIDKey.PrefixEnd()), }, { start: engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.StartKey)), end: engine.MakeMVCCMetadataKey(keys.MakeRangeKeyPrefix(d.EndKey)), }, { start: engine.MakeMVCCMetadataKey(dataStartKey), end: engine.MakeMVCCMetadataKey(d.EndKey.AsRawKey()), }, } }
// truncate restricts all contained requests to the given key range // and returns a new BatchRequest. // All requests contained in that batch are "truncated" to the given // span, inserting NoopRequest appropriately to replace requests which // are left without a key range to operate on. The number of non-noop // requests after truncation is returned. func truncate(ba roachpb.BatchRequest, rs roachpb.RSpan) (roachpb.BatchRequest, int, error) { truncateOne := func(args roachpb.Request) (bool, roachpb.Span, error) { if _, ok := args.(*roachpb.NoopRequest); ok { return true, emptySpan, nil } header := args.Header() if !roachpb.IsRange(args) { // This is a point request. if len(header.EndKey) > 0 { return false, emptySpan, errors.Errorf("%T is not a range command, but EndKey is set", args) } keyAddr, err := keys.Addr(header.Key) if err != nil { return false, emptySpan, err } if !rs.ContainsKey(keyAddr) { return false, emptySpan, nil } return true, header, nil } // We're dealing with a range-spanning request. local := false keyAddr, err := keys.Addr(header.Key) if err != nil { return false, emptySpan, err } endKeyAddr, err := keys.Addr(header.EndKey) if err != nil { return false, emptySpan, err } if l, r := !keyAddr.Equal(header.Key), !endKeyAddr.Equal(header.EndKey); l || r { if !l || !r { return false, emptySpan, errors.Errorf("local key mixed with global key in range") } local = true } if keyAddr.Less(rs.Key) { // rs.Key can't be local because it contains range split points, which // are never local. if !local { header.Key = rs.Key.AsRawKey() } else { // The local start key should be truncated to the boundary of local keys which // address to rs.Key. header.Key = keys.MakeRangeKeyPrefix(rs.Key) } } if !endKeyAddr.Less(rs.EndKey) { // rs.EndKey can't be local because it contains range split points, which // are never local. if !local { header.EndKey = rs.EndKey.AsRawKey() } else { // The local end key should be truncated to the boundary of local keys which // address to rs.EndKey. header.EndKey = keys.MakeRangeKeyPrefix(rs.EndKey) } } // Check whether the truncation has left any keys in the range. If not, // we need to cut it out of the request. if header.Key.Compare(header.EndKey) >= 0 { return false, emptySpan, nil } return true, header, nil } var numNoop int truncBA := ba truncBA.Requests = make([]roachpb.RequestUnion, len(ba.Requests)) for pos, arg := range ba.Requests { hasRequest, newHeader, err := truncateOne(arg.GetInner()) if !hasRequest { // We omit this one, i.e. replace it with a Noop. numNoop++ union := roachpb.RequestUnion{} union.MustSetInner(&noopRequest) truncBA.Requests[pos] = union } else { // Keep the old one. If we must adjust the header, must copy. if inner := ba.Requests[pos].GetInner(); newHeader.Equal(inner.Header()) { truncBA.Requests[pos] = ba.Requests[pos] } else { shallowCopy := inner.ShallowCopy() shallowCopy.SetHeader(newHeader) union := &truncBA.Requests[pos] // avoid operating on copy union.MustSetInner(shallowCopy) } } if err != nil { return roachpb.BatchRequest{}, 0, err } } return truncBA, len(ba.Requests) - numNoop, nil }
func TestTruncate(t *testing.T) { defer leaktest.AfterTest(t)() loc := func(s string) string { return string(keys.RangeDescriptorKey(roachpb.RKey(s))) } locPrefix := func(s string) string { return string(keys.MakeRangeKeyPrefix(roachpb.RKey(s))) } testCases := []struct { keys [][2]string expKeys [][2]string from, to string desc [2]string // optional, defaults to {from,to} err string }{ { // Keys inside of active range. keys: [][2]string{{"a", "q"}, {"c"}, {"b, e"}, {"q"}}, expKeys: [][2]string{{"a", "q"}, {"c"}, {"b, e"}, {"q"}}, from: "a", to: "q\x00", }, { // Keys outside of active range. keys: [][2]string{{"a"}, {"a", "b"}, {"q"}, {"q", "z"}}, expKeys: [][2]string{{}, {}, {}, {}}, from: "b", to: "q", }, { // Range-local keys inside of active range. keys: [][2]string{{loc("b")}, {loc("c")}}, expKeys: [][2]string{{loc("b")}, {loc("c")}}, from: "b", to: "e", }, { // Range-local key outside of active range. keys: [][2]string{{loc("a")}}, expKeys: [][2]string{{}}, from: "b", to: "e", }, { // Range-local range contained in active range. keys: [][2]string{{loc("b"), loc("e") + "\x00"}}, expKeys: [][2]string{{loc("b"), loc("e") + "\x00"}}, from: "b", to: "e\x00", }, { // Range-local range not contained in active range. keys: [][2]string{{loc("a"), loc("b")}}, expKeys: [][2]string{{}}, from: "c", to: "e", }, { // Range-local range not contained in active range. keys: [][2]string{{loc("a"), locPrefix("b")}, {loc("e"), loc("f")}}, expKeys: [][2]string{{}, {}}, from: "b", to: "e", }, { // Range-local range partially contained in active range. keys: [][2]string{{loc("a"), loc("b")}}, expKeys: [][2]string{{loc("a"), locPrefix("b")}}, from: "a", to: "b", }, { // Range-local range partially contained in active range. keys: [][2]string{{loc("a"), loc("b")}}, expKeys: [][2]string{{locPrefix("b"), loc("b")}}, from: "b", to: "e", }, { // Range-local range contained in active range. keys: [][2]string{{locPrefix("b"), loc("b")}}, expKeys: [][2]string{{locPrefix("b"), loc("b")}}, from: "b", to: "c", }, { // Mixed range-local vs global key range. keys: [][2]string{{loc("c"), "d\x00"}}, from: "b", to: "e", err: "local key mixed with global key", }, { // Key range touching and intersecting active range. keys: [][2]string{{"a", "b"}, {"a", "c"}, {"p", "q"}, {"p", "r"}, {"a", "z"}}, expKeys: [][2]string{{}, {"b", "c"}, {"p", "q"}, {"p", "q"}, {"b", "q"}}, from: "b", to: "q", }, // Active key range is intersection of descriptor and [from,to). { keys: [][2]string{{"c", "q"}}, expKeys: [][2]string{{"d", "p"}}, from: "a", to: "z", desc: [2]string{"d", "p"}, }, { keys: [][2]string{{"c", "q"}}, expKeys: [][2]string{{"d", "p"}}, from: "d", to: "p", desc: [2]string{"a", "z"}, }, } for i, test := range testCases { goldenOriginal := roachpb.BatchRequest{} for _, ks := range test.keys { if len(ks[1]) > 0 { u := uuid.MakeV4() goldenOriginal.Add(&roachpb.ResolveIntentRangeRequest{ Span: roachpb.Span{Key: roachpb.Key(ks[0]), EndKey: roachpb.Key(ks[1])}, IntentTxn: enginepb.TxnMeta{ID: &u}, }) } else { goldenOriginal.Add(&roachpb.GetRequest{ Span: roachpb.Span{Key: roachpb.Key(ks[0])}, }) } } original := roachpb.BatchRequest{Requests: make([]roachpb.RequestUnion, len(goldenOriginal.Requests))} for i, request := range goldenOriginal.Requests { original.Requests[i].SetValue(request.GetInner().ShallowCopy()) } desc := &roachpb.RangeDescriptor{ StartKey: roachpb.RKey(test.desc[0]), EndKey: roachpb.RKey(test.desc[1]), } if len(desc.StartKey) == 0 { desc.StartKey = roachpb.RKey(test.from) } if len(desc.EndKey) == 0 { desc.EndKey = roachpb.RKey(test.to) } rs := roachpb.RSpan{Key: roachpb.RKey(test.from), EndKey: roachpb.RKey(test.to)} rs, err := rs.Intersect(desc) if err != nil { t.Errorf("%d: intersection failure: %v", i, err) continue } ba, num, err := truncate(original, rs) if err != nil || test.err != "" { if !testutils.IsError(err, test.err) { t.Errorf("%d: %v (expected: %q)", i, err, test.err) } continue } var reqs int for j, arg := range ba.Requests { req := arg.GetInner() if _, ok := req.(*roachpb.NoopRequest); ok { continue } if h := req.Header(); !bytes.Equal(h.Key, roachpb.Key(test.expKeys[j][0])) || !bytes.Equal(h.EndKey, roachpb.Key(test.expKeys[j][1])) { t.Errorf("%d.%d: range mismatch: actual [%q,%q), wanted [%q,%q)", i, j, h.Key, h.EndKey, test.expKeys[j][0], test.expKeys[j][1]) } else if _, ok := req.(*roachpb.NoopRequest); ok != (len(h.Key) == 0) { t.Errorf("%d.%d: expected NoopRequest, got %T", i, j, req) } else if len(h.Key) != 0 { reqs++ } } if reqs != num { t.Errorf("%d: counted %d requests, but truncation indicated %d", i, reqs, num) } if !reflect.DeepEqual(original, goldenOriginal) { t.Errorf("%d: truncation mutated original:\nexpected: %s\nactual: %s", i, goldenOriginal, original) } } }
// processLocalKeyRange scans the local range key entries, consisting of // transaction records, queue last processed timestamps, and range descriptors. // // - Transaction entries: updates txnMap with those transactions which // are old and either PENDING or with intents registered. In the // first case we want to push the transaction so that it is aborted, // and in the second case we may have to resolve the intents // success- fully before GCing the entry. The transaction records // which can be gc'ed are returned separately and are not added to // txnMap nor intentSpanMap. // // - Queue last processed times: cleanup any entries which don't match // this range's start key. This can happen on range merges. func processLocalKeyRange( ctx context.Context, snap engine.Reader, desc *roachpb.RangeDescriptor, txnMap map[uuid.UUID]*roachpb.Transaction, cutoff hlc.Timestamp, infoMu *lockableGCInfo, resolveIntents resolveFunc, ) ([]roachpb.GCRequest_GCKey, error) { infoMu.Lock() defer infoMu.Unlock() var gcKeys []roachpb.GCRequest_GCKey handleOneTransaction := func(kv roachpb.KeyValue) error { var txn roachpb.Transaction if err := kv.Value.GetProto(&txn); err != nil { return err } infoMu.TransactionSpanTotal++ if !txn.LastActive().Less(cutoff) { return nil } txnID := *txn.ID // The transaction record should be considered for removal. switch txn.Status { case roachpb.PENDING: // Marked as running, so we need to push it to abort it but won't // try to GC it in this cycle (for convenience). // TODO(tschottdorf): refactor so that we can GC PENDING entries // in the same cycle, but keeping the calls to pushTxn in a central // location (keeping it easy to batch them up in the future). infoMu.TransactionSpanGCPending++ txnMap[txnID] = &txn return nil case roachpb.ABORTED: // If we remove this transaction, it effectively still counts as // ABORTED (by design). So this can be GC'ed even if we can't // resolve the intents. // Note: Most aborted transaction weren't aborted by their client, // but instead by the coordinator - those will not have any intents // persisted, though they still might exist in the system. infoMu.TransactionSpanGCAborted++ func() { infoMu.Unlock() // intentional defer infoMu.Lock() if err := resolveIntents(roachpb.AsIntents(txn.Intents, &txn), true /* wait */, false /* !poison */); err != nil { log.Warningf(ctx, "failed to resolve intents of aborted txn on gc: %s", err) } }() case roachpb.COMMITTED: // It's committed, so it doesn't need a push but we can only // GC it after its intents are resolved. if err := func() error { infoMu.Unlock() // intentional defer infoMu.Lock() return resolveIntents(roachpb.AsIntents(txn.Intents, &txn), true /* wait */, false /* !poison */) }(); err != nil { log.Warningf(ctx, "unable to resolve intents of committed txn on gc: %s", err) // Returning the error here would abort the whole GC run, and // we don't want that. Instead, we simply don't GC this entry. return nil } infoMu.TransactionSpanGCCommitted++ default: panic(fmt.Sprintf("invalid transaction state: %s", txn)) } gcKeys = append(gcKeys, roachpb.GCRequest_GCKey{Key: kv.Key}) // zero timestamp return nil } handleOneQueueLastProcessed := func(kv roachpb.KeyValue, rangeKey roachpb.RKey) error { if !rangeKey.Equal(desc.StartKey) { // Garbage collect the last processed timestamp if it doesn't match start key. gcKeys = append(gcKeys, roachpb.GCRequest_GCKey{Key: kv.Key}) // zero timestamp } return nil } handleOne := func(kv roachpb.KeyValue) error { rangeKey, suffix, _, err := keys.DecodeRangeKey(kv.Key) if err != nil { return err } if suffix.Equal(keys.LocalTransactionSuffix.AsRawKey()) { if err := handleOneTransaction(kv); err != nil { return err } } else if suffix.Equal(keys.LocalQueueLastProcessedSuffix.AsRawKey()) { if err := handleOneQueueLastProcessed(kv, roachpb.RKey(rangeKey)); err != nil { return err } } return nil } startKey := keys.MakeRangeKeyPrefix(desc.StartKey) endKey := keys.MakeRangeKeyPrefix(desc.EndKey) _, err := engine.MVCCIterate(ctx, snap, startKey, endKey, hlc.ZeroTimestamp, true /* consistent */, nil, /* txn */ false /* !reverse */, func(kv roachpb.KeyValue) (bool, error) { return false, handleOne(kv) }) return gcKeys, err }