// createRangeData creates sample range data in all possible areas of // the key space. Returns a slice of the encoded keys of all created // data. func createRangeData(t *testing.T, r *Replica) []engine.MVCCKey { ts0 := hlc.ZeroTimestamp ts := hlc.Timestamp{WallTime: 1} desc := r.Desc() keyTSs := []struct { key roachpb.Key ts hlc.Timestamp }{ {keys.AbortCacheKey(r.RangeID, testTxnID), ts0}, {keys.AbortCacheKey(r.RangeID, testTxnID2), ts0}, {keys.RangeFrozenStatusKey(r.RangeID), ts0}, {keys.RangeLastGCKey(r.RangeID), ts0}, {keys.RaftAppliedIndexKey(r.RangeID), ts0}, {keys.RaftTruncatedStateKey(r.RangeID), ts0}, {keys.RangeLeaseKey(r.RangeID), ts0}, {keys.LeaseAppliedIndexKey(r.RangeID), ts0}, {keys.RangeStatsKey(r.RangeID), ts0}, {keys.RangeTxnSpanGCThresholdKey(r.RangeID), ts0}, {keys.RaftHardStateKey(r.RangeID), ts0}, {keys.RaftLastIndexKey(r.RangeID), ts0}, {keys.RaftLogKey(r.RangeID, 1), ts0}, {keys.RaftLogKey(r.RangeID, 2), ts0}, {keys.RangeLastReplicaGCTimestampKey(r.RangeID), ts0}, {keys.RangeLastVerificationTimestampKeyDeprecated(r.RangeID), ts0}, {keys.RangeDescriptorKey(desc.StartKey), ts}, {keys.TransactionKey(roachpb.Key(desc.StartKey), uuid.MakeV4()), ts0}, {keys.TransactionKey(roachpb.Key(desc.StartKey.Next()), uuid.MakeV4()), ts0}, {keys.TransactionKey(fakePrevKey(desc.EndKey), uuid.MakeV4()), ts0}, // TODO(bdarnell): KeyMin.Next() results in a key in the reserved system-local space. // Once we have resolved https://github.com/cockroachdb/cockroach/issues/437, // replace this with something that reliably generates the first valid key in the range. //{r.Desc().StartKey.Next(), ts}, // The following line is similar to StartKey.Next() but adds more to the key to // avoid falling into the system-local space. {append(append([]byte{}, desc.StartKey...), '\x02'), ts}, {fakePrevKey(r.Desc().EndKey), ts}, } keys := []engine.MVCCKey{} for _, keyTS := range keyTSs { if err := engine.MVCCPut(context.Background(), r.store.Engine(), nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil { t.Fatal(err) } keys = append(keys, engine.MVCCKey{Key: keyTS.key, Timestamp: keyTS.ts}) } return keys }
// processTransactionTable scans the transaction table and updates txnMap with // those transactions which are old and either PENDING or with intents // registered. In the first case we want to push the transaction so that it is // aborted, and in the second case we may have to resolve the intents success- // fully before GCing the entry. The transaction records which can be gc'ed are // returned separately and are not added to txnMap nor intentSpanMap. func processTransactionTable( ctx context.Context, snap engine.Reader, desc *roachpb.RangeDescriptor, txnMap map[uuid.UUID]*roachpb.Transaction, cutoff hlc.Timestamp, infoMu *lockableGCInfo, resolveIntents resolveFunc, ) ([]roachpb.GCRequest_GCKey, error) { infoMu.Lock() defer infoMu.Unlock() var gcKeys []roachpb.GCRequest_GCKey handleOne := func(kv roachpb.KeyValue) error { var txn roachpb.Transaction if err := kv.Value.GetProto(&txn); err != nil { return err } infoMu.TransactionSpanTotal++ if !txn.LastActive().Less(cutoff) { return nil } txnID := *txn.ID // The transaction record should be considered for removal. switch txn.Status { case roachpb.PENDING: // Marked as running, so we need to push it to abort it but won't // try to GC it in this cycle (for convenience). // TODO(tschottdorf): refactor so that we can GC PENDING entries // in the same cycle, but keeping the calls to pushTxn in a central // location (keeping it easy to batch them up in the future). infoMu.TransactionSpanGCPending++ txnMap[txnID] = &txn return nil case roachpb.ABORTED: // If we remove this transaction, it effectively still counts as // ABORTED (by design). So this can be GC'ed even if we can't // resolve the intents. // Note: Most aborted transaction weren't aborted by their client, // but instead by the coordinator - those will not have any intents // persisted, though they still might exist in the system. infoMu.TransactionSpanGCAborted++ func() { infoMu.Unlock() // intentional defer infoMu.Lock() if err := resolveIntents(roachpb.AsIntents(txn.Intents, &txn), true /* wait */, false /* !poison */); err != nil { log.Warningf(ctx, "failed to resolve intents of aborted txn on gc: %s", err) } }() case roachpb.COMMITTED: // It's committed, so it doesn't need a push but we can only // GC it after its intents are resolved. if err := func() error { infoMu.Unlock() // intentional defer infoMu.Lock() return resolveIntents(roachpb.AsIntents(txn.Intents, &txn), true /* wait */, false /* !poison */) }(); err != nil { log.Warningf(ctx, "unable to resolve intents of committed txn on gc: %s", err) // Returning the error here would abort the whole GC run, and // we don't want that. Instead, we simply don't GC this entry. return nil } infoMu.TransactionSpanGCCommitted++ default: panic(fmt.Sprintf("invalid transaction state: %s", txn)) } gcKeys = append(gcKeys, roachpb.GCRequest_GCKey{Key: kv.Key}) // zero timestamp return nil } startKey := keys.TransactionKey(desc.StartKey.AsRawKey(), uuid.UUID{}) endKey := keys.TransactionKey(desc.EndKey.AsRawKey(), uuid.UUID{}) _, err := engine.MVCCIterate(ctx, snap, startKey, endKey, hlc.ZeroTimestamp, true /* consistent */, nil, /* txn */ false /* !reverse */, func(kv roachpb.KeyValue) (bool, error) { return false, handleOne(kv) }) return gcKeys, err }
func TestGCQueueTransactionTable(t *testing.T) { defer leaktest.AfterTest(t)() const now time.Duration = 3 * 24 * time.Hour const gcTxnAndAC = now - txnCleanupThreshold const gcACOnly = now - abortCacheAgeThreshold if gcTxnAndAC >= gcACOnly { t.Fatalf("test assumption violated due to changing constants; needs adjustment") } type spec struct { status roachpb.TransactionStatus orig time.Duration hb time.Duration // last heartbeat (none if ZeroTimestamp) newStatus roachpb.TransactionStatus // -1 for GCed failResolve bool // do we want to fail resolves in this trial? expResolve bool // expect attempt at removing txn-persisted intents? expAbortGC bool // expect abort cache entries removed? } // Describes the state of the Txn table before the test. // Many of the abort cache entries deleted wouldn't even be there, so don't // be confused by that. testCases := map[string]spec{ // Too young, should not touch. "aa": { status: roachpb.PENDING, orig: gcACOnly + 1, newStatus: roachpb.PENDING, }, // A little older, so the AbortCache gets cleaned up. "ab": { status: roachpb.PENDING, orig: gcTxnAndAC + 1, newStatus: roachpb.PENDING, expAbortGC: true, }, // Old and pending, but still heartbeat (so no Push attempted; it would succeed). // It's old enough to delete the abort cache entry though. "ba": { status: roachpb.PENDING, hb: gcTxnAndAC + 1, newStatus: roachpb.PENDING, expAbortGC: true, }, // Not old enough for Txn GC, but old enough to remove the abort cache entry. "bb": { status: roachpb.ABORTED, orig: gcACOnly - 1, newStatus: roachpb.ABORTED, expAbortGC: true, }, // Old, pending and abandoned. Should push and abort it successfully, // but not GC it just yet (this is an artifact of the implementation). // The abort cache gets cleaned up though. "c": { status: roachpb.PENDING, orig: gcTxnAndAC - 1, newStatus: roachpb.ABORTED, expAbortGC: true, }, // Old and aborted, should delete. "d": { status: roachpb.ABORTED, orig: gcTxnAndAC - 1, newStatus: -1, expResolve: true, expAbortGC: true, }, // Committed and fresh, so no action. But the abort cache entry is old // enough to be discarded. "e": { status: roachpb.COMMITTED, orig: gcTxnAndAC + 1, newStatus: roachpb.COMMITTED, expAbortGC: true, }, // Committed and old. It has an intent (like all tests here), which is // resolvable and hence we can GC. "f": { status: roachpb.COMMITTED, orig: gcTxnAndAC - 1, newStatus: -1, expResolve: true, expAbortGC: true, }, // Same as the previous one, but we've rigged things so that the intent // resolution here will fail and consequently no GC is expected. "g": { status: roachpb.COMMITTED, orig: gcTxnAndAC - 1, newStatus: roachpb.COMMITTED, failResolve: true, expResolve: true, expAbortGC: true, }, } resolved := map[string][]roachpb.Span{} tc := testContext{} tsc := TestStoreConfig() tsc.TestingKnobs.TestingCommandFilter = func(filterArgs storagebase.FilterArgs) *roachpb.Error { if resArgs, ok := filterArgs.Req.(*roachpb.ResolveIntentRequest); ok { id := string(resArgs.IntentTxn.Key) resolved[id] = append(resolved[id], roachpb.Span{ Key: resArgs.Key, EndKey: resArgs.EndKey, }) // We've special cased one test case. Note that the intent is still // counted in `resolved`. if testCases[id].failResolve { return roachpb.NewErrorWithTxn(errors.Errorf("boom"), filterArgs.Hdr.Txn) } } return nil } tc.StartWithStoreConfig(t, tsc) defer tc.Stop() tc.manualClock.Set(int64(now)) outsideKey := tc.rng.Desc().EndKey.Next().AsRawKey() testIntents := []roachpb.Span{{Key: roachpb.Key("intent")}} txns := map[string]roachpb.Transaction{} for strKey, test := range testCases { baseKey := roachpb.Key(strKey) txnClock := hlc.NewClock(hlc.NewManualClock(int64(test.orig)).UnixNano) txn := newTransaction("txn1", baseKey, 1, enginepb.SERIALIZABLE, txnClock) txn.Status = test.status txn.Intents = testIntents if test.hb > 0 { txn.LastHeartbeat = &hlc.Timestamp{WallTime: int64(test.hb)} } // Set a high Timestamp to make sure it does not matter. Only // OrigTimestamp (and heartbeat) are used for GC decisions. txn.Timestamp.Forward(hlc.MaxTimestamp) txns[strKey] = *txn for _, addrKey := range []roachpb.Key{baseKey, outsideKey} { key := keys.TransactionKey(addrKey, txn.ID) if err := engine.MVCCPutProto(context.Background(), tc.engine, nil, key, hlc.ZeroTimestamp, nil, txn); err != nil { t.Fatal(err) } } entry := roachpb.AbortCacheEntry{Key: txn.Key, Timestamp: txn.LastActive()} if err := tc.rng.abortCache.Put(context.Background(), tc.engine, nil, txn.ID, &entry); err != nil { t.Fatal(err) } } // Run GC. gcQ := newGCQueue(tc.store, tc.gossip) cfg, ok := tc.gossip.GetSystemConfig() if !ok { t.Fatal("config not set") } if err := gcQ.process(context.Background(), tc.clock.Now(), tc.rng, cfg); err != nil { t.Fatal(err) } util.SucceedsSoon(t, func() error { for strKey, sp := range testCases { txn := &roachpb.Transaction{} key := keys.TransactionKey(roachpb.Key(strKey), txns[strKey].ID) ok, err := engine.MVCCGetProto(context.Background(), tc.engine, key, hlc.ZeroTimestamp, true, nil, txn) if err != nil { return err } if expGC := (sp.newStatus == -1); expGC { if expGC != !ok { return fmt.Errorf("%s: expected gc: %t, but found %s\n%s", strKey, expGC, txn, roachpb.Key(strKey)) } } else if sp.newStatus != txn.Status { return fmt.Errorf("%s: expected status %s, but found %s", strKey, sp.newStatus, txn.Status) } var expIntents []roachpb.Span if sp.expResolve { expIntents = testIntents } if !reflect.DeepEqual(resolved[strKey], expIntents) { return fmt.Errorf("%s: unexpected intent resolutions:\nexpected: %s\nobserved: %s", strKey, expIntents, resolved[strKey]) } entry := &roachpb.AbortCacheEntry{} abortExists, err := tc.rng.abortCache.Get(context.Background(), tc.store.Engine(), txns[strKey].ID, entry) if err != nil { t.Fatal(err) } if abortExists == sp.expAbortGC { return fmt.Errorf("%s: expected abort cache gc: %t, found %+v", strKey, sp.expAbortGC, entry) } } return nil }) outsideTxnPrefix := keys.TransactionKey(outsideKey, uuid.EmptyUUID) outsideTxnPrefixEnd := keys.TransactionKey(outsideKey.Next(), uuid.EmptyUUID) var count int if _, err := engine.MVCCIterate(context.Background(), tc.store.Engine(), outsideTxnPrefix, outsideTxnPrefixEnd, hlc.ZeroTimestamp, true, nil, false, func(roachpb.KeyValue) (bool, error) { count++ return false, nil }); err != nil { t.Fatal(err) } if exp := len(testCases); exp != count { t.Fatalf("expected the %d external transaction entries to remain untouched, "+ "but only %d are left", exp, count) } batch := tc.engine.NewSnapshot() defer batch.Close() tc.rng.assertState(batch) // check that in-mem and on-disk state were updated tc.rng.mu.Lock() txnSpanThreshold := tc.rng.mu.state.TxnSpanGCThreshold tc.rng.mu.Unlock() // Verify that the new TxnSpanGCThreshold has reached the Replica. if expWT := int64(gcTxnAndAC); txnSpanThreshold.WallTime != expWT { t.Fatalf("expected TxnSpanGCThreshold.Walltime %d, got timestamp %s", expWT, txnSpanThreshold) } }
// processIntentsAsync asynchronously processes intents which were // encountered during another command but did not interfere with the // execution of that command. This occurs in two cases: inconsistent // reads and EndTransaction (which queues its own external intents for // processing via this method). The two cases are handled somewhat // differently and would be better served by different entry points, // but combining them simplifies the plumbing necessary in Replica. func (ir *intentResolver) processIntentsAsync(r *Replica, intents []intentsWithArg) { now := r.store.Clock().Now() ctx := context.TODO() stopper := r.store.Stopper() for _, item := range intents { if item.args.Method() != roachpb.EndTransaction { if err := stopper.RunLimitedAsyncTask( ctx, ir.sem, true /* wait */, func(ctx context.Context) { // Everything here is best effort; give up rather than waiting // too long (helps avoid deadlocks during test shutdown, // although this is imperfect due to the use of an // uninterruptible WaitGroup.Wait in beginCmds). ctxWithTimeout, cancel := context.WithTimeout(ctx, base.NetworkTimeout) defer cancel() h := roachpb.Header{Timestamp: now} resolveIntents, pushErr := ir.maybePushTransactions(ctxWithTimeout, item.intents, h, roachpb.PUSH_TOUCH, true /* skipInFlight */) // resolveIntents with poison=true because we're resolving // intents outside of the context of an EndTransaction. // // Naively, it doesn't seem like we need to poison the abort // cache since we're pushing with PUSH_TOUCH - meaning that // the primary way our Push leads to aborting intents is that // of the transaction having timed out (and thus presumably no // client being around any more, though at the time of writing // we don't guarantee that). But there are other paths in which // the Push comes back successful while the coordinating client // may still be active. Examples of this are when: // // - the transaction was aborted by someone else, but the // coordinating client may still be running. // - the transaction entry wasn't written yet, which at the // time of writing has our push abort it, leading to the // same situation as above. // // Thus, we must poison. if err := ir.resolveIntents(ctxWithTimeout, resolveIntents, true /* wait */, true /* poison */); err != nil { log.Warningf(ctx, "%s: failed to resolve intents: %s", r, err) return } if pushErr != nil { log.Warningf(ctx, "%s: failed to push during intent resolution: %s", r, pushErr) return } }); err != nil { log.Warningf(ctx, "failed to resolve intents: %s", err) return } } else { // EndTransaction if err := stopper.RunLimitedAsyncTask( ctx, ir.sem, true /* wait */, func(ctx context.Context) { ctxWithTimeout, cancel := context.WithTimeout(ctx, base.NetworkTimeout) defer cancel() // For EndTransaction, we know the transaction is finalized so // we can skip the push and go straight to the resolve. // // This mechanism assumes that when an EndTransaction fails, // the client makes no assumptions about the result. For // example, an attempt to explicitly rollback the transaction // may succeed (triggering this code path), but the result may // not make it back to the client. if err := ir.resolveIntents(ctxWithTimeout, item.intents, true /* wait */, false /* !poison */); err != nil { log.Warningf(ctx, "%s: failed to resolve intents: %s", r, err) return } // We successfully resolved the intents, so we're able to GC from // the txn span directly. b := &client.Batch{} txn := item.intents[0].Txn txnKey := keys.TransactionKey(txn.Key, *txn.ID) // This is pretty tricky. Transaction keys are range-local and // so they are encoded specially. The key range addressed by // (txnKey, txnKey.Next()) might be empty (since Next() does // not imply monotonicity on the address side). Instead, we // send this request to a range determined using the resolved // transaction anchor, i.e. if the txn is anchored on // /Local/RangeDescriptor/"a"/uuid, the key range below would // be ["a", "a\x00"). However, the first range is special again // because the above procedure results in KeyMin, but we need // at least KeyLocalMax. // // #7880 will address this by making GCRequest less special and // thus obviating the need to cook up an artificial range here. var gcArgs roachpb.GCRequest { key := keys.MustAddr(txn.Key) if localMax := keys.MustAddr(keys.LocalMax); key.Less(localMax) { key = localMax } endKey := key.Next() gcArgs.Span = roachpb.Span{ Key: key.AsRawKey(), EndKey: endKey.AsRawKey(), } } gcArgs.Keys = append(gcArgs.Keys, roachpb.GCRequest_GCKey{ Key: txnKey, }) b.AddRawRequest(&gcArgs) if err := ir.store.db.Run(ctx, b); err != nil { log.Warningf(ctx, "could not GC completed transaction anchored at %s: %s", roachpb.Key(txn.Key), err) return } }); err != nil { log.Warningf(ctx, "failed to resolve intents: %s", err) return } } } }