func injectErrors( req roachpb.Request, hdr roachpb.Header, magicVals *filterVals, ) error { magicVals.Lock() defer magicVals.Unlock() switch req := req.(type) { case *roachpb.ConditionalPutRequest: for key, count := range magicVals.restartCounts { checkCorrectTxn(string(req.Value.RawBytes), magicVals, hdr.Txn) if count > 0 && bytes.Contains(req.Value.RawBytes, []byte(key)) { magicVals.restartCounts[key]-- err := roachpb.NewReadWithinUncertaintyIntervalError( hlc.ZeroTimestamp, hlc.ZeroTimestamp) magicVals.failedValues[string(req.Value.RawBytes)] = failureRecord{err, hdr.Txn} return err } } for key, count := range magicVals.abortCounts { checkCorrectTxn(string(req.Value.RawBytes), magicVals, hdr.Txn) if count > 0 && bytes.Contains(req.Value.RawBytes, []byte(key)) { magicVals.abortCounts[key]-- err := roachpb.NewTransactionAbortedError() magicVals.failedValues[string(req.Value.RawBytes)] = failureRecord{err, hdr.Txn} return err } } // If we're writing a value that's marked for an EndTransaction failure, // keep track of the txn id so we can fail it later on. for key, count := range magicVals.endTxnRestartCounts { if count > 0 && bytes.Contains(req.Value.RawBytes, []byte(key)) { txnID := *hdr.Txn.TxnMeta.ID if _, found := magicVals.txnsToFail[txnID]; found { continue } magicVals.endTxnRestartCounts[key]-- magicVals.txnsToFail[txnID] = true } } return nil case *roachpb.EndTransactionRequest: txnID := *hdr.Txn.TxnMeta.ID if !magicVals.txnsToFail[txnID] { return nil } delete(magicVals.txnsToFail, txnID) // Note that we can't return TransactionAborted errors, although those are // more representative for the errors that EndTransaction might encounter, // because returning those would result in the txn's intents being left // around. return roachpb.NewTransactionRetryError() default: return nil } }
// TestRunTransactionRetryOnErrors verifies that the transaction // is retried on the correct errors. func TestRunTransactionRetryOnErrors(t *testing.T) { defer leaktest.AfterTest(t)() testCases := []struct { err error retry bool // Expect retry? }{ {roachpb.NewReadWithinUncertaintyIntervalError(hlc.ZeroTimestamp, hlc.ZeroTimestamp), true}, {&roachpb.TransactionAbortedError{}, true}, {&roachpb.TransactionPushError{}, true}, {&roachpb.TransactionRetryError{}, true}, {&roachpb.WriteTooOldError{}, true}, {&roachpb.RangeNotFoundError{}, false}, {&roachpb.RangeKeyMismatchError{}, false}, {&roachpb.TransactionStatusError{}, false}, } for i, test := range testCases { count := 0 dbCtx := DefaultDBContext() dbCtx.TxnRetryOptions.InitialBackoff = 1 * time.Millisecond db := NewDBWithContext(newTestSender( func(ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { if _, ok := ba.GetArg(roachpb.Put); ok { count++ if count == 1 { return nil, roachpb.NewErrorWithTxn(test.err, ba.Txn) } } return ba.CreateReply(), nil }, nil), dbCtx) err := db.Txn(context.TODO(), func(txn *Txn) error { return txn.Put("a", "b") }) if test.retry { if count != 2 { t.Errorf("%d: expected one retry; got %d", i, count-1) } if err != nil { t.Errorf("%d: expected success on retry; got %s", i, err) } } else { if count != 1 { t.Errorf("%d: expected no retries; got %d", i, count) } if reflect.TypeOf(err) != reflect.TypeOf(test.err) { t.Errorf("%d: expected error of type %T; got %T", i, test.err, err) } } } }
// TestAbortTransactionOnCommitErrors verifies that transactions are // aborted on the correct errors. func TestAbortTransactionOnCommitErrors(t *testing.T) { defer leaktest.AfterTest(t)() testCases := []struct { err error abort bool }{ {roachpb.NewReadWithinUncertaintyIntervalError(roachpb.ZeroTimestamp, roachpb.ZeroTimestamp), true}, {&roachpb.TransactionAbortedError{}, false}, {&roachpb.TransactionPushError{}, true}, {&roachpb.TransactionRetryError{}, true}, {&roachpb.RangeNotFoundError{}, true}, {&roachpb.RangeKeyMismatchError{}, true}, {&roachpb.TransactionStatusError{}, true}, } for _, test := range testCases { var commit, abort bool db := NewDB(newTestSender(func(ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { switch t := ba.Requests[0].GetInner().(type) { case *roachpb.EndTransactionRequest: if t.Commit { commit = true return nil, roachpb.NewError(test.err) } abort = true } return ba.CreateReply(), nil }, nil)) txn := NewTxn(context.Background(), *db) if pErr := txn.Put("a", "b"); pErr != nil { t.Fatalf("put failed: %s", pErr) } if pErr := txn.CommitOrCleanup(); pErr == nil { t.Fatalf("unexpected commit success") } if !commit { t.Errorf("%T: failed to find commit", test.err) } if test.abort && !abort { t.Errorf("%T: failed to find abort", test.err) } else if !test.abort && abort { t.Errorf("%T: found unexpected abort", test.err) } } }
func injectRetriableErrors( _ roachpb.StoreID, req roachpb.Request, hdr roachpb.Header, magicVals []string, restarts map[string]int) error { cput, ok := req.(*roachpb.ConditionalPutRequest) if !ok { return nil } for _, val := range magicVals { if restarts[val] < 2 && bytes.Contains(cput.Value.RawBytes, []byte(val)) { restarts[val]++ return roachpb.NewReadWithinUncertaintyIntervalError(roachpb.ZeroTimestamp, roachpb.ZeroTimestamp) } } return nil }
func injectRetriableErrors( _ roachpb.StoreID, req roachpb.Request, hdr roachpb.Header, magicVals *filterVals) error { magicVals.Lock() defer magicVals.Unlock() cput, ok := req.(*roachpb.ConditionalPutRequest) if !ok { return nil } for _, val := range magicVals.vals { if magicVals.restartCounts[val] < 2 && bytes.Contains(cput.Value.RawBytes, []byte(val)) { magicVals.restartCounts[val]++ return roachpb.NewReadWithinUncertaintyIntervalError( roachpb.ZeroTimestamp, roachpb.ZeroTimestamp) } } return nil }
// Verifies that an expired lease is released and a new lease is acquired on transaction // restart. // // This test triggers the above scenario by making ReadWithinUncertaintyIntervalError advance // the clock, so that the transaction timestamp exceeds the deadline of the EndTransactionRequest. func TestReacquireLeaseOnRestart(t *testing.T) { defer leaktest.AfterTest(t)() var cmdFilters CommandFilters cmdFilters.AppendFilter(checkEndTransactionTrigger, true) var clockUpdate int32 testKey := []byte("test_key") testingKnobs := &storage.StoreTestingKnobs{ TestingCommandFilter: cmdFilters.runFilters, ClockBeforeSend: func(c *hlc.Clock, ba roachpb.BatchRequest) { if atomic.LoadInt32(&clockUpdate) > 0 { return } // Hack to advance the transaction timestamp on a transaction restart. for _, union := range ba.Requests { if req, ok := union.GetInner().(*roachpb.ScanRequest); ok { if bytes.Contains(req.Key, testKey) { atomic.AddInt32(&clockUpdate, 1) now := c.Now() now.WallTime += int64(5 * sql.LeaseDuration) c.Update(now) break } } } }, } params, _ := createTestServerParams() params.Knobs.Store = testingKnobs s, sqlDB, _ := serverutils.StartServer(t, params) defer s.Stopper().Stop() var restartDone int32 cleanupFilter := cmdFilters.AppendFilter( func(args storagebase.FilterArgs) *roachpb.Error { if atomic.LoadInt32(&restartDone) > 0 { return nil } if req, ok := args.Req.(*roachpb.ScanRequest); ok { if bytes.Contains(req.Key, testKey) { atomic.AddInt32(&restartDone, 1) // Return ReadWithinUncertaintyIntervalError to update the transaction timestamp on retry. txn := args.Hdr.Txn txn.ResetObservedTimestamps() now := s.Clock().Now() txn.UpdateObservedTimestamp( s.(*server.TestServer).Gossip().GetNodeID(), now) return roachpb.NewErrorWithTxn(roachpb.NewReadWithinUncertaintyIntervalError(now, now), txn) } } return nil }, false) defer cleanupFilter() // Use a large max offset to avoid rejecting a transaction whose timestanp is in // future (as we will advance the transaction timestamp with ReadWithinUncertaintyIntervalError). s.Clock().SetMaxOffset(sql.LeaseDuration * 10) sqlDB.SetMaxOpenConns(1) if _, err := sqlDB.Exec(` CREATE DATABASE t; CREATE TABLE t.test (k TEXT PRIMARY KEY, v TEXT); INSERT INTO t.test (k, v) VALUES ('test_key', 'test_val'); `); err != nil { t.Fatal(err) } // Acquire the lease and enable the auto-retry. The first read attempt will trigger ReadWithinUncertaintyIntervalError // and advance the transaction timestamp. The transaction timestamp will exceed the lease expiration // time, and the second read attempt will re-acquire the lease. if _, err := sqlDB.Exec(` SELECT * from t.test WHERE k = 'test_key'; `); err != nil { t.Fatal(err) } if u := atomic.LoadInt32(&clockUpdate); u != 1 { t.Errorf("expected exacltly one clock update, but got %d", u) } if u := atomic.LoadInt32(&restartDone); u != 1 { t.Errorf("expected exactly one restart, but got %d", u) } }
// TestTxnCoordSenderTxnUpdatedOnError verifies that errors adjust the // response transaction's timestamp and priority as appropriate. func TestTxnCoordSenderTxnUpdatedOnError(t *testing.T) { defer leaktest.AfterTest(t)() origTS := makeTS(123, 0) plus10 := origTS.Add(10, 10) plus20 := plus10.Add(10, 0) testCases := []struct { pErr *roachpb.Error expEpoch uint32 expPri int32 expTS, expOrigTS roachpb.Timestamp nodeSeen bool }{ { // No error, so nothing interesting either. pErr: nil, expEpoch: 0, expPri: 1, expTS: origTS, expOrigTS: origTS, }, { // On uncertainty error, new epoch begins and node is seen. // Timestamp moves ahead of the existing write. pErr: func() *roachpb.Error { pErr := roachpb.NewErrorWithTxn( roachpb.NewReadWithinUncertaintyIntervalError(roachpb.ZeroTimestamp, roachpb.ZeroTimestamp), &roachpb.Transaction{}) const nodeID = 1 pErr.GetTxn().UpdateObservedTimestamp(nodeID, plus10) pErr.OriginNode = nodeID return pErr }(), expEpoch: 1, expPri: 1, expTS: plus10, expOrigTS: plus10, nodeSeen: true, }, { // On abort, nothing changes but we get a new priority to use for // the next attempt. pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionAbortedError{}, &roachpb.Transaction{ TxnMeta: roachpb.TxnMeta{Timestamp: plus20, Priority: 10}, }), expPri: 10, }, { // On failed push, new epoch begins just past the pushed timestamp. // Additionally, priority ratchets up to just below the pusher's. pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionPushError{ PusheeTxn: roachpb.Transaction{ TxnMeta: roachpb.TxnMeta{Timestamp: plus10, Priority: int32(10)}, }, }, &roachpb.Transaction{}), expEpoch: 1, expPri: 9, expTS: plus10, expOrigTS: plus10, }, { // On retry, restart with new epoch, timestamp and priority. pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionRetryError{}, &roachpb.Transaction{ TxnMeta: roachpb.TxnMeta{Timestamp: plus10, Priority: int32(10)}, }, ), expEpoch: 1, expPri: 10, expTS: plus10, expOrigTS: plus10, }, } for i, test := range testCases { stopper := stop.NewStopper() manual := hlc.NewManualClock(origTS.WallTime) clock := hlc.NewClock(manual.UnixNano) clock.SetMaxOffset(20) ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { var reply *roachpb.BatchResponse if test.pErr == nil { reply = ba.CreateReply() } return reply, test.pErr }), clock, false, tracing.NewTracer(), stopper, NewTxnMetrics(metric.NewRegistry())) db := client.NewDB(ts) txn := client.NewTxn(context.Background(), *db) txn.InternalSetPriority(1) txn.Proto.Name = "test txn" key := roachpb.Key("test-key") _, err := txn.Get(key) teardownHeartbeats(ts) stopper.Stop() if test.pErr != nil && err == nil { t.Fatalf("expected an error") } if txn.Proto.Epoch != test.expEpoch { t.Errorf("%d: expected epoch = %d; got %d", i, test.expEpoch, txn.Proto.Epoch) } if txn.Proto.Priority != test.expPri { t.Errorf("%d: expected priority = %d; got %d", i, test.expPri, txn.Proto.Priority) } if !txn.Proto.Timestamp.Equal(test.expTS) { t.Errorf("%d: expected timestamp to be %s; got %s", i, test.expTS, txn.Proto.Timestamp) } if !txn.Proto.OrigTimestamp.Equal(test.expOrigTS) { t.Errorf("%d: expected orig timestamp to be %s; got %s", i, test.expOrigTS, txn.Proto.OrigTimestamp) } if ns := txn.Proto.ObservedTimestamps; (len(ns) != 0) != test.nodeSeen { t.Errorf("%d: expected nodeSeen=%t, but list of hosts is %v", i, test.nodeSeen, ns) } } }
// TestPropagateTxnOnError verifies that DistSender.sendChunk properly // propagates the txn data to a next iteration. Use txn.Writing field to // verify that. func TestPropagateTxnOnError(t *testing.T) { defer leaktest.AfterTest(t)() // Set up a filter to so that the first CPut operation will // get a ReadWithinUncertaintyIntervalError. targetKey := roachpb.Key("b") var numGets int32 ctx := server.NewTestContext() ctx.TestingKnobs.StoreTestingKnobs.TestingCommandFilter = func(fArgs storageutils.FilterArgs) *roachpb.Error { _, ok := fArgs.Req.(*roachpb.ConditionalPutRequest) if ok && fArgs.Req.Header().Key.Equal(targetKey) { if atomic.AddInt32(&numGets, 1) == 1 { z := roachpb.ZeroTimestamp pErr := roachpb.NewReadWithinUncertaintyIntervalError(z, z) return roachpb.NewErrorWithTxn(pErr, fArgs.Hdr.Txn) } } return nil } s := server.StartTestServerWithContext(t, ctx) defer s.Stop() db := setupMultipleRanges(t, s, "b") // Set the initial value on the target key "b". origVal := "val" if pErr := db.Put(targetKey, origVal); pErr != nil { t.Fatal(pErr) } // The following txn creates a batch request that is split // into two requests: Put and CPut. The CPut operation will // get a ReadWithinUncertaintyIntervalError and the txn will be // retried. epoch := 0 if pErr := db.Txn(func(txn *client.Txn) *roachpb.Error { epoch++ if epoch >= 2 { // Writing must be true since we ran the BeginTransaction command. if !txn.Proto.Writing { t.Errorf("unexpected non-writing txn") } } else { // Writing must be false since we haven't run any write command. if txn.Proto.Writing { t.Errorf("unexpected writing txn") } } b := txn.NewBatch() b.Put("a", "val") b.CPut(targetKey, "new_val", origVal) pErr := txn.CommitInBatch(b) if epoch == 1 { if _, ok := pErr.GetDetail().(*roachpb.ReadWithinUncertaintyIntervalError); ok { if !pErr.GetTxn().Writing { t.Errorf("unexpected non-writing txn on error") } } else { t.Errorf("expected ReadWithinUncertaintyIntervalError, but got: %s", pErr) } } return pErr }); pErr != nil { t.Errorf("unexpected error on transactional Puts: %s", pErr) } if epoch != 2 { t.Errorf("unexpected epoch; the txn must be retried exactly once, but got %d", epoch) } }
// TestStoreRangeSplitRaceUninitializedRHS reproduces #7600 (before it was // fixed). While splits are happening, we simulate incoming messages for the // right-hand side to trigger a race between the creation of the proper replica // and the uninitialized replica reacting to messages. func TestStoreRangeSplitRaceUninitializedRHS(t *testing.T) { defer leaktest.AfterTest(t)() mtc := &multiTestContext{} storeCtx := storage.TestStoreContext() // An aggressive tick interval lets groups communicate more and thus // triggers test failures much more reliably. We can't go too aggressive // or race tests never make any progress. storeCtx.RaftTickInterval = 50 * time.Millisecond storeCtx.RaftElectionTimeoutTicks = 2 currentTrigger := make(chan *roachpb.SplitTrigger) seen := make(map[storagebase.CmdIDKey]struct{}) storeCtx.TestingKnobs.TestingCommandFilter = func(args storagebase.FilterArgs) *roachpb.Error { et, ok := args.Req.(*roachpb.EndTransactionRequest) if !ok || et.InternalCommitTrigger == nil { return nil } trigger := protoutil.Clone(et.InternalCommitTrigger.GetSplitTrigger()).(*roachpb.SplitTrigger) if trigger != nil && len(trigger.NewDesc.Replicas) == 2 && args.Hdr.Txn.Epoch == 0 && args.Sid == mtc.stores[0].StoreID() { if _, ok := seen[args.CmdID]; ok { return nil } // Without replay protection, a single reproposal locks up the // test. seen[args.CmdID] = struct{}{} currentTrigger <- trigger return roachpb.NewError(roachpb.NewReadWithinUncertaintyIntervalError(args.Hdr.Timestamp, args.Hdr.Timestamp)) } return nil } mtc.storeContext = &storeCtx mtc.Start(t, 2) defer mtc.Stop() leftRange := mtc.stores[0].LookupReplica(roachpb.RKey("a"), nil) // We'll fake messages from term 1, ..., .magicIters-1. The exact number // doesn't matter for anything but for its likelihood of triggering the // race. const magicIters = 5 // Replicate the left range onto the second node. We don't wait since we // don't actually care what the second node does. All we want is that the // first node isn't surprised by messages from that node. mtc.replicateRange(leftRange.RangeID, 1) for i := 0; i < 10; i++ { var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() // Split the data range. The split keys are chosen so that they move // towards "a" (so that the range being split is always the first // range). splitKey := roachpb.Key(encoding.EncodeVarintDescending([]byte("a"), int64(i))) splitArgs := adminSplitArgs(keys.SystemMax, splitKey) if _, pErr := client.SendWrapped(mtc.distSenders[0], nil, &splitArgs); pErr != nil { t.Fatal(pErr) } }() go func() { defer wg.Done() trigger := <-currentTrigger // our own copy // Make sure the first node is first for convenience. replicas := trigger.NewDesc.Replicas if replicas[0].NodeID > replicas[1].NodeID { tmp := replicas[1] replicas[1] = replicas[0] replicas[0] = tmp } // Send a few vote requests which look like they're from the other // node's right hand side of the split. This triggers a race which // is discussed in #7600 (briefly, the creation of the right hand // side in the split trigger was racing with the uninitialized // version for the same group, resulting in clobbered HardState). for term := uint64(1); term < magicIters; term++ { if err := mtc.stores[0].HandleRaftMessage(&storage.RaftMessageRequest{ RangeID: trigger.NewDesc.RangeID, ToReplica: replicas[0], FromReplica: replicas[1], Message: raftpb.Message{ Type: raftpb.MsgVote, To: uint64(replicas[0].ReplicaID), From: uint64(replicas[1].ReplicaID), Term: term, }, }); err != nil { t.Error(err) } } }() wg.Wait() } }