// TestEndWriteRestartReadOnlyTransaction verifies that if
// a transaction writes, then restarts and turns read-only,
// an explicit EndTransaction call is still sent if retry-
// able didn't, regardless of whether there is an error
// or not.
func TestEndWriteRestartReadOnlyTransaction(t *testing.T) {
defer leaktest.AfterTest(t)()
for _, success := range []bool{true, false} {
expCalls := []roachpb.Method{roachpb.BeginTransaction, roachpb.Put, roachpb.EndTransaction}
var calls []roachpb.Method
db := NewDB(newTestSender(func(ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
calls = append(calls, ba.Methods()...)
return ba.CreateReply(), nil
}, nil))
ok := false
if err := db.Txn(context.TODO(), func(txn *Txn) error {
if !ok {
if err := txn.Put("consider", "phlebas"); err != nil {
t.Fatal(err)
}
ok = true
// Return an immediate txn retry error. We need to go through the pErr
// and back to get a RetryableTxnError.
return roachpb.NewErrorWithTxn(roachpb.NewTransactionRetryError(), &txn.Proto).GoError()
}
if !success {
return errors.New("aborting on purpose")
}
return nil
}); err == nil != success {
t.Errorf("expected error: %t, got error: %v", !success, err)
}
if !reflect.DeepEqual(expCalls, calls) {
t.Fatalf("expected %v, got %v", expCalls, calls)
}
}
}
// TestTxnCoordSenderErrorWithIntent validates that if a transactional request
// returns an error but also indicates a Writing transaction, the coordinator
// tracks it just like a successful request.
func TestTxnCoordSenderErrorWithIntent(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
defer stopper.Stop()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
testCases := []struct {
roachpb.Error
errMsg string
}{
{*roachpb.NewError(roachpb.NewTransactionRetryError()), "retry txn"},
{*roachpb.NewError(roachpb.NewTransactionPushError(roachpb.Transaction{
TxnMeta: enginepb.TxnMeta{
ID: uuid.NewV4(),
}})), "failed to push"},
{*roachpb.NewErrorf("testError"), "testError"},
}
for i, test := range testCases {
func() {
senderFunc := func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
txn := ba.Txn.Clone()
txn.Writing = true
pErr := &roachpb.Error{}
*pErr = test.Error
pErr.SetTxn(&txn)
return nil, pErr
}
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
ts := NewTxnCoordSender(
ambient,
senderFn(senderFunc),
clock,
false,
stopper,
MakeTxnMetrics(metric.TestSampleInterval),
)
var ba roachpb.BatchRequest
key := roachpb.Key("test")
ba.Add(&roachpb.BeginTransactionRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.PutRequest{Span: roachpb.Span{Key: key}})
ba.Add(&roachpb.EndTransactionRequest{})
ba.Txn = &roachpb.Transaction{Name: "test"}
_, pErr := ts.Send(context.Background(), ba)
if !testutils.IsPError(pErr, test.errMsg) {
t.Errorf("%d: error did not match %s: %v", i, test.errMsg, pErr)
}
defer teardownHeartbeats(ts)
ts.Lock()
defer ts.Unlock()
if len(ts.txns) != 1 {
t.Errorf("%d: expected transaction to be tracked", i)
}
}()
}
}
// TestTransactionKeyNotChangedInRestart verifies that if the transaction already has a key (we're
// in a restart), the key in the begin transaction request is not changed.
func TestTransactionKeyNotChangedInRestart(t *testing.T) {
defer leaktest.AfterTest(t)()
tries := 0
db := NewDB(newTestSender(nil, func(ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
var bt *roachpb.BeginTransactionRequest
if args, ok := ba.GetArg(roachpb.BeginTransaction); ok {
bt = args.(*roachpb.BeginTransactionRequest)
} else {
t.Fatal("failed to find a begin transaction request")
}
// In the first try, the transaction key is the key of the first write command. Before the
// second try, the transaction key is set to txnKey by the test sender. In the second try, the
// transaction key is txnKey.
var expectedKey roachpb.Key
if tries == 1 {
expectedKey = testKey
} else {
expectedKey = txnKey
}
if !bt.Key.Equal(expectedKey) {
t.Fatalf("expected transaction key %v, got %v", expectedKey, bt.Key)
}
return ba.CreateReply(), nil
}))
if err := db.Txn(context.TODO(), func(txn *Txn) error {
tries++
b := txn.NewBatch()
b.Put("a", "b")
if err := txn.Run(b); err != nil {
t.Fatal(err)
}
if tries == 1 {
return roachpb.NewErrorWithTxn(roachpb.NewTransactionRetryError(), &txn.Proto).GoError()
}
return nil
}); err != nil {
t.Errorf("unexpected error on commit: %s", err)
}
minimumTries := 2
if tries < minimumTries {
t.Errorf("expected try count >= %d, got %d", minimumTries, tries)
}
}
// Test that a TransactionRetryError will retry the read until it succeeds. The
// test is designed so that if the proto timestamps are bumped during retry
// a failure will occur.
func TestAsOfRetry(t *testing.T) {
defer leaktest.AfterTest(t)()
params, cmdFilters := createTestServerParams()
// Disable one phase commits because they cannot be restarted.
params.Knobs.Store.(*storage.StoreTestingKnobs).DisableOnePhaseCommits = true
s, sqlDB, _ := serverutils.StartServer(t, params)
defer s.Stopper().Stop()
const val1 = 1
const val2 = 2
const name = "boulanger"
if _, err := sqlDB.Exec(`
CREATE DATABASE d;
CREATE TABLE d.t (s STRING PRIMARY KEY, a INT);
`); err != nil {
t.Fatal(err)
}
var tsStart string
if err := sqlDB.QueryRow(`
INSERT INTO d.t (s, a) VALUES ($1, $2)
RETURNING cluster_logical_timestamp();
`, name, val1).Scan(&tsStart); err != nil {
t.Fatal(err)
}
var tsVal2 string
if err := sqlDB.QueryRow("UPDATE d.t SET a = $1 RETURNING cluster_logical_timestamp()", val2).Scan(&tsVal2); err != nil {
t.Fatal(err)
}
walltime := new(inf.Dec)
if _, ok := walltime.SetString(tsVal2); !ok {
t.Fatalf("couldn't set decimal: %s", tsVal2)
}
oneTick := inf.NewDec(1, 0)
// Set tsVal1 to 1ns before tsVal2.
tsVal1 := walltime.Sub(walltime, oneTick).String()
// Set up error injection that causes retries.
magicVals := createFilterVals(nil, nil)
magicVals.restartCounts = map[string]int{
name: 5,
}
cleanupFilter := cmdFilters.AppendFilter(
func(args storagebase.FilterArgs) *roachpb.Error {
magicVals.Lock()
defer magicVals.Unlock()
switch req := args.Req.(type) {
case *roachpb.ScanRequest:
for key, count := range magicVals.restartCounts {
if err := checkCorrectTxn(string(req.Key), magicVals, args.Hdr.Txn); err != nil {
return roachpb.NewError(err)
}
if count > 0 && bytes.Contains(req.Key, []byte(key)) {
magicVals.restartCounts[key]--
err := roachpb.NewTransactionRetryError()
magicVals.failedValues[string(req.Key)] =
failureRecord{err, args.Hdr.Txn}
txn := args.Hdr.Txn.Clone()
txn.Timestamp = txn.Timestamp.Add(0, 1)
return roachpb.NewErrorWithTxn(err, &txn)
}
}
}
return nil
}, false)
var i int
// Query with tsVal1 which should return the first value. Since tsVal1 is just
// one nanosecond before tsVal2, any proto timestamp bumping will return val2
// and error.
// Must specify the WHERE here to trigger the injection errors.
if err := sqlDB.QueryRow(fmt.Sprintf("SELECT a FROM d.t AS OF SYSTEM TIME %s WHERE s = '%s'", tsVal1, name)).Scan(&i); err != nil {
t.Fatal(err)
} else if i != val1 {
t.Fatalf("unexpected val: %v", i)
}
cleanupFilter()
// Verify that the retry errors were injected.
checkRestarts(t, magicVals)
// Query with tsVal2 to ensure val2 is indeed present.
if err := sqlDB.QueryRow(fmt.Sprintf("SELECT a FROM d.t AS OF SYSTEM TIME %s", tsVal2)).Scan(&i); err != nil {
t.Fatal(err)
} else if i != val2 {
t.Fatalf("unexpected val: %v", i)
}
}