// TestRangeLookupWithOpenTransaction verifies that range lookups are
// done in such a way (e.g. using inconsistent reads) that they
// proceed in the event that a write intent is extant at the meta
// index record being read.
func TestRangeLookupWithOpenTransaction(t *testing.T) {
defer leaktest.AfterTest(t)
s := server.StartTestServer(t)
defer s.Stop()
db := createTestClient(t, s.Stopper(), s.ServingAddr())
// Create an intent on the meta1 record by writing directly to the
// engine.
key := testutils.MakeKey(keys.Meta1Prefix, roachpb.KeyMax)
now := s.Clock().Now()
txn := roachpb.NewTransaction("txn", roachpb.Key("foobar"), 0, roachpb.SERIALIZABLE, now, 0)
if err := engine.MVCCPutProto(s.Ctx.Engines[0], nil, key, now, txn, &roachpb.RangeDescriptor{}); err != nil {
t.Fatal(err)
}
// Now, with an intent pending, attempt (asynchronously) to read
// from an arbitrary key. This will cause the distributed sender to
// do a range lookup, which will encounter the intent. We're
// verifying here that the range lookup doesn't fail with a write
// intent error. If it did, it would go into a deadloop attempting
// to push the transaction, which in turn requires another range
// lookup, etc, ad nauseam.
if _, err := db.Get("a"); err != nil {
t.Fatal(err)
}
}
// newTxn begins a transaction. For testing purposes, this comes with a ID
// pre-initialized, but with the Writing flag set to false.
func newTxn(clock *hlc.Clock, baseKey roachpb.Key) *roachpb.Transaction {
f, l, fun := caller.Lookup(1)
name := fmt.Sprintf("%s:%d %s", f, l, fun)
txn := roachpb.NewTransaction("test", baseKey, 1, roachpb.SERIALIZABLE, clock.Now(), clock.MaxOffset().Nanoseconds())
txn.Name = name
return txn
}
// Verifies that an inner transaction in a nested transaction strips the transaction
// information in its error when propagating it to an other transaction.
func TestNestedTransaction(t *testing.T) {
defer leaktest.AfterTest(t)()
s, db := setup()
defer s.Stop()
pErr := db.Txn(func(txn1 *client.Txn) *roachpb.Error {
if pErr := txn1.Put("a", "1"); pErr != nil {
t.Fatalf("unexpected put error: %s", pErr)
}
iPErr := db.Txn(func(txn2 *client.Txn) *roachpb.Error {
txnProto := roachpb.NewTransaction("test", roachpb.Key("a"), 1, roachpb.SERIALIZABLE, roachpb.Timestamp{}, 0)
return roachpb.NewErrorWithTxn(util.Errorf("inner txn error"), txnProto)
})
if iPErr.GetTxn() != nil {
t.Errorf("error txn must be stripped: %s", iPErr)
}
return iPErr
})
if pErr == nil {
t.Fatal("unexpected success of txn")
}
if !testutils.IsPError(pErr, "inner txn error") {
t.Errorf("unexpected failure: %s", pErr)
}
}
// TestTxnMultipleCoord checks that a coordinator uses the Writing flag to
// enforce that only one coordinator can be used for transactional writes.
func TestTxnMultipleCoord(t *testing.T) {
defer leaktest.AfterTest(t)()
s, sender := createTestDB(t)
defer s.Stop()
testCases := []struct {
args roachpb.Request
writing bool
ok bool
}{
{roachpb.NewGet(roachpb.Key("a")), true, false},
{roachpb.NewGet(roachpb.Key("a")), false, true},
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), false, false}, // transactional write before begin
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), true, false}, // must have switched coordinators
}
for i, tc := range testCases {
txn := roachpb.NewTransaction("test", roachpb.Key("a"), 1, roachpb.SERIALIZABLE,
s.Clock.Now(), s.Clock.MaxOffset().Nanoseconds())
txn.Writing = tc.writing
reply, pErr := client.SendWrappedWith(sender, nil, roachpb.Header{
Txn: txn,
}, tc.args)
if pErr == nil != tc.ok {
t.Errorf("%d: %T (writing=%t): success_expected=%t, but got: %v",
i, tc.args, tc.writing, tc.ok, pErr)
}
if pErr != nil {
continue
}
txn = reply.Header().Txn
// The transaction should come back rw if it started rw or if we just
// wrote.
isWrite := roachpb.IsTransactionWrite(tc.args)
if (tc.writing || isWrite) != txn.Writing {
t.Errorf("%d: unexpected writing state: %s", i, txn)
}
if !isWrite {
continue
}
// Abort for clean shutdown.
if _, pErr := client.SendWrappedWith(sender, nil, roachpb.Header{
Txn: txn,
}, &roachpb.EndTransactionRequest{
Commit: false,
}); pErr != nil {
t.Fatal(pErr)
}
}
}
// maybeBeginTxn begins a new transaction if a txn has been specified
// in the request but has a nil ID. The new transaction is initialized
// using the name and isolation in the otherwise uninitialized txn.
// The Priority, if non-zero is used as a minimum.
//
// No transactional writes are allowed unless preceded by a begin
// transaction request within the same batch. The exception is if the
// transaction is already in state txn.Writing=true.
func (tc *TxnCoordSender) maybeBeginTxn(ba *roachpb.BatchRequest) error {
if ba.Txn == nil {
return nil
}
if len(ba.Requests) == 0 {
return util.Errorf("empty batch with txn")
}
if ba.Txn.ID == nil {
// Create transaction without a key. The key is set when a begin
// transaction request is received.
// The initial timestamp may be communicated by a higher layer.
// If so, use that. Otherwise make up a new one.
timestamp := ba.Txn.OrigTimestamp
if timestamp == roachpb.ZeroTimestamp {
timestamp = tc.clock.Now()
}
newTxn := roachpb.NewTransaction(ba.Txn.Name, nil, ba.UserPriority,
ba.Txn.Isolation, timestamp, tc.clock.MaxOffset().Nanoseconds())
// Use existing priority as a minimum. This is used on transaction
// aborts to ratchet priority when creating successor transaction.
if newTxn.Priority < ba.Txn.Priority {
newTxn.Priority = ba.Txn.Priority
}
ba.Txn = newTxn
}
// Check for a begin transaction to set txn key based on the key of
// the first transactional write. Also enforce that no transactional
// writes occur before a begin transaction.
var haveBeginTxn bool
for _, req := range ba.Requests {
args := req.GetInner()
if bt, ok := args.(*roachpb.BeginTransactionRequest); ok {
if haveBeginTxn || ba.Txn.Writing {
return util.Errorf("begin transaction requested twice in the same transaction: %s", ba.Txn)
}
haveBeginTxn = true
if ba.Txn.Key == nil {
ba.Txn.Key = bt.Key
}
}
if roachpb.IsTransactionWrite(args) && !haveBeginTxn && !ba.Txn.Writing {
return util.Errorf("transactional write before begin transaction")
}
}
return nil
}
// maybeBeginTxn begins a new transaction if a txn has been specified
// in the request but has a nil ID. The new transaction is initialized
// using the name and isolation in the otherwise uninitialized txn.
// The Priority, if non-zero is used as a minimum.
func (tc *TxnCoordSender) maybeBeginTxn(ba *roachpb.BatchRequest) {
if ba.Txn == nil {
return
}
if len(ba.Requests) == 0 {
panic("empty batch with txn")
}
if len(ba.Txn.ID) == 0 {
// TODO(tschottdorf): should really choose the first txn write here.
firstKey := ba.Requests[0].GetInner().Header().Key
newTxn := roachpb.NewTransaction(ba.Txn.Name, keys.KeyAddress(firstKey), ba.GetUserPriority(),
ba.Txn.Isolation, tc.clock.Now(), tc.clock.MaxOffset().Nanoseconds())
// Use existing priority as a minimum. This is used on transaction
// aborts to ratchet priority when creating successor transaction.
if newTxn.Priority < ba.Txn.Priority {
newTxn.Priority = ba.Txn.Priority
}
ba.Txn = newTxn
}
}
// Verifies that a nested transaction returns an error if an inner txn
// propagates an error to an outer txn.
func TestNestedTransaction(t *testing.T) {
defer leaktest.AfterTest(t)()
s, db := setup()
defer s.Stop()
txnProto := roachpb.NewTransaction("test", roachpb.Key("a"), 1, roachpb.SERIALIZABLE, roachpb.Timestamp{}, 0)
pErr := db.Txn(func(txn1 *client.Txn) *roachpb.Error {
if pErr := txn1.Put("a", "1"); pErr != nil {
t.Fatalf("unexpected put error: %s", pErr)
}
return db.Txn(func(txn2 *client.Txn) *roachpb.Error {
return roachpb.NewErrorWithTxn(util.Errorf("err"), txnProto)
})
})
if pErr == nil {
t.Fatal("unexpected success of txn")
}
if !testutils.IsPError(pErr, "mismatching transaction record in the error") {
t.Errorf("unexpected failure: %s", pErr)
}
}
// TestStoreRangeSplit executes a split of a range and verifies that the
// resulting ranges respond to the right key ranges and that their stats
// and sequence cache have been properly accounted for.
func TestStoreRangeSplitIdempotency(t *testing.T) {
defer leaktest.AfterTest(t)
store, stopper := createTestStore(t)
defer stopper.Stop()
rangeID := roachpb.RangeID(1)
splitKey := roachpb.Key("m")
content := roachpb.Key("asdvb")
// First, write some values left and right of the proposed split key.
pArgs := putArgs([]byte("c"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("x"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
// Increments are a good way of testing the sequence cache. Up here, we
// address them to the original range, then later to the one that contains
// the key.
txn := roachpb.NewTransaction("test", []byte("c"), 10, roachpb.SERIALIZABLE,
store.Clock().Now(), 0)
lIncArgs := incrementArgs([]byte("apoptosis"), 100)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: txn,
}, &lIncArgs); err != nil {
t.Fatal(err)
}
rIncArgs := incrementArgs([]byte("wobble"), 10)
txn.Sequence++
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: txn,
}, &rIncArgs); err != nil {
t.Fatal(err)
}
// Get the original stats for key and value bytes.
var ms engine.MVCCStats
if err := engine.MVCCGetRangeStats(store.Engine(), rangeID, &ms); err != nil {
t.Fatal(err)
}
keyBytes, valBytes := ms.KeyBytes, ms.ValBytes
// Split the range.
args := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
// Verify no intents remains on range descriptor keys.
for _, key := range []roachpb.Key{keys.RangeDescriptorKey(roachpb.RKeyMin), keys.RangeDescriptorKey(keys.Addr(splitKey))} {
if _, _, err := engine.MVCCGet(store.Engine(), key, store.Clock().Now(), true, nil); err != nil {
t.Fatal(err)
}
}
rng := store.LookupReplica(roachpb.RKeyMin, nil)
newRng := store.LookupReplica([]byte("m"), nil)
if !bytes.Equal(newRng.Desc().StartKey, splitKey) || !bytes.Equal(splitKey, rng.Desc().EndKey) {
t.Errorf("ranges mismatched, wanted %q=%q=%q", newRng.Desc().StartKey, splitKey, rng.Desc().EndKey)
}
if !bytes.Equal(newRng.Desc().EndKey, roachpb.RKeyMax) || !bytes.Equal(rng.Desc().StartKey, roachpb.RKeyMin) {
t.Errorf("new ranges do not cover KeyMin-KeyMax, but only %q-%q", rng.Desc().StartKey, newRng.Desc().EndKey)
}
// Try to get values from both left and right of where the split happened.
gArgs := getArgs([]byte("c"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("x"))
if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: newRng.Desc().RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Send out an increment request copied from above (same txn/sequence)
// which remains in the old range.
_, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: txn,
}, &lIncArgs)
if _, ok := err.(*roachpb.TransactionRetryError); !ok {
t.Fatalf("unexpected sequence cache miss: %v", err)
}
// Send out the same increment copied from above (same txn/sequence), but
// now to the newly created range (which should hold that key).
_, err = client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: newRng.Desc().RangeID,
Txn: txn,
}, &rIncArgs)
if _, ok := err.(*roachpb.TransactionRetryError); !ok {
t.Fatalf("unexpected sequence cache miss: %v", err)
}
// Compare stats of split ranges to ensure they are non zero and
// exceed the original range when summed.
var left, right engine.MVCCStats
if err := engine.MVCCGetRangeStats(store.Engine(), rangeID, &left); err != nil {
t.Fatal(err)
}
lKeyBytes, lValBytes := left.KeyBytes, left.ValBytes
if err := engine.MVCCGetRangeStats(store.Engine(), newRng.Desc().RangeID, &right); err != nil {
t.Fatal(err)
}
rKeyBytes, rValBytes := right.KeyBytes, right.ValBytes
if lKeyBytes == 0 || rKeyBytes == 0 {
t.Errorf("expected non-zero key bytes; got %d, %d", lKeyBytes, rKeyBytes)
}
if lValBytes == 0 || rValBytes == 0 {
t.Errorf("expected non-zero val bytes; got %d, %d", lValBytes, rValBytes)
}
if lKeyBytes+rKeyBytes <= keyBytes {
t.Errorf("left + right key bytes don't match; %d + %d <= %d", lKeyBytes, rKeyBytes, keyBytes)
}
if lValBytes+rValBytes <= valBytes {
t.Errorf("left + right val bytes don't match; %d + %d <= %d", lValBytes, rValBytes, valBytes)
}
}