// TestInternalPushTxnUpgradeExistingTxn verifies that pushing
// a transaction record with a new epoch upgrades the pushee's
// epoch and timestamp if greater. In all test cases, the
// priorities are set such that the push will succeed.
func TestInternalPushTxnUpgradeExistingTxn(t *testing.T) {
rng, _, clock, _ := createTestRangeWithClock(t)
defer rng.Stop()
ts1 := proto.Timestamp{WallTime: 1}
ts2 := proto.Timestamp{WallTime: 2}
testCases := []struct {
startEpoch, epoch, expEpoch int32
startTS, ts, expTS proto.Timestamp
}{
// Move epoch forward.
{0, 1, 1, ts1, ts1, ts1},
// Move timestamp forward.
{0, 0, 0, ts1, ts2, ts2},
// Move epoch backwards (has no effect).
{1, 0, 1, ts1, ts1, ts1},
// Move timestamp backwards (has no effect).
{0, 0, 0, ts2, ts1, ts2},
// Move both epoch & timestamp forward.
{0, 1, 1, ts1, ts2, ts2},
// Move both epoch & timestamp backward (has no effect).
{1, 0, 1, ts2, ts1, ts2},
}
for i, test := range testCases {
key := engine.Key(fmt.Sprintf("key-%d", i))
pusher := NewTransaction(engine.MakeKey(key, []byte{1}), 1, proto.SERIALIZABLE, clock)
pushee := NewTransaction(engine.MakeKey(key, []byte{2}), 1, proto.SERIALIZABLE, clock)
pushee.Priority = 1
pusher.Priority = 2 // Pusher will win.
// First, establish "start" of existing pushee's txn via heartbeat.
pushee.Epoch = test.startEpoch
pushee.Timestamp = test.startTS
hbArgs, hbReply := heartbeatArgs(pushee, 0)
hbArgs.Timestamp = pushee.Timestamp
if err := rng.ReadWriteCmd("InternalHeartbeatTxn", hbArgs, hbReply); err != nil {
t.Fatal(err)
}
// Now, attempt to push the transaction using updated values for epoch & timestamp.
pushee.Epoch = test.epoch
pushee.Timestamp = test.ts
args, reply := pushTxnArgs(pusher, pushee, true, 0)
if err := rng.ReadWriteCmd("InternalPushTxn", args, reply); err != nil {
t.Fatal(err)
}
expTxn := gogoproto.Clone(pushee).(*proto.Transaction)
expTxn.Epoch = test.expEpoch
expTxn.Timestamp = test.expTS
expTxn.Status = proto.ABORTED
expTxn.LastHeartbeat = &test.startTS
if !reflect.DeepEqual(expTxn, reply.PusheeTxn) {
t.Errorf("unexpected push txn in trial %d; expected %+v, got %+v", i, expTxn, reply.PusheeTxn)
}
}
}
func TestMerge(t *testing.T) {
for _, m := range mergeTests {
got := proto.Clone(m.dst)
proto.Merge(got, m.src)
if !proto.Equal(got, m.want) {
t.Errorf("Merge(%v, %v)\n got %v\nwant %v\n", m.dst, m.src, got, m.want)
}
}
}
func TestClone(t *testing.T) {
m := proto.Clone(cloneTestMessage).(*pb.MyMessage)
if !proto.Equal(m, cloneTestMessage) {
t.Errorf("Clone(%v) = %v", cloneTestMessage, m)
}
// Verify it was a deep copy.
*m.Inner.Port++
if proto.Equal(m, cloneTestMessage) {
t.Error("Mutating clone changed the original")
}
}
func TestCloneNil(t *testing.T) {
var m *pb.MyMessage
if c := proto.Clone(m); !proto.Equal(m, c) {
t.Errorf("Clone(%v) = %v", m, c)
}
}
// EndTransaction either commits or aborts (rolls back) an extant
// transaction according to the args.Commit parameter.
func (r *Range) EndTransaction(args *proto.EndTransactionRequest, reply *proto.EndTransactionResponse) {
// Create the actual key to the system-local transaction table.
key := engine.MakeKey(engine.KeyLocalTransactionPrefix, args.Key)
// Start with supplied transaction, then possibly load from txn record.
reply.Txn = gogoproto.Clone(args.Txn).(*proto.Transaction)
// Fetch existing transaction if possible.
existTxn := &proto.Transaction{}
ok, err := engine.GetProto(r.engine, key, existTxn)
if err != nil {
reply.SetGoError(err)
return
}
// If the transaction record already exists, verify that we can either
// commit it or abort it (according to args.Commit), and also that the
// Timestamp and Epoch have not suffered regression.
if ok {
if existTxn.Status == proto.COMMITTED {
reply.SetGoError(proto.NewTransactionStatusError(existTxn, "already committed"))
return
} else if existTxn.Status == proto.ABORTED {
reply.SetGoError(proto.NewTransactionStatusError(existTxn, "already aborted"))
return
} else if args.Txn.Epoch < existTxn.Epoch {
reply.SetGoError(proto.NewTransactionStatusError(existTxn, fmt.Sprintf("epoch regression: %d", args.Txn.Epoch)))
return
} else if existTxn.Timestamp.Less(args.Txn.Timestamp) {
// The transaction record can only ever be pushed forward, so it's an
// error if somehow the transaction record has an earlier timestamp
// than the transaction timestamp.
reply.SetGoError(proto.NewTransactionStatusError(existTxn, fmt.Sprintf("timestamp regression: %+v", args.Txn.Timestamp)))
return
}
// Use the persisted transaction record as final transaction.
gogoproto.Merge(reply.Txn, existTxn)
}
// Take max of requested timestamp and possibly "pushed" txn
// record timestamp as the final commit timestamp.
if reply.Txn.Timestamp.Less(args.Timestamp) {
reply.Txn.Timestamp = args.Timestamp
}
// Set transaction status to COMMITTED or ABORTED as per the
// args.Commit parameter.
if args.Commit {
// If the isolation level is SERIALIZABLE, return a transaction
// retry error if the commit timestamp isn't equal to the txn
// timestamp.
if args.Txn.Isolation == proto.SERIALIZABLE && !reply.Txn.Timestamp.Equal(args.Txn.Timestamp) {
reply.SetGoError(proto.NewTransactionRetryError(reply.Txn))
return
}
reply.Txn.Status = proto.COMMITTED
} else {
reply.Txn.Status = proto.ABORTED
}
// Persist the transaction record with updated status (& possibly timestmap).
if err := engine.PutProto(r.engine, key, reply.Txn); err != nil {
reply.SetGoError(err)
return
}
}
// makeTxn creates a new transaction using the specified base
// txn and timestamp.
func makeTxn(baseTxn *proto.Transaction, ts proto.Timestamp) *proto.Transaction {
txn := gogoproto.Clone(baseTxn).(*proto.Transaction)
txn.Timestamp = ts
return txn
}
// InternalPushTxn resolves conflicts between concurrent txns (or
// between a non-transactional reader or writer and a txn) in several
// ways depending on the statuses and priorities of the conflicting
// transactions. The InternalPushTxn operation is invoked by a
// "pusher" (the writer trying to abort a conflicting txn or the
// reader trying to push a conflicting txn's commit timestamp
// forward), who attempts to resolve a conflict with a "pushee"
// (args.PushTxn -- the pushee txn whose intent(s) caused the
// conflict).
//
// Txn already committed/aborted: If pushee txn is committed or
// aborted return success.
//
// Txn Timeout: If pushee txn entry isn't present or its LastHeartbeat
// timestamp isn't set, use PushTxn.Timestamp as LastHeartbeat. If
// current time - LastHeartbeat > 2 * DefaultHeartbeatInterval, then
// the pushee txn should be either pushed forward or aborted,
// depending on value of Request.Abort.
//
// Old Txn Epoch: If persisted pushee txn entry has a newer Epoch than
// PushTxn.Epoch, return success, as older epoch may be removed.
//
// Lower Txn Priority: If pushee txn has a lower priority than pusher,
// adjust pushee's persisted txn depending on value of args.Abort. If
// args.Abort is true, set txn.Status to ABORTED, and priority to one
// less than the pusher's priority and return success. If args.Abort
// is false, set txn.Timestamp to pusher's txn.Timestamp + 1.
//
// Higher Txn Priority: If pushee txn has a higher priority than
// pusher, return TransactionRetryError. Transaction will be retried
// with priority one less than the pushee's higher priority.
func (r *Range) InternalPushTxn(args *proto.InternalPushTxnRequest, reply *proto.InternalPushTxnResponse) {
if !bytes.Equal(args.Key, args.PusheeTxn.ID) {
reply.SetGoError(util.Errorf("request key %q should match pushee's txn ID %q", args.Key, args.PusheeTxn.ID))
return
}
// Create the actual key to the system-local transaction table.
key := engine.MakeKey(engine.KeyLocalTransactionPrefix, args.Key)
// Fetch existing transaction if possible.
existTxn := &proto.Transaction{}
ok, err := engine.GetProto(r.engine, key, existTxn)
if err != nil {
reply.SetGoError(err)
return
}
if ok {
// Start with the persisted transaction record as final transaction.
reply.PusheeTxn = gogoproto.Clone(existTxn).(*proto.Transaction)
// Upgrade the epoch and timestamp as necessary.
if reply.PusheeTxn.Epoch < args.PusheeTxn.Epoch {
reply.PusheeTxn.Epoch = args.PusheeTxn.Epoch
}
if reply.PusheeTxn.Timestamp.Less(args.PusheeTxn.Timestamp) {
reply.PusheeTxn.Timestamp = args.PusheeTxn.Timestamp
}
} else {
// Some sanity checks for case where we don't find a transaction record.
if args.PusheeTxn.LastHeartbeat != nil {
reply.SetGoError(proto.NewTransactionStatusError(&args.PusheeTxn,
"no txn persisted, yet intent has heartbeat"))
return
} else if args.PusheeTxn.Status != proto.PENDING {
reply.SetGoError(proto.NewTransactionStatusError(&args.PusheeTxn,
fmt.Sprintf("no txn persisted, yet intent has status %s", args.PusheeTxn.Status)))
return
}
// The transaction doesn't exist yet on disk; use the supplied version.
reply.PusheeTxn = gogoproto.Clone(&args.PusheeTxn).(*proto.Transaction)
}
// If already committed or aborted, return success.
if reply.PusheeTxn.Status != proto.PENDING {
// Trivial noop.
return
}
// If we're trying to move the timestamp forward, and it's already
// far enough forward, return success.
if !args.Abort && args.Timestamp.Less(reply.PusheeTxn.Timestamp) {
// Trivial noop.
return
}
// pusherWins bool is true in the event the pusher prevails.
var pusherWins bool
// Check for txn timeout.
if reply.PusheeTxn.LastHeartbeat == nil {
reply.PusheeTxn.LastHeartbeat = &reply.PusheeTxn.Timestamp
}
// Compute heartbeat expiration.
expiry := r.clock.Now()
expiry.WallTime -= 2 * DefaultHeartbeatInterval.Nanoseconds()
if reply.PusheeTxn.LastHeartbeat.Less(expiry) {
log.V(1).Infof("pushing expired txn %+v", reply.PusheeTxn)
pusherWins = true
} else if args.PusheeTxn.Epoch < reply.PusheeTxn.Epoch {
// Check for an intent from a prior epoch.
log.V(1).Infof("pushing intent from previous epoch for txn %+v", reply.PusheeTxn)
pusherWins = true
} else if reply.PusheeTxn.Priority < args.Txn.Priority ||
(reply.PusheeTxn.Priority == args.Txn.Priority && args.Txn.Timestamp.Less(reply.PusheeTxn.Timestamp)) {
// Finally, choose based on priority; if priorities are equal, order by lower txn timestamp.
log.V(1).Infof("pushing intent from txn with lower priority %+v vs %+v", reply.PusheeTxn, args.Txn)
pusherWins = true
}
if !pusherWins {
log.V(1).Infof("failed to push intent %+v vs %+v", reply.PusheeTxn, args.Txn)
reply.SetGoError(proto.NewTransactionRetryError(reply.PusheeTxn))
return
}
// If aborting transaction, set new status and return success.
if args.Abort {
reply.PusheeTxn.Status = proto.ABORTED
} else {
// Otherwise, update timestamp to be one greater than the request's timestamp.
reply.PusheeTxn.Timestamp = args.Timestamp
reply.PusheeTxn.Timestamp.Logical++
}
// Persist the pushed transaction.
if err := engine.PutProto(r.engine, key, reply.PusheeTxn); err != nil {
reply.SetGoError(err)
return
}
}