// TestTxnCoordSenderTxnUpdatedOnError verifies that errors adjust the
// response transaction's timestamp and priority as appropriate.
func TestTxnCoordSenderTxnUpdatedOnError(t *testing.T) {
defer leaktest.AfterTest(t)
t.Skip("TODO(tschottdorf): fix up and re-enable. It depends on each logical clock tick, so not fun.")
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(20)
testCases := []struct {
err error
expEpoch int32
expPri int32
expTS proto.Timestamp
expOrigTS proto.Timestamp
nodeSeen bool
}{
{nil, 0, 1, makeTS(0, 1), makeTS(0, 1), false},
{&proto.ReadWithinUncertaintyIntervalError{
ExistingTimestamp: makeTS(10, 10)}, 1, 1, makeTS(10, 11),
makeTS(10, 11), true},
{&proto.TransactionAbortedError{Txn: proto.Transaction{
Timestamp: makeTS(20, 10), Priority: 10}}, 0, 10, makeTS(20, 10),
makeTS(0, 1), false},
{&proto.TransactionPushError{PusheeTxn: proto.Transaction{
Timestamp: makeTS(10, 10), Priority: int32(10)}}, 1, 9,
makeTS(10, 11), makeTS(10, 11), false},
{&proto.TransactionRetryError{Txn: proto.Transaction{
Timestamp: makeTS(10, 10), Priority: int32(10)}}, 1, 10,
makeTS(10, 10), makeTS(10, 10), false},
}
var testPutReq = &proto.PutRequest{
RequestHeader: proto.RequestHeader{
Key: proto.Key("test-key"),
UserPriority: gogoproto.Int32(-1),
Txn: &proto.Transaction{
Name: "test txn",
},
Replica: proto.Replica{
NodeID: 12345,
},
},
}
for i, test := range testCases {
stopper := stop.NewStopper()
ts := NewTxnCoordSender(senderFn(func(_ context.Context, _ proto.BatchRequest) (*proto.BatchResponse, *proto.Error) {
return nil, proto.NewError(test.err)
}), clock, false, nil, stopper)
var reply *proto.PutResponse
if r, err := batchutil.SendWrapped(ts, gogoproto.Clone(testPutReq).(proto.Request)); err != nil {
t.Fatal(err)
} else {
reply = r.(*proto.PutResponse)
}
teardownHeartbeats(ts)
stopper.Stop()
if reflect.TypeOf(test.err) != reflect.TypeOf(reply.GoError()) {
t.Fatalf("%d: expected %T; got %T: %v", i, test.err, reply.GoError(), reply.GoError())
}
if reply.Txn.Epoch != test.expEpoch {
t.Errorf("%d: expected epoch = %d; got %d",
i, test.expEpoch, reply.Txn.Epoch)
}
if reply.Txn.Priority != test.expPri {
t.Errorf("%d: expected priority = %d; got %d",
i, test.expPri, reply.Txn.Priority)
}
if !reply.Txn.Timestamp.Equal(test.expTS) {
t.Errorf("%d: expected timestamp to be %s; got %s",
i, test.expTS, reply.Txn.Timestamp)
}
if !reply.Txn.OrigTimestamp.Equal(test.expOrigTS) {
t.Errorf("%d: expected orig timestamp to be %s + 1; got %s",
i, test.expOrigTS, reply.Txn.OrigTimestamp)
}
if nodes := reply.Txn.CertainNodes.Nodes; (len(nodes) != 0) != test.nodeSeen {
t.Errorf("%d: expected nodeSeen=%t, but list of hosts is %v",
i, test.nodeSeen, nodes)
}
}
}
// verifyUncertainty writes values to a key in 5ns intervals and then launches
// a transaction at each value's timestamp reading that value with
// the maximumOffset given, verifying in the process that the correct values
// are read (usually after one transaction restart).
func verifyUncertainty(concurrency int, maxOffset time.Duration, t *testing.T) {
db, _, clock, _, lSender, transport, err := createTestDB()
if err != nil {
t.Fatal(err)
}
defer transport.Close()
txnOpts := &client.TransactionOptions{
Name: "test",
}
key := []byte("key-test")
// wgStart waits for all transactions to line up, wgEnd has the main
// function wait for them to finish.
var wgStart, wgEnd sync.WaitGroup
wgStart.Add(concurrency + 1)
wgEnd.Add(concurrency)
// Initial high offset to allow for future writes.
clock.SetMaxOffset(999 * time.Nanosecond)
for i := 0; i < concurrency; i++ {
value := []byte(fmt.Sprintf("value-%d", i))
// Values will be written with 5ns spacing.
futureTS := clock.Now().Add(5, 0)
clock.Update(futureTS)
// Expected number of versions skipped.
skipCount := int(maxOffset) / 5
if i+skipCount >= concurrency {
skipCount = concurrency - i - 1
}
readValue := []byte(fmt.Sprintf("value-%d", i+skipCount))
pr := proto.PutResponse{}
db.Call(proto.Put, &proto.PutRequest{
RequestHeader: proto.RequestHeader{
Key: key,
},
Value: proto.Value{Bytes: value},
}, &pr)
if err := pr.GoError(); err != nil {
t.Errorf("%d: got write error: %v", i, err)
}
gr := proto.GetResponse{}
db.Call(proto.Get, &proto.GetRequest{
RequestHeader: proto.RequestHeader{
Key: key,
Timestamp: clock.Now(),
},
}, &gr)
if gr.GoError() != nil || gr.Value == nil || !bytes.Equal(gr.Value.Bytes, value) {
t.Fatalf("%d: expected success reading value %+v: %v", i, gr.Value, gr.GoError())
}
go func(i int) {
defer wgEnd.Done()
wgStart.Done()
// Wait until the other goroutines are running.
wgStart.Wait()
txnManual := hlc.NewManualClock(futureTS.WallTime)
txnClock := hlc.NewClock(txnManual.UnixNano)
// Make sure to incorporate the logical component if the wall time
// hasn't changed (i=0). The logical component will change
// internally in a way we can't track, but we want to be just
// ahead.
txnClock.Update(futureTS.Add(0, 999))
// The written values are spaced out in intervals of 5ns, so
// setting <5ns here should make do without any restarts while
// higher values require roughly offset/5 restarts.
txnClock.SetMaxOffset(maxOffset)
sender := NewTxnCoordSender(lSender, txnClock, false)
txnDB := client.NewKV(sender, nil)
txnDB.User = storage.UserRoot
if err := txnDB.RunTransaction(txnOpts, func(txn *client.KV) error {
// Read within the transaction.
gr := proto.GetResponse{}
txn.Call(proto.Get, &proto.GetRequest{
RequestHeader: proto.RequestHeader{
Key: key,
Timestamp: futureTS,
},
}, &gr)
if err := gr.GoError(); err != nil {
if _, ok := gr.GoError().(*proto.ReadWithinUncertaintyIntervalError); ok {
return err
}
return util.Errorf("unexpected read error of type %s: %v", reflect.TypeOf(err), err)
}
if gr.Value == nil || gr.Value.Bytes == nil {
return util.Errorf("no value read")
}
if !bytes.Equal(gr.Value.Bytes, readValue) {
return util.Errorf("%d: read wrong value %q at %v, wanted %q", i, gr.Value.Bytes, futureTS, readValue)
}
return nil
}); err != nil {
t.Error(err)
}
}(i)
}
// Kick the goroutines loose.
wgStart.Done()
// Wait for the goroutines to finish.
wgEnd.Wait()
}