func TestGet(t *testing.T) {
defer leaktest.AfterTest(t)
emptyKeys := []roachpb.KeyValue{}
someKeys := []roachpb.KeyValue{
plainKV("a", "vala"),
plainKV("c", "valc"),
plainKV("d", "vald"),
}
aVal := roachpb.MakeValueFromString("vala")
bVal := roachpb.MakeValueFromString("valc")
cVal := roachpb.MakeValueFromString("vald")
testCases := []struct {
values []roachpb.KeyValue
key string
value *roachpb.Value
}{
{emptyKeys, "a", nil},
{emptyKeys, "b", nil},
{emptyKeys, "c", nil},
{emptyKeys, "d", nil},
{emptyKeys, "e", nil},
{someKeys, "", nil},
{someKeys, "b", nil},
{someKeys, "e", nil},
{someKeys, "a0", nil},
{someKeys, "a", &aVal},
{someKeys, "c", &bVal},
{someKeys, "d", &cVal},
}
cfg := config.SystemConfig{}
for tcNum, tc := range testCases {
cfg.Values = tc.values
if val := cfg.GetValue([]byte(tc.key)); !proto.Equal(val, tc.value) {
t.Errorf("#%d: expected=%s, found=%s", tcNum, tc.value, val)
}
}
}
func TestOwnNodeCertain(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
const expNodeID = 42
nd := &roachpb.NodeDescriptor{
NodeID: expNodeID,
Address: util.MakeUnresolvedAddr("tcp", "foobar:1234"),
}
g.ResetNodeID(nd.NodeID)
if err := g.SetNodeDescriptor(nd); err != nil {
t.Fatal(err)
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(expNodeID), nd, time.Hour); err != nil {
t.Fatal(err)
}
act := make(map[roachpb.NodeID]roachpb.Timestamp)
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
ba roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
for k, v := range ba.Txn.ObservedTimestamps {
act[k] = v
}
return ba.CreateReply(), nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
expTS := roachpb.ZeroTimestamp.Add(1, 2)
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrappedWith(ds, nil, roachpb.Header{
// MaxTimestamp is set very high so that all uncertainty updates have
// effect.
Txn: &roachpb.Transaction{OrigTimestamp: expTS, MaxTimestamp: roachpb.MaxTimestamp},
}, put); err != nil {
t.Fatalf("put encountered error: %s", err)
}
exp := map[roachpb.NodeID]roachpb.Timestamp{
expNodeID: expTS,
}
if !reflect.DeepEqual(exp, act) {
t.Fatalf("wanted %v, got %v", exp, act)
}
}
// TestClockUpdateOnResponse verifies that the DistSender picks up
// the timestamp of the remote party embedded in responses.
func TestClockUpdateOnResponse(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
ctx := &DistSenderContext{
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
expectedErr := roachpb.NewError(errors.New("boom"))
// Prepare the test function
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value"))
doCheck := func(testFn rpcSendFn, fakeTime roachpb.Timestamp) {
ds.rpcSend = testFn
_, err := client.SendWrapped(ds, nil, put)
if err != nil && err != expectedErr {
t.Fatal(err)
}
newTime := ds.clock.Now()
if newTime.Less(fakeTime) {
t.Fatalf("clock was not advanced: expected >= %s; got %s", fakeTime, newTime)
}
}
// Test timestamp propagation on valid BatchResults.
fakeTime := ds.clock.Now().Add(10000000000 /*10s*/, 0)
replyNormal := func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
rb := args.CreateReply()
rb.Now = fakeTime
return rb, nil
}
doCheck(replyNormal, fakeTime)
// Test timestamp propagation on errors.
fakeTime = ds.clock.Now().Add(10000000000 /*10s*/, 0)
replyError := func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
rb := args.CreateReply()
rb.Error = expectedErr
rb.Error.Now = fakeTime
return rb, nil
}
doCheck(replyError, fakeTime)
}
func TestBatchProto(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
e := NewInMem(roachpb.Attributes{}, 1<<20, stopper)
b := e.NewBatch()
defer b.Close()
val := roachpb.MakeValueFromString("value")
if _, _, err := PutProto(b, MVCCKey("proto"), &val); err != nil {
t.Fatal(err)
}
getVal := &roachpb.Value{}
ok, keySize, valSize, err := b.GetProto(MVCCKey("proto"), getVal)
if !ok || err != nil {
t.Fatalf("expected GetProto to success ok=%t: %s", ok, err)
}
if keySize != 5 {
t.Errorf("expected key size 5; got %d", keySize)
}
data, err := val.Marshal()
if err != nil {
t.Fatal(err)
}
if valSize != int64(len(data)) {
t.Errorf("expected value size %d; got %d", len(data), valSize)
}
if !proto.Equal(getVal, &val) {
t.Errorf("expected %v; got %v", &val, getVal)
}
// Before commit, proto will not be available via engine.
if ok, _, _, err := e.GetProto(MVCCKey("proto"), getVal); ok || err != nil {
t.Fatalf("expected GetProto to fail ok=%t: %s", ok, err)
}
// Commit and verify the proto can be read directly from the engine.
if err := b.Commit(); err != nil {
t.Fatal(err)
}
if ok, _, _, err := e.GetProto(MVCCKey("proto"), getVal); !ok || err != nil {
t.Fatalf("expected GetProto to success ok=%t: %s", ok, err)
}
if !proto.Equal(getVal, &val) {
t.Errorf("expected %v; got %v", &val, getVal)
}
}
func TestOwnNodeCertain(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
const expNodeID = 42
nd := &roachpb.NodeDescriptor{
NodeID: expNodeID,
Address: util.MakeUnresolvedAddr("tcp", "foobar:1234"),
}
g.SetNodeID(nd.NodeID)
if err := g.SetNodeDescriptor(nd); err != nil {
t.Fatal(err)
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(expNodeID), nd, time.Hour); err != nil {
t.Fatal(err)
}
var act roachpb.NodeList
var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) proto.Message, _ func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
ba := getArgs(nil).(*roachpb.BatchRequest)
for _, nodeID := range ba.Txn.CertainNodes.Nodes {
act.Add(roachpb.NodeID(nodeID))
}
return []proto.Message{ba.CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrappedWith(ds, nil, roachpb.Header{
Txn: &roachpb.Transaction{},
}, put); err != nil {
t.Fatalf("put encountered error: %s", err)
}
if expNodes := []roachpb.NodeID{expNodeID}; !reflect.DeepEqual(act.Nodes, expNodes) {
t.Fatalf("got %v, expected %v", act.Nodes, expNodes)
}
}
// TestRetryOnDescriptorLookupError verifies that the DistSender retries a descriptor
// lookup on retryable errors.
func TestRetryOnDescriptorLookupError(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
return args.CreateReply(), nil
}
pErrs := []*roachpb.Error{
roachpb.NewError(errors.New("fatal boom")),
roachpb.NewError(&roachpb.RangeKeyMismatchError{}), // retryable
nil,
nil,
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(k roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
// Return next error and truncate the prefix of the errors array.
var pErr *roachpb.Error
if k != nil {
pErr = pErrs[0]
pErrs = pErrs[1:]
}
return []roachpb.RangeDescriptor{testRangeDescriptor}, pErr
}),
}
ds := NewDistSender(ctx, g)
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value"))
// Fatal error on descriptor lookup, propagated to reply.
if _, pErr := client.SendWrapped(ds, nil, put); pErr.String() != "fatal boom" {
t.Errorf("unexpected error: %s", pErr)
}
// Retryable error on descriptor lookup, second attempt successful.
if _, pErr := client.SendWrapped(ds, nil, put); pErr != nil {
t.Errorf("unexpected error: %s", pErr)
}
if len(pErrs) != 0 {
t.Fatalf("expected more descriptor lookups, leftover pErrs: %+v", pErrs)
}
}
// TestRetryOnDescriptorLookupError verifies that the DistSender retries a descriptor
// lookup on retryable errors.
func TestRetryOnDescriptorLookupError(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) proto.Message, _ func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil
}
errors := []error{
errors.New("fatal boom"),
&roachpb.RangeKeyMismatchError{}, // retryable
nil,
nil,
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(k roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, error) {
// Return next error and truncate the prefix of the errors array.
var err error
if k != nil {
err = errors[0]
errors = errors[1:]
}
return []roachpb.RangeDescriptor{testRangeDescriptor}, err
}),
}
ds := NewDistSender(ctx, g)
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value"))
// Fatal error on descriptor lookup, propagated to reply.
if _, err := client.SendWrapped(ds, nil, put); err.Error() != "fatal boom" {
t.Errorf("unexpected error: %s", err)
}
// Retryable error on descriptor lookup, second attempt successful.
if _, err := client.SendWrapped(ds, nil, put); err != nil {
t.Errorf("unexpected error: %s", err)
}
if len(errors) != 0 {
t.Fatalf("expected more descriptor lookups, leftover errors: %+v", errors)
}
}
// TestRetryOnNotLeaderError verifies that the DistSender correctly updates the
// leader cache and retries when receiving a NotLeaderError.
func TestRetryOnNotLeaderError(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
leader := roachpb.ReplicaDescriptor{
NodeID: 99,
StoreID: 999,
}
first := true
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
if first {
reply := &roachpb.BatchResponse{}
reply.Error = roachpb.NewError(
&roachpb.NotLeaderError{Leader: &leader, Replica: &roachpb.ReplicaDescriptor{}})
first = false
return reply, nil
}
return args.CreateReply(), nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrapped(ds, nil, put); err != nil {
t.Errorf("put encountered error: %s", err)
}
if first {
t.Errorf("The command did not retry")
}
if cur := ds.leaderCache.Lookup(1); cur.StoreID != leader.StoreID {
t.Errorf("leader cache was not updated: expected %v, got %v",
&leader, cur)
}
}
// createRangeData creates sample range data in all possible areas of
// the key space. Returns a slice of the encoded keys of all created
// data.
func createRangeData(r *Replica, t *testing.T) []roachpb.EncodedKey {
ts0 := roachpb.ZeroTimestamp
ts := roachpb.Timestamp{WallTime: 1}
keyTSs := []struct {
key roachpb.Key
ts roachpb.Timestamp
}{
{keys.ResponseCacheKey(r.Desc().RangeID, &roachpb.ClientCmdID{WallTime: 1, Random: 1}), ts0},
{keys.ResponseCacheKey(r.Desc().RangeID, &roachpb.ClientCmdID{WallTime: 2, Random: 2}), ts0},
{keys.RaftHardStateKey(r.Desc().RangeID), ts0},
{keys.RaftLogKey(r.Desc().RangeID, 1), ts0},
{keys.RaftLogKey(r.Desc().RangeID, 2), ts0},
{keys.RangeGCMetadataKey(r.Desc().RangeID), ts0},
{keys.RangeLastVerificationTimestampKey(r.Desc().RangeID), ts0},
{keys.RangeStatsKey(r.Desc().RangeID), ts0},
{keys.RangeDescriptorKey(r.Desc().StartKey), ts},
{keys.TransactionKey(roachpb.Key(r.Desc().StartKey), []byte("1234")), ts0},
{keys.TransactionKey(roachpb.Key(r.Desc().StartKey.Next()), []byte("5678")), ts0},
{keys.TransactionKey(fakePrevKey(r.Desc().EndKey), []byte("2468")), ts0},
// TODO(bdarnell): KeyMin.Next() results in a key in the reserved system-local space.
// Once we have resolved https://github.com/cockroachdb/cockroach/issues/437,
// replace this with something that reliably generates the first valid key in the range.
//{r.Desc().StartKey.Next(), ts},
// The following line is similar to StartKey.Next() but adds more to the key to
// avoid falling into the system-local space.
{append(append([]byte{}, r.Desc().StartKey...), '\x01'), ts},
{fakePrevKey(r.Desc().EndKey), ts},
}
keys := []roachpb.EncodedKey{}
for _, keyTS := range keyTSs {
if err := engine.MVCCPut(r.store.Engine(), nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil {
t.Fatal(err)
}
keys = append(keys, engine.MVCCEncodeKey(keyTS.key))
if !keyTS.ts.Equal(ts0) {
keys = append(keys, engine.MVCCEncodeVersionKey(keyTS.key, keyTS.ts))
}
}
return keys
}
// TestRetryOnNotLeaderError verifies that the DistSender correctly updates the
// leader cache and retries when receiving a NotLeaderError.
func TestRetryOnNotLeaderError(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
leader := roachpb.ReplicaDescriptor{
NodeID: 99,
StoreID: 999,
}
first := true
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if first {
reply := getReply()
reply.(*roachpb.BatchResponse).SetGoError(
&roachpb.NotLeaderError{Leader: &leader, Replica: &roachpb.ReplicaDescriptor{}})
first = false
return []proto.Message{reply}, nil
}
return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _ lookupOptions) ([]roachpb.RangeDescriptor, error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrapped(ds, nil, put); err != nil {
t.Errorf("put encountered error: %s", err)
}
if first {
t.Errorf("The command did not retry")
}
if cur := ds.leaderCache.Lookup(1); cur.StoreID != leader.StoreID {
t.Errorf("leader cache was not updated: expected %v, got %v",
&leader, cur)
}
}
// createRangeData creates sample range data in all possible areas of
// the key space. Returns a slice of the encoded keys of all created
// data.
func createRangeData(t *testing.T, r *Replica) []engine.MVCCKey {
ts0 := roachpb.ZeroTimestamp
ts := roachpb.Timestamp{WallTime: 1}
desc := r.Desc()
keyTSs := []struct {
key roachpb.Key
ts roachpb.Timestamp
}{
{keys.SequenceCacheKey(r.RangeID, testTxnID, testTxnEpoch, 2), ts0},
{keys.SequenceCacheKey(r.RangeID, testTxnID, testTxnEpoch, 1), ts0},
{keys.RangeStatsKey(r.RangeID), ts0},
{keys.RaftHardStateKey(r.RangeID), ts0},
{keys.RaftLogKey(r.RangeID, 1), ts0},
{keys.RaftLogKey(r.RangeID, 2), ts0},
{keys.RangeLastReplicaGCTimestampKey(r.RangeID), ts0},
{keys.RangeLastVerificationTimestampKey(r.RangeID), ts0},
{keys.RangeDescriptorKey(desc.StartKey), ts},
{keys.TransactionKey(roachpb.Key(desc.StartKey), uuid.NewV4()), ts0},
{keys.TransactionKey(roachpb.Key(desc.StartKey.Next()), uuid.NewV4()), ts0},
{keys.TransactionKey(fakePrevKey(desc.EndKey), uuid.NewV4()), ts0},
// TODO(bdarnell): KeyMin.Next() results in a key in the reserved system-local space.
// Once we have resolved https://github.com/cockroachdb/cockroach/issues/437,
// replace this with something that reliably generates the first valid key in the range.
//{r.Desc().StartKey.Next(), ts},
// The following line is similar to StartKey.Next() but adds more to the key to
// avoid falling into the system-local space.
{append(append([]byte{}, desc.StartKey...), '\x02'), ts},
{fakePrevKey(r.Desc().EndKey), ts},
}
keys := []engine.MVCCKey{}
for _, keyTS := range keyTSs {
if err := engine.MVCCPut(r.store.Engine(), nil, keyTS.key, keyTS.ts, roachpb.MakeValueFromString("value"), nil); err != nil {
t.Fatal(err)
}
keys = append(keys, engine.MVCCKey{Key: keyTS.key, Timestamp: keyTS.ts})
}
return keys
}
// TestSequenceUpdateOnMultiRangeQueryLoop reproduces #3206 and
// verifies that the sequence is updated in the DistSender
// multi-range-query loop.
//
// More specifically, the issue was that DistSender might send
// multiple batch requests to the same replica when it finds a
// post-split range descriptor in the cache while the split has not
// yet been fully completed. By giving a higher sequence to the second
// request, we can avoid an infinite txn restart error (otherwise
// caused by hitting the sequence cache).
func TestSequenceUpdateOnMultiRangeQueryLoop(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
// Fill mockRangeDescriptorDB with two descriptors.
var descriptor1 = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
var descriptor2 = roachpb.RangeDescriptor{
RangeID: 2,
StartKey: roachpb.RKey("b"),
EndKey: roachpb.RKey("c"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
desc := descriptor1
if key.Equal(roachpb.RKey("b")) {
desc = descriptor2
}
return []roachpb.RangeDescriptor{desc}, nil
})
// Define our rpcSend stub which checks the span of the batch
// requests. The first request should be the point request on
// "a". The second request should be on "b". The sequence of the
// second request will be incremented by one from that of the
// first request.
first := true
var firstSequence uint32
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method != "Node.Batch" {
return nil, util.Errorf("unexpected method %v", method)
}
ba := getArgs(testAddress).(*roachpb.BatchRequest)
rs := keys.Range(*ba)
if first {
if !(rs.Key.Equal(roachpb.RKey("a")) && rs.EndKey.Equal(roachpb.RKey("a").Next())) {
t.Errorf("unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
first = false
firstSequence = ba.Txn.Sequence
} else {
if !(rs.Key.Equal(roachpb.RKey("b")) && rs.EndKey.Equal(roachpb.RKey("b").Next())) {
t.Errorf("unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
if ba.Txn.Sequence != firstSequence+1 {
t.Errorf("unexpected sequence; expected %d, but got %d", firstSequence+1, ba.Txn.Sequence)
}
}
return []proto.Message{ba.CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send a batch request containing two puts.
var ba roachpb.BatchRequest
ba.Txn = &roachpb.Transaction{Name: "test"}
val := roachpb.MakeValueFromString("val")
ba.Add(roachpb.NewPut(roachpb.Key("a"), val).(*roachpb.PutRequest))
ba.Add(roachpb.NewPut(roachpb.Key("b"), val).(*roachpb.PutRequest))
_, pErr := ds.Send(context.Background(), ba)
if err := pErr.GoError(); err != nil {
t.Fatal(err)
}
}
// TestMultiRangeMergeStaleDescriptor simulates the situation in which the
// DistSender executes a multi-range scan which encounters the stale descriptor
// of a range which has since incorporated its right neighbor by means of a
// merge. It is verified that the DistSender scans the correct keyrange exactly
// once.
func TestMultiRangeMergeStaleDescriptor(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
// Assume we have two ranges, [a-b) and [b-KeyMax).
merged := false
// The stale first range descriptor which is unaware of the merge.
var FirstRange = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
// The merged descriptor, which will be looked up after having processed
// the stale range [a,b).
var mergedRange = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
// Assume we have two key-value pairs, a=1 and c=2.
existingKVs := []roachpb.KeyValue{
{Key: roachpb.Key("a"), Value: roachpb.MakeValueFromString("1")},
{Key: roachpb.Key("c"), Value: roachpb.MakeValueFromString("2")},
}
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method != "Node.Batch" {
t.Fatalf("unexpected method:%s", method)
}
ba := getArgs(testAddress).(*roachpb.BatchRequest)
rs := keys.Range(*ba)
batchReply := getReply().(*roachpb.BatchResponse)
reply := &roachpb.ScanResponse{}
batchReply.Add(reply)
results := []roachpb.KeyValue{}
for _, curKV := range existingKVs {
if rs.Key.Less(keys.Addr(curKV.Key).Next()) && keys.Addr(curKV.Key).Less(rs.EndKey) {
results = append(results, curKV)
}
}
reply.Rows = results
return []proto.Message{batchReply}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
if !merged {
// Assume a range merge operation happened.
merged = true
return []roachpb.RangeDescriptor{FirstRange}, nil
}
return []roachpb.RangeDescriptor{mergedRange}, nil
}),
}
ds := NewDistSender(ctx, g)
scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 10).(*roachpb.ScanRequest)
// Set the Txn info to avoid an OpRequiresTxnError.
reply, err := client.SendWrappedWith(ds, nil, roachpb.Header{
Txn: &roachpb.Transaction{},
}, scan)
if err != nil {
t.Fatalf("scan encountered error: %s", err)
}
sr := reply.(*roachpb.ScanResponse)
if !reflect.DeepEqual(existingKVs, sr.Rows) {
t.Fatalf("expect get %v, actual get %v", existingKVs, sr.Rows)
}
}
// TestTruncateWithSpanAndDescriptor verifies that a batch request is truncated with a
// range span and the range of a descriptor found in cache.
func TestTruncateWithSpanAndDescriptor(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
g.SetNodeID(1)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
// Fill mockRangeDescriptorDB with two descriptors. When a
// range descriptor is looked up by key "b", return the second
// descriptor whose range is ["a", "c") and partially overlaps
// with the first descriptor's range.
var descriptor1 = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
var descriptor2 = roachpb.RangeDescriptor{
RangeID: 2,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKey("c"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
desc := descriptor1
if key.Equal(roachpb.RKey("b")) {
desc = descriptor2
}
return []roachpb.RangeDescriptor{desc}, nil
})
// Define our rpcSend stub which checks the span of the batch
// requests. The first request should be the point request on
// "a". The second request should be on "b".
first := true
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method != "Node.Batch" {
return nil, util.Errorf("unexpected method %v", method)
}
ba := getArgs(testAddress).(*roachpb.BatchRequest)
rs := keys.Range(*ba)
if first {
if !(rs.Key.Equal(roachpb.RKey("a")) && rs.EndKey.Equal(roachpb.RKey("a").Next())) {
t.Errorf("Unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
first = false
} else {
if !(rs.Key.Equal(roachpb.RKey("b")) && rs.EndKey.Equal(roachpb.RKey("b").Next())) {
t.Errorf("Unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
}
batchReply := getReply().(*roachpb.BatchResponse)
reply := &roachpb.PutResponse{}
batchReply.Add(reply)
return []proto.Message{batchReply}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send a batch request contains two puts. In the first
// attempt, the range of the descriptor found in the cache is
// ["a", "b"). The request is truncated to contain only the put
// on "a".
//
// In the second attempt, The range of the descriptor found in
// the cache is ["a", c"), but the put on "a" will not be
// resent. The request is truncated to contain only the put on "b".
ba := roachpb.BatchRequest{}
ba.Txn = &roachpb.Transaction{Name: "test"}
val := roachpb.MakeValueFromString("val")
ba.Add(roachpb.NewPut(keys.RangeTreeNodeKey(roachpb.RKey("a")), val).(*roachpb.PutRequest))
ba.Add(roachpb.NewPut(keys.RangeTreeNodeKey(roachpb.RKey("b")), val).(*roachpb.PutRequest))
_, pErr := ds.Send(context.Background(), ba)
if err := pErr.GoError(); err != nil {
t.Fatal(err)
}
}
func TestEngineBatchStaleCachedIterator(t *testing.T) {
defer leaktest.AfterTest(t)()
// Prevent regression of a bug which caused spurious MVCC errors due to an
// invalid optimization which let an iterator return key-value pairs which
// had since been deleted from the underlying engine.
// Discovered in #6878.
runWithAllEngines(func(eng Engine, t *testing.T) {
// Focused failure mode: highlights the actual bug.
{
batch := eng.NewBatch()
defer batch.Close()
iter := batch.NewIterator(false)
key := MVCCKey{Key: roachpb.Key("b")}
if err := batch.Put(key, []byte("foo")); err != nil {
t.Fatal(err)
}
iter.Seek(key)
if err := batch.Clear(key); err != nil {
t.Fatal(err)
}
// Iterator should not reuse its cached result.
iter.Seek(key)
if iter.Valid() {
t.Fatalf("iterator unexpectedly valid: %v -> %v",
iter.unsafeKey(), iter.unsafeValue())
}
}
// Higher-level failure mode. Mostly for documentation.
{
batch := eng.NewBatch().(*rocksDBBatch)
defer batch.Close()
key := roachpb.Key("z")
// Put a value so that the deletion below finds a value to seek
// to.
if err := MVCCPut(context.Background(), batch, nil, key, roachpb.ZeroTimestamp,
roachpb.MakeValueFromString("x"), nil); err != nil {
t.Fatal(err)
}
// Seek the iterator to `key` and clear the value (but without
// telling the iterator about that).
if err := MVCCDelete(context.Background(), batch, nil, key,
roachpb.ZeroTimestamp, nil); err != nil {
t.Fatal(err)
}
// Trigger a seek on the cached iterator by seeking to the (now
// absent) key.
// The underlying iterator will already be in the right position
// due to a seek in MVCCDelete (followed by a Clear, which does not
// invalidate the iterator's cache), and if it reports its cached
// result back, we'll see the (newly deleted) value (due to the
// failure mode above).
if v, _, err := MVCCGet(context.Background(), batch, key,
roachpb.ZeroTimestamp, true, nil); err != nil {
t.Fatal(err)
} else if v != nil {
t.Fatalf("expected no value, got %+v", v)
}
}
}, t)
}
func TestEvictCacheOnError(t *testing.T) {
defer leaktest.AfterTest(t)
// if rpcError is true, the first attempt gets an RPC error, otherwise
// the RPC call succeeds but there is an error in the RequestHeader.
// Currently leader and cached range descriptor are treated equally.
testCases := []struct{ rpcError, retryable, shouldClearLeader, shouldClearReplica bool }{
{false, false, false, false}, // non-retryable replica error
{false, true, false, false}, // retryable replica error
{true, false, true, true}, // RPC error aka all nodes dead
{true, true, false, false}, // retryable RPC error
}
for i, tc := range testCases {
g, s := makeTestGossip(t)
defer s()
leader := roachpb.ReplicaDescriptor{
NodeID: 99,
StoreID: 999,
}
first := true
var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if !first {
return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil
}
first = false
if tc.rpcError {
return nil, rpc.NewSendError("boom", tc.retryable)
}
var err error
if tc.retryable {
err = &roachpb.RangeKeyMismatchError{}
} else {
err = errors.New("boom")
}
reply := getReply()
reply.(*roachpb.BatchResponse).SetGoError(err)
return []proto.Message{reply}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
ds.updateLeaderCache(1, leader)
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value")).(*roachpb.PutRequest)
if _, pErr := client.SendWrapped(ds, nil, put); pErr != nil && !testutils.IsError(pErr.GoError(), "boom") {
t.Errorf("put encountered unexpected error: %s", pErr)
}
if cur := ds.leaderCache.Lookup(1); reflect.DeepEqual(cur, &roachpb.ReplicaDescriptor{}) && !tc.shouldClearLeader {
t.Errorf("%d: leader cache eviction: shouldClearLeader=%t, but value is %v", i, tc.shouldClearLeader, cur)
}
_, cachedDesc := ds.rangeCache.getCachedRangeDescriptor(roachpb.RKey(put.Key), false /* !inclusive */)
if cachedDesc == nil != tc.shouldClearReplica {
t.Errorf("%d: unexpected second replica lookup behaviour: wanted=%t", i, tc.shouldClearReplica)
}
}
}
// TestMultiRangeSplitEndTransaction verifies that when a chunk of batch looks
// like it's going to be dispatched to more than one range, it will be split
// up if it it contains EndTransaction.
func TestMultiRangeSplitEndTransaction(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
testCases := []struct {
put1, put2, et roachpb.Key
exp [][]roachpb.Method
}{
{
// Everything hits the first range, so we get a 1PC txn.
roachpb.Key("a1"), roachpb.Key("a2"), roachpb.Key("a3"),
[][]roachpb.Method{{roachpb.Put, roachpb.Put, roachpb.EndTransaction}},
},
{
// Only EndTransaction hits the second range.
roachpb.Key("a1"), roachpb.Key("a2"), roachpb.Key("b"),
[][]roachpb.Method{{roachpb.Put, roachpb.Put}, {roachpb.EndTransaction}},
},
{
// One write hits the second range, so EndTransaction has to be split off.
// In this case, going in the usual order without splitting off
// would actually be fine, but it doesn't seem worth optimizing at
// this point.
roachpb.Key("a1"), roachpb.Key("b1"), roachpb.Key("a1"),
[][]roachpb.Method{{roachpb.Put, roachpb.Noop}, {roachpb.Noop, roachpb.Put}, {roachpb.EndTransaction}},
},
{
// Both writes go to the second range, but not EndTransaction.
roachpb.Key("b1"), roachpb.Key("b2"), roachpb.Key("a1"),
[][]roachpb.Method{{roachpb.Put, roachpb.Put}, {roachpb.EndTransaction}},
},
}
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
// Fill mockRangeDescriptorDB with two descriptors.
var descriptor1 = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
var descriptor2 = roachpb.RangeDescriptor{
RangeID: 2,
StartKey: roachpb.RKey("b"),
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
desc := descriptor1
if !key.Less(roachpb.RKey("b")) {
desc = descriptor2
}
return []roachpb.RangeDescriptor{desc}, nil
})
for _, test := range testCases {
var act [][]roachpb.Method
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, ga func(addr net.Addr) proto.Message, _ func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
ba := ga(testAddress).(*roachpb.BatchRequest)
var cur []roachpb.Method
for _, union := range ba.Requests {
cur = append(cur, union.GetInner().Method())
}
act = append(act, cur)
return []proto.Message{ba.CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send a batch request containing two puts.
var ba roachpb.BatchRequest
ba.Txn = &roachpb.Transaction{Name: "test"}
val := roachpb.MakeValueFromString("val")
ba.Add(roachpb.NewPut(roachpb.Key(test.put1), val).(*roachpb.PutRequest))
ba.Add(roachpb.NewPut(roachpb.Key(test.put2), val).(*roachpb.PutRequest))
ba.Add(&roachpb.EndTransactionRequest{Span: roachpb.Span{Key: test.et}})
_, pErr := ds.Send(context.Background(), ba)
if err := pErr.GoError(); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(test.exp, act) {
t.Fatalf("expected %v, got %v", test.exp, act)
}
}
}
func appender(s string) []byte {
val := roachpb.MakeValueFromString(s)
v := &enginepb.MVCCMetadata{RawBytes: val.RawBytes}
return mustMarshal(v)
}
func appender(s string) []byte {
val := roachpb.MakeValueFromString(s)
v := &MVCCMetadata{Value: &val}
return mustMarshal(v)
}
func TestNodeEventFeed(t *testing.T) {
defer leaktest.AfterTest(t)
nodeDesc := roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(99),
}
// A testCase corresponds to a single Store event type. Each case contains a
// method which publishes a single event to the given storeEventPublisher,
// and an expected result interface which should match the produced
// event.
testCases := []struct {
publishTo func(status.NodeEventFeed)
expected interface{}
}{
{
publishTo: func(nef status.NodeEventFeed) {
nef.StartNode(nodeDesc, 100)
},
expected: &status.StartNodeEvent{
Desc: nodeDesc,
StartedAt: 100,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewGet(roachpb.Key("abc"))), 0, nil)
},
expected: &status.CallSuccessEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Get,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewPut(roachpb.Key("abc"), roachpb.MakeValueFromString("def"))), 0, nil)
},
expected: &status.CallSuccessEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Put,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewGet(roachpb.Key("abc"))), 0, roachpb.NewErrorf("error"))
},
expected: &status.CallErrorEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Batch,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewGet(roachpb.Key("abc"))), time.Minute, &roachpb.Error{
Detail: &roachpb.ErrorDetail{
WriteIntent: &roachpb.WriteIntentError{},
},
Index: &roachpb.ErrPosition{Index: 0},
Message: "boo",
})
},
expected: &status.CallErrorEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Get,
Duration: time.Minute,
},
},
}
// Compile expected events into a single slice.
expectedEvents := make([]interface{}, len(testCases))
for i := range testCases {
expectedEvents[i] = testCases[i].expected
}
events := make([]interface{}, 0, len(expectedEvents))
// Run test cases directly through a feed.
stopper := stop.NewStopper()
defer stopper.Stop()
feed := util.NewFeed(stopper)
feed.Subscribe(func(event interface{}) {
events = append(events, event)
})
nodefeed := status.NewNodeEventFeed(roachpb.NodeID(1), feed)
for _, tc := range testCases {
tc.publishTo(nodefeed)
}
feed.Flush()
if a, e := events, expectedEvents; !reflect.DeepEqual(a, e) {
t.Errorf("received incorrect events.\nexpected: %v\nactual: %v", e, a)
}
}
// TestSequenceUpdate verifies txn sequence number is incremented
// on successive commands.
func TestSequenceUpdate(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{
{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
},
}, nil
})
var expSequence uint32
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice, ba roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
expSequence++
if expSequence != ba.Txn.Sequence {
t.Errorf("expected sequence %d; got %d", expSequence, ba.Txn.Sequence)
}
br := ba.CreateReply()
br.Txn = ba.Txn
return br, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send 5 puts and verify sequence number increase.
txn := &roachpb.Transaction{Name: "test"}
for i := 0; i < 5; i++ {
var ba roachpb.BatchRequest
ba.Txn = txn
ba.Add(roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("foo")).(*roachpb.PutRequest))
br, pErr := ds.Send(context.Background(), ba)
if pErr != nil {
t.Fatal(pErr)
}
txn = br.Txn
}
}