// TestTxnMultipleCoord checks that a coordinator uses the Writing flag to
// enforce that only one coordinator can be used for transactional writes.
func TestTxnMultipleCoord(t *testing.T) {
defer leaktest.AfterTest(t)()
s, sender := createTestDB(t)
defer s.Stop()
testCases := []struct {
args roachpb.Request
writing bool
ok bool
}{
{roachpb.NewGet(roachpb.Key("a")), true, false},
{roachpb.NewGet(roachpb.Key("a")), false, true},
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), false, false}, // transactional write before begin
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), true, false}, // must have switched coordinators
}
for i, tc := range testCases {
txn := roachpb.NewTransaction("test", roachpb.Key("a"), 1, roachpb.SERIALIZABLE,
s.Clock.Now(), s.Clock.MaxOffset().Nanoseconds())
txn.Writing = tc.writing
reply, pErr := client.SendWrappedWith(sender, nil, roachpb.Header{
Txn: txn,
}, tc.args)
if pErr == nil != tc.ok {
t.Errorf("%d: %T (writing=%t): success_expected=%t, but got: %v",
i, tc.args, tc.writing, tc.ok, pErr)
}
if pErr != nil {
continue
}
txn = reply.Header().Txn
// The transaction should come back rw if it started rw or if we just
// wrote.
isWrite := roachpb.IsTransactionWrite(tc.args)
if (tc.writing || isWrite) != txn.Writing {
t.Errorf("%d: unexpected writing state: %s", i, txn)
}
if !isWrite {
continue
}
// Abort for clean shutdown.
if _, pErr := client.SendWrappedWith(sender, nil, roachpb.Header{
Txn: txn,
}, &roachpb.EndTransactionRequest{
Commit: false,
}); pErr != nil {
t.Fatal(pErr)
}
}
}
// TestTxnMultipleCoord checks that a coordinator uses the Writing flag to
// enforce that only one coordinator can be used for transactional writes.
func TestTxnMultipleCoord(t *testing.T) {
defer leaktest.AfterTest(t)
s := createTestDB(t)
defer s.Stop()
for i, tc := range []struct {
args roachpb.Request
writing bool
ok bool
}{
{roachpb.NewGet(roachpb.Key("a")), true, true},
{roachpb.NewGet(roachpb.Key("a")), false, true},
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), false, true},
{roachpb.NewPut(roachpb.Key("a"), roachpb.Value{}), true, false},
} {
{
txn := newTxn(s.Clock, roachpb.Key("a"))
txn.Writing = tc.writing
tc.args.Header().Txn = txn
}
reply, err := client.SendWrapped(s.Sender, nil, tc.args)
if err == nil != tc.ok {
t.Errorf("%d: %T (writing=%t): success_expected=%t, but got: %v",
i, tc.args, tc.writing, tc.ok, err)
}
if err != nil {
continue
}
txn := reply.Header().Txn
// The transaction should come back rw if it started rw or if we just
// wrote.
isWrite := roachpb.IsTransactionWrite(tc.args)
if (tc.writing || isWrite) != txn.Writing {
t.Errorf("%d: unexpected writing state: %s", i, txn)
}
if !isWrite {
continue
}
// Abort for clean shutdown.
if _, err := client.SendWrapped(s.Sender, nil, &roachpb.EndTransactionRequest{
RequestHeader: roachpb.RequestHeader{
Txn: txn,
},
Commit: false,
}); err != nil {
t.Fatal(err)
}
}
}
// TestMultiRangeScanWithMaxResults tests that commands which access multiple
// ranges with MaxResults parameter are carried out properly.
func TestMultiRangeScanWithMaxResults(t *testing.T) {
defer leaktest.AfterTest(t)()
testCases := []struct {
splitKeys []roachpb.Key
keys []roachpb.Key
}{
{[]roachpb.Key{roachpb.Key("m")},
[]roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}},
{[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")},
[]roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"),
roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}},
}
for i, tc := range testCases {
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldDrain()
ds := kv.NewDistSender(&kv.DistSenderContext{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
tds := kv.NewTxnCoordSender(ds, ts.Clock(), ts.Ctx.Linearizable, tracing.NewTracer(),
ts.stopper, kv.NewTxnMetrics(metric.NewRegistry()))
for _, sk := range tc.splitKeys {
if err := ts.node.ctx.DB.AdminSplit(sk); err != nil {
t.Fatal(err)
}
}
for _, k := range tc.keys {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
if _, err := client.SendWrapped(tds, nil, put); err != nil {
t.Fatal(err)
}
}
// Try every possible ScanRequest startKey.
for start := 0; start < len(tc.keys); start++ {
// Try every possible maxResults, from 1 to beyond the size of key array.
for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ {
scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next(),
int64(maxResults))
reply, err := client.SendWrapped(tds, nil, scan)
if err != nil {
t.Fatal(err)
}
rows := reply.(*roachpb.ScanResponse).Rows
if start+maxResults <= len(tc.keys) && len(rows) != maxResults {
t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows))
} else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 {
t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows))
}
}
}
}
}
// TestMultiRangeScanWithMaxResults tests that commands which access multiple
// ranges with MaxResults parameter are carried out properly.
func TestMultiRangeScanWithMaxResults(t *testing.T) {
defer leaktest.AfterTest(t)
testCases := []struct {
splitKeys []roachpb.Key
keys []roachpb.Key
}{
{[]roachpb.Key{roachpb.Key("m")},
[]roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}},
{[]roachpb.Key{roachpb.Key("h"), roachpb.Key("q")},
[]roachpb.Key{roachpb.Key("b"), roachpb.Key("f"), roachpb.Key("k"),
roachpb.Key("r"), roachpb.Key("w"), roachpb.Key("y")}},
}
for i, tc := range testCases {
s := StartTestServer(t)
ds := kv.NewDistSender(&kv.DistSenderContext{Clock: s.Clock()}, s.Gossip())
tds := kv.NewTxnCoordSender(ds, s.Clock(), testContext.Linearizable, nil, s.stopper)
for _, sk := range tc.splitKeys {
if err := s.node.ctx.DB.AdminSplit(sk); err != nil {
t.Fatal(err)
}
}
for _, k := range tc.keys {
put := roachpb.NewPut(k, roachpb.Value{Bytes: k})
if _, err := client.SendWrapped(tds, nil, put); err != nil {
t.Fatal(err)
}
}
// Try every possible ScanRequest startKey.
for start := 0; start < len(tc.keys); start++ {
// Try every possible maxResults, from 1 to beyond the size of key array.
for maxResults := 1; maxResults <= len(tc.keys)-start+1; maxResults++ {
scan := roachpb.NewScan(tc.keys[start], tc.keys[len(tc.keys)-1].Next(),
int64(maxResults))
reply, err := client.SendWrapped(tds, nil, scan)
if err != nil {
t.Fatal(err)
}
rows := reply.(*roachpb.ScanResponse).Rows
if start+maxResults <= len(tc.keys) && len(rows) != maxResults {
t.Errorf("%d: start=%s: expected %d rows, but got %d", i, tc.keys[start], maxResults, len(rows))
} else if start+maxResults == len(tc.keys)+1 && len(rows) != maxResults-1 {
t.Errorf("%d: expected %d rows, but got %d", i, maxResults-1, len(rows))
}
}
}
defer s.Stop()
}
}
func TestOwnNodeCertain(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
const expNodeID = 42
nd := &roachpb.NodeDescriptor{
NodeID: expNodeID,
Address: util.MakeUnresolvedAddr("tcp", "foobar:1234"),
}
g.ResetNodeID(nd.NodeID)
if err := g.SetNodeDescriptor(nd); err != nil {
t.Fatal(err)
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(expNodeID), nd, time.Hour); err != nil {
t.Fatal(err)
}
act := make(map[roachpb.NodeID]roachpb.Timestamp)
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
ba roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
for k, v := range ba.Txn.ObservedTimestamps {
act[k] = v
}
return ba.CreateReply(), nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
expTS := roachpb.ZeroTimestamp.Add(1, 2)
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrappedWith(ds, nil, roachpb.Header{
// MaxTimestamp is set very high so that all uncertainty updates have
// effect.
Txn: &roachpb.Transaction{OrigTimestamp: expTS, MaxTimestamp: roachpb.MaxTimestamp},
}, put); err != nil {
t.Fatalf("put encountered error: %s", err)
}
exp := map[roachpb.NodeID]roachpb.Timestamp{
expNodeID: expTS,
}
if !reflect.DeepEqual(exp, act) {
t.Fatalf("wanted %v, got %v", exp, act)
}
}
// Put sets the value for a key.
//
// A new result will be appended to the batch which will contain a single row
// and Result.Err will indicate success or failure.
//
// key can be either a byte slice or a string. value can be any key type, a
// proto.Message or any Go primitive type (bool, int, etc).
func (b *Batch) Put(key, value interface{}) {
k, err := marshalKey(key)
if err != nil {
b.initResult(0, 1, err)
return
}
v, err := marshalValue(value)
if err != nil {
b.initResult(0, 1, err)
return
}
b.reqs = append(b.reqs, roachpb.NewPut(k, v))
b.initResult(1, 1, nil)
}
// TestClockUpdateOnResponse verifies that the DistSender picks up
// the timestamp of the remote party embedded in responses.
func TestClockUpdateOnResponse(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
ctx := &DistSenderContext{
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
expectedErr := roachpb.NewError(errors.New("boom"))
// Prepare the test function
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value"))
doCheck := func(testFn rpcSendFn, fakeTime roachpb.Timestamp) {
ds.rpcSend = testFn
_, err := client.SendWrapped(ds, nil, put)
if err != nil && err != expectedErr {
t.Fatal(err)
}
newTime := ds.clock.Now()
if newTime.Less(fakeTime) {
t.Fatalf("clock was not advanced: expected >= %s; got %s", fakeTime, newTime)
}
}
// Test timestamp propagation on valid BatchResults.
fakeTime := ds.clock.Now().Add(10000000000 /*10s*/, 0)
replyNormal := func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
rb := args.CreateReply()
rb.Now = fakeTime
return rb, nil
}
doCheck(replyNormal, fakeTime)
// Test timestamp propagation on errors.
fakeTime = ds.clock.Now().Add(10000000000 /*10s*/, 0)
replyError := func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
rb := args.CreateReply()
rb.Error = expectedErr
rb.Error.Now = fakeTime
return rb, nil
}
doCheck(replyError, fakeTime)
}
func TestOwnNodeCertain(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
const expNodeID = 42
nd := &roachpb.NodeDescriptor{
NodeID: expNodeID,
Address: util.MakeUnresolvedAddr("tcp", "foobar:1234"),
}
g.SetNodeID(nd.NodeID)
if err := g.SetNodeDescriptor(nd); err != nil {
t.Fatal(err)
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(expNodeID), nd, time.Hour); err != nil {
t.Fatal(err)
}
var act roachpb.NodeList
var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) proto.Message, _ func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
ba := getArgs(nil).(*roachpb.BatchRequest)
for _, nodeID := range ba.Txn.CertainNodes.Nodes {
act.Add(roachpb.NodeID(nodeID))
}
return []proto.Message{ba.CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrappedWith(ds, nil, roachpb.Header{
Txn: &roachpb.Transaction{},
}, put); err != nil {
t.Fatalf("put encountered error: %s", err)
}
if expNodes := []roachpb.NodeID{expNodeID}; !reflect.DeepEqual(act.Nodes, expNodes) {
t.Fatalf("got %v, expected %v", act.Nodes, expNodes)
}
}
func (b *Batch) put(key, value interface{}, inline bool) {
k, err := marshalKey(key)
if err != nil {
b.initResult(0, 1, notRaw, err)
return
}
v, err := marshalValue(value)
if err != nil {
b.initResult(0, 1, notRaw, err)
return
}
if inline {
b.appendReqs(roachpb.NewPutInline(k, v))
} else {
b.appendReqs(roachpb.NewPut(k, v))
}
b.initResult(1, 1, notRaw, nil)
}
// TestRetryOnDescriptorLookupError verifies that the DistSender retries a descriptor
// lookup on retryable errors.
func TestRetryOnDescriptorLookupError(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
return args.CreateReply(), nil
}
pErrs := []*roachpb.Error{
roachpb.NewError(errors.New("fatal boom")),
roachpb.NewError(&roachpb.RangeKeyMismatchError{}), // retryable
nil,
nil,
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(k roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
// Return next error and truncate the prefix of the errors array.
var pErr *roachpb.Error
if k != nil {
pErr = pErrs[0]
pErrs = pErrs[1:]
}
return []roachpb.RangeDescriptor{testRangeDescriptor}, pErr
}),
}
ds := NewDistSender(ctx, g)
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value"))
// Fatal error on descriptor lookup, propagated to reply.
if _, pErr := client.SendWrapped(ds, nil, put); pErr.String() != "fatal boom" {
t.Errorf("unexpected error: %s", pErr)
}
// Retryable error on descriptor lookup, second attempt successful.
if _, pErr := client.SendWrapped(ds, nil, put); pErr != nil {
t.Errorf("unexpected error: %s", pErr)
}
if len(pErrs) != 0 {
t.Fatalf("expected more descriptor lookups, leftover pErrs: %+v", pErrs)
}
}
// TestRetryOnNotLeaderError verifies that the DistSender correctly updates the
// leader cache and retries when receiving a NotLeaderError.
func TestRetryOnNotLeaderError(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
leader := roachpb.ReplicaDescriptor{
NodeID: 99,
StoreID: 999,
}
first := true
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
if first {
reply := &roachpb.BatchResponse{}
reply.Error = roachpb.NewError(
&roachpb.NotLeaderError{Leader: &leader, Replica: &roachpb.ReplicaDescriptor{}})
first = false
return reply, nil
}
return args.CreateReply(), nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrapped(ds, nil, put); err != nil {
t.Errorf("put encountered error: %s", err)
}
if first {
t.Errorf("The command did not retry")
}
if cur := ds.leaderCache.Lookup(1); cur.StoreID != leader.StoreID {
t.Errorf("leader cache was not updated: expected %v, got %v",
&leader, cur)
}
}
// TestRetryOnDescriptorLookupError verifies that the DistSender retries a descriptor
// lookup on retryable errors.
func TestRetryOnDescriptorLookupError(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) proto.Message, _ func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil
}
errors := []error{
errors.New("fatal boom"),
&roachpb.RangeKeyMismatchError{}, // retryable
nil,
nil,
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(k roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, error) {
// Return next error and truncate the prefix of the errors array.
var err error
if k != nil {
err = errors[0]
errors = errors[1:]
}
return []roachpb.RangeDescriptor{testRangeDescriptor}, err
}),
}
ds := NewDistSender(ctx, g)
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value"))
// Fatal error on descriptor lookup, propagated to reply.
if _, err := client.SendWrapped(ds, nil, put); err.Error() != "fatal boom" {
t.Errorf("unexpected error: %s", err)
}
// Retryable error on descriptor lookup, second attempt successful.
if _, err := client.SendWrapped(ds, nil, put); err != nil {
t.Errorf("unexpected error: %s", err)
}
if len(errors) != 0 {
t.Fatalf("expected more descriptor lookups, leftover errors: %+v", errors)
}
}
// TestRetryOnNotLeaderError verifies that the DistSender correctly updates the
// leader cache and retries when receiving a NotLeaderError.
func TestRetryOnNotLeaderError(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
leader := roachpb.ReplicaDescriptor{
NodeID: 99,
StoreID: 999,
}
first := true
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if first {
reply := getReply()
reply.(*roachpb.BatchResponse).SetGoError(
&roachpb.NotLeaderError{Leader: &leader, Replica: &roachpb.ReplicaDescriptor{}})
first = false
return []proto.Message{reply}, nil
}
return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _ lookupOptions) ([]roachpb.RangeDescriptor, error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
v := roachpb.MakeValueFromString("value")
put := roachpb.NewPut(roachpb.Key("a"), v)
if _, err := client.SendWrapped(ds, nil, put); err != nil {
t.Errorf("put encountered error: %s", err)
}
if first {
t.Errorf("The command did not retry")
}
if cur := ds.leaderCache.Lookup(1); cur.StoreID != leader.StoreID {
t.Errorf("leader cache was not updated: expected %v, got %v",
&leader, cur)
}
}
func TestEvictCacheOnError(t *testing.T) {
defer leaktest.AfterTest(t)
// if rpcError is true, the first attempt gets an RPC error, otherwise
// the RPC call succeeds but there is an error in the RequestHeader.
// Currently leader and cached range descriptor are treated equally.
testCases := []struct{ rpcError, retryable, shouldClearLeader, shouldClearReplica bool }{
{false, false, false, false}, // non-retryable replica error
{false, true, false, false}, // retryable replica error
{true, false, true, true}, // RPC error aka all nodes dead
{true, true, false, false}, // retryable RPC error
}
for i, tc := range testCases {
g, s := makeTestGossip(t)
defer s()
leader := roachpb.ReplicaDescriptor{
NodeID: 99,
StoreID: 999,
}
first := true
var testFn rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if !first {
return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil
}
first = false
if tc.rpcError {
return nil, rpc.NewSendError("boom", tc.retryable)
}
var err error
if tc.retryable {
err = &roachpb.RangeKeyMismatchError{}
} else {
err = errors.New("boom")
}
reply := getReply()
reply.(*roachpb.BatchResponse).SetGoError(err)
return []proto.Message{reply}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
ds.updateLeaderCache(1, leader)
put := roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("value")).(*roachpb.PutRequest)
if _, pErr := client.SendWrapped(ds, nil, put); pErr != nil && !testutils.IsError(pErr.GoError(), "boom") {
t.Errorf("put encountered unexpected error: %s", pErr)
}
if cur := ds.leaderCache.Lookup(1); reflect.DeepEqual(cur, &roachpb.ReplicaDescriptor{}) && !tc.shouldClearLeader {
t.Errorf("%d: leader cache eviction: shouldClearLeader=%t, but value is %v", i, tc.shouldClearLeader, cur)
}
_, cachedDesc := ds.rangeCache.getCachedRangeDescriptor(roachpb.RKey(put.Key), false /* !inclusive */)
if cachedDesc == nil != tc.shouldClearReplica {
t.Errorf("%d: unexpected second replica lookup behaviour: wanted=%t", i, tc.shouldClearReplica)
}
}
}
// TestMultiRangeSplitEndTransaction verifies that when a chunk of batch looks
// like it's going to be dispatched to more than one range, it will be split
// up if it it contains EndTransaction.
func TestMultiRangeSplitEndTransaction(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
testCases := []struct {
put1, put2, et roachpb.Key
exp [][]roachpb.Method
}{
{
// Everything hits the first range, so we get a 1PC txn.
roachpb.Key("a1"), roachpb.Key("a2"), roachpb.Key("a3"),
[][]roachpb.Method{{roachpb.Put, roachpb.Put, roachpb.EndTransaction}},
},
{
// Only EndTransaction hits the second range.
roachpb.Key("a1"), roachpb.Key("a2"), roachpb.Key("b"),
[][]roachpb.Method{{roachpb.Put, roachpb.Put}, {roachpb.EndTransaction}},
},
{
// One write hits the second range, so EndTransaction has to be split off.
// In this case, going in the usual order without splitting off
// would actually be fine, but it doesn't seem worth optimizing at
// this point.
roachpb.Key("a1"), roachpb.Key("b1"), roachpb.Key("a1"),
[][]roachpb.Method{{roachpb.Put, roachpb.Noop}, {roachpb.Noop, roachpb.Put}, {roachpb.EndTransaction}},
},
{
// Both writes go to the second range, but not EndTransaction.
roachpb.Key("b1"), roachpb.Key("b2"), roachpb.Key("a1"),
[][]roachpb.Method{{roachpb.Put, roachpb.Put}, {roachpb.EndTransaction}},
},
}
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
// Fill mockRangeDescriptorDB with two descriptors.
var descriptor1 = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
var descriptor2 = roachpb.RangeDescriptor{
RangeID: 2,
StartKey: roachpb.RKey("b"),
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
desc := descriptor1
if !key.Less(roachpb.RKey("b")) {
desc = descriptor2
}
return []roachpb.RangeDescriptor{desc}, nil
})
for _, test := range testCases {
var act [][]roachpb.Method
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, ga func(addr net.Addr) proto.Message, _ func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
ba := ga(testAddress).(*roachpb.BatchRequest)
var cur []roachpb.Method
for _, union := range ba.Requests {
cur = append(cur, union.GetInner().Method())
}
act = append(act, cur)
return []proto.Message{ba.CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send a batch request containing two puts.
var ba roachpb.BatchRequest
ba.Txn = &roachpb.Transaction{Name: "test"}
val := roachpb.MakeValueFromString("val")
ba.Add(roachpb.NewPut(roachpb.Key(test.put1), val).(*roachpb.PutRequest))
ba.Add(roachpb.NewPut(roachpb.Key(test.put2), val).(*roachpb.PutRequest))
ba.Add(&roachpb.EndTransactionRequest{Span: roachpb.Span{Key: test.et}})
_, pErr := ds.Send(context.Background(), ba)
if err := pErr.GoError(); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(test.exp, act) {
t.Fatalf("expected %v, got %v", test.exp, act)
}
}
}
// TestSequenceUpdate verifies txn sequence number is incremented
// on successive commands.
func TestSequenceUpdate(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{
{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
},
}, nil
})
var expSequence uint32
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice, ba roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
expSequence++
if expSequence != ba.Txn.Sequence {
t.Errorf("expected sequence %d; got %d", expSequence, ba.Txn.Sequence)
}
br := ba.CreateReply()
br.Txn = ba.Txn
return br, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send 5 puts and verify sequence number increase.
txn := &roachpb.Transaction{Name: "test"}
for i := 0; i < 5; i++ {
var ba roachpb.BatchRequest
ba.Txn = txn
ba.Add(roachpb.NewPut(roachpb.Key("a"), roachpb.MakeValueFromString("foo")).(*roachpb.PutRequest))
br, pErr := ds.Send(context.Background(), ba)
if pErr != nil {
t.Fatal(pErr)
}
txn = br.Txn
}
}
// TestMultiRangeScanDeleteRange tests that commands which access multiple
// ranges are carried out properly.
func TestMultiRangeScanDeleteRange(t *testing.T) {
defer leaktest.AfterTest(t)
s := StartTestServer(t)
defer s.Stop()
ds := kv.NewDistSender(&kv.DistSenderContext{Clock: s.Clock()}, s.Gossip())
tds := kv.NewTxnCoordSender(ds, s.Clock(), testContext.Linearizable, nil, s.stopper)
if err := s.node.ctx.DB.AdminSplit("m"); err != nil {
t.Fatal(err)
}
writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}
get := &roachpb.GetRequest{
Span: roachpb.Span{Key: writes[0]},
}
get.EndKey = writes[len(writes)-1]
if _, err := client.SendWrapped(tds, nil, get); err == nil {
t.Errorf("able to call Get with a key range: %v", get)
}
var delTS roachpb.Timestamp
for i, k := range writes {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
reply, err := client.SendWrapped(tds, nil, put)
if err != nil {
t.Fatal(err)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next(), 0).(*roachpb.ScanRequest)
// The Put ts may have been pushed by tsCache,
// so make sure we see their values in our Scan.
delTS = reply.(*roachpb.PutResponse).Timestamp
reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if sr.Txn != nil {
// This was the other way around at some point in the past.
// Same below for Delete, etc.
t.Errorf("expected no transaction in response header")
}
if rows := sr.Rows; len(rows) != i+1 {
t.Fatalf("expected %d rows, but got %d", i+1, len(rows))
}
}
del := &roachpb.DeleteRangeRequest{
Span: roachpb.Span{
Key: writes[0],
EndKey: roachpb.Key(writes[len(writes)-1]).Next(),
},
}
reply, err := client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, del)
if err != nil {
t.Fatal(err)
}
dr := reply.(*roachpb.DeleteRangeResponse)
if dr.Txn != nil {
t.Errorf("expected no transaction in response header")
}
if n := dr.NumDeleted; n != int64(len(writes)) {
t.Errorf("expected %d keys to be deleted, but got %d instead",
len(writes), n)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next(), 0).(*roachpb.ScanRequest)
txn := &roachpb.Transaction{Name: "MyTxn"}
reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Txn: txn}, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" {
t.Errorf("wanted Txn to persist, but it changed to %v", txn)
}
if rows := sr.Rows; len(rows) > 0 {
t.Fatalf("scan after delete returned rows: %v", rows)
}
}
// TestTruncateWithSpanAndDescriptor verifies that a batch request is truncated with a
// range span and the range of a descriptor found in cache.
func TestTruncateWithSpanAndDescriptor(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
g.SetNodeID(1)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
// Fill mockRangeDescriptorDB with two descriptors. When a
// range descriptor is looked up by key "b", return the second
// descriptor whose range is ["a", "c") and partially overlaps
// with the first descriptor's range.
var descriptor1 = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
var descriptor2 = roachpb.RangeDescriptor{
RangeID: 2,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKey("c"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
desc := descriptor1
if key.Equal(roachpb.RKey("b")) {
desc = descriptor2
}
return []roachpb.RangeDescriptor{desc}, nil
})
// Define our rpcSend stub which checks the span of the batch
// requests. The first request should be the point request on
// "a". The second request should be on "b".
first := true
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method != "Node.Batch" {
return nil, util.Errorf("unexpected method %v", method)
}
ba := getArgs(testAddress).(*roachpb.BatchRequest)
rs := keys.Range(*ba)
if first {
if !(rs.Key.Equal(roachpb.RKey("a")) && rs.EndKey.Equal(roachpb.RKey("a").Next())) {
t.Errorf("Unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
first = false
} else {
if !(rs.Key.Equal(roachpb.RKey("b")) && rs.EndKey.Equal(roachpb.RKey("b").Next())) {
t.Errorf("Unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
}
batchReply := getReply().(*roachpb.BatchResponse)
reply := &roachpb.PutResponse{}
batchReply.Add(reply)
return []proto.Message{batchReply}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send a batch request contains two puts. In the first
// attempt, the range of the descriptor found in the cache is
// ["a", "b"). The request is truncated to contain only the put
// on "a".
//
// In the second attempt, The range of the descriptor found in
// the cache is ["a", c"), but the put on "a" will not be
// resent. The request is truncated to contain only the put on "b".
ba := roachpb.BatchRequest{}
ba.Txn = &roachpb.Transaction{Name: "test"}
val := roachpb.MakeValueFromString("val")
ba.Add(roachpb.NewPut(keys.RangeTreeNodeKey(roachpb.RKey("a")), val).(*roachpb.PutRequest))
ba.Add(roachpb.NewPut(keys.RangeTreeNodeKey(roachpb.RKey("b")), val).(*roachpb.PutRequest))
_, pErr := ds.Send(context.Background(), ba)
if err := pErr.GoError(); err != nil {
t.Fatal(err)
}
}
// TestMultiRangeScanDeleteRange tests that commands which access multiple
// ranges are carried out properly.
func TestMultiRangeScanDeleteRange(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
ts := s.(*TestServer)
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = ts.stopper.ShouldQuiesce()
ds := kv.NewDistSender(&kv.DistSenderConfig{
Clock: s.Clock(),
RPCContext: s.RPCContext(),
RPCRetryOptions: &retryOpts,
}, ts.Gossip())
ctx := tracing.WithTracer(context.Background(), tracing.NewTracer())
tds := kv.NewTxnCoordSender(ctx, ds, s.Clock(), ts.Ctx.Linearizable,
ts.stopper, kv.MakeTxnMetrics())
if err := ts.node.ctx.DB.AdminSplit("m"); err != nil {
t.Fatal(err)
}
writes := []roachpb.Key{roachpb.Key("a"), roachpb.Key("z")}
get := &roachpb.GetRequest{
Span: roachpb.Span{Key: writes[0]},
}
get.EndKey = writes[len(writes)-1]
if _, err := client.SendWrapped(tds, nil, get); err == nil {
t.Errorf("able to call Get with a key range: %v", get)
}
var delTS hlc.Timestamp
for i, k := range writes {
put := roachpb.NewPut(k, roachpb.MakeValueFromBytes(k))
reply, err := client.SendWrapped(tds, nil, put)
if err != nil {
t.Fatal(err)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
reply, err = client.SendWrapped(tds, nil, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if sr.Txn != nil {
// This was the other way around at some point in the past.
// Same below for Delete, etc.
t.Errorf("expected no transaction in response header")
}
if rows := sr.Rows; len(rows) != i+1 {
t.Fatalf("expected %d rows, but got %d", i+1, len(rows))
}
}
del := &roachpb.DeleteRangeRequest{
Span: roachpb.Span{
Key: writes[0],
EndKey: roachpb.Key(writes[len(writes)-1]).Next(),
},
ReturnKeys: true,
}
reply, err := client.SendWrappedWith(tds, nil, roachpb.Header{Timestamp: delTS}, del)
if err != nil {
t.Fatal(err)
}
dr := reply.(*roachpb.DeleteRangeResponse)
if dr.Txn != nil {
t.Errorf("expected no transaction in response header")
}
if !reflect.DeepEqual(dr.Keys, writes) {
t.Errorf("expected %d keys to be deleted, but got %d instead", writes, dr.Keys)
}
scan := roachpb.NewScan(writes[0], writes[len(writes)-1].Next())
txn := &roachpb.Transaction{Name: "MyTxn"}
reply, err = client.SendWrappedWith(tds, nil, roachpb.Header{Txn: txn}, scan)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if txn := sr.Txn; txn == nil || txn.Name != "MyTxn" {
t.Errorf("wanted Txn to persist, but it changed to %v", txn)
}
if rows := sr.Rows; len(rows) > 0 {
t.Fatalf("scan after delete returned rows: %v", rows)
}
}
// TestSequenceUpdateOnMultiRangeQueryLoop reproduces #3206 and
// verifies that the sequence is updated in the DistSender
// multi-range-query loop.
//
// More specifically, the issue was that DistSender might send
// multiple batch requests to the same replica when it finds a
// post-split range descriptor in the cache while the split has not
// yet been fully completed. By giving a higher sequence to the second
// request, we can avoid an infinite txn restart error (otherwise
// caused by hitting the sequence cache).
func TestSequenceUpdateOnMultiRangeQueryLoop(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(1),
Address: util.MakeUnresolvedAddr(testAddress.Network(), testAddress.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(1)), nd, time.Hour); err != nil {
t.Fatal(err)
}
// Fill mockRangeDescriptorDB with two descriptors.
var descriptor1 = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
var descriptor2 = roachpb.RangeDescriptor{
RangeID: 2,
StartKey: roachpb.RKey("b"),
EndKey: roachpb.RKey("c"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
descDB := mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
desc := descriptor1
if key.Equal(roachpb.RKey("b")) {
desc = descriptor2
}
return []roachpb.RangeDescriptor{desc}, nil
})
// Define our rpcSend stub which checks the span of the batch
// requests. The first request should be the point request on
// "a". The second request should be on "b". The sequence of the
// second request will be incremented by one from that of the
// first request.
first := true
var firstSequence uint32
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method != "Node.Batch" {
return nil, util.Errorf("unexpected method %v", method)
}
ba := getArgs(testAddress).(*roachpb.BatchRequest)
rs := keys.Range(*ba)
if first {
if !(rs.Key.Equal(roachpb.RKey("a")) && rs.EndKey.Equal(roachpb.RKey("a").Next())) {
t.Errorf("unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
first = false
firstSequence = ba.Txn.Sequence
} else {
if !(rs.Key.Equal(roachpb.RKey("b")) && rs.EndKey.Equal(roachpb.RKey("b").Next())) {
t.Errorf("unexpected span [%s,%s)", rs.Key, rs.EndKey)
}
if ba.Txn.Sequence != firstSequence+1 {
t.Errorf("unexpected sequence; expected %d, but got %d", firstSequence+1, ba.Txn.Sequence)
}
}
return []proto.Message{ba.CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: descDB,
}
ds := NewDistSender(ctx, g)
// Send a batch request containing two puts.
var ba roachpb.BatchRequest
ba.Txn = &roachpb.Transaction{Name: "test"}
val := roachpb.MakeValueFromString("val")
ba.Add(roachpb.NewPut(roachpb.Key("a"), val).(*roachpb.PutRequest))
ba.Add(roachpb.NewPut(roachpb.Key("b"), val).(*roachpb.PutRequest))
_, pErr := ds.Send(context.Background(), ba)
if err := pErr.GoError(); err != nil {
t.Fatal(err)
}
}
func TestNodeEventFeed(t *testing.T) {
defer leaktest.AfterTest(t)
nodeDesc := roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(99),
}
// A testCase corresponds to a single Store event type. Each case contains a
// method which publishes a single event to the given storeEventPublisher,
// and an expected result interface which should match the produced
// event.
testCases := []struct {
publishTo func(status.NodeEventFeed)
expected interface{}
}{
{
publishTo: func(nef status.NodeEventFeed) {
nef.StartNode(nodeDesc, 100)
},
expected: &status.StartNodeEvent{
Desc: nodeDesc,
StartedAt: 100,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewGet(roachpb.Key("abc"))), 0, nil)
},
expected: &status.CallSuccessEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Get,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewPut(roachpb.Key("abc"), roachpb.MakeValueFromString("def"))), 0, nil)
},
expected: &status.CallSuccessEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Put,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewGet(roachpb.Key("abc"))), 0, roachpb.NewErrorf("error"))
},
expected: &status.CallErrorEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Batch,
},
},
{
publishTo: func(nef status.NodeEventFeed) {
nef.CallComplete(wrap(roachpb.NewGet(roachpb.Key("abc"))), time.Minute, &roachpb.Error{
Detail: &roachpb.ErrorDetail{
WriteIntent: &roachpb.WriteIntentError{},
},
Index: &roachpb.ErrPosition{Index: 0},
Message: "boo",
})
},
expected: &status.CallErrorEvent{
NodeID: roachpb.NodeID(1),
Method: roachpb.Get,
Duration: time.Minute,
},
},
}
// Compile expected events into a single slice.
expectedEvents := make([]interface{}, len(testCases))
for i := range testCases {
expectedEvents[i] = testCases[i].expected
}
events := make([]interface{}, 0, len(expectedEvents))
// Run test cases directly through a feed.
stopper := stop.NewStopper()
defer stopper.Stop()
feed := util.NewFeed(stopper)
feed.Subscribe(func(event interface{}) {
events = append(events, event)
})
nodefeed := status.NewNodeEventFeed(roachpb.NodeID(1), feed)
for _, tc := range testCases {
tc.publishTo(nodefeed)
}
feed.Flush()
if a, e := events, expectedEvents; !reflect.DeepEqual(a, e) {
t.Errorf("received incorrect events.\nexpected: %v\nactual: %v", e, a)
}
}
