// TestKVDBInternalMethods verifies no internal methods are available
// HTTP DB interface.
func TestKVDBInternalMethods(t *testing.T) {
defer leaktest.AfterTest(t)
s := server.StartTestServer(t)
defer s.Stop()
testCases := []struct {
args proto.Request
reply proto.Response
}{
{&proto.InternalRangeLookupRequest{}, &proto.InternalRangeLookupResponse{}},
{&proto.InternalGCRequest{}, &proto.InternalGCResponse{}},
{&proto.InternalHeartbeatTxnRequest{}, &proto.InternalHeartbeatTxnResponse{}},
{&proto.InternalPushTxnRequest{}, &proto.InternalPushTxnResponse{}},
{&proto.InternalResolveIntentRequest{}, &proto.InternalResolveIntentResponse{}},
{&proto.InternalResolveIntentRangeRequest{}, &proto.InternalResolveIntentRangeResponse{}},
{&proto.InternalMergeRequest{}, &proto.InternalMergeResponse{}},
{&proto.InternalTruncateLogRequest{}, &proto.InternalTruncateLogResponse{}},
}
// Verify non-public methods experience bad request errors.
db := createTestClient(t, s.ServingAddr())
for i, test := range testCases {
test.args.Header().Key = proto.Key("a")
if proto.IsRange(test.args) {
test.args.Header().EndKey = test.args.Header().Key.Next()
}
b := &client.Batch{}
b.InternalAddCall(proto.Call{Args: test.args, Reply: test.reply})
err := db.Run(b)
if err == nil {
t.Errorf("%d: unexpected success calling %s", i, test.args.Method())
} else if !strings.Contains(err.Error(), "404 Not Found") {
t.Errorf("%d: expected 404; got %s", i, err)
}
}
}
// TestKVDBInternalMethods verifies no internal methods are available
// HTTP DB interface.
func TestKVDBInternalMethods(t *testing.T) {
defer leaktest.AfterTest(t)
t.Skip("test broken & disabled; obsolete after after #2271")
s := server.StartTestServer(t)
defer s.Stop()
testCases := []proto.Request{
&proto.HeartbeatTxnRequest{},
&proto.GCRequest{},
&proto.PushTxnRequest{},
&proto.RangeLookupRequest{},
&proto.ResolveIntentRequest{},
&proto.ResolveIntentRangeRequest{},
&proto.MergeRequest{},
&proto.TruncateLogRequest{},
&proto.LeaderLeaseRequest{},
&proto.EndTransactionRequest{
InternalCommitTrigger: &proto.InternalCommitTrigger{},
},
}
// Verify internal methods experience bad request errors.
db := createTestClient(t, s.Stopper(), s.ServingAddr())
for i, args := range testCases {
args.Header().Key = proto.Key("a")
if proto.IsRange(args) {
args.Header().EndKey = args.Header().Key.Next()
}
b := &client.Batch{}
b.InternalAddCall(proto.Call{Args: args, Reply: args.CreateReply()})
err := db.Run(b).GoError()
if err == nil {
t.Errorf("%d: unexpected success calling %s", i, args.Method())
} else if !testutils.IsError(err, "(couldn't find method|contains commit trigger)") {
t.Errorf("%d: expected missing method %s; got %s", i, args.Method(), err)
}
// Verify same but within a Batch request.
ba := &proto.BatchRequest{}
ba.Add(args)
b = &client.Batch{}
b.InternalAddCall(proto.Call{Args: ba, Reply: &proto.BatchResponse{}})
if err := db.Run(b).GoError(); err == nil {
t.Errorf("%d: unexpected success calling %s", i, args.Method())
} else if !testutils.IsError(err, "(contains an internal request|contains commit trigger)") {
t.Errorf("%d: expected disallowed method error %s; got %s", i, args.Method(), err)
}
}
}
// truncate restricts all contained requests to the given key range.
// Even on error, the returned closure must be executed; it undoes any
// truncations performed.
// First, the boundaries of the truncation are obtained: This is the
// intersection between [from,to) and the descriptor's range.
// Secondly, all requests contained in the batch are "truncated" to
// the resulting range, inserting NoopRequest appropriately to
// replace requests which are left without a key range to operate on.
// The number of non-noop requests after truncation is returned along
// with a closure which must be executed to undo the truncation, even
// in case of an error.
// TODO(tschottdorf): Consider returning a new BatchRequest, which has more
// overhead in the common case of a batch which never needs truncation but is
// less magical.
func truncate(br *proto.BatchRequest, desc *proto.RangeDescriptor, from, to proto.Key) (func(), int, error) {
if !desc.ContainsKey(from) {
from = desc.StartKey
}
if !desc.ContainsKeyRange(desc.StartKey, to) || to == nil {
to = desc.EndKey
}
truncateOne := func(args proto.Request) (bool, []func(), error) {
if _, ok := args.(*proto.NoopRequest); ok {
return true, nil, nil
}
header := args.Header()
if !proto.IsRange(args) {
if len(header.EndKey) > 0 {
return false, nil, util.Errorf("%T is not a range command, but EndKey is set", args)
}
if !desc.ContainsKey(keys.KeyAddress(header.Key)) {
return true, nil, nil
}
return false, nil, nil
}
var undo []func()
key, endKey := header.Key, header.EndKey
keyAddr, endKeyAddr := keys.KeyAddress(key), keys.KeyAddress(endKey)
if keyAddr.Less(from) {
undo = append(undo, func() { header.Key = key })
header.Key = from
keyAddr = from
}
if !endKeyAddr.Less(to) {
undo = append(undo, func() { header.EndKey = endKey })
header.EndKey = to
endKeyAddr = to
}
// Check whether the truncation has left any keys in the range. If not,
// we need to cut it out of the request.
return !keyAddr.Less(endKeyAddr), undo, nil
}
var fns []func()
gUndo := func() {
for _, f := range fns {
f()
}
}
var numNoop int
for pos, arg := range br.Requests {
omit, undo, err := truncateOne(arg.GetValue().(proto.Request))
if omit {
numNoop++
nReq := &proto.RequestUnion{}
nReq.SetValue(&proto.NoopRequest{})
oReq := br.Requests[pos]
br.Requests[pos] = *nReq
posCpy := pos // for closure
undo = append(undo, func() {
br.Requests[posCpy] = oReq
})
}
fns = append(fns, undo...)
if err != nil {
return gUndo, 0, err
}
}
return gUndo, len(br.Requests) - numNoop, nil
}
// TestSendRPCOrder verifies that sendRPC correctly takes into account the
// leader, attributes and required consistency to determine where to send
// remote requests.
func TestSendRPCOrder(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
rangeID := proto.RangeID(99)
nodeAttrs := map[int32][]string{
1: {}, // The local node, set in each test case.
2: {"us", "west", "gpu"},
3: {"eu", "dublin", "pdu2", "gpu"},
4: {"us", "east", "gpu"},
5: {"us", "east", "gpu", "flaky"},
}
// Gets filled below to identify the replica by its address.
addrToNode := make(map[string]int32)
makeVerifier := func(expOrder rpc.OrderingPolicy,
expAddrs []int32) func(rpc.Options, []net.Addr) error {
return func(o rpc.Options, addrs []net.Addr) error {
if o.Ordering != expOrder {
return util.Errorf("unexpected ordering, wanted %v, got %v",
expOrder, o.Ordering)
}
var actualAddrs []int32
for i, a := range addrs {
if len(expAddrs) <= i {
return util.Errorf("got unexpected address: %s", a)
}
if expAddrs[i] == 0 {
actualAddrs = append(actualAddrs, 0)
} else {
actualAddrs = append(actualAddrs, addrToNode[a.String()])
}
}
if !reflect.DeepEqual(expAddrs, actualAddrs) {
return util.Errorf("expected %d, but found %d", expAddrs, actualAddrs)
}
return nil
}
}
testCases := []struct {
args proto.Request
attrs []string
order rpc.OrderingPolicy
expReplica []int32
leader int32 // 0 for not caching a leader.
// Naming is somewhat off, as eventually consistent reads usually
// do not have to go to the leader when a node has a read lease.
// Would really want CONSENSUS here, but that is not implemented.
// Likely a test setup here will never have a read lease, but good
// to keep in mind.
consistent bool
}{
// Inconsistent Scan without matching attributes.
{
args: &proto.ScanRequest{},
attrs: []string{},
order: rpc.OrderRandom,
expReplica: []int32{1, 2, 3, 4, 5},
},
// Inconsistent Scan with matching attributes.
// Should move the two nodes matching the attributes to the front and
// go stable.
{
args: &proto.ScanRequest{},
attrs: nodeAttrs[5],
order: rpc.OrderStable,
// Compare only the first two resulting addresses.
expReplica: []int32{5, 4, 0, 0, 0},
},
// Scan without matching attributes that requires but does not find
// a leader.
{
args: &proto.ScanRequest{},
attrs: []string{},
order: rpc.OrderRandom,
expReplica: []int32{1, 2, 3, 4, 5},
consistent: true,
},
// Put without matching attributes that requires but does not find leader.
// Should go random and not change anything.
{
args: &proto.PutRequest{},
attrs: []string{"nomatch"},
order: rpc.OrderRandom,
expReplica: []int32{1, 2, 3, 4, 5},
},
// Put with matching attributes but no leader.
// Should move the two nodes matching the attributes to the front and
// go stable.
{
args: &proto.PutRequest{},
attrs: append(nodeAttrs[5], "irrelevant"),
// Compare only the first two resulting addresses.
order: rpc.OrderStable,
expReplica: []int32{5, 4, 0, 0, 0},
},
// Put with matching attributes that finds the leader (node 3).
// Should address the leader and the two nodes matching the attributes
// (the last and second to last) in that order.
{
args: &proto.PutRequest{},
attrs: append(nodeAttrs[5], "irrelevant"),
// Compare only the first resulting addresses as we have a leader
// and that means we're only trying to send there.
order: rpc.OrderStable,
expReplica: []int32{2, 5, 4, 0, 0},
leader: 2,
},
// Inconsistent Get without matching attributes but leader (node 3). Should just
// go random as the leader does not matter.
{
args: &proto.GetRequest{},
attrs: []string{},
order: rpc.OrderRandom,
expReplica: []int32{1, 2, 3, 4, 5},
leader: 2,
},
}
descriptor := proto.RangeDescriptor{
StartKey: proto.KeyMin,
EndKey: proto.KeyMax,
RangeID: rangeID,
Replicas: nil,
}
// Stub to be changed in each test case.
var verifyCall func(rpc.Options, []net.Addr) error
var testFn rpcSendFn = func(opts rpc.Options, method string,
addrs []net.Addr, _ func(addr net.Addr) gogoproto.Message,
getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) {
if err := verifyCall(opts, addrs); err != nil {
return nil, err
}
return []gogoproto.Message{getReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(proto.Key, lookupOptions) ([]proto.RangeDescriptor, error) {
return []proto.RangeDescriptor{descriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
for n, tc := range testCases {
verifyCall = makeVerifier(tc.order, tc.expReplica)
descriptor.Replicas = nil // could do this once above, but more convenient here
for i := int32(1); i <= 5; i++ {
addr := util.MakeUnresolvedAddr("tcp", fmt.Sprintf("node%d", i))
addrToNode[addr.String()] = i
nd := &proto.NodeDescriptor{
NodeID: proto.NodeID(i),
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
Attrs: proto.Attributes{
Attrs: nodeAttrs[i],
},
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(proto.NodeID(i)), nd, time.Hour); err != nil {
t.Fatal(err)
}
descriptor.Replicas = append(descriptor.Replicas, proto.Replica{
NodeID: proto.NodeID(i),
StoreID: proto.StoreID(i),
})
}
{
// The local node needs to get its attributes during sendRPC.
nd := &proto.NodeDescriptor{
NodeID: 6,
Attrs: proto.Attributes{
Attrs: tc.attrs,
},
}
if err := g.SetNodeDescriptor(nd); err != nil {
t.Fatal(err)
}
}
ds.leaderCache.Update(proto.RangeID(rangeID), proto.Replica{})
if tc.leader > 0 {
ds.leaderCache.Update(proto.RangeID(rangeID), descriptor.Replicas[tc.leader-1])
}
args := tc.args
args.Header().RangeID = rangeID // Not used in this test, but why not.
args.Header().Key = proto.Key("a")
if proto.IsRange(args) {
args.Header().EndKey = proto.Key("b")
}
if !tc.consistent {
args.Header().ReadConsistency = proto.INCONSISTENT
}
// Kill the cached NodeDescriptor, enforcing a lookup from Gossip.
ds.nodeDescriptor = nil
if _, err := batchutil.SendWrapped(ds, args); err != nil {
t.Errorf("%d: %s", n, err)
}
}
}