func testResolverStreamGeneric(t *testing.T, name string, action func(res *Resolver) (*sqltypes.Result, error)) {
// successful execute
s := createSandbox(name)
hc := discovery.NewFakeHealthCheck()
res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil)
sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
_, err := action(res)
if err != nil {
t.Errorf("want nil, got %v", err)
}
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// failure
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
sbc0.MustFailRetry = 1
_, err = action(res)
want := fmt.Sprintf("shard, host: %s.-20.master, alias:<cell:\"aa\" > hostname:\"-20\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"-20\" type:MASTER , retry: err", name, name)
if err == nil || err.Error() != want {
t.Errorf("want\n%s\ngot\n%v", want, err)
}
// Ensure that we tried only once.
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// Ensure that we tried topo only once
if s.SrvKeyspaceCounter != 1 {
t.Errorf("want 1, got %v", s.SrvKeyspaceCounter)
}
}
func TestDiscoveryGatewayGetTablets(t *testing.T) {
keyspace := "ks"
shard := "0"
hc := discovery.NewFakeHealthCheck()
dg := createDiscoveryGateway(hc, topo.Server{}, nil, "local", 2).(*discoveryGateway)
// replica should only use local ones
hc.Reset()
dg.tsc.ResetForTesting()
hc.AddTestTablet("remote", "1.1.1.1", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil)
ep1 := hc.AddTestTablet("local", "2.2.2.2", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil).Tablet()
tsl := dg.tsc.GetHealthyTabletStats(keyspace, shard, topodatapb.TabletType_REPLICA)
if len(tsl) != 1 || !topo.TabletEquality(tsl[0].Tablet, ep1) {
t.Errorf("want %+v, got %+v", ep1, tsl)
}
// master should use the one with newer timestamp regardless of cell
hc.Reset()
dg.tsc.ResetForTesting()
hc.AddTestTablet("remote", "1.1.1.1", 1001, keyspace, shard, topodatapb.TabletType_MASTER, true, 5, nil)
ep1 = hc.AddTestTablet("remote", "2.2.2.2", 1001, keyspace, shard, topodatapb.TabletType_MASTER, true, 10, nil).Tablet()
tsl = dg.tsc.GetHealthyTabletStats(keyspace, shard, topodatapb.TabletType_MASTER)
if len(tsl) != 1 || !topo.TabletEquality(tsl[0].Tablet, ep1) {
t.Errorf("want %+v, got %+v", ep1, tsl)
}
}
func testDiscoveryGatewayTransact(t *testing.T, streaming bool, f func(dg Gateway, keyspace, shard string, tabletType topodatapb.TabletType) error) {
keyspace := "ks"
shard := "0"
tabletType := topodatapb.TabletType_REPLICA
hc := discovery.NewFakeHealthCheck()
dg := createDiscoveryGateway(hc, topo.Server{}, nil, "cell", 2).(*discoveryGateway)
// retry error - no retry
hc.Reset()
dg.tsc.ResetForTesting()
sc1 := hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil)
sc2 := hc.AddTestTablet("cell", "1.1.1.1", 1002, keyspace, shard, tabletType, true, 10, nil)
sc1.MustFailRetry = 1
sc2.MustFailRetry = 1
ep1 := sc1.Tablet()
ep2 := sc2.Tablet()
wants := map[string]int{
fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep1): 0,
fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep2): 0,
}
err := f(dg, keyspace, shard, tabletType)
if _, ok := wants[fmt.Sprintf("%v", err)]; !ok {
t.Errorf("wanted error: %+v, got error: %v", wants, err)
}
// conn error - no retry
hc.Reset()
dg.tsc.ResetForTesting()
sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil)
sc1.MustFailConn = 1
ep1 = sc1.Tablet()
want := fmt.Sprintf(`shard, host: ks.0.replica, %+v, error: conn`, ep1)
err = f(dg, keyspace, shard, tabletType)
verifyShardError(t, err, want, vtrpcpb.ErrorCode_UNKNOWN_ERROR)
}
func TestScatterConnRollback(t *testing.T) {
createSandbox("TestScatterConnRollback")
hc := discovery.NewFakeHealthCheck()
sbc0 := hc.AddTestTablet("aa", "0", 1, "TestScatterConnRollback", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 := hc.AddTestTablet("aa", "1", 1, "TestScatterConnRollback", "1", topodatapb.TabletType_REPLICA, true, 1, nil)
stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
// Sequence the executes to ensure commit order
session := NewSafeSession(&vtgatepb.Session{InTransaction: true})
stc.Execute(context.Background(), "query1", nil, "TestScatterConnRollback", []string{"0"}, topodatapb.TabletType_REPLICA, session, false)
stc.Execute(context.Background(), "query1", nil, "TestScatterConnRollback", []string{"0", "1"}, topodatapb.TabletType_REPLICA, session, false)
err := stc.Rollback(context.Background(), session)
if err != nil {
t.Errorf("want nil, got %v", err)
}
wantSession := vtgatepb.Session{}
if !reflect.DeepEqual(wantSession, *session.Session) {
t.Errorf("want\n%#v, got\n%#v", wantSession, *session.Session)
}
if rollbackCount := sbc0.RollbackCount.Get(); rollbackCount != 1 {
t.Errorf("want 1, got %d", rollbackCount)
}
if rollbackCount := sbc1.RollbackCount.Get(); rollbackCount != 1 {
t.Errorf("want 1, got %d", rollbackCount)
}
}
func TestMultiExecs(t *testing.T) {
createSandbox("TestMultiExecs")
hc := discovery.NewFakeHealthCheck()
sbc0 := hc.AddTestTablet("aa", "0", 1, "TestMultiExecs", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 := hc.AddTestTablet("aa", "1", 1, "TestMultiExecs", "1", topodatapb.TabletType_REPLICA, true, 1, nil)
stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
shardVars := map[string]map[string]interface{}{
"0": {
"bv0": 0,
},
"1": {
"bv1": 1,
},
}
_, _ = stc.ExecuteMulti(context.Background(), "query", "TestMultiExecs", shardVars, topodatapb.TabletType_REPLICA, nil, false)
if !reflect.DeepEqual(sbc0.Queries[0].BindVariables, shardVars["0"]) {
t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["0"])
}
if !reflect.DeepEqual(sbc1.Queries[0].BindVariables, shardVars["1"]) {
t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["1"])
}
sbc0.Queries = nil
sbc1.Queries = nil
_ = stc.StreamExecuteMulti(context.Background(), "query", "TestMultiExecs", shardVars, topodatapb.TabletType_REPLICA, func(*sqltypes.Result) error {
return nil
})
if !reflect.DeepEqual(sbc0.Queries[0].BindVariables, shardVars["0"]) {
t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["0"])
}
if !reflect.DeepEqual(sbc1.Queries[0].BindVariables, shardVars["1"]) {
t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["1"])
}
}
func TestSelectScatter(t *testing.T) {
// Special setup: Don't use createRouterEnv.
cell := "aa"
hc := discovery.NewFakeHealthCheck()
s := createSandbox("TestRouter")
s.VSchema = routerVSchema
getSandbox(KsTestUnsharded).VSchema = unshardedVSchema
serv := new(sandboxTopo)
scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil)
shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}
var conns []*sandboxconn.SandboxConn
for _, shard := range shards {
sbc := hc.AddTestTablet(cell, shard, 1, "TestRouter", shard, topodatapb.TabletType_MASTER, true, 1, nil)
conns = append(conns, sbc)
}
router := NewRouter(context.Background(), serv, cell, "", scatterConn)
_, err := routerExec(router, "select id from user", nil)
if err != nil {
t.Error(err)
}
wantQueries := []querytypes.BoundQuery{{
Sql: "select id from user",
BindVariables: map[string]interface{}{},
}}
for _, conn := range conns {
if !reflect.DeepEqual(conn.Queries, wantQueries) {
t.Errorf("conn.Queries = %#v, want %#v", conn.Queries, wantQueries)
}
}
}
func init() {
getSandbox(KsTestUnsharded).VSchema = `
{
"Sharded": false,
"Tables": {
"t1": {}
}
}
`
hcVTGateTest = discovery.NewFakeHealthCheck()
Init(context.Background(), hcVTGateTest, topo.Server{}, new(sandboxTopo), "aa", 10, nil)
}
func TestScatterConnStreamExecuteSendError(t *testing.T) {
createSandbox("TestScatterConnStreamExecuteSendError")
hc := discovery.NewFakeHealthCheck()
hc.AddTestTablet("aa", "0", 1, "TestScatterConnStreamExecuteSendError", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
err := stc.StreamExecute(context.Background(), "query", nil, "TestScatterConnStreamExecuteSendError", []string{"0"}, topodatapb.TabletType_REPLICA, func(*sqltypes.Result) error {
return fmt.Errorf("send error")
})
want := "send error"
// Ensure that we handle send errors.
if err == nil || err.Error() != want {
t.Errorf("want %s, got %v", want, err)
}
}
func TestScatterCommitRollbackIncorrectSession(t *testing.T) {
createSandbox("TestScatterCommitRollbackIncorrectSession")
hc := discovery.NewFakeHealthCheck()
hc.AddTestTablet("aa", "0", 1, "TestScatterCommitRollbackIncorrectSession", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
// nil session
err := stc.Commit(context.Background(), nil)
verifyErrorCode(t, err, vtrpcpb.ErrorCode_BAD_INPUT)
err = stc.Rollback(context.Background(), nil)
if err != nil {
t.Errorf("want nil, got %v", err)
}
// not in transaction
session := NewSafeSession(&vtgatepb.Session{})
err = stc.Commit(context.Background(), session)
verifyErrorCode(t, err, vtrpcpb.ErrorCode_NOT_IN_TX)
}
func TestResolverDmlOnMultipleKeyspaceIds(t *testing.T) {
keyspace := "TestResolverDmlOnMultipleKeyspaceIds"
createSandbox(keyspace)
hc := discovery.NewFakeHealthCheck()
res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil)
hc.AddTestTablet("aa", "1.1.1.1", 1001, keyspace, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
hc.AddTestTablet("aa", "1.1.1.1", 1002, keyspace, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
errStr := "DML should not span multiple keyspace_ids"
_, err := res.ExecuteKeyspaceIds(context.Background(),
"update table set a = b",
nil,
keyspace,
[][]byte{{0x10}, {0x25}},
topodatapb.TabletType_MASTER,
nil,
false)
if err == nil {
t.Errorf("want %v, got nil", errStr)
}
}
func createRouterEnv() (router *Router, sbc1, sbc2, sbclookup *sandboxconn.SandboxConn) {
cell := "aa"
hc := discovery.NewFakeHealthCheck()
s := createSandbox("TestRouter")
s.VSchema = routerVSchema
sbc1 = hc.AddTestTablet(cell, "-20", 1, "TestRouter", "-20", topodatapb.TabletType_MASTER, true, 1, nil)
sbc2 = hc.AddTestTablet(cell, "40-60", 1, "TestRouter", "40-60", topodatapb.TabletType_MASTER, true, 1, nil)
createSandbox(KsTestUnsharded)
sbclookup = hc.AddTestTablet(cell, "0", 1, KsTestUnsharded, "0", topodatapb.TabletType_MASTER, true, 1, nil)
bad := createSandbox("TestBadSharding")
bad.VSchema = badVSchema
getSandbox(KsTestUnsharded).VSchema = unshardedVSchema
serv := new(sandboxTopo)
scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil)
router = NewRouter(context.Background(), serv, cell, "", scatterConn)
return router, sbc1, sbc2, sbclookup
}
func TestStreamSelectScatter(t *testing.T) {
// Special setup: Don't use createRouterEnv.
cell := "aa"
hc := discovery.NewFakeHealthCheck()
s := createSandbox("TestRouter")
s.VSchema = routerVSchema
getSandbox(KsTestUnsharded).VSchema = unshardedVSchema
serv := new(sandboxTopo)
scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil)
shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}
var conns []*sandboxconn.SandboxConn
for _, shard := range shards {
sbc := hc.AddTestTablet(cell, shard, 1, "TestRouter", shard, topodatapb.TabletType_MASTER, true, 1, nil)
conns = append(conns, sbc)
}
router := NewRouter(context.Background(), serv, cell, "", scatterConn)
sql := "select id from user"
result, err := routerStream(router, sql)
if err != nil {
t.Error(err)
}
wantResult := &sqltypes.Result{
Fields: sandboxconn.SingleRowResult.Fields,
Rows: [][]sqltypes.Value{
sandboxconn.SingleRowResult.Rows[0],
sandboxconn.SingleRowResult.Rows[0],
sandboxconn.SingleRowResult.Rows[0],
sandboxconn.SingleRowResult.Rows[0],
sandboxconn.SingleRowResult.Rows[0],
sandboxconn.SingleRowResult.Rows[0],
sandboxconn.SingleRowResult.Rows[0],
sandboxconn.SingleRowResult.Rows[0],
},
RowsAffected: 8,
}
if !reflect.DeepEqual(result, wantResult) {
t.Errorf("result: %+v, want %+v", result, wantResult)
}
}
func TestResolverExecBatchAsTransaction(t *testing.T) {
keyspace := "TestResolverExecBatchAsTransaction"
createSandbox(keyspace)
hc := discovery.NewFakeHealthCheck()
res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil)
sbc := hc.AddTestTablet("aa", "0", 1, keyspace, "0", topodatapb.TabletType_MASTER, true, 1, nil)
sbc.MustFailRetry = 20
callcount := 0
buildBatchRequest := func() (*scatterBatchRequest, error) {
callcount++
queries := []*vtgatepb.BoundShardQuery{{
Query: &querypb.BoundQuery{
Sql: "query",
BindVariables: nil,
},
Keyspace: keyspace,
Shards: []string{"0"},
}}
return boundShardQueriesToScatterBatchRequest(queries)
}
_, err := res.ExecuteBatch(context.Background(), topodatapb.TabletType_MASTER, true, nil, buildBatchRequest)
want := "shard, host: TestResolverExecBatchAsTransaction.0.master, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"TestResolverExecBatchAsTransaction\" shard:\"0\" type:MASTER , retry: err"
if err == nil || err.Error() != want {
t.Errorf("want %v, got %v", want, err)
}
// Ensure scatter did not re-resolve
if callcount != 1 {
t.Errorf("want 1, got %v", callcount)
}
if count := sbc.AsTransactionCount.Get(); count != 1 {
t.Errorf("want 1, got %v", count)
return
}
}
func TestSelectScatterFail(t *testing.T) {
// Special setup: Don't use createRouterEnv.
cell := "aa"
hc := discovery.NewFakeHealthCheck()
s := createSandbox("TestRouter")
s.VSchema = routerVSchema
getSandbox(KsTestUnsharded).VSchema = unshardedVSchema
s.SrvKeyspaceMustFail = 1
shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"}
var conns []*sandboxconn.SandboxConn
for _, shard := range shards {
sbc := hc.AddTestTablet(cell, shard, 1, "TestRouter", shard, topodatapb.TabletType_MASTER, true, 1, nil)
conns = append(conns, sbc)
}
serv := new(sandboxTopo)
scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil)
router := NewRouter(context.Background(), serv, cell, "", scatterConn)
_, err := routerExec(router, "select id from user", nil)
want := "paramsSelectScatter: keyspace TestRouter fetch error: topo error GetSrvKeyspace"
if err == nil || err.Error() != want {
t.Errorf("routerExec: %v, want %v", err, want)
}
}
func TestScatterConnQueryNotInTransaction(t *testing.T) {
s := createSandbox("TestScatterConnQueryNotInTransaction")
hc := discovery.NewFakeHealthCheck()
// case 1: read query (not in transaction) followed by write query, not in the same shard.
hc.Reset()
sbc0 := hc.AddTestTablet("aa", "0", 1, "TestScatterConnQueryNotInTransaction", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 := hc.AddTestTablet("aa", "1", 1, "TestScatterConnQueryNotInTransaction", "1", topodatapb.TabletType_REPLICA, true, 1, nil)
stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
session := NewSafeSession(&vtgatepb.Session{InTransaction: true})
stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0"}, topodatapb.TabletType_REPLICA, session, true)
stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"1"}, topodatapb.TabletType_REPLICA, session, false)
wantSession := vtgatepb.Session{
InTransaction: true,
ShardSessions: []*vtgatepb.Session_ShardSession{{
Target: &querypb.Target{
Keyspace: "TestScatterConnQueryNotInTransaction",
Shard: "1",
TabletType: topodatapb.TabletType_REPLICA,
},
TransactionId: 1,
}},
}
if !reflect.DeepEqual(wantSession, *session.Session) {
t.Errorf("want\n%+v\ngot\n%+v", wantSession, *session.Session)
}
stc.Commit(context.Background(), session)
{
execCount0 := sbc0.ExecCount.Get()
execCount1 := sbc1.ExecCount.Get()
if execCount0 != 1 || execCount1 != 1 {
t.Errorf("want 1/1, got %d/%d", execCount0, execCount1)
}
}
if commitCount := sbc0.CommitCount.Get(); commitCount != 0 {
t.Errorf("want 0, got %d", commitCount)
}
if commitCount := sbc1.CommitCount.Get(); commitCount != 1 {
t.Errorf("want 1, got %d", commitCount)
}
// case 2: write query followed by read query (not in transaction), not in the same shard.
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "0", 1, "TestScatterConnQueryNotInTransaction", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "1", 1, "TestScatterConnQueryNotInTransaction", "1", topodatapb.TabletType_REPLICA, true, 1, nil)
stc = NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
session = NewSafeSession(&vtgatepb.Session{InTransaction: true})
stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0"}, topodatapb.TabletType_REPLICA, session, false)
stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"1"}, topodatapb.TabletType_REPLICA, session, true)
wantSession = vtgatepb.Session{
InTransaction: true,
ShardSessions: []*vtgatepb.Session_ShardSession{{
Target: &querypb.Target{
Keyspace: "TestScatterConnQueryNotInTransaction",
Shard: "0",
TabletType: topodatapb.TabletType_REPLICA,
},
TransactionId: 1,
}},
}
if !reflect.DeepEqual(wantSession, *session.Session) {
t.Errorf("want\n%+v\ngot\n%+v", wantSession, *session.Session)
}
stc.Commit(context.Background(), session)
{
execCount0 := sbc0.ExecCount.Get()
execCount1 := sbc1.ExecCount.Get()
if execCount0 != 1 || execCount1 != 1 {
t.Errorf("want 1/1, got %d/%d", execCount0, execCount1)
}
}
if commitCount := sbc0.CommitCount.Get(); commitCount != 1 {
t.Errorf("want 1, got %d", commitCount)
}
if commitCount := sbc1.CommitCount.Get(); commitCount != 0 {
t.Errorf("want 0, got %d", commitCount)
}
// case 3: write query followed by read query, in the same shard.
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "0", 1, "TestScatterConnQueryNotInTransaction", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "1", 1, "TestScatterConnQueryNotInTransaction", "1", topodatapb.TabletType_REPLICA, true, 1, nil)
stc = NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
session = NewSafeSession(&vtgatepb.Session{InTransaction: true})
stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0"}, topodatapb.TabletType_REPLICA, session, false)
stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0", "1"}, topodatapb.TabletType_REPLICA, session, true)
wantSession = vtgatepb.Session{
InTransaction: true,
ShardSessions: []*vtgatepb.Session_ShardSession{{
Target: &querypb.Target{
Keyspace: "TestScatterConnQueryNotInTransaction",
Shard: "0",
TabletType: topodatapb.TabletType_REPLICA,
},
TransactionId: 1,
}},
}
if !reflect.DeepEqual(wantSession, *session.Session) {
t.Errorf("want\n%+v\ngot\n%+v", wantSession, *session.Session)
}
stc.Commit(context.Background(), session)
{
execCount0 := sbc0.ExecCount.Get()
execCount1 := sbc1.ExecCount.Get()
if execCount0 != 2 || execCount1 != 1 {
t.Errorf("want 2/1, got %d/%d", execCount0, execCount1)
}
}
if commitCount := sbc0.CommitCount.Get(); commitCount != 1 {
t.Errorf("want 1, got %d", commitCount)
}
if commitCount := sbc1.CommitCount.Get(); commitCount != 0 {
t.Errorf("want 0, got %d", commitCount)
}
}
func testDiscoveryGatewayGeneric(t *testing.T, streaming bool, f func(dg Gateway, keyspace, shard string, tabletType topodatapb.TabletType) error) {
keyspace := "ks"
shard := "0"
tabletType := topodatapb.TabletType_REPLICA
hc := discovery.NewFakeHealthCheck()
dg := createDiscoveryGateway(hc, topo.Server{}, nil, "cell", 2).(*discoveryGateway)
// no tablet
hc.Reset()
dg.tsc.ResetForTesting()
want := "shard, host: ks.0.replica, no valid tablet"
err := f(dg, keyspace, shard, tabletType)
verifyShardError(t, err, want, vtrpcpb.ErrorCode_INTERNAL_ERROR)
// tablet with error
hc.Reset()
dg.tsc.ResetForTesting()
hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, false, 10, fmt.Errorf("no connection"))
want = "shard, host: ks.0.replica, no valid tablet"
err = f(dg, keyspace, shard, tabletType)
verifyShardError(t, err, want, vtrpcpb.ErrorCode_INTERNAL_ERROR)
// tablet without connection
hc.Reset()
dg.tsc.ResetForTesting()
ep1 := hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, false, 10, nil).Tablet()
want = fmt.Sprintf(`shard, host: ks.0.replica, no valid tablet`)
err = f(dg, keyspace, shard, tabletType)
verifyShardError(t, err, want, vtrpcpb.ErrorCode_INTERNAL_ERROR)
// retry error
hc.Reset()
dg.tsc.ResetForTesting()
sc1 := hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil)
sc2 := hc.AddTestTablet("cell", "1.1.1.1", 1002, keyspace, shard, tabletType, true, 10, nil)
sc1.MustFailRetry = 1
sc2.MustFailRetry = 1
ep1 = sc1.Tablet()
ep2 := sc2.Tablet()
wants := map[string]int{
fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep1): 0,
fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep2): 0,
}
err = f(dg, keyspace, shard, tabletType)
if _, ok := wants[fmt.Sprintf("%v", err)]; !ok {
t.Errorf("wanted error: %+v, got error: %v", wants, err)
}
// fatal error
hc.Reset()
dg.tsc.ResetForTesting()
sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil)
sc2 = hc.AddTestTablet("cell", "1.1.1.1", 1002, keyspace, shard, tabletType, true, 10, nil)
sc1.MustFailFatal = 1
sc2.MustFailFatal = 1
ep1 = sc1.Tablet()
ep2 = sc2.Tablet()
wants = map[string]int{
fmt.Sprintf(`shard, host: ks.0.replica, %+v, fatal: err`, ep1): 0,
fmt.Sprintf(`shard, host: ks.0.replica, %+v, fatal: err`, ep2): 0,
}
err = f(dg, keyspace, shard, tabletType)
if _, ok := wants[fmt.Sprintf("%v", err)]; !ok {
t.Errorf("wanted error: %+v, got error: %v", wants, err)
}
// server error - no retry
hc.Reset()
dg.tsc.ResetForTesting()
sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil)
sc1.MustFailServer = 1
ep1 = sc1.Tablet()
want = fmt.Sprintf(`shard, host: ks.0.replica, %+v, error: err`, ep1)
err = f(dg, keyspace, shard, tabletType)
verifyShardError(t, err, want, vtrpcpb.ErrorCode_BAD_INPUT)
// conn error - no retry
hc.Reset()
dg.tsc.ResetForTesting()
sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil)
sc1.MustFailConn = 1
ep1 = sc1.Tablet()
want = fmt.Sprintf(`shard, host: ks.0.replica, %+v, error: conn`, ep1)
err = f(dg, keyspace, shard, tabletType)
verifyShardError(t, err, want, vtrpcpb.ErrorCode_UNKNOWN_ERROR)
// no failure
hc.Reset()
dg.tsc.ResetForTesting()
hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil)
err = f(dg, keyspace, shard, tabletType)
if err != nil {
t.Errorf("want nil, got %v", err)
}
}
func TestScatterConnCommitSuccess(t *testing.T) {
createSandbox("TestScatterConnCommitSuccess")
hc := discovery.NewFakeHealthCheck()
sbc0 := hc.AddTestTablet("aa", "0", 1, "TestScatterConnCommitSuccess", "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 := hc.AddTestTablet("aa", "1", 1, "TestScatterConnCommitSuccess", "1", topodatapb.TabletType_REPLICA, true, 1, nil)
stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil)
// Sequence the executes to ensure commit order
session := NewSafeSession(&vtgatepb.Session{InTransaction: true})
stc.Execute(context.Background(), "query1", nil, "TestScatterConnCommitSuccess", []string{"0"}, topodatapb.TabletType_REPLICA, session, false)
wantSession := vtgatepb.Session{
InTransaction: true,
ShardSessions: []*vtgatepb.Session_ShardSession{{
Target: &querypb.Target{
Keyspace: "TestScatterConnCommitSuccess",
Shard: "0",
TabletType: topodatapb.TabletType_REPLICA,
},
TransactionId: 1,
}},
}
if !reflect.DeepEqual(wantSession, *session.Session) {
t.Errorf("want\n%+v, got\n%+v", wantSession, *session.Session)
}
stc.Execute(context.Background(), "query1", nil, "TestScatterConnCommitSuccess", []string{"0", "1"}, topodatapb.TabletType_REPLICA, session, false)
wantSession = vtgatepb.Session{
InTransaction: true,
ShardSessions: []*vtgatepb.Session_ShardSession{{
Target: &querypb.Target{
Keyspace: "TestScatterConnCommitSuccess",
Shard: "0",
TabletType: topodatapb.TabletType_REPLICA,
},
TransactionId: 1,
}, {
Target: &querypb.Target{
Keyspace: "TestScatterConnCommitSuccess",
Shard: "1",
TabletType: topodatapb.TabletType_REPLICA,
},
TransactionId: 1,
}},
}
if !reflect.DeepEqual(wantSession, *session.Session) {
t.Errorf("want\n%+v, got\n%+v", wantSession, *session.Session)
}
sbc0.MustFailServer = 1
err := stc.Commit(context.Background(), session)
if err == nil {
t.Errorf("want error, got nil")
}
wantSession = vtgatepb.Session{}
if !reflect.DeepEqual(wantSession, *session.Session) {
t.Errorf("want\n%+v, got\n%+v", wantSession, *session.Session)
}
if commitCount := sbc0.CommitCount.Get(); commitCount != 1 {
t.Errorf("want 1, got %d", commitCount)
}
if rollbackCount := sbc1.RollbackCount.Get(); rollbackCount != 1 {
t.Errorf("want 1, got %d", rollbackCount)
}
}
func testResolverGeneric(t *testing.T, name string, action func(res *Resolver) (*sqltypes.Result, error)) {
// successful execute
s := createSandbox(name)
hc := discovery.NewFakeHealthCheck()
res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil)
sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
_, err := action(res)
if err != nil {
t.Errorf("want nil, got %v", err)
}
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// non-retryable failure
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
sbc0.MustFailServer = 1
sbc1.MustFailRetry = 1
_, err = action(res)
want1 := fmt.Sprintf("shard, host: %s.-20.master, alias:<cell:\"aa\" > hostname:\"-20\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"-20\" type:MASTER , error: err", name, name)
want2 := fmt.Sprintf("shard, host: %s.20-40.master, alias:<cell:\"aa\" > hostname:\"20-40\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"20-40\" type:MASTER , retry: err", name, name)
want := []string{want1, want2}
sort.Strings(want)
if err == nil {
t.Errorf("want\n%v\ngot\n%v", want, err)
} else {
got := strings.Split(err.Error(), "\n")
sort.Strings(got)
if !reflect.DeepEqual(want, got) {
t.Errorf("want\n%v\ngot\n%v", want, got)
}
}
// Ensure that we tried only once
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// Ensure that we tried topo only once when mapping KeyspaceId/KeyRange to shards
if s.SrvKeyspaceCounter != 1 {
t.Errorf("want 1, got %v", s.SrvKeyspaceCounter)
}
// retryable failure, no sharding event
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
sbc0.MustFailRetry = 1
sbc1.MustFailFatal = 1
_, err = action(res)
want1 = fmt.Sprintf("shard, host: %s.-20.master, alias:<cell:\"aa\" > hostname:\"-20\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"-20\" type:MASTER , retry: err", name, name)
want2 = fmt.Sprintf("shard, host: %s.20-40.master, alias:<cell:\"aa\" > hostname:\"20-40\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"20-40\" type:MASTER , fatal: err", name, name)
want = []string{want1, want2}
sort.Strings(want)
if err == nil {
t.Errorf("want\n%v\ngot\n%v", want, err)
} else {
got := strings.Split(err.Error(), "\n")
sort.Strings(got)
if !reflect.DeepEqual(want, got) {
t.Errorf("want\n%v\ngot\n%v", want, got)
}
}
// Ensure that we tried only once.
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// Ensure that we tried topo only twice.
if s.SrvKeyspaceCounter != 2 {
t.Errorf("want 2, got %v", s.SrvKeyspaceCounter)
}
// no failure, initial vertical resharding
s.Reset()
addSandboxServedFrom(name, name+"ServedFrom0")
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
s0 := createSandbox(name + "ServedFrom0") // make sure we have a fresh copy
s0.ShardSpec = "-80-"
sbc2 := hc.AddTestTablet("aa", "1.1.1.1", 1003, name+"ServedFrom0", "-80", topodatapb.TabletType_MASTER, true, 1, nil)
_, err = action(res)
if err != nil {
t.Errorf("want nil, got %v", err)
}
// Ensure original keyspace is not used.
if execCount := sbc0.ExecCount.Get(); execCount != 0 {
t.Errorf("want 0, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 0 {
t.Errorf("want 0, got %v", execCount)
}
// Ensure redirected keyspace is accessed once.
if execCount := sbc2.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// Ensure that we tried each keyspace only once.
if s.SrvKeyspaceCounter != 1 {
t.Errorf("want 1, got %v", s.SrvKeyspaceCounter)
}
if s0.SrvKeyspaceCounter != 1 {
t.Errorf("want 1, got %v", s0.SrvKeyspaceCounter)
}
s0.Reset()
// retryable failure, vertical resharding
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1.MustFailFatal = 1
i := 0
s.SrvKeyspaceCallback = func() {
if i == 1 {
addSandboxServedFrom(name, name+"ServedFrom")
hc.Reset()
hc.AddTestTablet("aa", "1.1.1.1", 1001, name+"ServedFrom", "-20", topodatapb.TabletType_MASTER, true, 1, nil)
hc.AddTestTablet("aa", "1.1.1.1", 1002, name+"ServedFrom", "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
}
i++
}
_, err = action(res)
if err != nil {
t.Errorf("want nil, got %v", err)
}
// Ensure that we tried only once on the original conn.
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// Ensure that we tried topo only 3 times.
if s.SrvKeyspaceCounter != 3 {
t.Errorf("want 3, got %v", s.SrvKeyspaceCounter)
}
// retryable failure, horizontal resharding
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil)
sbc1.MustFailRetry = 1
i = 0
s.SrvKeyspaceCallback = func() {
if i == 1 {
s.ShardSpec = "-20-30-40-60-80-a0-c0-e0-"
hc.Reset()
hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil)
hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-30", topodatapb.TabletType_MASTER, true, 1, nil)
}
i++
}
_, err = action(res)
if err != nil {
t.Errorf("want nil, got %v", err)
}
// Ensure that we tried only once on original guy.
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// Ensure that we tried topo only twice.
if s.SrvKeyspaceCounter != 2 {
t.Errorf("want 2, got %v", s.SrvKeyspaceCounter)
}
}
func testScatterConnGeneric(t *testing.T, name string, f func(hc discovery.HealthCheck, shards []string) (*sqltypes.Result, error)) {
hc := discovery.NewFakeHealthCheck()
// no shard
s := createSandbox(name)
qr, err := f(hc, nil)
if qr.RowsAffected != 0 {
t.Errorf("want 0, got %v", qr.RowsAffected)
}
if err != nil {
t.Errorf("want nil, got %v", err)
}
// single shard
s.Reset()
sbc := hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc.MustFailServer = 1
qr, err = f(hc, []string{"0"})
want := fmt.Sprintf("shard, host: %v.0.replica, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"0\" type:REPLICA , error: err", name, name)
// Verify server error string.
if err == nil || err.Error() != want {
t.Errorf("want %s, got %v", want, err)
}
// Ensure that we tried only once.
if execCount := sbc.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// two shards
s.Reset()
hc.Reset()
sbc0 := hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 := hc.AddTestTablet("aa", "1", 1, name, "1", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc0.MustFailServer = 1
sbc1.MustFailServer = 1
_, err = f(hc, []string{"0", "1"})
// Verify server errors are consolidated.
want = fmt.Sprintf("shard, host: %v.0.replica, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"0\" type:REPLICA , error: err\nshard, host: %v.1.replica, alias:<cell:\"aa\" > hostname:\"1\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"1\" type:REPLICA , error: err", name, name, name, name)
verifyScatterConnError(t, err, want, vtrpcpb.ErrorCode_BAD_INPUT)
// Ensure that we tried only once.
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// two shards with different errors
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "1", 1, name, "1", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc0.MustFailServer = 1
sbc1.MustFailTxPool = 1
_, err = f(hc, []string{"0", "1"})
// Verify server errors are consolidated.
want = fmt.Sprintf("shard, host: %v.0.replica, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"0\" type:REPLICA , error: err\nshard, host: %v.1.replica, alias:<cell:\"aa\" > hostname:\"1\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"1\" type:REPLICA , tx_pool_full: err", name, name, name, name)
// We should only surface the higher priority error code
verifyScatterConnError(t, err, want, vtrpcpb.ErrorCode_BAD_INPUT)
// Ensure that we tried only once.
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// duplicate shards
s.Reset()
hc.Reset()
sbc = hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil)
qr, err = f(hc, []string{"0", "0"})
// Ensure that we executed only once.
if execCount := sbc.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
// no errors
s.Reset()
hc.Reset()
sbc0 = hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil)
sbc1 = hc.AddTestTablet("aa", "1", 1, name, "1", topodatapb.TabletType_REPLICA, true, 1, nil)
qr, err = f(hc, []string{"0", "1"})
if err != nil {
t.Errorf("want nil, got %v", err)
}
if execCount := sbc0.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if execCount := sbc1.ExecCount.Get(); execCount != 1 {
t.Errorf("want 1, got %v", execCount)
}
if qr.RowsAffected != 2 {
t.Errorf("want 2, got %v", qr.RowsAffected)
}
if len(qr.Rows) != 2 {
t.Errorf("want 2, got %v", len(qr.Rows))
}
}
