func testResolverStreamGeneric(t *testing.T, name string, action func(res *Resolver) (*sqltypes.Result, error)) { // successful execute s := createSandbox(name) hc := discovery.NewFakeHealthCheck() res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil) sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) _, err := action(res) if err != nil { t.Errorf("want nil, got %v", err) } if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // failure s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) sbc0.MustFailRetry = 1 _, err = action(res) want := fmt.Sprintf("shard, host: %s.-20.master, alias:<cell:\"aa\" > hostname:\"-20\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"-20\" type:MASTER , retry: err", name, name) if err == nil || err.Error() != want { t.Errorf("want\n%s\ngot\n%v", want, err) } // Ensure that we tried only once. if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // Ensure that we tried topo only once if s.SrvKeyspaceCounter != 1 { t.Errorf("want 1, got %v", s.SrvKeyspaceCounter) } }
func TestDiscoveryGatewayGetTablets(t *testing.T) { keyspace := "ks" shard := "0" hc := discovery.NewFakeHealthCheck() dg := createDiscoveryGateway(hc, topo.Server{}, nil, "local", 2).(*discoveryGateway) // replica should only use local ones hc.Reset() dg.tsc.ResetForTesting() hc.AddTestTablet("remote", "1.1.1.1", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil) ep1 := hc.AddTestTablet("local", "2.2.2.2", 1001, keyspace, shard, topodatapb.TabletType_REPLICA, true, 10, nil).Tablet() tsl := dg.tsc.GetHealthyTabletStats(keyspace, shard, topodatapb.TabletType_REPLICA) if len(tsl) != 1 || !topo.TabletEquality(tsl[0].Tablet, ep1) { t.Errorf("want %+v, got %+v", ep1, tsl) } // master should use the one with newer timestamp regardless of cell hc.Reset() dg.tsc.ResetForTesting() hc.AddTestTablet("remote", "1.1.1.1", 1001, keyspace, shard, topodatapb.TabletType_MASTER, true, 5, nil) ep1 = hc.AddTestTablet("remote", "2.2.2.2", 1001, keyspace, shard, topodatapb.TabletType_MASTER, true, 10, nil).Tablet() tsl = dg.tsc.GetHealthyTabletStats(keyspace, shard, topodatapb.TabletType_MASTER) if len(tsl) != 1 || !topo.TabletEquality(tsl[0].Tablet, ep1) { t.Errorf("want %+v, got %+v", ep1, tsl) } }
func testDiscoveryGatewayTransact(t *testing.T, streaming bool, f func(dg Gateway, keyspace, shard string, tabletType topodatapb.TabletType) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA hc := discovery.NewFakeHealthCheck() dg := createDiscoveryGateway(hc, topo.Server{}, nil, "cell", 2).(*discoveryGateway) // retry error - no retry hc.Reset() dg.tsc.ResetForTesting() sc1 := hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil) sc2 := hc.AddTestTablet("cell", "1.1.1.1", 1002, keyspace, shard, tabletType, true, 10, nil) sc1.MustFailRetry = 1 sc2.MustFailRetry = 1 ep1 := sc1.Tablet() ep2 := sc2.Tablet() wants := map[string]int{ fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep1): 0, fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep2): 0, } err := f(dg, keyspace, shard, tabletType) if _, ok := wants[fmt.Sprintf("%v", err)]; !ok { t.Errorf("wanted error: %+v, got error: %v", wants, err) } // conn error - no retry hc.Reset() dg.tsc.ResetForTesting() sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil) sc1.MustFailConn = 1 ep1 = sc1.Tablet() want := fmt.Sprintf(`shard, host: ks.0.replica, %+v, error: conn`, ep1) err = f(dg, keyspace, shard, tabletType) verifyShardError(t, err, want, vtrpcpb.ErrorCode_UNKNOWN_ERROR) }
func TestScatterConnRollback(t *testing.T) { createSandbox("TestScatterConnRollback") hc := discovery.NewFakeHealthCheck() sbc0 := hc.AddTestTablet("aa", "0", 1, "TestScatterConnRollback", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1", 1, "TestScatterConnRollback", "1", topodatapb.TabletType_REPLICA, true, 1, nil) stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) // Sequence the executes to ensure commit order session := NewSafeSession(&vtgatepb.Session{InTransaction: true}) stc.Execute(context.Background(), "query1", nil, "TestScatterConnRollback", []string{"0"}, topodatapb.TabletType_REPLICA, session, false) stc.Execute(context.Background(), "query1", nil, "TestScatterConnRollback", []string{"0", "1"}, topodatapb.TabletType_REPLICA, session, false) err := stc.Rollback(context.Background(), session) if err != nil { t.Errorf("want nil, got %v", err) } wantSession := vtgatepb.Session{} if !reflect.DeepEqual(wantSession, *session.Session) { t.Errorf("want\n%#v, got\n%#v", wantSession, *session.Session) } if rollbackCount := sbc0.RollbackCount.Get(); rollbackCount != 1 { t.Errorf("want 1, got %d", rollbackCount) } if rollbackCount := sbc1.RollbackCount.Get(); rollbackCount != 1 { t.Errorf("want 1, got %d", rollbackCount) } }
func TestMultiExecs(t *testing.T) { createSandbox("TestMultiExecs") hc := discovery.NewFakeHealthCheck() sbc0 := hc.AddTestTablet("aa", "0", 1, "TestMultiExecs", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1", 1, "TestMultiExecs", "1", topodatapb.TabletType_REPLICA, true, 1, nil) stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) shardVars := map[string]map[string]interface{}{ "0": { "bv0": 0, }, "1": { "bv1": 1, }, } _, _ = stc.ExecuteMulti(context.Background(), "query", "TestMultiExecs", shardVars, topodatapb.TabletType_REPLICA, nil, false) if !reflect.DeepEqual(sbc0.Queries[0].BindVariables, shardVars["0"]) { t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["0"]) } if !reflect.DeepEqual(sbc1.Queries[0].BindVariables, shardVars["1"]) { t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["1"]) } sbc0.Queries = nil sbc1.Queries = nil _ = stc.StreamExecuteMulti(context.Background(), "query", "TestMultiExecs", shardVars, topodatapb.TabletType_REPLICA, func(*sqltypes.Result) error { return nil }) if !reflect.DeepEqual(sbc0.Queries[0].BindVariables, shardVars["0"]) { t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["0"]) } if !reflect.DeepEqual(sbc1.Queries[0].BindVariables, shardVars["1"]) { t.Errorf("got %+v, want %+v", sbc0.Queries[0].BindVariables, shardVars["1"]) } }
func TestSelectScatter(t *testing.T) { // Special setup: Don't use createRouterEnv. cell := "aa" hc := discovery.NewFakeHealthCheck() s := createSandbox("TestRouter") s.VSchema = routerVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema serv := new(sandboxTopo) scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil) shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"} var conns []*sandboxconn.SandboxConn for _, shard := range shards { sbc := hc.AddTestTablet(cell, shard, 1, "TestRouter", shard, topodatapb.TabletType_MASTER, true, 1, nil) conns = append(conns, sbc) } router := NewRouter(context.Background(), serv, cell, "", scatterConn) _, err := routerExec(router, "select id from user", nil) if err != nil { t.Error(err) } wantQueries := []querytypes.BoundQuery{{ Sql: "select id from user", BindVariables: map[string]interface{}{}, }} for _, conn := range conns { if !reflect.DeepEqual(conn.Queries, wantQueries) { t.Errorf("conn.Queries = %#v, want %#v", conn.Queries, wantQueries) } } }
func init() { getSandbox(KsTestUnsharded).VSchema = ` { "Sharded": false, "Tables": { "t1": {} } } ` hcVTGateTest = discovery.NewFakeHealthCheck() Init(context.Background(), hcVTGateTest, topo.Server{}, new(sandboxTopo), "aa", 10, nil) }
func TestScatterConnStreamExecuteSendError(t *testing.T) { createSandbox("TestScatterConnStreamExecuteSendError") hc := discovery.NewFakeHealthCheck() hc.AddTestTablet("aa", "0", 1, "TestScatterConnStreamExecuteSendError", "0", topodatapb.TabletType_REPLICA, true, 1, nil) stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) err := stc.StreamExecute(context.Background(), "query", nil, "TestScatterConnStreamExecuteSendError", []string{"0"}, topodatapb.TabletType_REPLICA, func(*sqltypes.Result) error { return fmt.Errorf("send error") }) want := "send error" // Ensure that we handle send errors. if err == nil || err.Error() != want { t.Errorf("want %s, got %v", want, err) } }
func TestScatterCommitRollbackIncorrectSession(t *testing.T) { createSandbox("TestScatterCommitRollbackIncorrectSession") hc := discovery.NewFakeHealthCheck() hc.AddTestTablet("aa", "0", 1, "TestScatterCommitRollbackIncorrectSession", "0", topodatapb.TabletType_REPLICA, true, 1, nil) stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) // nil session err := stc.Commit(context.Background(), nil) verifyErrorCode(t, err, vtrpcpb.ErrorCode_BAD_INPUT) err = stc.Rollback(context.Background(), nil) if err != nil { t.Errorf("want nil, got %v", err) } // not in transaction session := NewSafeSession(&vtgatepb.Session{}) err = stc.Commit(context.Background(), session) verifyErrorCode(t, err, vtrpcpb.ErrorCode_NOT_IN_TX) }
func TestResolverDmlOnMultipleKeyspaceIds(t *testing.T) { keyspace := "TestResolverDmlOnMultipleKeyspaceIds" createSandbox(keyspace) hc := discovery.NewFakeHealthCheck() res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil) hc.AddTestTablet("aa", "1.1.1.1", 1001, keyspace, "-20", topodatapb.TabletType_MASTER, true, 1, nil) hc.AddTestTablet("aa", "1.1.1.1", 1002, keyspace, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) errStr := "DML should not span multiple keyspace_ids" _, err := res.ExecuteKeyspaceIds(context.Background(), "update table set a = b", nil, keyspace, [][]byte{{0x10}, {0x25}}, topodatapb.TabletType_MASTER, nil, false) if err == nil { t.Errorf("want %v, got nil", errStr) } }
func createRouterEnv() (router *Router, sbc1, sbc2, sbclookup *sandboxconn.SandboxConn) { cell := "aa" hc := discovery.NewFakeHealthCheck() s := createSandbox("TestRouter") s.VSchema = routerVSchema sbc1 = hc.AddTestTablet(cell, "-20", 1, "TestRouter", "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc2 = hc.AddTestTablet(cell, "40-60", 1, "TestRouter", "40-60", topodatapb.TabletType_MASTER, true, 1, nil) createSandbox(KsTestUnsharded) sbclookup = hc.AddTestTablet(cell, "0", 1, KsTestUnsharded, "0", topodatapb.TabletType_MASTER, true, 1, nil) bad := createSandbox("TestBadSharding") bad.VSchema = badVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema serv := new(sandboxTopo) scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil) router = NewRouter(context.Background(), serv, cell, "", scatterConn) return router, sbc1, sbc2, sbclookup }
func TestStreamSelectScatter(t *testing.T) { // Special setup: Don't use createRouterEnv. cell := "aa" hc := discovery.NewFakeHealthCheck() s := createSandbox("TestRouter") s.VSchema = routerVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema serv := new(sandboxTopo) scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil) shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"} var conns []*sandboxconn.SandboxConn for _, shard := range shards { sbc := hc.AddTestTablet(cell, shard, 1, "TestRouter", shard, topodatapb.TabletType_MASTER, true, 1, nil) conns = append(conns, sbc) } router := NewRouter(context.Background(), serv, cell, "", scatterConn) sql := "select id from user" result, err := routerStream(router, sql) if err != nil { t.Error(err) } wantResult := &sqltypes.Result{ Fields: sandboxconn.SingleRowResult.Fields, Rows: [][]sqltypes.Value{ sandboxconn.SingleRowResult.Rows[0], sandboxconn.SingleRowResult.Rows[0], sandboxconn.SingleRowResult.Rows[0], sandboxconn.SingleRowResult.Rows[0], sandboxconn.SingleRowResult.Rows[0], sandboxconn.SingleRowResult.Rows[0], sandboxconn.SingleRowResult.Rows[0], sandboxconn.SingleRowResult.Rows[0], }, RowsAffected: 8, } if !reflect.DeepEqual(result, wantResult) { t.Errorf("result: %+v, want %+v", result, wantResult) } }
func TestResolverExecBatchAsTransaction(t *testing.T) { keyspace := "TestResolverExecBatchAsTransaction" createSandbox(keyspace) hc := discovery.NewFakeHealthCheck() res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil) sbc := hc.AddTestTablet("aa", "0", 1, keyspace, "0", topodatapb.TabletType_MASTER, true, 1, nil) sbc.MustFailRetry = 20 callcount := 0 buildBatchRequest := func() (*scatterBatchRequest, error) { callcount++ queries := []*vtgatepb.BoundShardQuery{{ Query: &querypb.BoundQuery{ Sql: "query", BindVariables: nil, }, Keyspace: keyspace, Shards: []string{"0"}, }} return boundShardQueriesToScatterBatchRequest(queries) } _, err := res.ExecuteBatch(context.Background(), topodatapb.TabletType_MASTER, true, nil, buildBatchRequest) want := "shard, host: TestResolverExecBatchAsTransaction.0.master, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"TestResolverExecBatchAsTransaction\" shard:\"0\" type:MASTER , retry: err" if err == nil || err.Error() != want { t.Errorf("want %v, got %v", want, err) } // Ensure scatter did not re-resolve if callcount != 1 { t.Errorf("want 1, got %v", callcount) } if count := sbc.AsTransactionCount.Get(); count != 1 { t.Errorf("want 1, got %v", count) return } }
func TestSelectScatterFail(t *testing.T) { // Special setup: Don't use createRouterEnv. cell := "aa" hc := discovery.NewFakeHealthCheck() s := createSandbox("TestRouter") s.VSchema = routerVSchema getSandbox(KsTestUnsharded).VSchema = unshardedVSchema s.SrvKeyspaceMustFail = 1 shards := []string{"-20", "20-40", "40-60", "60-80", "80-a0", "a0-c0", "c0-e0", "e0-"} var conns []*sandboxconn.SandboxConn for _, shard := range shards { sbc := hc.AddTestTablet(cell, shard, 1, "TestRouter", shard, topodatapb.TabletType_MASTER, true, 1, nil) conns = append(conns, sbc) } serv := new(sandboxTopo) scatterConn := NewScatterConn(hc, topo.Server{}, serv, "", cell, 10, nil) router := NewRouter(context.Background(), serv, cell, "", scatterConn) _, err := routerExec(router, "select id from user", nil) want := "paramsSelectScatter: keyspace TestRouter fetch error: topo error GetSrvKeyspace" if err == nil || err.Error() != want { t.Errorf("routerExec: %v, want %v", err, want) } }
func TestScatterConnQueryNotInTransaction(t *testing.T) { s := createSandbox("TestScatterConnQueryNotInTransaction") hc := discovery.NewFakeHealthCheck() // case 1: read query (not in transaction) followed by write query, not in the same shard. hc.Reset() sbc0 := hc.AddTestTablet("aa", "0", 1, "TestScatterConnQueryNotInTransaction", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1", 1, "TestScatterConnQueryNotInTransaction", "1", topodatapb.TabletType_REPLICA, true, 1, nil) stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) session := NewSafeSession(&vtgatepb.Session{InTransaction: true}) stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0"}, topodatapb.TabletType_REPLICA, session, true) stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"1"}, topodatapb.TabletType_REPLICA, session, false) wantSession := vtgatepb.Session{ InTransaction: true, ShardSessions: []*vtgatepb.Session_ShardSession{{ Target: &querypb.Target{ Keyspace: "TestScatterConnQueryNotInTransaction", Shard: "1", TabletType: topodatapb.TabletType_REPLICA, }, TransactionId: 1, }}, } if !reflect.DeepEqual(wantSession, *session.Session) { t.Errorf("want\n%+v\ngot\n%+v", wantSession, *session.Session) } stc.Commit(context.Background(), session) { execCount0 := sbc0.ExecCount.Get() execCount1 := sbc1.ExecCount.Get() if execCount0 != 1 || execCount1 != 1 { t.Errorf("want 1/1, got %d/%d", execCount0, execCount1) } } if commitCount := sbc0.CommitCount.Get(); commitCount != 0 { t.Errorf("want 0, got %d", commitCount) } if commitCount := sbc1.CommitCount.Get(); commitCount != 1 { t.Errorf("want 1, got %d", commitCount) } // case 2: write query followed by read query (not in transaction), not in the same shard. s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "0", 1, "TestScatterConnQueryNotInTransaction", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1", 1, "TestScatterConnQueryNotInTransaction", "1", topodatapb.TabletType_REPLICA, true, 1, nil) stc = NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) session = NewSafeSession(&vtgatepb.Session{InTransaction: true}) stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0"}, topodatapb.TabletType_REPLICA, session, false) stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"1"}, topodatapb.TabletType_REPLICA, session, true) wantSession = vtgatepb.Session{ InTransaction: true, ShardSessions: []*vtgatepb.Session_ShardSession{{ Target: &querypb.Target{ Keyspace: "TestScatterConnQueryNotInTransaction", Shard: "0", TabletType: topodatapb.TabletType_REPLICA, }, TransactionId: 1, }}, } if !reflect.DeepEqual(wantSession, *session.Session) { t.Errorf("want\n%+v\ngot\n%+v", wantSession, *session.Session) } stc.Commit(context.Background(), session) { execCount0 := sbc0.ExecCount.Get() execCount1 := sbc1.ExecCount.Get() if execCount0 != 1 || execCount1 != 1 { t.Errorf("want 1/1, got %d/%d", execCount0, execCount1) } } if commitCount := sbc0.CommitCount.Get(); commitCount != 1 { t.Errorf("want 1, got %d", commitCount) } if commitCount := sbc1.CommitCount.Get(); commitCount != 0 { t.Errorf("want 0, got %d", commitCount) } // case 3: write query followed by read query, in the same shard. s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "0", 1, "TestScatterConnQueryNotInTransaction", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1", 1, "TestScatterConnQueryNotInTransaction", "1", topodatapb.TabletType_REPLICA, true, 1, nil) stc = NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) session = NewSafeSession(&vtgatepb.Session{InTransaction: true}) stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0"}, topodatapb.TabletType_REPLICA, session, false) stc.Execute(context.Background(), "query1", nil, "TestScatterConnQueryNotInTransaction", []string{"0", "1"}, topodatapb.TabletType_REPLICA, session, true) wantSession = vtgatepb.Session{ InTransaction: true, ShardSessions: []*vtgatepb.Session_ShardSession{{ Target: &querypb.Target{ Keyspace: "TestScatterConnQueryNotInTransaction", Shard: "0", TabletType: topodatapb.TabletType_REPLICA, }, TransactionId: 1, }}, } if !reflect.DeepEqual(wantSession, *session.Session) { t.Errorf("want\n%+v\ngot\n%+v", wantSession, *session.Session) } stc.Commit(context.Background(), session) { execCount0 := sbc0.ExecCount.Get() execCount1 := sbc1.ExecCount.Get() if execCount0 != 2 || execCount1 != 1 { t.Errorf("want 2/1, got %d/%d", execCount0, execCount1) } } if commitCount := sbc0.CommitCount.Get(); commitCount != 1 { t.Errorf("want 1, got %d", commitCount) } if commitCount := sbc1.CommitCount.Get(); commitCount != 0 { t.Errorf("want 0, got %d", commitCount) } }
func testDiscoveryGatewayGeneric(t *testing.T, streaming bool, f func(dg Gateway, keyspace, shard string, tabletType topodatapb.TabletType) error) { keyspace := "ks" shard := "0" tabletType := topodatapb.TabletType_REPLICA hc := discovery.NewFakeHealthCheck() dg := createDiscoveryGateway(hc, topo.Server{}, nil, "cell", 2).(*discoveryGateway) // no tablet hc.Reset() dg.tsc.ResetForTesting() want := "shard, host: ks.0.replica, no valid tablet" err := f(dg, keyspace, shard, tabletType) verifyShardError(t, err, want, vtrpcpb.ErrorCode_INTERNAL_ERROR) // tablet with error hc.Reset() dg.tsc.ResetForTesting() hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, false, 10, fmt.Errorf("no connection")) want = "shard, host: ks.0.replica, no valid tablet" err = f(dg, keyspace, shard, tabletType) verifyShardError(t, err, want, vtrpcpb.ErrorCode_INTERNAL_ERROR) // tablet without connection hc.Reset() dg.tsc.ResetForTesting() ep1 := hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, false, 10, nil).Tablet() want = fmt.Sprintf(`shard, host: ks.0.replica, no valid tablet`) err = f(dg, keyspace, shard, tabletType) verifyShardError(t, err, want, vtrpcpb.ErrorCode_INTERNAL_ERROR) // retry error hc.Reset() dg.tsc.ResetForTesting() sc1 := hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil) sc2 := hc.AddTestTablet("cell", "1.1.1.1", 1002, keyspace, shard, tabletType, true, 10, nil) sc1.MustFailRetry = 1 sc2.MustFailRetry = 1 ep1 = sc1.Tablet() ep2 := sc2.Tablet() wants := map[string]int{ fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep1): 0, fmt.Sprintf(`shard, host: ks.0.replica, %+v, retry: err`, ep2): 0, } err = f(dg, keyspace, shard, tabletType) if _, ok := wants[fmt.Sprintf("%v", err)]; !ok { t.Errorf("wanted error: %+v, got error: %v", wants, err) } // fatal error hc.Reset() dg.tsc.ResetForTesting() sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil) sc2 = hc.AddTestTablet("cell", "1.1.1.1", 1002, keyspace, shard, tabletType, true, 10, nil) sc1.MustFailFatal = 1 sc2.MustFailFatal = 1 ep1 = sc1.Tablet() ep2 = sc2.Tablet() wants = map[string]int{ fmt.Sprintf(`shard, host: ks.0.replica, %+v, fatal: err`, ep1): 0, fmt.Sprintf(`shard, host: ks.0.replica, %+v, fatal: err`, ep2): 0, } err = f(dg, keyspace, shard, tabletType) if _, ok := wants[fmt.Sprintf("%v", err)]; !ok { t.Errorf("wanted error: %+v, got error: %v", wants, err) } // server error - no retry hc.Reset() dg.tsc.ResetForTesting() sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil) sc1.MustFailServer = 1 ep1 = sc1.Tablet() want = fmt.Sprintf(`shard, host: ks.0.replica, %+v, error: err`, ep1) err = f(dg, keyspace, shard, tabletType) verifyShardError(t, err, want, vtrpcpb.ErrorCode_BAD_INPUT) // conn error - no retry hc.Reset() dg.tsc.ResetForTesting() sc1 = hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil) sc1.MustFailConn = 1 ep1 = sc1.Tablet() want = fmt.Sprintf(`shard, host: ks.0.replica, %+v, error: conn`, ep1) err = f(dg, keyspace, shard, tabletType) verifyShardError(t, err, want, vtrpcpb.ErrorCode_UNKNOWN_ERROR) // no failure hc.Reset() dg.tsc.ResetForTesting() hc.AddTestTablet("cell", "1.1.1.1", 1001, keyspace, shard, tabletType, true, 10, nil) err = f(dg, keyspace, shard, tabletType) if err != nil { t.Errorf("want nil, got %v", err) } }
func TestScatterConnCommitSuccess(t *testing.T) { createSandbox("TestScatterConnCommitSuccess") hc := discovery.NewFakeHealthCheck() sbc0 := hc.AddTestTablet("aa", "0", 1, "TestScatterConnCommitSuccess", "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1", 1, "TestScatterConnCommitSuccess", "1", topodatapb.TabletType_REPLICA, true, 1, nil) stc := NewScatterConn(hc, topo.Server{}, new(sandboxTopo), "", "aa", retryCount, nil) // Sequence the executes to ensure commit order session := NewSafeSession(&vtgatepb.Session{InTransaction: true}) stc.Execute(context.Background(), "query1", nil, "TestScatterConnCommitSuccess", []string{"0"}, topodatapb.TabletType_REPLICA, session, false) wantSession := vtgatepb.Session{ InTransaction: true, ShardSessions: []*vtgatepb.Session_ShardSession{{ Target: &querypb.Target{ Keyspace: "TestScatterConnCommitSuccess", Shard: "0", TabletType: topodatapb.TabletType_REPLICA, }, TransactionId: 1, }}, } if !reflect.DeepEqual(wantSession, *session.Session) { t.Errorf("want\n%+v, got\n%+v", wantSession, *session.Session) } stc.Execute(context.Background(), "query1", nil, "TestScatterConnCommitSuccess", []string{"0", "1"}, topodatapb.TabletType_REPLICA, session, false) wantSession = vtgatepb.Session{ InTransaction: true, ShardSessions: []*vtgatepb.Session_ShardSession{{ Target: &querypb.Target{ Keyspace: "TestScatterConnCommitSuccess", Shard: "0", TabletType: topodatapb.TabletType_REPLICA, }, TransactionId: 1, }, { Target: &querypb.Target{ Keyspace: "TestScatterConnCommitSuccess", Shard: "1", TabletType: topodatapb.TabletType_REPLICA, }, TransactionId: 1, }}, } if !reflect.DeepEqual(wantSession, *session.Session) { t.Errorf("want\n%+v, got\n%+v", wantSession, *session.Session) } sbc0.MustFailServer = 1 err := stc.Commit(context.Background(), session) if err == nil { t.Errorf("want error, got nil") } wantSession = vtgatepb.Session{} if !reflect.DeepEqual(wantSession, *session.Session) { t.Errorf("want\n%+v, got\n%+v", wantSession, *session.Session) } if commitCount := sbc0.CommitCount.Get(); commitCount != 1 { t.Errorf("want 1, got %d", commitCount) } if rollbackCount := sbc1.RollbackCount.Get(); rollbackCount != 1 { t.Errorf("want 1, got %d", rollbackCount) } }
func testResolverGeneric(t *testing.T, name string, action func(res *Resolver) (*sqltypes.Result, error)) { // successful execute s := createSandbox(name) hc := discovery.NewFakeHealthCheck() res := NewResolver(hc, topo.Server{}, new(sandboxTopo), "", "aa", 0, nil) sbc0 := hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) _, err := action(res) if err != nil { t.Errorf("want nil, got %v", err) } if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // non-retryable failure s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) sbc0.MustFailServer = 1 sbc1.MustFailRetry = 1 _, err = action(res) want1 := fmt.Sprintf("shard, host: %s.-20.master, alias:<cell:\"aa\" > hostname:\"-20\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"-20\" type:MASTER , error: err", name, name) want2 := fmt.Sprintf("shard, host: %s.20-40.master, alias:<cell:\"aa\" > hostname:\"20-40\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"20-40\" type:MASTER , retry: err", name, name) want := []string{want1, want2} sort.Strings(want) if err == nil { t.Errorf("want\n%v\ngot\n%v", want, err) } else { got := strings.Split(err.Error(), "\n") sort.Strings(got) if !reflect.DeepEqual(want, got) { t.Errorf("want\n%v\ngot\n%v", want, got) } } // Ensure that we tried only once if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // Ensure that we tried topo only once when mapping KeyspaceId/KeyRange to shards if s.SrvKeyspaceCounter != 1 { t.Errorf("want 1, got %v", s.SrvKeyspaceCounter) } // retryable failure, no sharding event s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "-20", 1, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "20-40", 1, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) sbc0.MustFailRetry = 1 sbc1.MustFailFatal = 1 _, err = action(res) want1 = fmt.Sprintf("shard, host: %s.-20.master, alias:<cell:\"aa\" > hostname:\"-20\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"-20\" type:MASTER , retry: err", name, name) want2 = fmt.Sprintf("shard, host: %s.20-40.master, alias:<cell:\"aa\" > hostname:\"20-40\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"20-40\" type:MASTER , fatal: err", name, name) want = []string{want1, want2} sort.Strings(want) if err == nil { t.Errorf("want\n%v\ngot\n%v", want, err) } else { got := strings.Split(err.Error(), "\n") sort.Strings(got) if !reflect.DeepEqual(want, got) { t.Errorf("want\n%v\ngot\n%v", want, got) } } // Ensure that we tried only once. if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // Ensure that we tried topo only twice. if s.SrvKeyspaceCounter != 2 { t.Errorf("want 2, got %v", s.SrvKeyspaceCounter) } // no failure, initial vertical resharding s.Reset() addSandboxServedFrom(name, name+"ServedFrom0") hc.Reset() sbc0 = hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) s0 := createSandbox(name + "ServedFrom0") // make sure we have a fresh copy s0.ShardSpec = "-80-" sbc2 := hc.AddTestTablet("aa", "1.1.1.1", 1003, name+"ServedFrom0", "-80", topodatapb.TabletType_MASTER, true, 1, nil) _, err = action(res) if err != nil { t.Errorf("want nil, got %v", err) } // Ensure original keyspace is not used. if execCount := sbc0.ExecCount.Get(); execCount != 0 { t.Errorf("want 0, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 0 { t.Errorf("want 0, got %v", execCount) } // Ensure redirected keyspace is accessed once. if execCount := sbc2.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // Ensure that we tried each keyspace only once. if s.SrvKeyspaceCounter != 1 { t.Errorf("want 1, got %v", s.SrvKeyspaceCounter) } if s0.SrvKeyspaceCounter != 1 { t.Errorf("want 1, got %v", s0.SrvKeyspaceCounter) } s0.Reset() // retryable failure, vertical resharding s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) sbc1.MustFailFatal = 1 i := 0 s.SrvKeyspaceCallback = func() { if i == 1 { addSandboxServedFrom(name, name+"ServedFrom") hc.Reset() hc.AddTestTablet("aa", "1.1.1.1", 1001, name+"ServedFrom", "-20", topodatapb.TabletType_MASTER, true, 1, nil) hc.AddTestTablet("aa", "1.1.1.1", 1002, name+"ServedFrom", "20-40", topodatapb.TabletType_MASTER, true, 1, nil) } i++ } _, err = action(res) if err != nil { t.Errorf("want nil, got %v", err) } // Ensure that we tried only once on the original conn. if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // Ensure that we tried topo only 3 times. if s.SrvKeyspaceCounter != 3 { t.Errorf("want 3, got %v", s.SrvKeyspaceCounter) } // retryable failure, horizontal resharding s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-40", topodatapb.TabletType_MASTER, true, 1, nil) sbc1.MustFailRetry = 1 i = 0 s.SrvKeyspaceCallback = func() { if i == 1 { s.ShardSpec = "-20-30-40-60-80-a0-c0-e0-" hc.Reset() hc.AddTestTablet("aa", "1.1.1.1", 1001, name, "-20", topodatapb.TabletType_MASTER, true, 1, nil) hc.AddTestTablet("aa", "1.1.1.1", 1002, name, "20-30", topodatapb.TabletType_MASTER, true, 1, nil) } i++ } _, err = action(res) if err != nil { t.Errorf("want nil, got %v", err) } // Ensure that we tried only once on original guy. if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // Ensure that we tried topo only twice. if s.SrvKeyspaceCounter != 2 { t.Errorf("want 2, got %v", s.SrvKeyspaceCounter) } }
func testScatterConnGeneric(t *testing.T, name string, f func(hc discovery.HealthCheck, shards []string) (*sqltypes.Result, error)) { hc := discovery.NewFakeHealthCheck() // no shard s := createSandbox(name) qr, err := f(hc, nil) if qr.RowsAffected != 0 { t.Errorf("want 0, got %v", qr.RowsAffected) } if err != nil { t.Errorf("want nil, got %v", err) } // single shard s.Reset() sbc := hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc.MustFailServer = 1 qr, err = f(hc, []string{"0"}) want := fmt.Sprintf("shard, host: %v.0.replica, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"%s\" shard:\"0\" type:REPLICA , error: err", name, name) // Verify server error string. if err == nil || err.Error() != want { t.Errorf("want %s, got %v", want, err) } // Ensure that we tried only once. if execCount := sbc.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // two shards s.Reset() hc.Reset() sbc0 := hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 := hc.AddTestTablet("aa", "1", 1, name, "1", topodatapb.TabletType_REPLICA, true, 1, nil) sbc0.MustFailServer = 1 sbc1.MustFailServer = 1 _, err = f(hc, []string{"0", "1"}) // Verify server errors are consolidated. want = fmt.Sprintf("shard, host: %v.0.replica, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"0\" type:REPLICA , error: err\nshard, host: %v.1.replica, alias:<cell:\"aa\" > hostname:\"1\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"1\" type:REPLICA , error: err", name, name, name, name) verifyScatterConnError(t, err, want, vtrpcpb.ErrorCode_BAD_INPUT) // Ensure that we tried only once. if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // two shards with different errors s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1", 1, name, "1", topodatapb.TabletType_REPLICA, true, 1, nil) sbc0.MustFailServer = 1 sbc1.MustFailTxPool = 1 _, err = f(hc, []string{"0", "1"}) // Verify server errors are consolidated. want = fmt.Sprintf("shard, host: %v.0.replica, alias:<cell:\"aa\" > hostname:\"0\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"0\" type:REPLICA , error: err\nshard, host: %v.1.replica, alias:<cell:\"aa\" > hostname:\"1\" port_map:<key:\"vt\" value:1 > keyspace:\"%v\" shard:\"1\" type:REPLICA , tx_pool_full: err", name, name, name, name) // We should only surface the higher priority error code verifyScatterConnError(t, err, want, vtrpcpb.ErrorCode_BAD_INPUT) // Ensure that we tried only once. if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // duplicate shards s.Reset() hc.Reset() sbc = hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil) qr, err = f(hc, []string{"0", "0"}) // Ensure that we executed only once. if execCount := sbc.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } // no errors s.Reset() hc.Reset() sbc0 = hc.AddTestTablet("aa", "0", 1, name, "0", topodatapb.TabletType_REPLICA, true, 1, nil) sbc1 = hc.AddTestTablet("aa", "1", 1, name, "1", topodatapb.TabletType_REPLICA, true, 1, nil) qr, err = f(hc, []string{"0", "1"}) if err != nil { t.Errorf("want nil, got %v", err) } if execCount := sbc0.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if execCount := sbc1.ExecCount.Get(); execCount != 1 { t.Errorf("want 1, got %v", execCount) } if qr.RowsAffected != 2 { t.Errorf("want 2, got %v", qr.RowsAffected) } if len(qr.Rows) != 2 { t.Errorf("want 2, got %v", len(qr.Rows)) } }