func agentRPCTestHealthStream(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
c, errFunc, err := client.HealthStream(ctx, ti)
if err != nil {
t.Fatalf("HealthStream failed: %v", err)
}
// channel should have one response, then closed
hsr, ok := <-c
if !ok {
t.Fatalf("HealthStream got no response")
}
// close HealthStreamSynchronization so server side knows we
// got the response, and it can send the error
close(HealthStreamSynchronization)
_, ok = <-c
if ok {
t.Fatalf("HealthStream wasn't closed")
}
err = errFunc()
if !strings.Contains(err.Error(), testRegisterHealthStreamError) {
t.Fatalf("HealthStream failed with the wrong error: %v", err)
}
compareError(t, "HealthStream", nil, *hsr, *testHealthStreamHealthStreamReply)
}
func agentRPCTestExecuteFetchPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
_, err := client.ExecuteFetchAsDba(ctx, ti, testExecuteFetchQuery, testExecuteFetchMaxRows, true, true, false)
expectRPCWrapPanic(t, err)
_, err = client.ExecuteFetchAsApp(ctx, ti, testExecuteFetchQuery, testExecuteFetchMaxRows, true)
expectRPCWrapPanic(t, err)
}
func agentRPCTestBackup(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
stream, err := client.Backup(ctx, ti, testBackupConcurrency)
if err != nil {
t.Fatalf("Backup failed: %v", err)
}
err = compareLoggedStuff(t, "Backup", stream, 10)
compareError(t, "Backup", err, true, testBackupCalled)
}
func agentRPCTestRestoreFromBackup(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
stream, err := client.RestoreFromBackup(ctx, tablet)
if err != nil {
t.Fatalf("RestoreFromBackup failed: %v", err)
}
err = compareLoggedStuff(t, "RestoreFromBackup", stream, 10)
compareError(t, "RestoreFromBackup", err, true, testRestoreFromBackupCalled)
}
func agentRPCTestRefreshState(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
err := client.RefreshState(ctx, tablet)
if err != nil {
t.Errorf("RefreshState failed: %v", err)
}
if !testRefreshStateCalled {
t.Errorf("RefreshState didn't call the server side")
}
}
func agentRPCTestReloadSchema(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
err := client.ReloadSchema(ctx, tablet, "")
if err != nil {
t.Errorf("ReloadSchema failed: %v", err)
}
if !testReloadSchemaCalled {
t.Errorf("ReloadSchema didn't call the server side")
}
}
func agentRpcTestMultiRestore(t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
logChannel, errFunc, err := client.MultiRestore(ti, testMultiRestoreArgs, time.Minute)
if err != nil {
t.Fatalf("MultiRestore failed: %v", err)
}
compareLoggedStuff(t, "MultiRestore", logChannel, 1000)
err = errFunc()
compareError(t, "MultiRestore", err, true, testMultiRestoreCalled)
}
func agentRpcTestMultiSnapshot(t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
logChannel, errFunc, err := client.MultiSnapshot(ti, testMultiSnapshotArgs, time.Minute)
if err != nil {
t.Fatalf("MultiSnapshot failed: %v", err)
}
compareLoggedStuff(t, "MultiSnapshot", logChannel, 100)
sr, err := errFunc()
compareError(t, "MultiSnapshot", err, sr, testMultiSnapshotReply)
}
func agentRpcTestReloadSchema(t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
err := client.ReloadSchema(ti, time.Minute)
if err != nil {
t.Errorf("ReloadSchema failed: %v", err)
}
if !testReloadSchemaCalled {
t.Errorf("ReloadSchema didn't call the server side")
}
}
func agentRPCTestBackupPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
stream, err := client.Backup(ctx, ti, testBackupConcurrency)
if err != nil {
t.Fatalf("Backup failed: %v", err)
}
e, err := stream.Recv()
if err == nil {
t.Fatalf("Unexpected Backup logs: %v", e)
}
expectRPCWrapLockActionPanic(t, err)
}
func agentRPCTestBackupPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
logChannel, errFunc, err := client.Backup(ctx, ti, testBackupConcurrency)
if err != nil {
t.Fatalf("Backup failed: %v", err)
}
if e, ok := <-logChannel; ok {
t.Fatalf("Unexpected Backup logs: %v", e)
}
err = errFunc()
expectRPCWrapLockActionPanic(t, err)
}
func agentRPCTestBackupPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
stream, err := client.Backup(ctx, tablet, testBackupConcurrency)
if err != nil {
t.Fatalf("Backup failed: %v", err)
}
e, err := stream.Recv()
if err == nil {
t.Fatalf("Unexpected Backup logs: %v", e)
}
expectHandleRPCPanic(t, "Backup", true /*verbose*/, err)
}
func agentRPCTestRestoreFromBackupPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
stream, err := client.RestoreFromBackup(ctx, tablet)
if err != nil {
t.Fatalf("RestoreFromBackup failed: %v", err)
}
e, err := stream.Recv()
if err == nil {
t.Fatalf("Unexpected RestoreFromBackup logs: %v", e)
}
expectRPCWrapLockActionPanic(t, err)
}
func agentRpcTestSetReadOnly(t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
testSetReadOnlyExpectedValue = true
err := client.SetReadOnly(ti, time.Minute)
if err != nil {
t.Errorf("SetReadOnly failed: %v", err)
}
testSetReadOnlyExpectedValue = false
err = client.SetReadWrite(ti, time.Minute)
if err != nil {
t.Errorf("SetReadWrite failed: %v", err)
}
}
func agentRPCTestSetReadOnly(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
testSetReadOnlyExpectedValue = true
err := client.SetReadOnly(ctx, tablet)
if err != nil {
t.Errorf("SetReadOnly failed: %v", err)
}
testSetReadOnlyExpectedValue = false
err = client.SetReadWrite(ctx, tablet)
if err != nil {
t.Errorf("SetReadWrite failed: %v", err)
}
}
// agentRPCTestIsTimeoutErrorDialExpiredContext verifies that
// client.IsTimeoutError() returns true for RPCs failed due to an expired
// context before .Dial().
func agentRPCTestIsTimeoutErrorDialExpiredContext(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
// Using a timeout of 0 here such that .Dial() will fail immediately.
expiredCtx, cancel := context.WithTimeout(ctx, 0)
defer cancel()
err := client.Ping(expiredCtx, ti)
if err == nil {
t.Fatal("agentRPCTestIsTimeoutErrorDialExpiredContext: RPC with expired context did not fail")
}
if !client.IsTimeoutError(err) {
t.Errorf("agentRPCTestIsTimeoutErrorDialExpiredContext: want: IsTimeoutError() = true. error: %v", err)
}
}
func agentRPCTestHealthStreamPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
c, errFunc, err := client.HealthStream(ctx, ti)
if err != nil {
t.Fatalf("HealthStream failed: %v", err)
}
// channel should have no response, just closed
_, ok := <-c
if ok {
t.Fatalf("HealthStream wasn't closed")
}
err = errFunc()
expectRPCWrapPanic(t, err)
}
func agentRPCTestExecuteFetchPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
// using pool
_, err := client.ExecuteFetchAsDba(ctx, tablet, true, testExecuteFetchQuery, testExecuteFetchMaxRows, true, false)
expectHandleRPCPanic(t, "ExecuteFetchAsDba", false /*verbose*/, err)
_, err = client.ExecuteFetchAsApp(ctx, tablet, true, testExecuteFetchQuery, testExecuteFetchMaxRows)
expectHandleRPCPanic(t, "ExecuteFetchAsApp", false /*verbose*/, err)
// not using pool
_, err = client.ExecuteFetchAsDba(ctx, tablet, false, testExecuteFetchQuery, testExecuteFetchMaxRows, true, false)
expectHandleRPCPanic(t, "ExecuteFetchAsDba", false /*verbose*/, err)
_, err = client.ExecuteFetchAsApp(ctx, tablet, false, testExecuteFetchQuery, testExecuteFetchMaxRows)
expectHandleRPCPanic(t, "ExecuteFetchAsApp", false /*verbose*/, err)
_, err = client.ExecuteFetchAsAllPrivs(ctx, tablet, testExecuteFetchQuery, testExecuteFetchMaxRows, false)
expectHandleRPCPanic(t, "ExecuteFetchAsAllPrivs", false /*verbose*/, err)
}
func agentRPCTestExecuteFetch(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
// using pool
qr, err := client.ExecuteFetchAsDba(ctx, tablet, true, testExecuteFetchQuery, testExecuteFetchMaxRows, true, true)
compareError(t, "ExecuteFetchAsDba", err, qr, testExecuteFetchResult)
qr, err = client.ExecuteFetchAsApp(ctx, tablet, true, testExecuteFetchQuery, testExecuteFetchMaxRows)
compareError(t, "ExecuteFetchAsApp", err, qr, testExecuteFetchResult)
// not using pool
qr, err = client.ExecuteFetchAsDba(ctx, tablet, false, testExecuteFetchQuery, testExecuteFetchMaxRows, true, true)
compareError(t, "ExecuteFetchAsDba", err, qr, testExecuteFetchResult)
qr, err = client.ExecuteFetchAsApp(ctx, tablet, false, testExecuteFetchQuery, testExecuteFetchMaxRows)
compareError(t, "ExecuteFetchAsApp", err, qr, testExecuteFetchResult)
qr, err = client.ExecuteFetchAsAllPrivs(ctx, tablet, testExecuteFetchQuery, testExecuteFetchMaxRows, true)
compareError(t, "ExecuteFetchAsAllPrivs", err, qr, testExecuteFetchResult)
}
// agentRPCTestIsTimeoutErrorDialTimeout verifies that client.IsTimeoutError()
// returns true for RPCs failed due to a connect timeout during .Dial().
func agentRPCTestIsTimeoutErrorDialTimeout(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo) {
// Connect to a non-existing tablet.
// For example, this provokes gRPC to return error grpc.ErrClientConnTimeout.
invalidTi := topo.NewTabletInfo(ti.Tablet, ti.Version())
invalidTi.Tablet = proto.Clone(invalidTi.Tablet).(*topodatapb.Tablet)
invalidTi.Tablet.Hostname = "Non-Existent.Server"
shortCtx, cancel := context.WithTimeout(ctx, time.Millisecond)
defer cancel()
err := client.Ping(shortCtx, invalidTi)
if err == nil {
t.Fatal("agentRPCTestIsTimeoutErrorDialTimeout: connect to non-existant tablet did not fail")
}
if !client.IsTimeoutError(err) {
t.Errorf("agentRPCTestIsTimeoutErrorDialTimeout: want: IsTimeoutError() = true. error: %v", err)
}
}
// agentRPCTestIsTimeoutErrorRPC verifies that client.IsTimeoutError() returns
// true for RPCs failed due to an expired context during RPC execution.
func agentRPCTestIsTimeoutErrorRPC(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, ti *topo.TabletInfo, fakeAgent *fakeRPCAgent) {
// We must use a timeout > 0 such that the context deadline hasn't expired
// yet in grpctmclient.Client.dial().
// NOTE: This might still race e.g. when test execution takes too long the
// context will be expired in dial() already. In such cases coverage
// will be reduced but the test will not flake.
shortCtx, cancel := context.WithTimeout(ctx, time.Millisecond)
defer cancel()
fakeAgent.slow = true
defer func() { fakeAgent.slow = false }()
err := client.Ping(shortCtx, ti)
if err == nil {
t.Fatal("agentRPCTestIsTimeoutErrorRPC: RPC with expired context did not fail")
}
if !client.IsTimeoutError(err) {
t.Errorf("agentRPCTestIsTimeoutErrorRPC: want: IsTimeoutError() = true. error: %v", err)
}
}
// agentRPCTestDialExpiredContext verifies that
// the context returns the right DeadlineExceeded Err() for
// RPCs failed due to an expired context before .Dial().
func agentRPCTestDialExpiredContext(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
// Using a timeout of 0 here such that .Dial() will fail immediately.
expiredCtx, cancel := context.WithTimeout(ctx, 0)
defer cancel()
err := client.Ping(expiredCtx, tablet)
if err == nil {
t.Fatal("agentRPCTestDialExpiredContext: RPC with expired context did not fail")
}
// The context was already expired when we created it. Here we only verify that it returns the expected error.
select {
case <-expiredCtx.Done():
if err := expiredCtx.Err(); err != context.DeadlineExceeded {
t.Errorf("agentRPCTestDialExpiredContext: got %v want context.DeadlineExceeded", err)
}
default:
t.Errorf("agentRPCTestDialExpiredContext: context.Done() not closed")
}
}
func agentRPCTestExecuteFetchPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
// using pool
_, err := client.ExecuteFetchAsDba(ctx, tablet, true, testExecuteFetchQuery, testExecuteFetchMaxRows, true, false)
expectRPCWrapPanic(t, err)
_, err = client.ExecuteFetchAsApp(ctx, tablet, true, testExecuteFetchQuery, testExecuteFetchMaxRows)
expectRPCWrapPanic(t, err)
// not using pool
_, err = client.ExecuteFetchAsDba(ctx, tablet, false, testExecuteFetchQuery, testExecuteFetchMaxRows, true, false)
expectRPCWrapPanic(t, err)
_, err = client.ExecuteFetchAsApp(ctx, tablet, false, testExecuteFetchQuery, testExecuteFetchMaxRows)
expectRPCWrapPanic(t, err)
}
// agentRPCTestRPCTimeout verifies that
// the context returns the right DeadlineExceeded Err() for
// RPCs failed due to an expired context during execution.
func agentRPCTestRPCTimeout(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet, fakeAgent *fakeRPCAgent) {
// We must use a timeout > 0 such that the context deadline hasn't expired
// yet in grpctmclient.Client.dial().
// NOTE: This might still race e.g. when test execution takes too long the
// context will be expired in dial() already. In such cases coverage
// will be reduced but the test will not flake.
shortCtx, cancel := context.WithTimeout(ctx, 10*time.Millisecond)
defer cancel()
fakeAgent.setSlow(true)
defer func() { fakeAgent.setSlow(false) }()
err := client.Ping(shortCtx, tablet)
if err == nil {
t.Fatal("agentRPCTestRPCTimeout: RPC with expired context did not fail")
}
select {
case <-shortCtx.Done():
if err := shortCtx.Err(); err != context.DeadlineExceeded {
t.Errorf("agentRPCTestRPCTimeout: got %v want context.DeadlineExceeded", err)
}
default:
t.Errorf("agentRPCTestRPCTimeout: context.Done() not closed")
}
}
func agentRPCTestPromoteSlaveWhenCaughtUp(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
rp, err := client.PromoteSlaveWhenCaughtUp(ctx, tablet, testReplicationPosition)
compareError(t, "PromoteSlaveWhenCaughtUp", err, rp, testReplicationPositionReturned)
}
func agentRPCTestDemoteMasterPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
_, err := client.DemoteMaster(ctx, tablet)
expectRPCWrapLockActionPanic(t, err)
}
func agentRPCTestDemoteMaster(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
rp, err := client.DemoteMaster(ctx, tablet)
compareError(t, "DemoteMaster", err, rp, testReplicationPosition)
}
func agentRPCTestInitSlavePanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
err := client.InitSlave(ctx, tablet, testMasterAlias, testReplicationPosition, testTimeCreatedNS)
expectRPCWrapLockActionPanic(t, err)
}
func agentRPCTestInitSlave(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
err := client.InitSlave(ctx, tablet, testMasterAlias, testReplicationPosition, testTimeCreatedNS)
compareError(t, "InitSlave", err, true, testInitSlaveCalled)
}
func agentRPCTestPopulateReparentJournalPanic(ctx context.Context, t *testing.T, client tmclient.TabletManagerClient, tablet *topodatapb.Tablet) {
err := client.PopulateReparentJournal(ctx, tablet, testTimeCreatedNS, testActionName, testMasterAlias, testReplicationPosition)
expectRPCWrapPanic(t, err)
}