func agentTestEnv(t *testing.T) (*Agent, func()) {
var cleanup []func()
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
cleanup = append(cleanup, func() { tc.Stop() })
agentSecurityConfig, err := tc.NewNodeConfig(ca.AgentRole)
assert.NoError(t, err)
addr := "localhost:4949"
remotes := picker.NewRemotes(api.Peer{Addr: addr})
conn, err := grpc.Dial(addr,
grpc.WithPicker(picker.NewPicker(remotes, addr)),
grpc.WithTransportCredentials(agentSecurityConfig.ClientTLSCreds))
assert.NoError(t, err)
db, cleanupStorage := storageTestEnv(t)
cleanup = append(cleanup, func() { cleanupStorage() })
agent, err := New(&Config{
Executor: &NoopExecutor{},
Managers: remotes,
Conn: conn,
DB: db,
})
return agent, func() {
for i := len(cleanup) - 1; i > 0; i-- {
cleanup[i]()
}
}
}
func TestGetRemoteCAInvalidHash(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
_, err := ca.GetRemoteCA(tc.Context, "sha256:2d2f968475269f0dde5299427cf74348ee1d6115b95c6e3f283e5a4de8da445b", tc.Picker)
assert.Error(t, err)
}
func TestLoadOrCreateSecurityConfigInvalidKeyWithValidTempKey(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, nil)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
assert.NotNil(t, nodeConfig.RootCA().Signer)
// Write some garbage to the Key
assert.NoError(t, os.Rename(tc.Paths.Node.Key, filepath.Dir(tc.Paths.Node.Key)+"."+filepath.Base(tc.Paths.Node.Key)))
ioutil.WriteFile(tc.Paths.Node.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n
some random garbage\n
-----END EC PRIVATE KEY-----`), 0644)
nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, nil, nil)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
assert.NotNil(t, nodeConfig.RootCA().Signer)
}
func TestLoadOrCreateSecurityConfigNoCerts(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Remove only the node certificates form the directory, and attest that we get
// new certificates that are locally signed
os.RemoveAll(tc.Paths.Node.Cert)
nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, nil)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
assert.NotNil(t, nodeConfig.RootCA().Signer)
assert.True(t, nodeConfig.RootCA().CanSign())
info := make(chan api.IssueNodeCertificateResponse, 1)
// Remove only the node certificates form the directory, and attest that we get
// new certificates that are issued by the remote CA
os.RemoveAll(tc.Paths.RootCA.Key)
os.RemoveAll(tc.Paths.Node.Cert)
nodeConfig, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, info)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
assert.Nil(t, nodeConfig.RootCA().Signer)
assert.False(t, nodeConfig.RootCA().CanSign())
assert.NotEmpty(t, <-info)
}
func TestForceRenewTLSConfig(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Get a new managerConfig with a TLS cert that has 15 minutes to live
nodeConfig, err := tc.WriteNewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
var success, timeout bool
renew := make(chan struct{}, 1)
updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Picker, renew)
for {
renew <- struct{}{}
select {
case <-time.After(2 * time.Second):
timeout = true
case certUpdate := <-updates:
assert.NoError(t, certUpdate.Err)
assert.NotNil(t, certUpdate)
assert.Equal(t, certUpdate.Role, ca.ManagerRole)
success = true
}
if timeout {
assert.Fail(t, "TestForceRenewTLSConfig timed-out")
break
}
if success {
break
}
}
}
func TestGetRemoteSignedCertificateAutoAccept(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Create a new CSR to be signed
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
certs, err := ca.GetRemoteSignedCertificate(context.Background(), csr, ca.ManagerRole, "", tc.RootCA.Pool, tc.Picker, nil, nil)
assert.NoError(t, err)
assert.NotNil(t, certs)
// Test the expiration for a manager certificate
parsedCerts, err := helpers.ParseCertificatesPEM(certs)
assert.NoError(t, err)
assert.Len(t, parsedCerts, 2)
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, -1).Before(parsedCerts[0].NotAfter))
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, 1).After(parsedCerts[0].NotAfter))
assert.Equal(t, parsedCerts[0].Subject.OrganizationalUnit[0], ca.ManagerRole)
// Test the expiration for an agent certificate
certs, err = ca.GetRemoteSignedCertificate(tc.Context, csr, ca.AgentRole, "", tc.RootCA.Pool, tc.Picker, nil, nil)
assert.NoError(t, err)
assert.NotNil(t, certs)
parsedCerts, err = helpers.ParseCertificatesPEM(certs)
assert.NoError(t, err)
assert.Len(t, parsedCerts, 2)
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, -1).Before(parsedCerts[0].NotAfter))
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, 1).After(parsedCerts[0].NotAfter))
assert.Equal(t, parsedCerts[0].Subject.OrganizationalUnit[0], ca.AgentRole)
}
func TestRenewTLSConfigManager(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Get a new nodeConfig with a TLS cert that has the default Cert duration
nodeConfig, err := tc.WriteNewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
// Create a new RootCA, and change the policy to issue 6 minute certificates
newRootCA, err := ca.NewRootCA(tc.RootCA.Cert, tc.RootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
newRootCA.Signer.SetPolicy(&cfconfig.Signing{
Default: &cfconfig.SigningProfile{
Usage: []string{"signing", "key encipherment", "server auth", "client auth"},
Expiry: 6 * time.Minute,
},
})
// Create a new CSR and overwrite the key on disk
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
// Issue a new certificate with the same details as the current config, but with 6 min expiration time
c := nodeConfig.ClientTLSCreds
signedCert, err := newRootCA.ParseValidateAndSignCSR(csr, c.NodeID(), c.Role(), c.Organization())
assert.NoError(t, err)
assert.NotNil(t, signedCert)
// Overwrite the certificate on disk with one that expires in 1 minute
err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644)
assert.NoError(t, err)
// Get a new nodeConfig with a TLS cert that has 6 minutes to live
var success, timeout bool
renew := make(chan struct{})
updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Picker, renew)
for {
select {
case <-time.After(2 * time.Second):
timeout = true
case certUpdate := <-updates:
assert.NoError(t, certUpdate.Err)
assert.NotNil(t, certUpdate)
assert.Equal(t, ca.ManagerRole, certUpdate.Role)
success = true
}
if timeout {
assert.Fail(t, "TestRenewTLSConfig timed-out")
break
}
if success {
break
}
}
}
func TestCanSign(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
assert.True(t, tc.RootCA.CanSign())
tc.RootCA.Signer = nil
assert.False(t, tc.RootCA.CanSign())
}
func TestLoadOrCreateSecurityConfigNoCertsAndNoRemote(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Remove the certificate from the temp dir and try loading with a new manager
os.Remove(tc.Paths.Node.Cert)
os.Remove(tc.Paths.RootCA.Key)
_, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, nil, nil)
assert.EqualError(t, err, "valid remote address picker required")
}
func createManagersCluster(t *testing.T, managersCount, agentsCount int) *managersCluster {
tc := catestutils.NewTestCA(t, catestutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
mc := &managersCluster{tc: tc}
require.NoError(t, mc.addManagers(t, managersCount))
time.Sleep(5 * time.Second)
require.NoError(t, mc.addAgents(agentsCount))
time.Sleep(10 * time.Second)
return mc
}
func TestLoadOrCreateSecurityConfigNoLocalCACertNoRemote(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Delete the root CA file so that LoadOrCreateSecurityConfig falls
// back to using the remote.
assert.Nil(t, os.Remove(tc.Paths.RootCA.Cert))
nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, nil, nil)
assert.EqualError(t, err, "valid remote address picker required")
assert.Nil(t, nodeConfig)
}
func TestLoadOrCreateSecurityConfigInvalidCAKey(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Write some garbage to the root key
ioutil.WriteFile(tc.Paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n
some random garbage\n
-----END EC PRIVATE KEY-----`), 0644)
// We should get an error when the local ca private key is invalid.
_, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, nil)
assert.Error(t, err)
}
func TestGetRemoteSignedCertificateNodeInfo(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Create a new CSR to be signed
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
info := make(chan api.IssueNodeCertificateResponse, 1)
cert, err := ca.GetRemoteSignedCertificate(context.Background(), csr, ca.ManagerRole, "", tc.RootCA.Pool, tc.Picker, nil, info)
assert.NoError(t, err)
assert.NotNil(t, cert)
assert.NotEmpty(t, <-info)
}
func TestRequestAndSaveNewCertificates(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
info := make(chan api.IssueNodeCertificateResponse, 1)
// Copy the current RootCA without the signer
rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool}
cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.Paths.Node, ca.ManagerRole, "", tc.Picker, nil, info)
assert.NoError(t, err)
assert.NotNil(t, cert)
perms, err := permbits.Stat(tc.Paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
assert.NotEmpty(t, <-info)
}
func TestGetRemoteCA(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
shaHash := sha256.New()
shaHash.Write(tc.RootCA.Cert)
md := shaHash.Sum(nil)
mdStr := hex.EncodeToString(md)
d, err := digest.ParseDigest("sha256:" + mdStr)
assert.NoError(t, err)
cert, err := ca.GetRemoteCA(tc.Context, d, tc.Picker)
assert.NoError(t, err)
assert.NotNil(t, cert)
}
func TestListClusters(t *testing.T) {
ts := newTestServer(t)
r, err := ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
assert.NoError(t, err)
assert.Empty(t, r.Clusters)
policy := testutils.AcceptancePolicy(true, true, "")
createCluster(t, ts, "id1", "name1", policy)
r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
assert.NoError(t, err)
assert.Equal(t, 1, len(r.Clusters))
createCluster(t, ts, "id2", "name2", policy)
createCluster(t, ts, "id3", "name3", policy)
r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
assert.NoError(t, err)
assert.Equal(t, 3, len(r.Clusters))
}
func TestLoadOrCreateSecurityConfigEmptyDir(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
info := make(chan api.IssueNodeCertificateResponse, 1)
// Remove all the contents from the temp dir and try again with a new node
os.RemoveAll(tc.TempDir)
nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, info)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
assert.Nil(t, nodeConfig.RootCA().Signer)
assert.False(t, nodeConfig.RootCA().CanSign())
assert.NotEmpty(t, <-info)
}
func TestLoadOrCreateSecurityConfigInvalidCert(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Write some garbage to the cert
ioutil.WriteFile(tc.Paths.Node.Cert, []byte(`-----BEGIN CERTIFICATE-----\n
some random garbage\n
-----END CERTIFICATE-----`), 0644)
nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, nil)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
assert.NotNil(t, nodeConfig.RootCA().Signer)
}
func TestLoadOrCreateSecurityConfigInvalidCACert(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// First load the current nodeConfig. We'll verify that after we corrupt
// the certificate, another subsquent call with get us new certs
nodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, nil)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
// We have a valid signer because we bootstrapped with valid root key-material
assert.NotNil(t, nodeConfig.RootCA().Signer)
assert.True(t, nodeConfig.RootCA().CanSign())
// Write some garbage to the CA cert
ioutil.WriteFile(tc.Paths.RootCA.Cert, []byte(`-----BEGIN CERTIFICATE-----\n
some random garbage\n
-----END CERTIFICATE-----`), 0644)
// We should get an error when the CA cert is invalid.
_, err = ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, nil)
assert.Error(t, err)
// Not having a local cert should cause us to fallback to using the
// picker to get a remote.
assert.Nil(t, os.Remove(tc.Paths.RootCA.Cert))
// Validate we got a new valid state
newNodeConfig, err := ca.LoadOrCreateSecurityConfig(tc.Context, tc.TempDir, "", "", ca.AgentRole, tc.Picker, nil)
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.NotNil(t, nodeConfig.RootCA().Pool)
assert.NotNil(t, nodeConfig.RootCA().Cert)
assert.NotNil(t, nodeConfig.RootCA().Signer)
assert.True(t, nodeConfig.RootCA().CanSign())
// Ensure that we have the same certificate as before
assert.Equal(t, nodeConfig.RootCA().Cert, newNodeConfig.RootCA().Cert)
}
func TestGetClusterWithSecret(t *testing.T) {
ts := newTestServer(t)
_, err := ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
_, err = ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{ClusterID: "invalid"})
assert.Error(t, err)
assert.Equal(t, codes.NotFound, grpc.Code(err))
policy := testutils.AcceptancePolicy(true, true, "secret")
cluster := createCluster(t, ts, "name", "name", policy)
r, err := ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{ClusterID: cluster.ID})
assert.NoError(t, err)
cluster.Meta.Version = r.Cluster.Meta.Version
assert.NotEqual(t, cluster, r.Cluster)
assert.NotContains(t, r.Cluster.String(), "secret")
assert.NotContains(t, r.Cluster.String(), "PRIVATE")
assert.NotNil(t, r.Cluster.Spec.AcceptancePolicy.Policies[0].Secret.Data)
}
func TestAgentStartStop(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
agentSecurityConfig, err := tc.NewNodeConfig(ca.AgentRole)
assert.NoError(t, err)
addr := "localhost:4949"
remotes := picker.NewRemotes(api.Peer{Addr: addr})
conn, err := grpc.Dial(addr,
grpc.WithPicker(picker.NewPicker(remotes, addr)),
grpc.WithTransportCredentials(agentSecurityConfig.ClientTLSCreds))
assert.NoError(t, err)
db, cleanup := storageTestEnv(t)
defer cleanup()
agent, err := New(&Config{
Executor: &NoopExecutor{},
Managers: remotes,
Conn: conn,
DB: db,
})
assert.NoError(t, err)
assert.NotNil(t, agent)
ctx, _ := context.WithTimeout(context.Background(), 5000*time.Millisecond)
assert.Equal(t, errAgentNotStarted, agent.Stop(ctx))
assert.NoError(t, agent.Start(ctx))
if err := agent.Start(ctx); err != errAgentStarted {
t.Fatalf("expected agent started error: %v", err)
}
assert.NoError(t, agent.Stop(ctx))
}
func TestIssueAndSaveNewCertificates(t *testing.T) {
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
// Copy the current RootCA without the signer
cert, err := tc.RootCA.IssueAndSaveNewCertificates(tc.Paths.Node, "CN", ca.ManagerRole, tc.Organization)
assert.NoError(t, err)
assert.NotNil(t, cert)
perms, err := permbits.Stat(tc.Paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
certBytes, err := ioutil.ReadFile(tc.Paths.Node.Cert)
assert.NoError(t, err)
certs, err := helpers.ParseCertificatesPEM(certBytes)
assert.NoError(t, err)
assert.Len(t, certs, 2)
assert.Equal(t, "CN", certs[0].Subject.CommonName)
assert.Equal(t, ca.ManagerRole, certs[0].Subject.OrganizationalUnit[0])
assert.Equal(t, tc.Organization, certs[0].Subject.Organization[0])
assert.Equal(t, "swarm-test-CA", certs[1].Subject.CommonName)
}
func TestListClustersWithSecrets(t *testing.T) {
ts := newTestServer(t)
r, err := ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
assert.NoError(t, err)
assert.Empty(t, r.Clusters)
policy := testutils.AcceptancePolicy(true, true, "secret")
createCluster(t, ts, "id1", "name1", policy)
r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
assert.NoError(t, err)
assert.Equal(t, 1, len(r.Clusters))
createCluster(t, ts, "id2", "name2", policy)
createCluster(t, ts, "id3", "name3", policy)
r, err = ts.Client.ListClusters(context.Background(), &api.ListClustersRequest{})
assert.NoError(t, err)
assert.Equal(t, 3, len(r.Clusters))
for _, cluster := range r.Clusters {
assert.NotContains(t, cluster.String(), policy.Policies[0].Secret)
assert.NotContains(t, cluster.String(), "PRIVATE")
assert.NotNil(t, cluster.Spec.AcceptancePolicy.Policies[0].Secret.Data)
}
}
func TestGetCluster(t *testing.T) {
ts := newTestServer(t)
_, err := ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
_, err = ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{ClusterID: "invalid"})
assert.Error(t, err)
assert.Equal(t, codes.NotFound, grpc.Code(err))
cluster := createCluster(t, ts, "name", "name", testutils.AcceptancePolicy(true, true, ""))
r, err := ts.Client.GetCluster(context.Background(), &api.GetClusterRequest{ClusterID: cluster.ID})
assert.NoError(t, err)
cluster.Meta.Version = r.Cluster.Meta.Version
// Only public fields should be available
assert.Equal(t, cluster.ID, r.Cluster.ID)
assert.Equal(t, cluster.Meta, r.Cluster.Meta)
assert.Equal(t, cluster.Spec, r.Cluster.Spec)
assert.Equal(t, cluster.RootCA.CACert, r.Cluster.RootCA.CACert)
assert.Equal(t, cluster.RootCA.CACertHash, r.Cluster.RootCA.CACertHash)
// CAKey and network keys should be nil
assert.Nil(t, r.Cluster.RootCA.CAKey)
assert.Nil(t, r.Cluster.NetworkBootstrapKeys)
}
func TestUpdateNodeDemote(t *testing.T) {
tc := cautils.NewTestCA(nil, cautils.AcceptancePolicy(true, true, ""))
ts := newTestServer(t)
nodes, clockSource := raftutils.NewRaftCluster(t, tc)
defer raftutils.TeardownCluster(t, nodes)
// Assign one of the raft node to the test server
ts.Server.raft = nodes[1].Node
ts.Server.store = nodes[1].MemoryStore()
// Create a node object for each of the managers
assert.NoError(t, nodes[1].MemoryStore().Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNode(tx, &api.Node{
ID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: api.NodeSpec{
Role: api.NodeRoleManager,
Membership: api.NodeMembershipAccepted,
},
}))
assert.NoError(t, store.CreateNode(tx, &api.Node{
ID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: api.NodeSpec{
Role: api.NodeRoleManager,
Membership: api.NodeMembershipAccepted,
},
}))
assert.NoError(t, store.CreateNode(tx, &api.Node{
ID: nodes[3].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: api.NodeSpec{
Role: api.NodeRoleManager,
Membership: api.NodeMembershipAccepted,
},
}))
return nil
}))
// Stop Node 3 (1 node out of 3)
nodes[3].Server.Stop()
nodes[3].Shutdown()
// Node 3 should be listed as Unreachable
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
members := nodes[1].GetMemberlist()
if len(members) != 3 {
return fmt.Errorf("expected 3 nodes, got %d", len(members))
}
if members[nodes[3].Config.ID].Status.Reachability == api.RaftMemberStatus_REACHABLE {
return fmt.Errorf("expected node 3 to be unreachable")
}
return nil
}))
// Try to demote Node 2, this should fail because of the quorum safeguard
r, err := ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID()})
assert.NoError(t, err)
spec := r.Node.Spec.Copy()
spec.Role = api.NodeRoleWorker
version := &r.Node.Meta.Version
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: spec,
NodeVersion: version,
})
assert.Error(t, err)
assert.Equal(t, codes.FailedPrecondition, grpc.Code(err))
// Restart Node 3
nodes[3] = raftutils.RestartNode(t, clockSource, nodes[3], false)
raftutils.WaitForCluster(t, clockSource, nodes)
// Node 3 should be listed as Reachable
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
members := nodes[1].GetMemberlist()
if len(members) != 3 {
return fmt.Errorf("expected 3 nodes, got %d", len(members))
}
if members[nodes[3].Config.ID].Status.Reachability == api.RaftMemberStatus_UNREACHABLE {
return fmt.Errorf("expected node 3 to be reachable")
}
return nil
}))
// Try to demote Node 3, this should succeed
r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodes[3].SecurityConfig.ClientTLSCreds.NodeID()})
assert.NoError(t, err)
spec = r.Node.Spec.Copy()
spec.Role = api.NodeRoleWorker
version = &r.Node.Meta.Version
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodes[3].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: spec,
NodeVersion: version,
})
assert.NoError(t, err)
newCluster := map[uint64]*raftutils.TestNode{
1: nodes[1],
2: nodes[2],
}
raftutils.WaitForCluster(t, clockSource, newCluster)
// Server should list 2 members
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
members := nodes[1].GetMemberlist()
if len(members) != 2 {
return fmt.Errorf("expected 2 nodes, got %d", len(members))
}
return nil
}))
// Try to demote Node 2
r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID()})
assert.NoError(t, err)
spec = r.Node.Spec.Copy()
spec.Role = api.NodeRoleWorker
version = &r.Node.Meta.Version
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: spec,
NodeVersion: version,
})
assert.NoError(t, err)
newCluster = map[uint64]*raftutils.TestNode{
1: nodes[1],
}
raftutils.WaitForCluster(t, clockSource, newCluster)
// New server should list 1 member
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
members := nodes[1].GetMemberlist()
if len(members) != 1 {
return fmt.Errorf("expected 1 node, got %d", len(members))
}
return nil
}))
// Make sure we can't demote the last manager.
r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID()})
assert.NoError(t, err)
spec = r.Node.Spec.Copy()
spec.Role = api.NodeRoleWorker
version = &r.Node.Meta.Version
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: spec,
NodeVersion: version,
})
assert.Error(t, err)
assert.Equal(t, codes.FailedPrecondition, grpc.Code(err))
}
func TestListManagerNodes(t *testing.T) {
tc := cautils.NewTestCA(nil, cautils.AcceptancePolicy(true, true, ""))
ts := newTestServer(t)
nodes, clockSource := raftutils.NewRaftCluster(t, tc)
defer raftutils.TeardownCluster(t, nodes)
// Create a node object for each of the managers
assert.NoError(t, nodes[1].MemoryStore().Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNode(tx, &api.Node{ID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID()}))
assert.NoError(t, store.CreateNode(tx, &api.Node{ID: nodes[2].SecurityConfig.ClientTLSCreds.NodeID()}))
assert.NoError(t, store.CreateNode(tx, &api.Node{ID: nodes[3].SecurityConfig.ClientTLSCreds.NodeID()}))
return nil
}))
// Assign one of the raft node to the test server
ts.Server.raft = nodes[1].Node
ts.Server.store = nodes[1].MemoryStore()
// There should be 3 reachable managers listed
r, err := ts.Client.ListNodes(context.Background(), &api.ListNodesRequest{})
assert.NoError(t, err)
assert.NotNil(t, r)
managers := getMap(t, r.Nodes)
assert.Len(t, ts.Server.raft.GetMemberlist(), 3)
assert.Len(t, r.Nodes, 3)
// Node 1 should be the leader
for i := 1; i <= 3; i++ {
if i == 1 {
assert.True(t, managers[nodes[uint64(i)].Config.ID].Leader)
continue
}
assert.False(t, managers[nodes[uint64(i)].Config.ID].Leader)
}
// All nodes should be reachable
for i := 1; i <= 3; i++ {
assert.Equal(t, api.RaftMemberStatus_REACHABLE, managers[nodes[uint64(i)].Config.ID].Reachability)
}
// Add two more nodes to the cluster
raftutils.AddRaftNode(t, clockSource, nodes, tc)
raftutils.AddRaftNode(t, clockSource, nodes, tc)
raftutils.WaitForCluster(t, clockSource, nodes)
// Add node entries for these
assert.NoError(t, nodes[1].MemoryStore().Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNode(tx, &api.Node{ID: nodes[4].SecurityConfig.ClientTLSCreds.NodeID()}))
assert.NoError(t, store.CreateNode(tx, &api.Node{ID: nodes[5].SecurityConfig.ClientTLSCreds.NodeID()}))
return nil
}))
// There should be 5 reachable managers listed
r, err = ts.Client.ListNodes(context.Background(), &api.ListNodesRequest{})
assert.NoError(t, err)
assert.NotNil(t, r)
managers = getMap(t, r.Nodes)
assert.Len(t, ts.Server.raft.GetMemberlist(), 5)
assert.Len(t, r.Nodes, 5)
for i := 1; i <= 5; i++ {
assert.Equal(t, api.RaftMemberStatus_REACHABLE, managers[nodes[uint64(i)].Config.ID].Reachability)
}
// Stops 2 nodes
nodes[4].Server.Stop()
nodes[4].Shutdown()
nodes[5].Server.Stop()
nodes[5].Shutdown()
// Node 4 and Node 5 should be listed as Unreachable
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
r, err = ts.Client.ListNodes(context.Background(), &api.ListNodesRequest{})
if err != nil {
return err
}
managers = getMap(t, r.Nodes)
if len(r.Nodes) != 5 {
return fmt.Errorf("expected 5 nodes, got %d", len(r.Nodes))
}
if managers[nodes[4].Config.ID].Reachability == api.RaftMemberStatus_REACHABLE {
return fmt.Errorf("expected node 4 to be unreachable")
}
if managers[nodes[5].Config.ID].Reachability == api.RaftMemberStatus_REACHABLE {
return fmt.Errorf("expected node 5 to be unreachable")
}
return nil
}))
// Restart the 2 nodes
nodes[4] = raftutils.RestartNode(t, clockSource, nodes[4], false)
nodes[5] = raftutils.RestartNode(t, clockSource, nodes[5], false)
raftutils.WaitForCluster(t, clockSource, nodes)
// All the nodes should be reachable again
r, err = ts.Client.ListNodes(context.Background(), &api.ListNodesRequest{})
assert.NoError(t, err)
assert.NotNil(t, r)
managers = getMap(t, r.Nodes)
assert.Len(t, ts.Server.raft.GetMemberlist(), 5)
assert.Len(t, r.Nodes, 5)
for i := 1; i <= 5; i++ {
assert.Equal(t, api.RaftMemberStatus_REACHABLE, managers[nodes[uint64(i)].Config.ID].Reachability)
}
// Switch the raft node used by the server
ts.Server.raft = nodes[2].Node
// Stop node 1 (leader)
nodes[1].Stop()
nodes[1].Server.Stop()
newCluster := map[uint64]*raftutils.TestNode{
2: nodes[2],
3: nodes[3],
4: nodes[4],
5: nodes[5],
}
// Wait for the re-election to occur
raftutils.WaitForCluster(t, clockSource, newCluster)
// Node 1 should not be the leader anymore
assert.NoError(t, raftutils.PollFunc(clockSource, func() error {
r, err = ts.Client.ListNodes(context.Background(), &api.ListNodesRequest{})
if err != nil {
return err
}
managers = getMap(t, r.Nodes)
if managers[nodes[1].Config.ID].Leader {
return fmt.Errorf("expected node 1 not to be the leader")
}
if managers[nodes[1].Config.ID].Reachability == api.RaftMemberStatus_REACHABLE {
return fmt.Errorf("expected node 1 to be unreachable")
}
return nil
}))
// Restart node 1
nodes[1].Shutdown()
nodes[1] = raftutils.RestartNode(t, clockSource, nodes[1], false)
raftutils.WaitForCluster(t, clockSource, nodes)
// Ensure that node 1 is not the leader
assert.False(t, managers[nodes[uint64(1)].Config.ID].Leader)
// Check that another node got the leader status
var leader uint64
leaderCount := 0
for i := 1; i <= 5; i++ {
if managers[nodes[uint64(i)].Config.ID].Leader {
leader = nodes[uint64(i)].Config.ID
leaderCount++
}
}
// There should be only one leader after node 1 recovery and it
// should be different than node 1
assert.Equal(t, 1, leaderCount)
assert.NotEqual(t, leader, nodes[1].Config.ID)
}
func TestManager(t *testing.T) {
ctx := context.TODO()
store := store.NewMemoryStore(nil)
assert.NotNil(t, store)
temp, err := ioutil.TempFile("", "test-socket")
assert.NoError(t, err)
assert.NoError(t, temp.Close())
assert.NoError(t, os.Remove(temp.Name()))
lunix, err := net.Listen("unix", temp.Name())
assert.NoError(t, err)
ltcp, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
stateDir, err := ioutil.TempDir("", "test-raft")
assert.NoError(t, err)
defer os.RemoveAll(stateDir)
tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
defer tc.Stop()
agentSecurityConfig, err := tc.NewNodeConfig(ca.AgentRole)
assert.NoError(t, err)
agentDiffOrgSecurityConfig, err := tc.NewNodeConfigOrg(ca.AgentRole, "another-org")
assert.NoError(t, err)
managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
m, err := manager.New(&manager.Config{
ProtoListener: map[string]net.Listener{"unix": lunix, "tcp": ltcp},
StateDir: stateDir,
SecurityConfig: managerSecurityConfig,
})
assert.NoError(t, err)
assert.NotNil(t, m)
done := make(chan error)
defer close(done)
go func() {
done <- m.Run(ctx)
}()
opts := []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(agentSecurityConfig.ClientTLSCreds),
}
conn, err := grpc.Dial(ltcp.Addr().String(), opts...)
assert.NoError(t, err)
defer func() {
assert.NoError(t, conn.Close())
}()
// We have to send a dummy request to verify if the connection is actually up.
client := api.NewDispatcherClient(conn)
_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
assert.Equal(t, dispatcher.ErrNodeNotRegistered.Error(), grpc.ErrorDesc(err))
// Try to have a client in a different org access this manager
opts = []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(agentDiffOrgSecurityConfig.ClientTLSCreds),
}
conn2, err := grpc.Dial(ltcp.Addr().String(), opts...)
assert.NoError(t, err)
defer func() {
assert.NoError(t, conn2.Close())
}()
// We have to send a dummy request to verify if the connection is actually up.
client = api.NewDispatcherClient(conn2)
_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
assert.Contains(t, grpc.ErrorDesc(err), "Permission denied: unauthorized peer role: rpc error: code = 7 desc = Permission denied: remote certificate not part of organization")
// Verify that requests to the various GRPC services running on TCP
// are rejected if they don't have certs.
opts = []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})),
}
noCertConn, err := grpc.Dial(ltcp.Addr().String(), opts...)
assert.NoError(t, err)
defer func() {
assert.NoError(t, noCertConn.Close())
}()
client = api.NewDispatcherClient(noCertConn)
_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")
controlClient := api.NewControlClient(noCertConn)
_, err = controlClient.ListNodes(context.Background(), &api.ListNodesRequest{})
assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")
raftClient := api.NewRaftMembershipClient(noCertConn)
_, err = raftClient.Join(context.Background(), &api.JoinRequest{})
assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")
m.Stop(ctx)
// After stopping we should NOT receive an error from ListenAndServe.
assert.NoError(t, <-done)
}
func TestUpdateNode(t *testing.T) {
tc := cautils.NewTestCA(nil, cautils.AcceptancePolicy(true, true, ""))
ts := newTestServer(t)
nodes := make(map[uint64]*raftutils.TestNode)
nodes[1], _ = raftutils.NewInitNode(t, tc, nil)
defer raftutils.TeardownCluster(t, nodes)
nodeID := nodes[1].SecurityConfig.ClientTLSCreds.NodeID()
// Assign one of the raft node to the test server
ts.Server.raft = nodes[1].Node
ts.Server.store = nodes[1].MemoryStore()
_, err := ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodeID,
Spec: &api.NodeSpec{
Availability: api.NodeAvailabilityDrain,
},
NodeVersion: &api.Version{},
})
assert.Error(t, err)
assert.Equal(t, codes.NotFound, grpc.Code(err))
// Create a node object for the manager
assert.NoError(t, nodes[1].MemoryStore().Update(func(tx store.Tx) error {
assert.NoError(t, store.CreateNode(tx, &api.Node{
ID: nodes[1].SecurityConfig.ClientTLSCreds.NodeID(),
Spec: api.NodeSpec{
Role: api.NodeRoleManager,
Membership: api.NodeMembershipAccepted,
},
}))
return nil
}))
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: "invalid", Spec: &api.NodeSpec{}, NodeVersion: &api.Version{}})
assert.Error(t, err)
assert.Equal(t, codes.NotFound, grpc.Code(err))
r, err := ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodeID})
assert.NoError(t, err)
if !assert.NotNil(t, r) {
assert.FailNow(t, "got unexpected nil response from GetNode")
}
assert.NotNil(t, r.Node)
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: nodeID})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
spec := r.Node.Spec.Copy()
spec.Availability = api.NodeAvailabilityDrain
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodeID,
Spec: spec,
})
assert.Error(t, err)
assert.Equal(t, codes.InvalidArgument, grpc.Code(err))
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{
NodeID: nodeID,
Spec: spec,
NodeVersion: &r.Node.Meta.Version,
})
assert.NoError(t, err)
r, err = ts.Client.GetNode(context.Background(), &api.GetNodeRequest{NodeID: nodeID})
assert.NoError(t, err)
if !assert.NotNil(t, r) {
assert.FailNow(t, "got unexpected nil response from GetNode")
}
assert.NotNil(t, r.Node)
assert.NotNil(t, r.Node.Spec)
assert.Equal(t, api.NodeAvailabilityDrain, r.Node.Spec.Availability)
version := &r.Node.Meta.Version
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: nodeID, Spec: &r.Node.Spec, NodeVersion: version})
assert.NoError(t, err)
// Perform an update with the "old" version.
_, err = ts.Client.UpdateNode(context.Background(), &api.UpdateNodeRequest{NodeID: nodeID, Spec: &r.Node.Spec, NodeVersion: version})
assert.Error(t, err)
}
func init() {
grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags))
logrus.SetOutput(ioutil.Discard)
tc = cautils.NewTestCA(nil, cautils.AcceptancePolicy(true, true, ""))
}
func startDispatcher(c *Config) (*grpcDispatcher, error) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
return nil, err
}
tca := testutils.NewTestCA(nil, testutils.AcceptancePolicy(true, true, ""))
agentSecurityConfig1, err := tca.NewNodeConfig(ca.AgentRole)
if err != nil {
return nil, err
}
agentSecurityConfig2, err := tca.NewNodeConfig(ca.AgentRole)
if err != nil {
return nil, err
}
managerSecurityConfig, err := tca.NewNodeConfig(ca.ManagerRole)
if err != nil {
return nil, err
}
serverOpts := []grpc.ServerOption{grpc.Creds(managerSecurityConfig.ServerTLSCreds)}
s := grpc.NewServer(serverOpts...)
tc := &testCluster{addr: l.Addr().String(), store: tca.MemoryStore}
d := New(tc, c)
authorize := func(ctx context.Context, roles []string) error {
_, err := ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, tca.Organization)
return err
}
authenticatedDispatcherAPI := api.NewAuthenticatedWrapperDispatcherServer(d, authorize)
api.RegisterDispatcherServer(s, authenticatedDispatcherAPI)
go func() {
// Serve will always return an error (even when properly stopped).
// Explicitly ignore it.
_ = s.Serve(l)
}()
go d.Run(context.Background())
if err := raftutils.PollFuncWithTimeout(nil, func() error {
d.mu.Lock()
defer d.mu.Unlock()
if !d.isRunning() {
return fmt.Errorf("dispatcher is not running")
}
return nil
}, 5*time.Second); err != nil {
return nil, err
}
clientOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
clientOpts1 := append(clientOpts, grpc.WithTransportCredentials(agentSecurityConfig1.ClientTLSCreds))
clientOpts2 := append(clientOpts, grpc.WithTransportCredentials(agentSecurityConfig2.ClientTLSCreds))
clientOpts3 := append(clientOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
conn1, err := grpc.Dial(l.Addr().String(), clientOpts1...)
if err != nil {
return nil, err
}
conn2, err := grpc.Dial(l.Addr().String(), clientOpts2...)
if err != nil {
return nil, err
}
conn3, err := grpc.Dial(l.Addr().String(), clientOpts3...)
if err != nil {
return nil, err
}
clients := []api.DispatcherClient{api.NewDispatcherClient(conn1), api.NewDispatcherClient(conn2), api.NewDispatcherClient(conn3)}
securityConfigs := []*ca.SecurityConfig{agentSecurityConfig1, agentSecurityConfig2, managerSecurityConfig}
conns := []*grpc.ClientConn{conn1, conn2, conn3}
return &grpcDispatcher{
Clients: clients,
SecurityConfigs: securityConfigs,
Store: tc.MemoryStore(),
dispatcherServer: d,
conns: conns,
grpcServer: s,
testCA: tca,
}, nil
}