// GetRemoteCA returns the remote endpoint's CA certificate
func GetRemoteCA(ctx context.Context, d digest.Digest, picker *picker.Picker) (RootCA, error) {
// We need a valid picker to be able to Dial to a remote CA
if picker == nil {
return RootCA{}, fmt.Errorf("valid remote address picker required")
}
// This TLS Config is intentionally using InsecureSkipVerify. Either we're
// doing TOFU, in which case we don't validate the remote CA, or we're using
// a user supplied hash to check the integrity of the CA certificate.
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecureCreds),
grpc.WithBackoffMaxDelay(10 * time.Second),
grpc.WithPicker(picker)}
firstAddr, err := picker.PickAddr()
if err != nil {
return RootCA{}, err
}
conn, err := grpc.Dial(firstAddr, opts...)
if err != nil {
return RootCA{}, err
}
defer conn.Close()
client := api.NewCAClient(conn)
response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
if err != nil {
return RootCA{}, err
}
if d != "" {
verifier, err := digest.NewDigestVerifier(d)
if err != nil {
return RootCA{}, fmt.Errorf("unexpected error getting digest verifier: %v", err)
}
io.Copy(verifier, bytes.NewReader(response.Certificate))
if !verifier.Verified() {
return RootCA{}, fmt.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())
}
}
// Check the validity of the remote Cert
_, err = helpers.ParseCertificatePEM(response.Certificate)
if err != nil {
return RootCA{}, err
}
// Create a Pool with our RootCACertificate
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(response.Certificate) {
return RootCA{}, fmt.Errorf("failed to append certificate to cert pool")
}
return RootCA{Cert: response.Certificate, Pool: pool}, nil
}
// GetUnlockKey returns the unlock key for the swarm.
func (c *Cluster) GetUnlockKey() (string, error) {
c.RLock()
defer c.RUnlock()
if !c.isActiveManager() {
return "", c.errNoManager()
}
ctx, cancel := c.getRequestContext()
defer cancel()
client := swarmapi.NewCAClient(c.conn)
r, err := client.GetUnlockKey(ctx, &swarmapi.GetUnlockKeyRequest{})
if err != nil {
return "", err
}
if len(r.UnlockKey) == 0 {
// no key
return "", nil
}
return encryption.HumanReadableKey(r.UnlockKey), nil
}
// GetRemoteCA returns the remote endpoint's CA certificate
func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) {
// This TLS Config is intentionally using InsecureSkipVerify. We use the
// digest instead to check the integrity of the CA certificate.
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
conn, peer, err := getGRPCConnection(insecureCreds, r)
if err != nil {
return RootCA{}, err
}
defer conn.Close()
client := api.NewCAClient(conn)
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
defer func() {
if err != nil {
r.Observe(peer, -remotes.DefaultObservationWeight)
return
}
r.Observe(peer, remotes.DefaultObservationWeight)
}()
response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
if err != nil {
return RootCA{}, err
}
if d != "" {
verifier, err := digest.NewDigestVerifier(d)
if err != nil {
return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier")
}
io.Copy(verifier, bytes.NewReader(response.Certificate))
if !verifier.Verified() {
return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())
}
}
// Check the validity of the remote Cert
_, err = helpers.ParseCertificatePEM(response.Certificate)
if err != nil {
return RootCA{}, err
}
// Create a Pool with our RootCACertificate
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(response.Certificate) {
return RootCA{}, errors.New("failed to append certificate to cert pool")
}
return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil
}
func displayUnlockKey(cmd *cobra.Command) error {
conn, err := common.DialConn(cmd)
if err != nil {
return err
}
defer conn.Close()
resp, err := api.NewCAClient(conn).GetUnlockKey(common.Context(cmd), &api.GetUnlockKeyRequest{})
if err != nil {
return err
}
if len(resp.UnlockKey) == 0 {
fmt.Printf("Managers not auto-locked")
}
fmt.Printf("Managers auto-locked. Unlock key: %s\n", encryption.HumanReadableKey(resp.UnlockKey))
return nil
}
// GetRemoteCA returns the remote endpoint's CA certificate bundle
func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbroker.Broker) (RootCA, error) {
// This TLS Config is intentionally using InsecureSkipVerify. We use the
// digest instead to check the integrity of the CA certificate.
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
conn, err := getGRPCConnection(insecureCreds, connBroker, false)
if err != nil {
return RootCA{}, err
}
client := api.NewCAClient(conn.ClientConn)
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
defer func() {
conn.Close(err == nil)
}()
response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
if err != nil {
return RootCA{}, err
}
// If a bundle of certificates are provided, the digest covers the entire bundle and not just
// one of the certificates in the bundle. Otherwise, a node can be MITMed while joining if
// the MITM CA provides a single certificate which matches the digest, and providing arbitrary
// other non-verified root certs that the manager certificate actually chains up to.
if d != "" {
verifier := d.Verifier()
if err != nil {
return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier")
}
io.Copy(verifier, bytes.NewReader(response.Certificate))
if !verifier.Verified() {
return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())
}
}
// NewRootCA will validate that the certificates are otherwise valid and create a RootCA object.
// Since there is no key, the certificate expiry does not matter and will not be used.
return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration)
}
func (rca *RootCA) getKEKUpdate(ctx context.Context, cert *x509.Certificate, keypair tls.Certificate, r remotes.Remotes) (*KEKData, error) {
var managerRole bool
for _, ou := range cert.Subject.OrganizationalUnit {
if ou == ManagerRole {
managerRole = true
break
}
}
if managerRole {
mtlsCreds := credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rca.Pool, Certificates: []tls.Certificate{keypair}})
conn, peer, err := getGRPCConnection(mtlsCreds, r)
if err != nil {
return nil, err
}
defer conn.Close()
client := api.NewCAClient(conn)
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
response, err := client.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{})
if err != nil {
if grpc.Code(err) == codes.Unimplemented { // if the server does not support keks, return as if no encryption key was specified
return &KEKData{}, nil
}
r.Observe(peer, -remotes.DefaultObservationWeight)
return nil, err
}
r.Observe(peer, remotes.DefaultObservationWeight)
return &KEKData{KEK: response.UnlockKey, Version: response.Version.Index}, nil
}
// If this is a worker, set to never encrypt. We always want to set to the lock key to nil,
// in case this was a manager that was demoted to a worker.
return &KEKData{}, nil
}
// NewTestCA is a helper method that creates a TestCA and a bunch of default
// connections and security configs.
func NewTestCA(t *testing.T) *TestCA {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
s := store.NewMemoryStore(nil)
paths := ca.NewConfigPaths(tempBaseDir)
organization := identity.NewID()
rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
var (
externalSigningServer *ExternalSigningServer
externalCAs []*api.ExternalCA
)
if External {
// Start the CA API server.
externalSigningServer, err = NewExternalSigningServer(rootCA, tempBaseDir)
assert.NoError(t, err)
externalCAs = []*api.ExternalCA{
{
Protocol: api.ExternalCA_CAProtocolCFSSL,
URL: externalSigningServer.URL,
},
}
}
managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "", External)
assert.NoError(t, err)
managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "", External)
assert.NoError(t, err)
agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "", External)
assert.NoError(t, err)
l, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds))
managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds))
managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds))
conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...)
assert.NoError(t, err)
conn2, err := grpc.Dial(l.Addr().String(), clientOpts...)
assert.NoError(t, err)
conn3, err := grpc.Dial(l.Addr().String(), managerOpts...)
assert.NoError(t, err)
conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...)
assert.NoError(t, err)
serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)}
grpcServer := grpc.NewServer(serverOpts...)
managerToken := ca.GenerateJoinToken(&rootCA)
workerToken := ca.GenerateJoinToken(&rootCA)
createClusterObject(t, s, organization, workerToken, managerToken, externalCAs...)
caServer := ca.NewServer(s, managerConfig)
api.RegisterCAServer(grpcServer, caServer)
api.RegisterNodeCAServer(grpcServer, caServer)
ctx := context.Background()
go grpcServer.Serve(l)
go caServer.Run(ctx)
// Wait for caServer to be ready to serve
<-caServer.Ready()
remotes := remotes.NewRemotes(api.Peer{Addr: l.Addr().String()})
caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)}
nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)}
conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4}
return &TestCA{
RootCA: rootCA,
ExternalSigningServer: externalSigningServer,
MemoryStore: s,
TempDir: tempBaseDir,
Organization: organization,
Paths: paths,
Context: ctx,
CAClients: caClients,
NodeCAClients: nodeCAClients,
Conns: conns,
Server: grpcServer,
CAServer: caServer,
WorkerToken: workerToken,
ManagerToken: managerToken,
Remotes: remotes,
}
}
// GetRemoteCA returns the remote endpoint's CA certificate
func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) {
// This TLS Config is intentionally using InsecureSkipVerify. Either we're
// doing TOFU, in which case we don't validate the remote CA, or we're using
// a user supplied hash to check the integrity of the CA certificate.
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecureCreds),
grpc.WithTimeout(5 * time.Second),
grpc.WithBackoffMaxDelay(5 * time.Second),
}
peer, err := r.Select()
if err != nil {
return RootCA{}, err
}
conn, err := grpc.Dial(peer.Addr, opts...)
if err != nil {
return RootCA{}, err
}
defer conn.Close()
client := api.NewCAClient(conn)
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
defer func() {
if err != nil {
r.Observe(peer, -remotes.DefaultObservationWeight)
return
}
r.Observe(peer, remotes.DefaultObservationWeight)
}()
response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
if err != nil {
return RootCA{}, err
}
if d != "" {
verifier, err := digest.NewDigestVerifier(d)
if err != nil {
return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier")
}
io.Copy(verifier, bytes.NewReader(response.Certificate))
if !verifier.Verified() {
return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())
}
}
// Check the validity of the remote Cert
_, err = helpers.ParseCertificatePEM(response.Certificate)
if err != nil {
return RootCA{}, err
}
// Create a Pool with our RootCACertificate
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(response.Certificate) {
return RootCA{}, errors.New("failed to append certificate to cert pool")
}
return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil
}
// Tests locking and unlocking the manager and key rotations
func TestManagerLockUnlock(t *testing.T) {
ctx := context.Background()
temp, err := ioutil.TempFile("", "test-manager-lock")
require.NoError(t, err)
require.NoError(t, temp.Close())
require.NoError(t, os.Remove(temp.Name()))
defer os.RemoveAll(temp.Name())
stateDir, err := ioutil.TempDir("", "test-raft")
require.NoError(t, err)
defer os.RemoveAll(stateDir)
tc := testutils.NewTestCA(t)
defer tc.Stop()
managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
require.NoError(t, err)
_, _, err = managerSecurityConfig.KeyReader().Read()
require.NoError(t, err)
m, err := New(&Config{
RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"},
ControlAPI: temp.Name(),
StateDir: stateDir,
SecurityConfig: managerSecurityConfig,
// start off without any encryption
})
require.NoError(t, err)
require.NotNil(t, m)
done := make(chan error)
defer close(done)
go func() {
done <- m.Run(ctx)
}()
opts := []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(managerSecurityConfig.ClientTLSCreds),
}
conn, err := grpc.Dial(m.Addr(), opts...)
require.NoError(t, err)
defer func() {
require.NoError(t, conn.Close())
}()
// check that there is no kek currently - we are using the API because this
// lets us wait until the manager is up and listening, as well
var cluster *api.Cluster
client := api.NewControlClient(conn)
require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error {
resp, err := client.ListClusters(ctx, &api.ListClustersRequest{})
if err != nil {
return err
}
if len(resp.Clusters) == 0 {
return fmt.Errorf("no clusters yet")
}
cluster = resp.Clusters[0]
return nil
}, 1*time.Second))
require.Nil(t, cluster.UnlockKeys)
// tls key is unencrypted, but there is a DEK
key, err := ioutil.ReadFile(tc.Paths.Node.Key)
require.NoError(t, err)
keyBlock, _ := pem.Decode(key)
require.NotNil(t, keyBlock)
require.False(t, x509.IsEncryptedPEMBlock(keyBlock))
require.Len(t, keyBlock.Headers, 2)
currentDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil)
require.NoError(t, err)
require.NotEmpty(t, currentDEK)
// update the lock key - this may fail due to update out of sequence errors, so try again
for {
getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID})
require.NoError(t, err)
cluster = getResp.Cluster
spec := cluster.Spec.Copy()
spec.EncryptionConfig.AutoLockManagers = true
updateResp, err := client.UpdateCluster(ctx, &api.UpdateClusterRequest{
ClusterID: cluster.ID,
ClusterVersion: &cluster.Meta.Version,
Spec: spec,
})
if grpc.ErrorDesc(err) == "update out of sequence" {
continue
}
// if there is any other type of error, this should fail
if err == nil {
cluster = updateResp.Cluster
}
break
}
require.NoError(t, err)
caConn := api.NewCAClient(conn)
unlockKeyResp, err := caConn.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{})
require.NoError(t, err)
// this should update the TLS key, rotate the DEK, and finish snapshotting
var updatedKey []byte
require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error {
updatedKey, err = ioutil.ReadFile(tc.Paths.Node.Key)
require.NoError(t, err) // this should never error due to atomic writes
if bytes.Equal(key, updatedKey) {
return fmt.Errorf("TLS key should have been re-encrypted at least")
}
keyBlock, _ = pem.Decode(updatedKey)
require.NotNil(t, keyBlock) // this should never error due to atomic writes
if !x509.IsEncryptedPEMBlock(keyBlock) {
return fmt.Errorf("Key not encrypted")
}
// we don't check that the TLS key has been rotated, because that may take
// a little bit, and is best effort only
currentDEKString, ok := keyBlock.Headers[pemHeaderRaftDEK]
require.True(t, ok) // there should never NOT be a current header
nowCurrentDEK, err := decodePEMHeaderValue(currentDEKString, unlockKeyResp.UnlockKey)
require.NoError(t, err) // it should always be encrypted
if bytes.Equal(currentDEK, nowCurrentDEK) {
return fmt.Errorf("snapshot has not been finished yet")
}
currentDEK = nowCurrentDEK
return nil
}, 1*time.Second))
_, ok := keyBlock.Headers[pemHeaderRaftPendingDEK]
require.False(t, ok) // once the snapshot is do
_, ok = keyBlock.Headers[pemHeaderRaftDEKNeedsRotation]
require.False(t, ok)
// verify that the snapshot is readable with the new DEK
encrypter, decrypter := encryption.Defaults(currentDEK)
// we can't use the raftLogger, because the WALs are still locked while the raft node is up. And once we remove
// the manager, they'll be deleted.
snapshot, err := storage.NewSnapFactory(encrypter, decrypter).New(filepath.Join(stateDir, "raft", "snap-v3-encrypted")).Load()
require.NoError(t, err)
require.NotNil(t, snapshot)
// update the lock key to nil
for i := 0; i < 3; i++ {
getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID})
require.NoError(t, err)
cluster = getResp.Cluster
spec := cluster.Spec.Copy()
spec.EncryptionConfig.AutoLockManagers = false
_, err = client.UpdateCluster(ctx, &api.UpdateClusterRequest{
ClusterID: cluster.ID,
ClusterVersion: &cluster.Meta.Version,
Spec: spec,
})
if grpc.ErrorDesc(err) == "update out of sequence" {
continue
}
require.NoError(t, err)
}
// this should update the TLS key
var unlockedKey []byte
require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error {
unlockedKey, err = ioutil.ReadFile(tc.Paths.Node.Key)
if err != nil {
return err
}
if bytes.Equal(unlockedKey, updatedKey) {
return fmt.Errorf("TLS key should have been rotated")
}
return nil
}, 1*time.Second))
// the new key should not be encrypted, and the DEK should also be unencrypted
// but not rotated
keyBlock, _ = pem.Decode(unlockedKey)
require.NotNil(t, keyBlock)
require.False(t, x509.IsEncryptedPEMBlock(keyBlock))
unencryptedDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil)
require.NoError(t, err)
require.NotNil(t, unencryptedDEK)
require.Equal(t, currentDEK, unencryptedDEK)
m.Stop(ctx)
// After stopping we should MAY receive an error from ListenAndServe if
// all this happened before WaitForLeader completed, so don't check the
// error.
<-done
}
// NewTestCA is a helper method that creates a TestCA and a bunch of default
// connections and security configs
func NewTestCA(t *testing.T, policy api.AcceptancePolicy) *TestCA {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
s := store.NewMemoryStore(nil)
paths := ca.NewConfigPaths(tempBaseDir)
organization := identity.NewID()
rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "")
assert.NoError(t, err)
managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "")
assert.NoError(t, err)
agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "")
assert.NoError(t, err)
l, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds))
managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds))
managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds))
conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...)
assert.NoError(t, err)
conn2, err := grpc.Dial(l.Addr().String(), clientOpts...)
assert.NoError(t, err)
conn3, err := grpc.Dial(l.Addr().String(), managerOpts...)
assert.NoError(t, err)
conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...)
assert.NoError(t, err)
serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)}
grpcServer := grpc.NewServer(serverOpts...)
createClusterObject(t, s, policy)
caServer := ca.NewServer(s, managerConfig)
api.RegisterCAServer(grpcServer, caServer)
api.RegisterNodeCAServer(grpcServer, caServer)
ctx := context.Background()
go grpcServer.Serve(l)
go caServer.Run(ctx)
// Wait for caServer to be ready to serve
<-caServer.Ready()
remotes := picker.NewRemotes(api.Peer{Addr: l.Addr().String()})
picker := picker.NewPicker(remotes, l.Addr().String())
caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)}
nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)}
conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4}
return &TestCA{
RootCA: rootCA,
MemoryStore: s,
Picker: picker,
TempDir: tempBaseDir,
Organization: organization,
Paths: paths,
Context: ctx,
CAClients: caClients,
NodeCAClients: nodeCAClients,
Conns: conns,
CAServer: caServer,
}
}
