// GetRemoteCA returns the remote endpoint's CA certificate func GetRemoteCA(ctx context.Context, d digest.Digest, picker *picker.Picker) (RootCA, error) { // We need a valid picker to be able to Dial to a remote CA if picker == nil { return RootCA{}, fmt.Errorf("valid remote address picker required") } // This TLS Config is intentionally using InsecureSkipVerify. Either we're // doing TOFU, in which case we don't validate the remote CA, or we're using // a user supplied hash to check the integrity of the CA certificate. insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecureCreds), grpc.WithBackoffMaxDelay(10 * time.Second), grpc.WithPicker(picker)} firstAddr, err := picker.PickAddr() if err != nil { return RootCA{}, err } conn, err := grpc.Dial(firstAddr, opts...) if err != nil { return RootCA{}, err } defer conn.Close() client := api.NewCAClient(conn) response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{}) if err != nil { return RootCA{}, err } if d != "" { verifier, err := digest.NewDigestVerifier(d) if err != nil { return RootCA{}, fmt.Errorf("unexpected error getting digest verifier: %v", err) } io.Copy(verifier, bytes.NewReader(response.Certificate)) if !verifier.Verified() { return RootCA{}, fmt.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex()) } } // Check the validity of the remote Cert _, err = helpers.ParseCertificatePEM(response.Certificate) if err != nil { return RootCA{}, err } // Create a Pool with our RootCACertificate pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(response.Certificate) { return RootCA{}, fmt.Errorf("failed to append certificate to cert pool") } return RootCA{Cert: response.Certificate, Pool: pool}, nil }
// GetUnlockKey returns the unlock key for the swarm. func (c *Cluster) GetUnlockKey() (string, error) { c.RLock() defer c.RUnlock() if !c.isActiveManager() { return "", c.errNoManager() } ctx, cancel := c.getRequestContext() defer cancel() client := swarmapi.NewCAClient(c.conn) r, err := client.GetUnlockKey(ctx, &swarmapi.GetUnlockKeyRequest{}) if err != nil { return "", err } if len(r.UnlockKey) == 0 { // no key return "", nil } return encryption.HumanReadableKey(r.UnlockKey), nil }
// GetRemoteCA returns the remote endpoint's CA certificate func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) { // This TLS Config is intentionally using InsecureSkipVerify. We use the // digest instead to check the integrity of the CA certificate. insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) conn, peer, err := getGRPCConnection(insecureCreds, r) if err != nil { return RootCA{}, err } defer conn.Close() client := api.NewCAClient(conn) ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() defer func() { if err != nil { r.Observe(peer, -remotes.DefaultObservationWeight) return } r.Observe(peer, remotes.DefaultObservationWeight) }() response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{}) if err != nil { return RootCA{}, err } if d != "" { verifier, err := digest.NewDigestVerifier(d) if err != nil { return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier") } io.Copy(verifier, bytes.NewReader(response.Certificate)) if !verifier.Verified() { return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex()) } } // Check the validity of the remote Cert _, err = helpers.ParseCertificatePEM(response.Certificate) if err != nil { return RootCA{}, err } // Create a Pool with our RootCACertificate pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(response.Certificate) { return RootCA{}, errors.New("failed to append certificate to cert pool") } return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil }
func displayUnlockKey(cmd *cobra.Command) error { conn, err := common.DialConn(cmd) if err != nil { return err } defer conn.Close() resp, err := api.NewCAClient(conn).GetUnlockKey(common.Context(cmd), &api.GetUnlockKeyRequest{}) if err != nil { return err } if len(resp.UnlockKey) == 0 { fmt.Printf("Managers not auto-locked") } fmt.Printf("Managers auto-locked. Unlock key: %s\n", encryption.HumanReadableKey(resp.UnlockKey)) return nil }
// GetRemoteCA returns the remote endpoint's CA certificate bundle func GetRemoteCA(ctx context.Context, d digest.Digest, connBroker *connectionbroker.Broker) (RootCA, error) { // This TLS Config is intentionally using InsecureSkipVerify. We use the // digest instead to check the integrity of the CA certificate. insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) conn, err := getGRPCConnection(insecureCreds, connBroker, false) if err != nil { return RootCA{}, err } client := api.NewCAClient(conn.ClientConn) ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() defer func() { conn.Close(err == nil) }() response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{}) if err != nil { return RootCA{}, err } // If a bundle of certificates are provided, the digest covers the entire bundle and not just // one of the certificates in the bundle. Otherwise, a node can be MITMed while joining if // the MITM CA provides a single certificate which matches the digest, and providing arbitrary // other non-verified root certs that the manager certificate actually chains up to. if d != "" { verifier := d.Verifier() if err != nil { return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier") } io.Copy(verifier, bytes.NewReader(response.Certificate)) if !verifier.Verified() { return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex()) } } // NewRootCA will validate that the certificates are otherwise valid and create a RootCA object. // Since there is no key, the certificate expiry does not matter and will not be used. return NewRootCA(response.Certificate, nil, DefaultNodeCertExpiration) }
func (rca *RootCA) getKEKUpdate(ctx context.Context, cert *x509.Certificate, keypair tls.Certificate, r remotes.Remotes) (*KEKData, error) { var managerRole bool for _, ou := range cert.Subject.OrganizationalUnit { if ou == ManagerRole { managerRole = true break } } if managerRole { mtlsCreds := credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rca.Pool, Certificates: []tls.Certificate{keypair}}) conn, peer, err := getGRPCConnection(mtlsCreds, r) if err != nil { return nil, err } defer conn.Close() client := api.NewCAClient(conn) ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() response, err := client.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{}) if err != nil { if grpc.Code(err) == codes.Unimplemented { // if the server does not support keks, return as if no encryption key was specified return &KEKData{}, nil } r.Observe(peer, -remotes.DefaultObservationWeight) return nil, err } r.Observe(peer, remotes.DefaultObservationWeight) return &KEKData{KEK: response.UnlockKey, Version: response.Version.Index}, nil } // If this is a worker, set to never encrypt. We always want to set to the lock key to nil, // in case this was a manager that was demoted to a worker. return &KEKData{}, nil }
// NewTestCA is a helper method that creates a TestCA and a bunch of default // connections and security configs. func NewTestCA(t *testing.T) *TestCA { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) s := store.NewMemoryStore(nil) paths := ca.NewConfigPaths(tempBaseDir) organization := identity.NewID() rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration) assert.NoError(t, err) var ( externalSigningServer *ExternalSigningServer externalCAs []*api.ExternalCA ) if External { // Start the CA API server. externalSigningServer, err = NewExternalSigningServer(rootCA, tempBaseDir) assert.NoError(t, err) externalCAs = []*api.ExternalCA{ { Protocol: api.ExternalCA_CAProtocolCFSSL, URL: externalSigningServer.URL, }, } } managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "", External) assert.NoError(t, err) managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "", External) assert.NoError(t, err) agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "", External) assert.NoError(t, err) l, err := net.Listen("tcp", "127.0.0.1:0") assert.NoError(t, err) baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)} insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}))) clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds)) managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds)) managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds)) conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...) assert.NoError(t, err) conn2, err := grpc.Dial(l.Addr().String(), clientOpts...) assert.NoError(t, err) conn3, err := grpc.Dial(l.Addr().String(), managerOpts...) assert.NoError(t, err) conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...) assert.NoError(t, err) serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)} grpcServer := grpc.NewServer(serverOpts...) managerToken := ca.GenerateJoinToken(&rootCA) workerToken := ca.GenerateJoinToken(&rootCA) createClusterObject(t, s, organization, workerToken, managerToken, externalCAs...) caServer := ca.NewServer(s, managerConfig) api.RegisterCAServer(grpcServer, caServer) api.RegisterNodeCAServer(grpcServer, caServer) ctx := context.Background() go grpcServer.Serve(l) go caServer.Run(ctx) // Wait for caServer to be ready to serve <-caServer.Ready() remotes := remotes.NewRemotes(api.Peer{Addr: l.Addr().String()}) caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)} nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)} conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4} return &TestCA{ RootCA: rootCA, ExternalSigningServer: externalSigningServer, MemoryStore: s, TempDir: tempBaseDir, Organization: organization, Paths: paths, Context: ctx, CAClients: caClients, NodeCAClients: nodeCAClients, Conns: conns, Server: grpcServer, CAServer: caServer, WorkerToken: workerToken, ManagerToken: managerToken, Remotes: remotes, } }
// GetRemoteCA returns the remote endpoint's CA certificate func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) { // This TLS Config is intentionally using InsecureSkipVerify. Either we're // doing TOFU, in which case we don't validate the remote CA, or we're using // a user supplied hash to check the integrity of the CA certificate. insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}) opts := []grpc.DialOption{ grpc.WithTransportCredentials(insecureCreds), grpc.WithTimeout(5 * time.Second), grpc.WithBackoffMaxDelay(5 * time.Second), } peer, err := r.Select() if err != nil { return RootCA{}, err } conn, err := grpc.Dial(peer.Addr, opts...) if err != nil { return RootCA{}, err } defer conn.Close() client := api.NewCAClient(conn) ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() defer func() { if err != nil { r.Observe(peer, -remotes.DefaultObservationWeight) return } r.Observe(peer, remotes.DefaultObservationWeight) }() response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{}) if err != nil { return RootCA{}, err } if d != "" { verifier, err := digest.NewDigestVerifier(d) if err != nil { return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier") } io.Copy(verifier, bytes.NewReader(response.Certificate)) if !verifier.Verified() { return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex()) } } // Check the validity of the remote Cert _, err = helpers.ParseCertificatePEM(response.Certificate) if err != nil { return RootCA{}, err } // Create a Pool with our RootCACertificate pool := x509.NewCertPool() if !pool.AppendCertsFromPEM(response.Certificate) { return RootCA{}, errors.New("failed to append certificate to cert pool") } return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil }
// Tests locking and unlocking the manager and key rotations func TestManagerLockUnlock(t *testing.T) { ctx := context.Background() temp, err := ioutil.TempFile("", "test-manager-lock") require.NoError(t, err) require.NoError(t, temp.Close()) require.NoError(t, os.Remove(temp.Name())) defer os.RemoveAll(temp.Name()) stateDir, err := ioutil.TempDir("", "test-raft") require.NoError(t, err) defer os.RemoveAll(stateDir) tc := testutils.NewTestCA(t) defer tc.Stop() managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole) require.NoError(t, err) _, _, err = managerSecurityConfig.KeyReader().Read() require.NoError(t, err) m, err := New(&Config{ RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"}, ControlAPI: temp.Name(), StateDir: stateDir, SecurityConfig: managerSecurityConfig, // start off without any encryption }) require.NoError(t, err) require.NotNil(t, m) done := make(chan error) defer close(done) go func() { done <- m.Run(ctx) }() opts := []grpc.DialOption{ grpc.WithTimeout(10 * time.Second), grpc.WithTransportCredentials(managerSecurityConfig.ClientTLSCreds), } conn, err := grpc.Dial(m.Addr(), opts...) require.NoError(t, err) defer func() { require.NoError(t, conn.Close()) }() // check that there is no kek currently - we are using the API because this // lets us wait until the manager is up and listening, as well var cluster *api.Cluster client := api.NewControlClient(conn) require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { resp, err := client.ListClusters(ctx, &api.ListClustersRequest{}) if err != nil { return err } if len(resp.Clusters) == 0 { return fmt.Errorf("no clusters yet") } cluster = resp.Clusters[0] return nil }, 1*time.Second)) require.Nil(t, cluster.UnlockKeys) // tls key is unencrypted, but there is a DEK key, err := ioutil.ReadFile(tc.Paths.Node.Key) require.NoError(t, err) keyBlock, _ := pem.Decode(key) require.NotNil(t, keyBlock) require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) require.Len(t, keyBlock.Headers, 2) currentDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) require.NoError(t, err) require.NotEmpty(t, currentDEK) // update the lock key - this may fail due to update out of sequence errors, so try again for { getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) require.NoError(t, err) cluster = getResp.Cluster spec := cluster.Spec.Copy() spec.EncryptionConfig.AutoLockManagers = true updateResp, err := client.UpdateCluster(ctx, &api.UpdateClusterRequest{ ClusterID: cluster.ID, ClusterVersion: &cluster.Meta.Version, Spec: spec, }) if grpc.ErrorDesc(err) == "update out of sequence" { continue } // if there is any other type of error, this should fail if err == nil { cluster = updateResp.Cluster } break } require.NoError(t, err) caConn := api.NewCAClient(conn) unlockKeyResp, err := caConn.GetUnlockKey(ctx, &api.GetUnlockKeyRequest{}) require.NoError(t, err) // this should update the TLS key, rotate the DEK, and finish snapshotting var updatedKey []byte require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { updatedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) require.NoError(t, err) // this should never error due to atomic writes if bytes.Equal(key, updatedKey) { return fmt.Errorf("TLS key should have been re-encrypted at least") } keyBlock, _ = pem.Decode(updatedKey) require.NotNil(t, keyBlock) // this should never error due to atomic writes if !x509.IsEncryptedPEMBlock(keyBlock) { return fmt.Errorf("Key not encrypted") } // we don't check that the TLS key has been rotated, because that may take // a little bit, and is best effort only currentDEKString, ok := keyBlock.Headers[pemHeaderRaftDEK] require.True(t, ok) // there should never NOT be a current header nowCurrentDEK, err := decodePEMHeaderValue(currentDEKString, unlockKeyResp.UnlockKey) require.NoError(t, err) // it should always be encrypted if bytes.Equal(currentDEK, nowCurrentDEK) { return fmt.Errorf("snapshot has not been finished yet") } currentDEK = nowCurrentDEK return nil }, 1*time.Second)) _, ok := keyBlock.Headers[pemHeaderRaftPendingDEK] require.False(t, ok) // once the snapshot is do _, ok = keyBlock.Headers[pemHeaderRaftDEKNeedsRotation] require.False(t, ok) // verify that the snapshot is readable with the new DEK encrypter, decrypter := encryption.Defaults(currentDEK) // we can't use the raftLogger, because the WALs are still locked while the raft node is up. And once we remove // the manager, they'll be deleted. snapshot, err := storage.NewSnapFactory(encrypter, decrypter).New(filepath.Join(stateDir, "raft", "snap-v3-encrypted")).Load() require.NoError(t, err) require.NotNil(t, snapshot) // update the lock key to nil for i := 0; i < 3; i++ { getResp, err := client.GetCluster(ctx, &api.GetClusterRequest{ClusterID: cluster.ID}) require.NoError(t, err) cluster = getResp.Cluster spec := cluster.Spec.Copy() spec.EncryptionConfig.AutoLockManagers = false _, err = client.UpdateCluster(ctx, &api.UpdateClusterRequest{ ClusterID: cluster.ID, ClusterVersion: &cluster.Meta.Version, Spec: spec, }) if grpc.ErrorDesc(err) == "update out of sequence" { continue } require.NoError(t, err) } // this should update the TLS key var unlockedKey []byte require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error { unlockedKey, err = ioutil.ReadFile(tc.Paths.Node.Key) if err != nil { return err } if bytes.Equal(unlockedKey, updatedKey) { return fmt.Errorf("TLS key should have been rotated") } return nil }, 1*time.Second)) // the new key should not be encrypted, and the DEK should also be unencrypted // but not rotated keyBlock, _ = pem.Decode(unlockedKey) require.NotNil(t, keyBlock) require.False(t, x509.IsEncryptedPEMBlock(keyBlock)) unencryptedDEK, err := decodePEMHeaderValue(keyBlock.Headers[pemHeaderRaftDEK], nil) require.NoError(t, err) require.NotNil(t, unencryptedDEK) require.Equal(t, currentDEK, unencryptedDEK) m.Stop(ctx) // After stopping we should MAY receive an error from ListenAndServe if // all this happened before WaitForLeader completed, so don't check the // error. <-done }
// NewTestCA is a helper method that creates a TestCA and a bunch of default // connections and security configs func NewTestCA(t *testing.T, policy api.AcceptancePolicy) *TestCA { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) s := store.NewMemoryStore(nil) paths := ca.NewConfigPaths(tempBaseDir) organization := identity.NewID() rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration) assert.NoError(t, err) managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "") assert.NoError(t, err) managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "") assert.NoError(t, err) agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "") assert.NoError(t, err) l, err := net.Listen("tcp", "127.0.0.1:0") assert.NoError(t, err) baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)} insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}))) clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds)) managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds)) managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds)) conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...) assert.NoError(t, err) conn2, err := grpc.Dial(l.Addr().String(), clientOpts...) assert.NoError(t, err) conn3, err := grpc.Dial(l.Addr().String(), managerOpts...) assert.NoError(t, err) conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...) assert.NoError(t, err) serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)} grpcServer := grpc.NewServer(serverOpts...) createClusterObject(t, s, policy) caServer := ca.NewServer(s, managerConfig) api.RegisterCAServer(grpcServer, caServer) api.RegisterNodeCAServer(grpcServer, caServer) ctx := context.Background() go grpcServer.Serve(l) go caServer.Run(ctx) // Wait for caServer to be ready to serve <-caServer.Ready() remotes := picker.NewRemotes(api.Peer{Addr: l.Addr().String()}) picker := picker.NewPicker(remotes, l.Addr().String()) caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)} nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)} conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4} return &TestCA{ RootCA: rootCA, MemoryStore: s, Picker: picker, TempDir: tempBaseDir, Organization: organization, Paths: paths, Context: ctx, CAClients: caClients, NodeCAClients: nodeCAClients, Conns: conns, CAServer: caServer, } }