// NewExternalSigningServer creates and runs a new ExternalSigningServer which
// uses the given rootCA to sign node certificates. A server key and cert are
// generated and saved into the given basedir and then a TLS listener is
// started on a random available port. On success, an HTTPS server will be
// running in a separate goroutine. The URL of the singing endpoint is
// available in the returned *ExternalSignerServer value. Calling the Close()
// method will stop the server.
func NewExternalSigningServer(rootCA ca.RootCA, basedir string) (*ExternalSigningServer, error) {
serverCN := "external-ca-example-server"
serverOU := "localhost" // Make a valid server cert for localhost.
// Create TLS credentials for the external CA server which we will run.
serverPaths := ca.CertPaths{
Cert: filepath.Join(basedir, "server.crt"),
Key: filepath.Join(basedir, "server.key"),
}
serverCert, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(serverPaths, nil, nil), serverCN, serverOU, "")
if err != nil {
return nil, errors.Wrap(err, "unable to get TLS server certificate")
}
serverTLSConfig := &tls.Config{
Certificates: []tls.Certificate{*serverCert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: rootCA.Pool,
}
tlsListener, err := tls.Listen("tcp", "localhost:0", serverTLSConfig)
if err != nil {
return nil, errors.Wrap(err, "unable to create TLS connection listener")
}
assignedPort := tlsListener.Addr().(*net.TCPAddr).Port
signURL := url.URL{
Scheme: "https",
Host: net.JoinHostPort("localhost", strconv.Itoa(assignedPort)),
Path: "/sign",
}
ess := &ExternalSigningServer{
listener: tlsListener,
URL: signURL.String(),
}
mux := http.NewServeMux()
handler := &signHandler{
numIssued: &ess.NumIssued,
rootCA: rootCA,
flaky: &ess.flaky,
}
mux.Handle(signURL.Path, handler)
server := &http.Server{
Handler: mux,
}
go server.Serve(tlsListener)
return ess, nil
}
func TestRequestAndSaveNewCertificates(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
info := make(chan api.IssueNodeCertificateResponse, 1)
// Copy the current RootCA without the signer
rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool}
cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.Paths.Node, tc.WorkerToken, tc.Remotes, nil, info)
assert.NoError(t, err)
assert.NotNil(t, cert)
perms, err := permbits.Stat(tc.Paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
assert.NotEmpty(t, <-info)
}
// newNode creates new node with specific role(manager or agent) and joins to
// existing cluster. if joinAddr is empty string, then new cluster will be initialized.
// It uses TestExecutor as executor. If lateBind is set, the remote API port is not
// bound. If rootCA is set, this root is used to bootstrap the node's TLS certs.
func newTestNode(joinAddr, joinToken string, lateBind bool, rootCA *ca.RootCA) (*testNode, error) {
tmpDir, err := ioutil.TempDir("", "swarmkit-integration-")
if err != nil {
return nil, err
}
cAddr := filepath.Join(tmpDir, "control.sock")
cfg := &node.Config{
ListenControlAPI: cAddr,
JoinAddr: joinAddr,
StateDir: tmpDir,
Executor: &TestExecutor{},
JoinToken: joinToken,
}
if !lateBind {
cfg.ListenRemoteAPI = "127.0.0.1:0"
}
if rootCA != nil {
certDir := filepath.Join(tmpDir, "certificates")
if err := os.MkdirAll(certDir, 0700); err != nil {
return nil, err
}
certPaths := ca.NewConfigPaths(certDir)
if err := ioutil.WriteFile(certPaths.RootCA.Cert, rootCA.Cert, 0644); err != nil {
return nil, err
}
if err := ioutil.WriteFile(certPaths.RootCA.Key, rootCA.Key, 0600); err != nil {
return nil, err
}
// generate TLS certs for this manager for bootstrapping, else the node will generate its own CA
_, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(certPaths.Node, nil, nil),
identity.NewID(), ca.ManagerRole, identity.NewID())
if err != nil {
return nil, err
}
}
node, err := node.New(cfg)
if err != nil {
return nil, err
}
return &testNode{
config: cfg,
node: node,
stateDir: tmpDir,
}, nil
}
func TestRequestAndSaveNewCertificates(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Copy the current RootCA without the signer
rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool}
cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.ManagerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, cert)
perms, err := permbits.Stat(tc.Paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
// there was no encryption config in the remote, so the key should be unencrypted
unencryptedKeyReader := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
// the worker token is also unencrypted
cert, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, cert)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
// If there is a different kek in the remote store, when TLS certs are renewed the new key will
// be encrypted with that kek
assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
cluster := store.GetCluster(tx, tc.Organization)
cluster.Spec.EncryptionConfig.AutoLockManagers = true
cluster.UnlockKeys = []*api.EncryptionKey{{
Subsystem: ca.ManagerRole,
Key: []byte("kek!"),
}}
return store.UpdateCluster(tx, cluster)
}))
assert.NoError(t, os.RemoveAll(tc.Paths.Node.Cert))
assert.NoError(t, os.RemoveAll(tc.Paths.Node.Key))
_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.ManagerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
// key can no longer be read without a kek
_, _, err = unencryptedKeyReader.Read()
require.Error(t, err)
_, _, err = ca.NewKeyReadWriter(tc.Paths.Node, []byte("kek!"), nil).Read()
require.NoError(t, err)
// if it's a worker though, the key is always unencrypted, even though the manager key is encrypted
_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
}
func genSecurityConfig(s *store.MemoryStore, rootCA ca.RootCA, role, org, tmpDir string, nonSigningRoot bool) (*ca.SecurityConfig, error) {
req := &cfcsr.CertificateRequest{
KeyRequest: cfcsr.NewBasicKeyRequest(),
}
csr, key, err := cfcsr.ParseRequest(req)
if err != nil {
return nil, err
}
// Obtain a signed Certificate
nodeID := identity.NewID()
// All managers get added the subject-alt-name of CA, so they can be used for cert issuance
hosts := []string{role}
if role == ca.ManagerRole {
hosts = append(hosts, ca.CARole)
}
cert, err := rootCA.Signer.Sign(cfsigner.SignRequest{
Request: string(csr),
// OU is used for Authentication of the node type. The CN has the random
// node ID.
Subject: &cfsigner.Subject{CN: nodeID, Names: []cfcsr.Name{{OU: role, O: org}}},
// Adding ou as DNS alt name, so clients can connect to ManagerRole and CARole
Hosts: hosts,
})
if err != nil {
return nil, err
}
// Append the root CA Key to the certificate, to create a valid chain
certChain := append(cert, rootCA.Cert...)
// If we were instructed to persist the files
if tmpDir != "" {
paths := ca.NewConfigPaths(tmpDir)
if err := ioutil.WriteFile(paths.Node.Cert, certChain, 0644); err != nil {
return nil, err
}
if err := ioutil.WriteFile(paths.Node.Key, key, 0600); err != nil {
return nil, err
}
}
// Load a valid tls.Certificate from the chain and the key
nodeCert, err := tls.X509KeyPair(certChain, key)
if err != nil {
return nil, err
}
nodeServerTLSCreds, err := rootCA.NewServerTLSCredentials(&nodeCert)
if err != nil {
return nil, err
}
nodeClientTLSCreds, err := rootCA.NewClientTLSCredentials(&nodeCert, ca.ManagerRole)
if err != nil {
return nil, err
}
err = createNode(s, nodeID, role, csr, cert)
if err != nil {
return nil, err
}
if nonSigningRoot {
rootCA = ca.RootCA{
Cert: rootCA.Cert,
Digest: rootCA.Digest,
Pool: rootCA.Pool,
}
}
return ca.NewSecurityConfig(&rootCA, nodeClientTLSCreds, nodeServerTLSCreds), nil
}