func (n *Node) loadCertificates() error {
certDir := filepath.Join(n.config.StateDir, "certificates")
rootCA, err := ca.GetLocalRootCA(certDir)
if err != nil {
if err == ca.ErrNoLocalRootCA {
return nil
}
return err
}
configPaths := ca.NewConfigPaths(certDir)
clientTLSCreds, _, err := ca.LoadTLSCreds(rootCA, configPaths.Node)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return errors.Wrapf(err, "error while loading TLS Certificate in %s", configPaths.Node.Cert)
}
// todo: try csr if no cert or store nodeID/role in some other way
n.Lock()
n.role = clientTLSCreds.Role()
n.nodeID = clientTLSCreds.NodeID()
n.nodeMembership = api.NodeMembershipAccepted
n.roleCond.Broadcast()
n.Unlock()
return nil
}
func TestCreateSecurityConfigNoCerts(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Remove only the node certificates form the directory, and attest that we get
// new certificates that are locally signed
os.RemoveAll(tc.Paths.Node.Cert)
krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
// Remove only the node certificates form the directory, get a new rootCA, and attest that we get
// new certificates that are issued by the remote CA
os.RemoveAll(tc.Paths.Node.Cert)
rootCA, err := ca.GetLocalRootCA(tc.Paths.RootCA)
assert.NoError(t, err)
nodeConfig, err = rootCA.CreateSecurityConfig(tc.Context, krw,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.Equal(t, rootCA, *nodeConfig.RootCA())
}
func TestGetLocalRootCA(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// First, try to load the local Root CA with the certificate missing.
_, err = ca.GetLocalRootCA(tempBaseDir)
assert.Equal(t, ca.ErrNoLocalRootCA, err)
// Create the local Root CA to ensure that we can reload it correctly.
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.True(t, rootCA.CanSign())
assert.NoError(t, err)
rootCA2, err := ca.GetLocalRootCA(tempBaseDir)
assert.NoError(t, err)
assert.True(t, rootCA2.CanSign())
assert.Equal(t, rootCA, rootCA2)
// Try again, this time without a private key.
assert.NoError(t, os.Remove(paths.RootCA.Key))
rootCA3, err := ca.GetLocalRootCA(tempBaseDir)
assert.NoError(t, err)
assert.False(t, rootCA3.CanSign())
assert.Equal(t, rootCA.Cert, rootCA3.Cert)
// Try with a private key that does not match the CA cert public key.
privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
assert.NoError(t, err)
privKeyBytes, err := x509.MarshalECPrivateKey(privKey)
assert.NoError(t, err)
privKeyPem := pem.EncodeToMemory(&pem.Block{
Type: "EC PRIVATE KEY",
Bytes: privKeyBytes,
})
assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, privKeyPem, os.FileMode(0600)))
_, err = ca.GetLocalRootCA(tempBaseDir)
assert.EqualError(t, err, "certificate key mismatch")
}
func TestGetLocalRootCAInvalidCert(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// Write some garbage to the CA cert
require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, []byte(`-----BEGIN CERTIFICATE-----\n
some random garbage\n
-----END CERTIFICATE-----`), 0644))
_, err = ca.GetLocalRootCA(paths.RootCA)
require.Error(t, err)
}
func TestNewRootCABundle(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// Write one root CA to disk, keep the bytes
secondRootCA, err := ca.CreateAndWriteRootCA("rootCN2", paths.RootCA)
assert.NoError(t, err)
// Overwrite the first root CA on disk
firstRootCA, err := ca.CreateAndWriteRootCA("rootCN1", paths.RootCA)
assert.NoError(t, err)
// Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle
bundle := append(firstRootCA.Cert, secondRootCA.Cert...)
err = ioutil.WriteFile(paths.RootCA.Cert, bundle, 0644)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(bundle, firstRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, bundle, newRootCA.Cert)
assert.Equal(t, 2, len(newRootCA.Pool.Subjects()))
// Now load the bundle from disk
diskRootCA, err := ca.GetLocalRootCA(tempBaseDir)
assert.NoError(t, err)
assert.Equal(t, bundle, diskRootCA.Cert)
assert.Equal(t, 2, len(diskRootCA.Pool.Subjects()))
// If I use GenerateAndSignNewTLSCert to sign certs, I'll get the correct CA in the chain
_, err = ca.GenerateAndSignNewTLSCert(diskRootCA, "CN", "OU", "ORG", paths.Node)
assert.NoError(t, err)
certBytes, err := ioutil.ReadFile(paths.Node.Cert)
assert.NoError(t, err)
certs, err := helpers.ParseCertificatesPEM(certBytes)
assert.NoError(t, err)
assert.Len(t, certs, 2)
assert.Equal(t, "CN", certs[0].Subject.CommonName)
assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0])
assert.Equal(t, "ORG", certs[0].Subject.Organization[0])
assert.Equal(t, "rootCN1", certs[1].Subject.CommonName)
}
func TestGetLocalRootCAInvalidKey(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// Create the local Root CA to ensure that we can reload it correctly.
_, err = ca.CreateRootCA("rootCN", paths.RootCA)
require.NoError(t, err)
// Write some garbage to the root key - this will cause the loading to fail
require.NoError(t, ioutil.WriteFile(paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n
some random garbage\n
-----END EC PRIVATE KEY-----`), 0600))
_, err = ca.GetLocalRootCA(paths.RootCA)
require.Error(t, err)
}
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
var securityConfig *ca.SecurityConfig
krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
if err := krw.Migrate(); err != nil {
return nil, err
}
// Check if we already have a valid certificates on disk.
rootCA, err := ca.GetLocalRootCA(paths.RootCA)
if err != nil && err != ca.ErrNoLocalRootCA {
return nil, err
}
if err == nil {
securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
if err != nil {
_, isInvalidKEK := errors.Cause(err).(ca.ErrInvalidKEK)
if isInvalidKEK {
return nil, ErrInvalidUnlockKey
} else if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
}
}
}
if securityConfig == nil {
if n.config.JoinAddr == "" {
// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
n.unlockKey = nil
if n.config.AutoLockManagers {
n.unlockKey = encryption.GenerateSecretKey()
}
krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
if err != nil {
return nil, err
}
log.G(ctx).Debug("generated CA key and certificate")
} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
if err != nil {
return nil, err
}
log.G(ctx).Debug("downloaded CA certificate")
}
// Obtain new certs and setup TLS certificates renewal for this node:
// - If certificates weren't present on disk, we call CreateSecurityConfig, which blocks
// until a valid certificate has been issued.
// - We wait for CreateSecurityConfig to finish since we need a certificate to operate.
// Attempt to load certificate from disk
securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
if err == nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": securityConfig.ClientTLSCreds.NodeID(),
}).Debugf("loaded TLS certificate")
} else {
if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
return nil, ErrInvalidUnlockKey
}
log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target())
securityConfig, err = rootCA.CreateSecurityConfig(ctx, krw, ca.CertificateRequestConfig{
Token: n.config.JoinToken,
Availability: n.config.Availability,
Remotes: n.remotes,
})
if err != nil {
return nil, err
}
}
}
n.Lock()
n.role = securityConfig.ClientTLSCreds.Role()
n.nodeID = securityConfig.ClientTLSCreds.NodeID()
n.roleCond.Broadcast()
n.Unlock()
return securityConfig, nil
}
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
var securityConfig *ca.SecurityConfig
krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
if err := krw.Migrate(); err != nil {
return nil, err
}
// Check if we already have a valid certificates on disk.
rootCA, err := ca.GetLocalRootCA(paths.RootCA)
if err != nil && err != ca.ErrNoLocalRootCA {
return nil, err
}
if err == nil {
clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw)
_, ok := errors.Cause(err).(ca.ErrInvalidKEK)
switch {
case err == nil:
securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds)
log.G(ctx).Debug("loaded CA and TLS certificates")
case ok:
return nil, ErrInvalidUnlockKey
case os.IsNotExist(err):
break
default:
return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
}
}
if securityConfig == nil {
if n.config.JoinAddr == "" {
// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
n.unlockKey = nil
if n.config.AutoLockManagers {
n.unlockKey = encryption.GenerateSecretKey()
}
krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
if err != nil {
return nil, err
}
log.G(ctx).Debug("generated CA key and certificate")
} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
if err != nil {
return nil, err
}
log.G(ctx).Debug("downloaded CA certificate")
}
// Obtain new certs and setup TLS certificates renewal for this node:
// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
// us to display the ID before the certificate gets issued (for potential approval).
// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
// up to date.
issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
go func() {
select {
case <-ctx.Done():
case resp := <-issueResponseChan:
log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
"node.id": resp.NodeID,
}).Debugf("loaded TLS certificate")
n.Lock()
n.nodeID = resp.NodeID
n.nodeMembership = resp.NodeMembership
n.Unlock()
close(n.certificateRequested)
}
}()
// LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS
// certificates and write them to disk
securityConfig, err = ca.LoadOrCreateSecurityConfig(
ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw)
if err != nil {
if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
return nil, ErrInvalidUnlockKey
}
return nil, err
}
}
n.Lock()
n.role = securityConfig.ClientTLSCreds.Role()
n.nodeID = securityConfig.ClientTLSCreds.NodeID()
n.nodeMembership = api.NodeMembershipAccepted
n.roleCond.Broadcast()
n.Unlock()
return securityConfig, nil
}
