// If there are CAs and TLS certs on disk, it tries to load and fails if there
// are any errors, even if a join token is provided.
func TestLoadSecurityConfigLoadFromDisk(t *testing.T) {
tempdir, err := ioutil.TempDir("", "test-load-node-tls")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
tc := cautils.NewTestCA(t)
defer tc.Stop()
peer, err := tc.ConnBroker.Remotes().Select()
require.NoError(t, err)
// Load successfully with valid passphrase
rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
krw := ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil)
require.NoError(t, err)
_, err = rootCA.IssueAndSaveNewCertificates(krw, identity.NewID(), ca.WorkerRole, identity.NewID())
require.NoError(t, err)
node, err := New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
UnlockKey: []byte("passphrase"),
})
require.NoError(t, err)
securityConfig, err := node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
// Invalid passphrase
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.Equal(t, ErrInvalidUnlockKey, err)
// Invalid CA
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
UnlockKey: []byte("passphrase"),
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.IsType(t, x509.UnknownAuthorityError{}, errors.Cause(err))
}
func TestParseValidateAndSignCSR(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
csr, _, err := ca.GenerateNewCSR()
assert.NoError(t, err)
signedCert, err := rootCA.ParseValidateAndSignCSR(csr, "CN", "OU", "ORG")
assert.NoError(t, err)
assert.NotNil(t, signedCert)
parsedCert, err := helpers.ParseCertificatesPEM(signedCert)
assert.NoError(t, err)
assert.Equal(t, 2, len(parsedCert))
assert.Equal(t, "CN", parsedCert[0].Subject.CommonName)
assert.Equal(t, 1, len(parsedCert[0].Subject.OrganizationalUnit))
assert.Equal(t, "OU", parsedCert[0].Subject.OrganizationalUnit[0])
assert.Equal(t, 3, len(parsedCert[0].Subject.Names))
assert.Equal(t, "ORG", parsedCert[0].Subject.Organization[0])
assert.Equal(t, "rootCN", parsedCert[1].Subject.CommonName)
}
func TestNewRootCABundle(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// make one rootCA
secondRootCA, err := ca.CreateRootCA("rootCN2", paths.RootCA)
assert.NoError(t, err)
// make a second root CA
firstRootCA, err := ca.CreateRootCA("rootCN1", paths.RootCA)
assert.NoError(t, err)
// Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle
bundle := append(firstRootCA.Cert, secondRootCA.Cert...)
err = ioutil.WriteFile(paths.RootCA.Cert, bundle, 0644)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(bundle, firstRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, bundle, newRootCA.Cert)
assert.Equal(t, 2, len(newRootCA.Pool.Subjects()))
// If I use newRootCA's IssueAndSaveNewCertificates to sign certs, I'll get the correct CA in the chain
kw := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, err = newRootCA.IssueAndSaveNewCertificates(kw, "CN", "OU", "ORG")
assert.NoError(t, err)
certBytes, err := ioutil.ReadFile(paths.Node.Cert)
assert.NoError(t, err)
certs, err := helpers.ParseCertificatesPEM(certBytes)
assert.NoError(t, err)
assert.Len(t, certs, 2)
assert.Equal(t, "CN", certs[0].Subject.CommonName)
assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0])
assert.Equal(t, "ORG", certs[0].Subject.Organization[0])
assert.Equal(t, "rootCN1", certs[1].Subject.CommonName)
}
func TestNewRootCAWithPassphrase(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
defer os.Setenv(ca.PassphraseENVVar, "")
defer os.Setenv(ca.PassphraseENVVarPrev, "")
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
// Ensure that we're encrypting the Key bytes out of NewRoot if there
// is a passphrase set as an env Var
os.Setenv(ca.PassphraseENVVar, "password1")
newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.NotEqual(t, rootCA.Key, newRootCA.Key)
assert.Equal(t, rootCA.Cert, newRootCA.Cert)
assert.NotContains(t, string(rootCA.Key), string(newRootCA.Key))
assert.Contains(t, string(newRootCA.Key), "Proc-Type: 4,ENCRYPTED")
// Ensure that we're decrypting the Key bytes out of NewRoot if there
// is a passphrase set as an env Var
anotherNewRootCA, err := ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, newRootCA, anotherNewRootCA)
assert.NotContains(t, string(rootCA.Key), string(anotherNewRootCA.Key))
assert.Contains(t, string(anotherNewRootCA.Key), "Proc-Type: 4,ENCRYPTED")
// Ensure that we cant decrypt the Key bytes out of NewRoot if there
// is a wrong passphrase set as an env Var
os.Setenv(ca.PassphraseENVVar, "password2")
anotherNewRootCA, err = ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.Error(t, err)
// Ensure that we cant decrypt the Key bytes out of NewRoot if there
// is a wrong passphrase set as an env Var
os.Setenv(ca.PassphraseENVVarPrev, "password2")
anotherNewRootCA, err = ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.Error(t, err)
// Ensure that we can decrypt the Key bytes out of NewRoot if there
// is a wrong passphrase set as an env Var, but a valid as Prev
os.Setenv(ca.PassphraseENVVarPrev, "password1")
anotherNewRootCA, err = ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, newRootCA, anotherNewRootCA)
assert.NotContains(t, string(rootCA.Key), string(anotherNewRootCA.Key))
assert.Contains(t, string(anotherNewRootCA.Key), "Proc-Type: 4,ENCRYPTED")
}
func TestNewRootCA(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, rootCA, newRootCA)
}
func TestGetLocalRootCAInvalidKey(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// Create the local Root CA to ensure that we can reload it correctly.
_, err = ca.CreateRootCA("rootCN", paths.RootCA)
require.NoError(t, err)
// Write some garbage to the root key - this will cause the loading to fail
require.NoError(t, ioutil.WriteFile(paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n
some random garbage\n
-----END EC PRIVATE KEY-----`), 0600))
_, err = ca.GetLocalRootCA(paths.RootCA)
require.Error(t, err)
}
func TestCreateRootCAExpiry(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
// Convert the certificate into an object to create a RootCA
parsedCert, err := helpers.ParseCertificatePEM(rootCA.Cert)
assert.NoError(t, err)
duration, err := time.ParseDuration(ca.RootCAExpiration)
assert.NoError(t, err)
assert.True(t, time.Now().Add(duration).AddDate(0, -1, 0).Before(parsedCert.NotAfter))
}
func TestCreateRootCA(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
_, err = ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
perms, err := permbits.Stat(paths.RootCA.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
_, err = permbits.Stat(paths.RootCA.Key)
assert.True(t, os.IsNotExist(err))
}
func TestGetLocalRootCA(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// First, try to load the local Root CA with the certificate missing.
_, err = ca.GetLocalRootCA(paths.RootCA)
assert.Equal(t, ca.ErrNoLocalRootCA, err)
// Create the local Root CA to ensure that we can reload it correctly.
rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA)
assert.True(t, rootCA.CanSign())
assert.NoError(t, err)
// No private key here
rootCA2, err := ca.GetLocalRootCA(paths.RootCA)
assert.NoError(t, err)
assert.Equal(t, rootCA.Cert, rootCA2.Cert)
assert.False(t, rootCA2.CanSign())
// write private key and assert we can load it and sign
assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, rootCA.Key, os.FileMode(0600)))
rootCA3, err := ca.GetLocalRootCA(paths.RootCA)
assert.NoError(t, err)
assert.Equal(t, rootCA.Cert, rootCA3.Cert)
assert.True(t, rootCA3.CanSign())
// Try with a private key that does not match the CA cert public key.
privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
assert.NoError(t, err)
privKeyBytes, err := x509.MarshalECPrivateKey(privKey)
assert.NoError(t, err)
privKeyPem := pem.EncodeToMemory(&pem.Block{
Type: "EC PRIVATE KEY",
Bytes: privKeyBytes,
})
assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, privKeyPem, os.FileMode(0600)))
_, err = ca.GetLocalRootCA(paths.RootCA)
assert.EqualError(t, err, "certificate key mismatch")
}
func main() {
// Create root material within the current directory.
rootPaths := ca.CertPaths{
Cert: filepath.Join("ca", "root.crt"),
Key: filepath.Join("ca", "root.key"),
}
// Initialize the Root CA.
rootCA, err := ca.CreateRootCA("external-ca-example", rootPaths)
if err != nil {
logrus.Fatalf("unable to initialize Root CA: %s", err)
}
// Create the initial manager node credentials.
nodeConfigPaths := ca.NewConfigPaths("certificates")
clusterID := identity.NewID()
nodeID := identity.NewID()
kw := ca.NewKeyReadWriter(nodeConfigPaths.Node, nil, nil)
if _, err := rootCA.IssueAndSaveNewCertificates(kw, nodeID, ca.ManagerRole, clusterID); err != nil {
logrus.Fatalf("unable to create initial manager node credentials: %s", err)
}
// And copy the Root CA certificate into the node config path for its
// CA.
ioutil.WriteFile(nodeConfigPaths.RootCA.Cert, rootCA.Cert, os.FileMode(0644))
server, err := testutils.NewExternalSigningServer(rootCA, "ca")
if err != nil {
logrus.Fatalf("unable to start server: %s", err)
}
defer server.Stop()
logrus.Infof("Now run: swarmd -d . --listen-control-api ./swarmd.sock --external-ca protocol=cfssl,url=%s", server.URL)
sigC := make(chan os.Signal, 1)
signal.Notify(sigC, syscall.SIGTERM, syscall.SIGINT)
<-sigC
}
func TestParseValidateAndSignMaliciousCSR(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
req := &cfcsr.CertificateRequest{
Names: []cfcsr.Name{
{
O: "maliciousOrg",
OU: "maliciousOU",
L: "maliciousLocality",
},
},
CN: "maliciousCN",
Hosts: []string{"docker.com"},
KeyRequest: &cfcsr.BasicKeyRequest{A: "ecdsa", S: 256},
}
csr, _, err := cfcsr.ParseRequest(req)
assert.NoError(t, err)
signedCert, err := rootCA.ParseValidateAndSignCSR(csr, "CN", "OU", "ORG")
assert.NoError(t, err)
assert.NotNil(t, signedCert)
parsedCert, err := helpers.ParseCertificatesPEM(signedCert)
assert.NoError(t, err)
assert.Equal(t, 2, len(parsedCert))
assert.Equal(t, "CN", parsedCert[0].Subject.CommonName)
assert.Equal(t, 1, len(parsedCert[0].Subject.OrganizationalUnit))
assert.Equal(t, "OU", parsedCert[0].Subject.OrganizationalUnit[0])
assert.Equal(t, 3, len(parsedCert[0].Subject.Names))
assert.Empty(t, parsedCert[0].Subject.Locality)
assert.Equal(t, "ORG", parsedCert[0].Subject.Organization[0])
assert.Equal(t, "rootCN", parsedCert[1].Subject.CommonName)
}
// If there's only a root CA on disk (no TLS certs), and no join addr, we create a new CA
// and a new set of TLS certs. Similarly if there's only a TLS cert and key, and no CA.
func TestLoadSecurityConfigPartialCertsOnDisk(t *testing.T) {
tempdir, err := ioutil.TempDir("", "test-new-node")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
node, err := New(&Config{
StateDir: tempdir,
})
require.NoError(t, err)
securityConfig, err := node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
cert, key, err := securityConfig.KeyReader().Read()
require.NoError(t, err)
// a new CA was generated because no existing TLS certs were present
require.NotEqual(t, rootCA.Cert, securityConfig.RootCA().Cert)
// if the TLS key and cert are on disk, but there's no CA, a new CA and TLS
// key+cert are generated
require.NoError(t, os.RemoveAll(paths.RootCA.Cert))
node, err = New(&Config{
StateDir: tempdir,
})
require.NoError(t, err)
securityConfig, err = node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
newCert, newKey, err := securityConfig.KeyReader().Read()
require.NoError(t, err)
require.NotEqual(t, cert, newCert)
require.NotEqual(t, key, newKey)
}
func TestNewRootCANonDefaultExpiry(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, 1*time.Hour)
assert.NoError(t, err)
// Create and sign a new CSR
csr, _, err := ca.GenerateNewCSR()
assert.NoError(t, err)
cert, err := newRootCA.ParseValidateAndSignCSR(csr, "CN", ca.ManagerRole, "ORG")
assert.NoError(t, err)
parsedCerts, err := helpers.ParseCertificatesPEM(cert)
assert.NoError(t, err)
assert.Len(t, parsedCerts, 2)
assert.True(t, time.Now().Add(time.Minute*59).Before(parsedCerts[0].NotAfter))
assert.True(t, time.Now().Add(time.Hour).Add(time.Minute).After(parsedCerts[0].NotAfter))
// Sign the same CSR again, this time with a 59 Minute expiration RootCA (under the 60 minute minimum).
// This should use the default of 3 months
newRootCA, err = ca.NewRootCA(rootCA.Cert, rootCA.Key, 59*time.Minute)
assert.NoError(t, err)
cert, err = newRootCA.ParseValidateAndSignCSR(csr, "CN", ca.ManagerRole, "ORG")
assert.NoError(t, err)
parsedCerts, err = helpers.ParseCertificatesPEM(cert)
assert.NoError(t, err)
assert.Len(t, parsedCerts, 2)
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, -1).Before(parsedCerts[0].NotAfter))
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, 1).After(parsedCerts[0].NotAfter))
}
func TestForceNewCluster(t *testing.T) {
t.Parallel()
// create an external CA so that we can use it to generate expired certificates
tempDir, err := ioutil.TempDir("", "external-ca")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
rootCA, err := ca.CreateRootCA("externalRoot", ca.NewConfigPaths(tempDir).RootCA)
require.NoError(t, err)
// start a new cluster with the external CA bootstrapped
numWorker, numManager := 0, 1
cl := newTestCluster()
defer func() {
require.NoError(t, cl.Stop())
}()
require.NoError(t, cl.AddManager(false, &rootCA), "manager number 1")
pollClusterReady(t, cl, numWorker, numManager)
leader, err := cl.Leader()
require.NoError(t, err)
sid, err := cl.CreateService("test_service", 2)
require.NoError(t, err)
pollServiceReady(t, cl, sid)
// generate an expired certificate
rootKey, err := helpers.ParsePrivateKeyPEM(rootCA.Key)
require.NoError(t, err)
rootCert, err := helpers.ParseCertificatePEM(rootCA.Cert)
require.NoError(t, err)
managerCertFile := filepath.Join(leader.stateDir, "certificates", "swarm-node.crt")
certBytes, err := ioutil.ReadFile(managerCertFile)
require.NoError(t, err)
managerCerts, err := helpers.ParseCertificatesPEM(certBytes)
require.NoError(t, err)
expiredCertTemplate := managerCerts[0]
expiredCertTemplate.NotBefore = time.Now().Add(time.Hour * -5)
expiredCertTemplate.NotAfter = time.Now().Add(time.Hour * -3)
expiredCertDERBytes, err := x509.CreateCertificate(rand.Reader, expiredCertTemplate, rootCert, expiredCertTemplate.PublicKey, rootKey)
require.NoError(t, err)
expiredCertPEM := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: expiredCertDERBytes,
})
// restart node with an expired certificate while forcing a new cluster - it should start without error and the certificate should be renewed
nodeID := leader.node.NodeID()
require.NoError(t, leader.Pause(true))
require.NoError(t, ioutil.WriteFile(managerCertFile, expiredCertPEM, 0644))
require.NoError(t, cl.StartNode(nodeID))
pollClusterReady(t, cl, numWorker, numManager)
pollServiceReady(t, cl, sid)
err = raftutils.PollFuncWithTimeout(nil, func() error {
certBytes, err := ioutil.ReadFile(managerCertFile)
if err != nil {
return err
}
managerCerts, err := helpers.ParseCertificatesPEM(certBytes)
if err != nil {
return err
}
if managerCerts[0].NotAfter.Before(time.Now()) {
return errors.New("certificate hasn't been renewed yet")
}
return nil
}, opsTimeout)
require.NoError(t, err)
// restart node with an expired certificate without forcing a new cluster - it should error on start
require.NoError(t, leader.Pause(true))
require.NoError(t, ioutil.WriteFile(managerCertFile, expiredCertPEM, 0644))
require.Error(t, cl.StartNode(nodeID))
}
func TestGetRemoteCA(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
shaHash := sha256.New()
shaHash.Write(tc.RootCA.Cert)
md := shaHash.Sum(nil)
mdStr := hex.EncodeToString(md)
d, err := digest.Parse("sha256:" + mdStr)
require.NoError(t, err)
downloadedRootCA, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
require.NoError(t, err)
require.Equal(t, downloadedRootCA.Cert, tc.RootCA.Cert)
// update the test CA to include a multi-certificate bundle as the root - the digest
// we use to verify with must be the digest of the whole bundle
tmpDir, err := ioutil.TempDir("", "GetRemoteCA")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
paths := ca.NewConfigPaths(tmpDir)
otherRootCA, err := ca.CreateRootCA("other", paths.RootCA)
require.NoError(t, err)
comboCertBundle := append(tc.RootCA.Cert, otherRootCA.Cert...)
require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
cluster := store.GetCluster(tx, tc.Organization)
cluster.RootCA.CACert = comboCertBundle
cluster.RootCA.CAKey = tc.RootCA.Key
return store.UpdateCluster(tx, cluster)
}))
require.NoError(t, raftutils.PollFunc(nil, func() error {
_, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
if err == nil {
return fmt.Errorf("testca's rootca hasn't updated yet")
}
require.Contains(t, err.Error(), "remote CA does not match fingerprint")
return nil
}))
// If we provide the right digest, the root CA is updated and we can validate
// certs signed by either one
d = digest.FromBytes(comboCertBundle)
downloadedRootCA, err = ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
require.NoError(t, err)
require.Equal(t, comboCertBundle, downloadedRootCA.Cert)
require.Equal(t, 2, len(downloadedRootCA.Pool.Subjects()))
for _, rootCA := range []ca.RootCA{tc.RootCA, otherRootCA} {
krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, err := rootCA.IssueAndSaveNewCertificates(krw, "cn", "ou", "org")
require.NoError(t, err)
certPEM, _, err := krw.Read()
require.NoError(t, err)
cert, err := helpers.ParseCertificatesPEM(certPEM)
require.NoError(t, err)
chains, err := cert[0].Verify(x509.VerifyOptions{
Roots: downloadedRootCA.Pool,
})
require.NoError(t, err)
require.Len(t, chains, 1)
}
}
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
var securityConfig *ca.SecurityConfig
krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
if err := krw.Migrate(); err != nil {
return nil, err
}
// Check if we already have a valid certificates on disk.
rootCA, err := ca.GetLocalRootCA(paths.RootCA)
if err != nil && err != ca.ErrNoLocalRootCA {
return nil, err
}
if err == nil {
clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw)
_, ok := errors.Cause(err).(ca.ErrInvalidKEK)
switch {
case err == nil:
securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds)
log.G(ctx).Debug("loaded CA and TLS certificates")
case ok:
return nil, ErrInvalidUnlockKey
case os.IsNotExist(err):
break
default:
return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
}
}
if securityConfig == nil {
if n.config.JoinAddr == "" {
// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
n.unlockKey = nil
if n.config.AutoLockManagers {
n.unlockKey = encryption.GenerateSecretKey()
}
krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
if err != nil {
return nil, err
}
log.G(ctx).Debug("generated CA key and certificate")
} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
if err != nil {
return nil, err
}
log.G(ctx).Debug("downloaded CA certificate")
}
// Obtain new certs and setup TLS certificates renewal for this node:
// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
// us to display the ID before the certificate gets issued (for potential approval).
// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
// up to date.
issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
go func() {
select {
case <-ctx.Done():
case resp := <-issueResponseChan:
log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
"node.id": resp.NodeID,
}).Debugf("loaded TLS certificate")
n.Lock()
n.nodeID = resp.NodeID
n.nodeMembership = resp.NodeMembership
n.Unlock()
close(n.certificateRequested)
}
}()
// LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS
// certificates and write them to disk
securityConfig, err = ca.LoadOrCreateSecurityConfig(
ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw)
if err != nil {
if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
return nil, ErrInvalidUnlockKey
}
return nil, err
}
}
n.Lock()
n.role = securityConfig.ClientTLSCreds.Role()
n.nodeID = securityConfig.ClientTLSCreds.NodeID()
n.nodeMembership = api.NodeMembershipAccepted
n.roleCond.Broadcast()
n.Unlock()
return securityConfig, nil
}
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
var securityConfig *ca.SecurityConfig
krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
if err := krw.Migrate(); err != nil {
return nil, err
}
// Check if we already have a valid certificates on disk.
rootCA, err := ca.GetLocalRootCA(paths.RootCA)
if err != nil && err != ca.ErrNoLocalRootCA {
return nil, err
}
if err == nil {
securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
if err != nil {
_, isInvalidKEK := errors.Cause(err).(ca.ErrInvalidKEK)
if isInvalidKEK {
return nil, ErrInvalidUnlockKey
} else if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
}
}
}
if securityConfig == nil {
if n.config.JoinAddr == "" {
// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
n.unlockKey = nil
if n.config.AutoLockManagers {
n.unlockKey = encryption.GenerateSecretKey()
}
krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
if err != nil {
return nil, err
}
log.G(ctx).Debug("generated CA key and certificate")
} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
if err != nil {
return nil, err
}
log.G(ctx).Debug("downloaded CA certificate")
}
// Obtain new certs and setup TLS certificates renewal for this node:
// - If certificates weren't present on disk, we call CreateSecurityConfig, which blocks
// until a valid certificate has been issued.
// - We wait for CreateSecurityConfig to finish since we need a certificate to operate.
// Attempt to load certificate from disk
securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
if err == nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": securityConfig.ClientTLSCreds.NodeID(),
}).Debugf("loaded TLS certificate")
} else {
if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
return nil, ErrInvalidUnlockKey
}
log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target())
securityConfig, err = rootCA.CreateSecurityConfig(ctx, krw, ca.CertificateRequestConfig{
Token: n.config.JoinToken,
Availability: n.config.Availability,
Remotes: n.remotes,
})
if err != nil {
return nil, err
}
}
}
n.Lock()
n.role = securityConfig.ClientTLSCreds.Role()
n.nodeID = securityConfig.ClientTLSCreds.NodeID()
n.roleCond.Broadcast()
n.Unlock()
return securityConfig, nil
}