// CreateAndWriteRootCA creates a Certificate authority for a new Swarm Cluster, potentially
// overwriting any existing CAs.
func CreateAndWriteRootCA(rootCN string, paths CertPaths) (RootCA, error) {
// Create a simple CSR for the CA using the default CA validator and policy
req := cfcsr.CertificateRequest{
CN: rootCN,
KeyRequest: &cfcsr.BasicKeyRequest{A: RootKeyAlgo, S: RootKeySize},
CA: &cfcsr.CAConfig{Expiry: RootCAExpiration},
}
// Generate the CA and get the certificate and private key
cert, _, key, err := initca.New(&req)
if err != nil {
return RootCA{}, err
}
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return RootCA{}, err
}
// Write the Private Key and Certificate to disk, using decent permissions
if err := ioutils.AtomicWriteFile(paths.Cert, cert, 0644); err != nil {
return RootCA{}, err
}
if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
return RootCA{}, err
}
return NewRootCA(cert, key, DefaultNodeCertExpiration)
}
// GenerateAndSignNewTLSCert creates a new keypair, signs the certificate using signer,
// and saves the certificate and key to disk. This method is used to bootstrap the first
// manager TLS certificates.
func GenerateAndSignNewTLSCert(rootCA RootCA, cn, ou, org string, paths CertPaths) (*tls.Certificate, error) {
// Generate and new keypair and CSR
csr, key, err := generateNewCSR()
if err != nil {
return nil, err
}
// Obtain a signed Certificate
certChain, err := rootCA.ParseValidateAndSignCSR(csr, cn, ou, org)
if err != nil {
return nil, errors.Wrap(err, "failed to sign node certificate")
}
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return nil, err
}
// Write both the chain and key to disk
if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil {
return nil, err
}
if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
return nil, err
}
// Load a valid tls.Certificate from the chain and the key
serverCert, err := tls.X509KeyPair(certChain, key)
if err != nil {
return nil, err
}
return &serverCert, nil
}
func TestRenewTLSConfigWithNoNode(t *testing.T) {
t.Parallel()
tc := testutils.NewTestCA(t)
defer tc.Stop()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Get a new nodeConfig with a TLS cert that has the default Cert duration
nodeConfig, err := tc.WriteNewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
// Create a new RootCA, and change the policy to issue 6 minute certificates.
// Because of the default backdate of 5 minutes, this issues certificates
// valid for 1 minute.
newRootCA, err := ca.NewRootCA(tc.RootCA.Cert, tc.RootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
newRootCA.Signer.SetPolicy(&cfconfig.Signing{
Default: &cfconfig.SigningProfile{
Usage: []string{"signing", "key encipherment", "server auth", "client auth"},
Expiry: 6 * time.Minute,
},
})
// Create a new CSR and overwrite the key on disk
csr, key, err := ca.GenerateNewCSR()
assert.NoError(t, err)
// Issue a new certificate with the same details as the current config, but with 1 min expiration time
c := nodeConfig.ClientTLSCreds
signedCert, err := newRootCA.ParseValidateAndSignCSR(csr, c.NodeID(), c.Role(), c.Organization())
assert.NoError(t, err)
assert.NotNil(t, signedCert)
// Overwrite the certificate on disk with one that expires in 1 minute
err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644)
assert.NoError(t, err)
err = ioutils.AtomicWriteFile(tc.Paths.Node.Key, key, 0600)
assert.NoError(t, err)
// Delete the node from the backend store
err = tc.MemoryStore.Update(func(tx store.Tx) error {
node := store.GetNode(tx, nodeConfig.ClientTLSCreds.NodeID())
assert.NotNil(t, node)
return store.DeleteNode(tx, nodeConfig.ClientTLSCreds.NodeID())
})
assert.NoError(t, err)
renew := make(chan struct{})
updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.Remotes, renew)
select {
case <-time.After(10 * time.Second):
assert.Fail(t, "TestRenewTLSConfig timed-out")
case certUpdate := <-updates:
assert.Error(t, certUpdate.Err)
assert.Contains(t, certUpdate.Err.Error(), "not found when attempting to renew certificate")
}
}
// createAndWriteca.RootCA creates a Certificate authority for a new Swarm Cluster.
// We're copying CreateAndWriteca.RootCA, so we can have smaller key-sizes for tests
func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duration) (ca.RootCA, error) {
// Create a simple CSR for the CA using the default CA validator and policy
req := cfcsr.CertificateRequest{
CN: rootCN,
KeyRequest: cfcsr.NewBasicKeyRequest(),
CA: &cfcsr.CAConfig{Expiry: ca.RootCAExpiration},
}
// Generate the CA and get the certificate and private key
cert, _, key, err := initca.New(&req)
if err != nil {
return ca.RootCA{}, err
}
// Convert the key given by initca to an object to create a ca.RootCA
parsedKey, err := helpers.ParsePrivateKeyPEM(key)
if err != nil {
log.Errorf("failed to parse private key: %v", err)
return ca.RootCA{}, err
}
// Convert the certificate into an object to create a ca.RootCA
parsedCert, err := helpers.ParseCertificatePEM(cert)
if err != nil {
return ca.RootCA{}, err
}
// Create a Signer out of the private key
signer, err := local.NewSigner(parsedKey, parsedCert, cfsigner.DefaultSigAlgo(parsedKey), ca.SigningPolicy(expiry))
if err != nil {
log.Errorf("failed to create signer: %v", err)
return ca.RootCA{}, err
}
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return ca.RootCA{}, err
}
// Write the Private Key and Certificate to disk, using decent permissions
if err := ioutils.AtomicWriteFile(paths.Cert, cert, 0644); err != nil {
return ca.RootCA{}, err
}
if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
return ca.RootCA{}, err
}
// Create a Pool with our Root CA Certificate
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(cert) {
return ca.RootCA{}, fmt.Errorf("failed to append certificate to cert pool")
}
return ca.RootCA{Signer: signer, Key: key, Cert: cert, Pool: pool}, nil
}
// createAndWriteRootCA creates a Certificate authority for a new Swarm Cluster.
// We're copying ca.CreateRootCA, so we can have smaller key-sizes for tests
func createAndWriteRootCA(rootCN string, paths ca.CertPaths, expiry time.Duration) (ca.RootCA, error) {
cert, key, err := CreateRootCertAndKey(rootCN)
if err != nil {
return ca.RootCA{}, err
}
// Convert the key given by initca to an object to create a ca.RootCA
parsedKey, err := helpers.ParsePrivateKeyPEM(key)
if err != nil {
log.Errorf("failed to parse private key: %v", err)
return ca.RootCA{}, err
}
// Convert the certificate into an object to create a ca.RootCA
parsedCert, err := helpers.ParseCertificatePEM(cert)
if err != nil {
return ca.RootCA{}, err
}
// Create a Signer out of the private key
signer, err := local.NewSigner(parsedKey, parsedCert, cfsigner.DefaultSigAlgo(parsedKey), ca.SigningPolicy(expiry))
if err != nil {
log.Errorf("failed to create signer: %v", err)
return ca.RootCA{}, err
}
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return ca.RootCA{}, err
}
// Write the Private Key and Certificate to disk, using decent permissions
if err := ioutils.AtomicWriteFile(paths.Cert, cert, 0644); err != nil {
return ca.RootCA{}, err
}
if err := ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
return ca.RootCA{}, err
}
// Create a Pool with our Root CA Certificate
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(cert) {
return ca.RootCA{}, errors.New("failed to append certificate to cert pool")
}
return ca.RootCA{
Signer: signer,
Key: key,
Cert: cert,
Pool: pool,
Digest: digest.FromBytes(cert),
}, nil
}
// IssueAndSaveNewCertificates generates a new key-pair, signs it with the local root-ca, and returns a
// tls certificate
func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org string) (*tls.Certificate, error) {
csr, key, err := GenerateAndWriteNewKey(paths)
if err != nil {
return nil, errors.Wrap(err, "error when generating new node certs")
}
if !rca.CanSign() {
return nil, ErrNoValidSigner
}
// Obtain a signed Certificate
certChain, err := rca.ParseValidateAndSignCSR(csr, cn, ou, org)
if err != nil {
return nil, errors.Wrap(err, "failed to sign node certificate")
}
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return nil, err
}
// Write the chain to disk
if err := ioutils.AtomicWriteFile(paths.Cert, certChain, 0644); err != nil {
return nil, err
}
// Create a valid TLSKeyPair out of the PEM encoded private key and certificate
tlsKeyPair, err := tls.X509KeyPair(certChain, key)
if err != nil {
return nil, err
}
return &tlsKeyPair, nil
}
// writeKey takes an unencrypted keyblock and, if the kek is not nil, encrypts it before
// writing it to disk. If the kek is nil, writes it to disk unencrypted.
func (k *KeyReadWriter) writeKey(keyBlock *pem.Block, kekData KEKData, pkh PEMKeyHeaders) error {
if kekData.KEK != nil {
encryptedPEMBlock, err := x509.EncryptPEMBlock(rand.Reader,
keyBlock.Type,
keyBlock.Bytes,
kekData.KEK,
x509.PEMCipherAES256)
if err != nil {
return err
}
if encryptedPEMBlock.Headers == nil {
return errors.New("unable to encrypt key - invalid PEM file produced")
}
keyBlock = encryptedPEMBlock
}
if pkh != nil {
headers, err := pkh.MarshalHeaders(kekData)
if err != nil {
return err
}
mergePEMHeaders(keyBlock.Headers, headers)
}
keyBlock.Headers[versionHeader] = strconv.FormatUint(kekData.Version, 10)
if err := ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil {
return err
}
k.kekData = kekData
k.headersObj = pkh
return nil
}
// Write attempts write a cert and key to text. This can also optionally update
// the KEK while writing, if an updated KEK is provided. If the pointer to the
// update KEK is nil, then we don't update. If the updated KEK itself is nil,
// then we update the KEK to be nil (data should be unencrypted).
func (k *KeyReadWriter) Write(certBytes, plaintextKeyBytes []byte, kekData *KEKData) error {
k.mu.Lock()
defer k.mu.Unlock()
// current assumption is that the cert and key will be in the same directory
if err := os.MkdirAll(filepath.Dir(k.paths.Key), 0755); err != nil {
return err
}
// Ensure that we will have a keypair on disk at all times by writing the cert to a
// temp path first. This is because we want to have only a single copy of the key
// for rotation and header modification.
tmpPaths := k.genTempPaths()
if err := ioutils.AtomicWriteFile(tmpPaths.Cert, certBytes, certPerms); err != nil {
return err
}
keyBlock, _ := pem.Decode(plaintextKeyBytes)
if keyBlock == nil {
return errors.New("invalid PEM-encoded private key")
}
if kekData == nil {
kekData = &k.kekData
}
pkh := k.headersObj
if k.headersObj != nil {
pkh = k.headersObj.UpdateKEK(k.kekData, *kekData)
}
if err := k.writeKey(keyBlock, *kekData, pkh); err != nil {
return err
}
return os.Rename(tmpPaths.Cert, k.paths.Cert)
}
func TestRenewTLSConfigManager(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Get a new nodeConfig with a TLS cert that has the default Cert duration
nodeConfig, err := tc.WriteNewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
// Create a new RootCA, and change the policy to issue 6 minute certificates
newRootCA, err := ca.NewRootCA(tc.RootCA.Cert, tc.RootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
newRootCA.Signer.SetPolicy(&cfconfig.Signing{
Default: &cfconfig.SigningProfile{
Usage: []string{"signing", "key encipherment", "server auth", "client auth"},
Expiry: 6 * time.Minute,
},
})
// Create a new CSR and overwrite the key on disk
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
// Issue a new certificate with the same details as the current config, but with 6 min expiration time
c := nodeConfig.ClientTLSCreds
signedCert, err := newRootCA.ParseValidateAndSignCSR(csr, c.NodeID(), c.Role(), c.Organization())
assert.NoError(t, err)
assert.NotNil(t, signedCert)
// Overwrite the certificate on disk with one that expires in 1 minute
err = ioutils.AtomicWriteFile(tc.Paths.Node.Cert, signedCert, 0644)
assert.NoError(t, err)
// Get a new nodeConfig with a TLS cert that has 6 minutes to live
var success, timeout bool
renew := make(chan struct{})
updates := ca.RenewTLSConfig(ctx, nodeConfig, tc.TempDir, tc.Picker, renew)
for {
select {
case <-time.After(2 * time.Second):
timeout = true
case certUpdate := <-updates:
assert.NoError(t, certUpdate.Err)
assert.NotNil(t, certUpdate)
assert.Equal(t, ca.ManagerRole, certUpdate.Role)
success = true
}
if timeout {
assert.Fail(t, "TestRenewTLSConfig timed-out")
break
}
if success {
break
}
}
}
func saveRootCA(rootCA RootCA, paths CertPaths) error {
// Make sure the necessary dirs exist and they are writable
err := os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return err
}
// If the root certificate got returned successfully, save the rootCA to disk.
return ioutils.AtomicWriteFile(paths.Cert, rootCA.Cert, 0644)
}
func (rca *RootCA) saveCertificate() error {
if rca.Cert == nil {
return errors.New("no valid certificate bundle found")
}
if rca.Path.Cert == "" {
return errors.New("no path found for this root CA")
}
// Make sure the necessary dirs exist and they are writable
err := os.MkdirAll(filepath.Dir(rca.Path.Cert), 0755)
if err != nil {
return err
}
return ioutils.AtomicWriteFile(rca.Path.Cert, rca.Cert, 0644)
}
func (s *persistentRemotes) save() error {
weights := s.Weights()
remotes := make([]api.Peer, 0, len(weights))
for r := range weights {
remotes = append(remotes, r)
}
sort.Sort(sortablePeers(remotes))
if reflect.DeepEqual(remotes, s.lastSavedState) {
return nil
}
dt, err := json.Marshal(remotes)
if err != nil {
return err
}
s.lastSavedState = remotes
return ioutils.AtomicWriteFile(s.storePath, dt, 0600)
}
// GenerateAndWriteNewKey generates a new pub/priv key pair, writes it to disk
// and returns the CSR and the private key material
func GenerateAndWriteNewKey(paths CertPaths) (csr, key []byte, err error) {
// Generate a new key pair
csr, key, err = generateNewCSR()
if err != nil {
return
}
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Key), 0755)
if err != nil {
return
}
if err = ioutils.AtomicWriteFile(paths.Key, key, 0600); err != nil {
return
}
return
}
// IssueAndSaveNewCertificates generates a new key-pair, signs it with the local root-ca, and returns a
// tls certificate
func (rca *RootCA) IssueAndSaveNewCertificates(paths CertPaths, cn, ou, org string) (*tls.Certificate, error) {
csr, key, err := GenerateAndWriteNewKey(paths)
if err != nil {
log.Debugf("error when generating new node certs: %v", err)
return nil, err
}
var signedCert []byte
if !rca.CanSign() {
return nil, ErrNoValidSigner
}
// Obtain a signed Certificate
signedCert, err = rca.ParseValidateAndSignCSR(csr, cn, ou, org)
if err != nil {
log.Debugf("failed to sign node certificate: %v", err)
return nil, err
}
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return nil, err
}
// Write the chain to disk
if err := ioutils.AtomicWriteFile(paths.Cert, signedCert, 0644); err != nil {
return nil, err
}
// Create a valid TLSKeyPair out of the PEM encoded private key and certificate
tlsKeyPair, err := tls.X509KeyPair(signedCert, key)
if err != nil {
return nil, err
}
log.Debugf("locally issued new TLS certificate for node ID: %s and role: %s", cn, ou)
return &tlsKeyPair, nil
}
// ViewAndUpdateHeaders updates the header manager, and updates any headers on the existing key
func (k *KeyReadWriter) ViewAndUpdateHeaders(cb func(PEMKeyHeaders) (PEMKeyHeaders, error)) error {
k.mu.Lock()
defer k.mu.Unlock()
pkh, err := cb(k.headersObj)
if err != nil {
return err
}
keyBlock, err := k.readKeyblock()
if err != nil {
return err
}
headers := make(map[string]string)
if pkh != nil {
var err error
headers, err = pkh.MarshalHeaders(k.kekData)
if err != nil {
return err
}
}
// we WANT any original encryption headers
for key, value := range keyBlock.Headers {
normalizedKey := strings.TrimSpace(strings.ToLower(key))
if normalizedKey == "proc-type" || normalizedKey == "dek-info" {
headers[key] = value
}
}
headers[versionHeader] = strconv.FormatUint(k.kekData.Version, 10)
keyBlock.Headers = headers
if err = ioutils.AtomicWriteFile(k.paths.Key, pem.EncodeToMemory(keyBlock), keyPerms); err != nil {
return err
}
k.headersObj = pkh
return nil
}
// RequestAndSaveNewCertificates gets new certificates issued, either by signing them locally if a signer is
// available, or by requesting them from the remote server at remoteAddr.
func (rca *RootCA) RequestAndSaveNewCertificates(ctx context.Context, paths CertPaths, role, secret string, picker *picker.Picker, transport credentials.TransportAuthenticator, nodeInfo chan<- api.IssueNodeCertificateResponse) (*tls.Certificate, error) {
// Create a new key/pair and CSR for the new manager
// Write the new CSR and the new key to a temporary location so we can survive crashes on rotation
tempPaths := genTempPaths(paths)
csr, key, err := GenerateAndWriteNewKey(tempPaths)
if err != nil {
log.Debugf("error when generating new node certs: %v", err)
return nil, err
}
// Get the remote manager to issue a CA signed certificate for this node
// Retry up to 5 times in case the manager we first try to contact isn't
// responding properly (for example, it may have just been demoted).
var signedCert []byte
for i := 0; i != 5; i++ {
signedCert, err = GetRemoteSignedCertificate(ctx, csr, role, secret, rca.Pool, picker, transport, nodeInfo)
if err == nil {
break
}
log.Warningf("error fetching signed node certificate: %v", err)
}
if err != nil {
return nil, err
}
// Доверяй, но проверяй.
// Before we overwrite our local certificate, let's make sure the server gave us one that is valid
// Create an X509Cert so we can .Verify()
certBlock, _ := pem.Decode(signedCert)
if certBlock == nil {
return nil, fmt.Errorf("failed to parse certificate PEM")
}
X509Cert, err := x509.ParseCertificate(certBlock.Bytes)
if err != nil {
return nil, err
}
// Include our current root pool
opts := x509.VerifyOptions{
Roots: rca.Pool,
}
// Check to see if this certificate was signed by our CA, and isn't expired
if _, err := X509Cert.Verify(opts); err != nil {
return nil, err
}
log.Infof("Downloaded new TLS credentials with role: %s.", role)
// Ensure directory exists
err = os.MkdirAll(filepath.Dir(paths.Cert), 0755)
if err != nil {
return nil, err
}
// Write the chain to disk
if err := ioutils.AtomicWriteFile(paths.Cert, signedCert, 0644); err != nil {
return nil, err
}
// Move the new key to the final location
if err := os.Rename(tempPaths.Key, paths.Key); err != nil {
return nil, err
}
// Create a valid TLSKeyPair out of the PEM encoded private key and certificate
tlsKeyPair, err := tls.X509KeyPair(signedCert, key)
if err != nil {
return nil, err
}
return &tlsKeyPair, nil
}