// The TLS KEK and the KEK for the headers should be in sync, and so failing
// to decrypt the TLS key should be mean we won't be able to decrypt the headers.
// However, the TLS Key encryption uses AES-256-CBC (golang as of 1.7.x does not seem
// to support GCM, so no cipher modes with digests) so sometimes decrypting with
// the wrong passphrase will not result in an error. This means we will ultimately
// have to rely on the header encryption mechanism, which does include a digest, to
// determine if the KEK is valid.
func TestDecryptTLSKeyFalsePositive(t *testing.T) {
badKey := []byte(`
-----BEGIN EC PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-256-CBC,e7927e79e748233776c03c2eb7275f09
kek-version: 392
raft-dek: CAESMBrzZ0gNVPe3FRs42743q8RtkUBrK1ICQpHWX2vdQ8iqSKt1WoKdFDFD2r28LYAVLxoYQguwHbijMx9k+BALUNBAI3s199S5tvnr
JfGenNvzm++AvsOh+UmcBY+JgI6lnfzaCB68agmlmEZYLYi5tqtAU7gif6VIJpCW
+Pj23Fzkw8sKKOOBeapSC5lp+Cjx9OsCci/R9xrdx+uxnnzKJNxOB/qzqcQfZDMh
id2LxdliFcPEk/Yj5gNGpT0UMFJ4G52enbOwOru46f0=
-----END EC PRIVATE KEY-----
`)
// not actually a real swarm cert - generated a cert corresponding to the key that expires in 20 years
matchingCert := []byte(`
-----BEGIN CERTIFICATE-----
MIIB9jCCAZygAwIBAgIRAIdzF3Z9VT2OXbRvEw5cR68wCgYIKoZIzj0EAwIwYDEi
MCAGA1UEChMZbWRwMXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dh
cm0tbWFuYWdlcjEiMCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTAg
GA8wMDAxMDEwMTAwMDAwMFoXDTM2MTEwODA2MjMwMlowYDEiMCAGA1UEChMZbWRw
MXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dhcm0tbWFuYWdlcjEi
MCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTBZMBMGByqGSM49AgEG
CCqGSM49AwEHA0IABGOivD25E/zcupRFQdKOKbPHS9Mx7JlUhlWnl0iR0K5VhVIU
XjUHt98GuX6gDjs4yUzEKSGxYPsSYlnG9zQqbQSjNTAzMA4GA1UdDwEB/wQEAwIF
oDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMAoGCCqGSM49BAMC
A0gAMEUCIQDWtjg1ITGznQILipaEe70G/NgZAOtFfuPXTVkUl3el+wIgSVOVKB/Q
O0T3aXuZGYNyh//KqAoA3erCmh6HauMz84Y=
-----END CERTIFICATE-----
`)
var wrongKEK []byte // empty passphrase doesn't decrypt without errors
falsePositiveKEK, err := base64.RawStdEncoding.DecodeString("bIQgLAAMoGCrHdjMLVhEVqnYTAM7ZNF2xWMiwtw7AiQ")
require.NoError(t, err)
realKEK, err := base64.RawStdEncoding.DecodeString("fDg9YejLnMjU+FpulWR62oJLzVpkD2j7VQuP5xiK9QA")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter-false-positive-decryption")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(tempdir)
require.NoError(t, ioutil.WriteFile(path.Node.Key, badKey, 0600))
require.NoError(t, ioutil.WriteFile(path.Node.Cert, matchingCert, 0644))
krw := ca.NewKeyReadWriter(path.Node, wrongKEK, RaftDEKData{})
_, _, err = krw.Read()
require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err))
krw = ca.NewKeyReadWriter(path.Node, falsePositiveKEK, RaftDEKData{})
_, _, err = krw.Read()
require.Error(t, err)
require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err))
krw = ca.NewKeyReadWriter(path.Node, realKEK, RaftDEKData{})
_, _, err = krw.Read()
require.NoError(t, err)
}
func TestLoadSecurityConfigIncorrectPassphrase(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
paths := ca.NewConfigPaths(tc.TempDir)
_, err := tc.RootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(paths.Node, []byte("kek"), nil),
"nodeID", ca.WorkerRole, tc.Organization)
require.NoError(t, err)
_, err = ca.LoadSecurityConfig(tc.Context, tc.RootCA, ca.NewKeyReadWriter(paths.Node, nil, nil))
require.IsType(t, ca.ErrInvalidKEK{}, err)
}
func TestKeyReadWriterViewAndRotateKEK(t *testing.T) {
cert, key, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir))
// write a key with headers to the key to make sure it gets passed when reading/writing headers
keyBlock, _ := pem.Decode(key)
require.NotNil(t, keyBlock)
keyBlock.Headers = map[string]string{"hello": "world"}
key = pem.EncodeToMemory(keyBlock)
require.NoError(t, ca.NewKeyReadWriter(path.Node, nil, nil).Write(cert, key, nil))
// if if getting new kek and headers fail, rotating a KEK fails, and the kek does not rotate
k := ca.NewKeyReadWriter(path.Node, nil, nil)
require.Error(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
require.Equal(t, ca.KEKData{}, k)
require.Nil(t, h)
return ca.KEKData{}, nil, fmt.Errorf("Nope")
}))
// writing new headers will write a key that has the headers returned by the header update function
k = ca.NewKeyReadWriter(path.Node, []byte("oldKEK"), nil)
require.NoError(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
require.Equal(t, ca.KEKData{KEK: []byte("oldKEK")}, k)
require.Nil(t, h)
return ca.KEKData{KEK: []byte("newKEK"), Version: uint64(2)},
testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
require.Equal(t, []byte("newKEK"), kek.KEK)
return map[string]string{"updated": "headers"}, nil
}}, nil
}))
// ensure the key has been re-encrypted and we can read it
k = ca.NewKeyReadWriter(path.Node, nil, nil)
_, _, err = k.Read()
require.Error(t, err)
var headers map[string]string
k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, _ ca.KEKData) (ca.PEMKeyHeaders, error) {
headers = h
return testHeaders{}, nil
}})
_, _, err = k.Read()
require.NoError(t, err)
require.Equal(t, map[string]string{"updated": "headers"}, headers)
}
func TestKeyReadWriterMigrate(t *testing.T) {
cert, key, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir))
// if the key exists in an old location, migrate it from there.
tempKeyPath := filepath.Join(filepath.Dir(path.Node.Key), "."+filepath.Base(path.Node.Key))
require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644))
require.NoError(t, ioutil.WriteFile(tempKeyPath, key, 0600))
krw := ca.NewKeyReadWriter(path.Node, nil, nil)
require.NoError(t, krw.Migrate())
_, err = os.Stat(tempKeyPath)
require.True(t, os.IsNotExist(err)) // it's been moved to the right place
_, _, err = krw.Read()
require.NoError(t, err)
// migrate does not affect any existing files
dirList, err := ioutil.ReadDir(filepath.Dir(path.Node.Key))
require.NoError(t, err)
require.NoError(t, krw.Migrate())
dirList2, err := ioutil.ReadDir(filepath.Dir(path.Node.Key))
require.NoError(t, err)
require.Equal(t, dirList, dirList2)
_, _, err = krw.Read()
require.NoError(t, err)
}
// If there is nothing on disk and no join addr, we create a new CA and a new set of TLS certs.
// If AutoLockManagers is enabled, the TLS key is encrypted with a randomly generated lock key.
func TestLoadSecurityConfigNewNode(t *testing.T) {
for _, autoLockManagers := range []bool{true, false} {
tempdir, err := ioutil.TempDir("", "test-new-node")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
node, err := New(&Config{
StateDir: tempdir,
AutoLockManagers: autoLockManagers,
})
require.NoError(t, err)
securityConfig, err := node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
unencryptedReader := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, _, err = unencryptedReader.Read()
if !autoLockManagers {
require.NoError(t, err)
} else {
require.IsType(t, ca.ErrInvalidKEK{}, err)
}
}
}
func TestLoadSecurityConfigInvalidCert(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Write some garbage to the cert
ioutil.WriteFile(tc.Paths.Node.Cert, []byte(`-----BEGIN CERTIFICATE-----\n
some random garbage\n
-----END CERTIFICATE-----`), 0644)
krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
_, err := ca.LoadSecurityConfig(tc.Context, tc.RootCA, krw)
assert.Error(t, err)
nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
ca.CertificateRequestConfig{
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
}
func TestCreateSecurityConfigNoCerts(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Remove only the node certificates form the directory, and attest that we get
// new certificates that are locally signed
os.RemoveAll(tc.Paths.Node.Cert)
krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
// Remove only the node certificates form the directory, get a new rootCA, and attest that we get
// new certificates that are issued by the remote CA
os.RemoveAll(tc.Paths.Node.Cert)
rootCA, err := ca.GetLocalRootCA(tc.Paths.RootCA)
assert.NoError(t, err)
nodeConfig, err = rootCA.CreateSecurityConfig(tc.Context, krw,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.Equal(t, rootCA, *nodeConfig.RootCA())
}
// If we abort in the middle of writing the key and cert, such that only the key is written
// to the final location, when we read we can still read the cert from the temporary
// location.
func TestTwoPhaseReadWrite(t *testing.T) {
cert1, _, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
cert2, key2, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir))
krw := ca.NewKeyReadWriter(path.Node, nil, nil)
// put a directory in the location where the cert goes, so we can't actually move
// the cert from the temporary location to the final location.
require.NoError(t, os.Mkdir(filepath.Join(path.Node.Cert), 0755))
require.Error(t, krw.Write(cert2, key2, nil))
// the temp cert file should exist
tempCertPath := filepath.Join(filepath.Dir(path.Node.Cert), "."+filepath.Base(path.Node.Cert))
readCert, err := ioutil.ReadFile(tempCertPath)
require.NoError(t, err)
require.Equal(t, cert2, readCert)
// remove the directory, to simulate it failing to write the first time
os.RemoveAll(path.Node.Cert)
readCert, readKey, err := krw.Read()
require.NoError(t, err)
require.Equal(t, cert2, readCert)
require.Equal(t, key2, readKey)
// the cert should have been moved to its proper location
_, err = os.Stat(tempCertPath)
require.True(t, os.IsNotExist(err))
// If the cert in the proper location doesn't match the key, the temp location is checked
require.NoError(t, ioutil.WriteFile(tempCertPath, cert2, 0644))
require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert1, 0644))
readCert, readKey, err = krw.Read()
require.NoError(t, err)
require.Equal(t, cert2, readCert)
require.Equal(t, key2, readKey)
// the cert should have been moved to its proper location
_, err = os.Stat(tempCertPath)
require.True(t, os.IsNotExist(err))
// If the cert in the temp location also doesn't match, the failure matching the
// correctly-located cert is returned
require.NoError(t, os.Remove(path.Node.Cert))
require.NoError(t, ioutil.WriteFile(tempCertPath, cert1, 0644)) // mismatching cert
_, _, err = krw.Read()
require.True(t, os.IsNotExist(err))
// the cert should have been removed
_, err = os.Stat(tempCertPath)
require.True(t, os.IsNotExist(err))
}
// If there are CAs and TLS certs on disk, it tries to load and fails if there
// are any errors, even if a join token is provided.
func TestLoadSecurityConfigLoadFromDisk(t *testing.T) {
tempdir, err := ioutil.TempDir("", "test-load-node-tls")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
tc := cautils.NewTestCA(t)
defer tc.Stop()
peer, err := tc.ConnBroker.Remotes().Select()
require.NoError(t, err)
// Load successfully with valid passphrase
rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
krw := ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil)
require.NoError(t, err)
_, err = rootCA.IssueAndSaveNewCertificates(krw, identity.NewID(), ca.WorkerRole, identity.NewID())
require.NoError(t, err)
node, err := New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
UnlockKey: []byte("passphrase"),
})
require.NoError(t, err)
securityConfig, err := node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
// Invalid passphrase
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.Equal(t, ErrInvalidUnlockKey, err)
// Invalid CA
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
UnlockKey: []byte("passphrase"),
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.IsType(t, x509.UnknownAuthorityError{}, errors.Cause(err))
}
// NewExternalSigningServer creates and runs a new ExternalSigningServer which
// uses the given rootCA to sign node certificates. A server key and cert are
// generated and saved into the given basedir and then a TLS listener is
// started on a random available port. On success, an HTTPS server will be
// running in a separate goroutine. The URL of the singing endpoint is
// available in the returned *ExternalSignerServer value. Calling the Close()
// method will stop the server.
func NewExternalSigningServer(rootCA ca.RootCA, basedir string) (*ExternalSigningServer, error) {
serverCN := "external-ca-example-server"
serverOU := "localhost" // Make a valid server cert for localhost.
// Create TLS credentials for the external CA server which we will run.
serverPaths := ca.CertPaths{
Cert: filepath.Join(basedir, "server.crt"),
Key: filepath.Join(basedir, "server.key"),
}
serverCert, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(serverPaths, nil, nil), serverCN, serverOU, "")
if err != nil {
return nil, errors.Wrap(err, "unable to get TLS server certificate")
}
serverTLSConfig := &tls.Config{
Certificates: []tls.Certificate{*serverCert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: rootCA.Pool,
}
tlsListener, err := tls.Listen("tcp", "localhost:0", serverTLSConfig)
if err != nil {
return nil, errors.Wrap(err, "unable to create TLS connection listener")
}
assignedPort := tlsListener.Addr().(*net.TCPAddr).Port
signURL := url.URL{
Scheme: "https",
Host: net.JoinHostPort("localhost", strconv.Itoa(assignedPort)),
Path: "/sign",
}
ess := &ExternalSigningServer{
listener: tlsListener,
URL: signURL.String(),
}
mux := http.NewServeMux()
handler := &signHandler{
numIssued: &ess.NumIssued,
rootCA: rootCA,
flaky: &ess.flaky,
}
mux.Handle(signURL.Path, handler)
server := &http.Server{
Handler: mux,
}
go server.Serve(tlsListener)
return ess, nil
}
func TestDecrypt(t *testing.T) {
tempdir, err := ioutil.TempDir("", "rafttool")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
kek := []byte("kek")
dek := []byte("dek")
unlockKey := encryption.HumanReadableKey(kek)
// write a key to disk, else we won't be able to decrypt anything
paths := certPaths(tempdir)
krw := ca.NewKeyReadWriter(paths.Node, kek,
manager.RaftDEKData{EncryptionKeys: raft.EncryptionKeys{CurrentDEK: dek}})
cert, key, err := testutils.CreateRootCertAndKey("not really a root, just need cert and key")
require.NoError(t, err)
require.NoError(t, krw.Write(cert, key, nil))
// create the encrypted v3 directory
origSnapshot := raftpb.Snapshot{
Data: []byte("snapshot"),
Metadata: raftpb.SnapshotMetadata{
Index: 1,
Term: 1,
},
}
e, d := encryption.Defaults(dek)
writeFakeRaftData(t, tempdir, &origSnapshot, storage.NewWALFactory(e, d), storage.NewSnapFactory(e, d))
outdir := filepath.Join(tempdir, "outdir")
// if we use the wrong unlock key, we can't actually decrypt anything. The output directory won't get created.
err = decryptRaftData(tempdir, outdir, "")
require.IsType(t, ca.ErrInvalidKEK{}, err)
require.False(t, fileutil.Exist(outdir))
// Using the right unlock key, we produce data that is unencrypted
require.NoError(t, decryptRaftData(tempdir, outdir, unlockKey))
require.True(t, fileutil.Exist(outdir))
// The snapshot directory is readable by the regular snapshotter
snapshot, err := storage.OriginalSnap.New(filepath.Join(outdir, "snap-decrypted")).Load()
require.NoError(t, err)
require.NotNil(t, snapshot)
require.Equal(t, origSnapshot, *snapshot)
// The wals are readable by the regular wal
walreader, err := storage.OriginalWAL.Open(filepath.Join(outdir, "wal-decrypted"), walpb.Snapshot{Index: 1, Term: 1})
require.NoError(t, err)
metadata, _, entries, err := walreader.ReadAll()
require.NoError(t, err)
require.Equal(t, []byte("v3metadata"), metadata)
require.Len(t, entries, 5)
}
func TestRaftDEKManagerUpdateKeys(t *testing.T) {
tempDir, err := ioutil.TempDir("", "manager-update-keys-")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
paths := ca.NewConfigPaths(tempDir)
cert, key, err := cautils.CreateRootCertAndKey("cn")
require.NoError(t, err)
keys := raft.EncryptionKeys{
CurrentDEK: []byte("key1"),
PendingDEK: []byte("key2"),
}
krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{
EncryptionKeys: keys,
NeedsRotation: true,
})
require.NoError(t, krw.Write(cert, key, nil))
dekManager, err := NewRaftDEKManager(krw)
require.NoError(t, err)
newKeys := raft.EncryptionKeys{
CurrentDEK: []byte("new current"),
}
require.NoError(t, dekManager.UpdateKeys(newKeys))
// don't run GetKeys, because NeedsRotation is true and it'd just generate a new one
h, _ := krw.GetCurrentState()
dekData, ok := h.(RaftDEKData)
require.True(t, ok)
require.True(t, dekData.NeedsRotation)
// UpdateKeys so there is no CurrentDEK: all the headers should be wiped out
require.NoError(t, dekManager.UpdateKeys(raft.EncryptionKeys{}))
require.Equal(t, raft.EncryptionKeys{}, dekManager.GetKeys())
require.False(t, dekManager.NeedsRotation())
h, _ = krw.GetCurrentState()
require.Nil(t, h)
keyBytes, err := ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
keyBlock, _ := pem.Decode(keyBytes)
require.NotNil(t, keyBlock)
// the only header remaining should be the kek version
require.Len(t, keyBlock.Headers, 1)
require.Contains(t, keyBlock.Headers, "kek-version")
}
// newNode creates new node with specific role(manager or agent) and joins to
// existing cluster. if joinAddr is empty string, then new cluster will be initialized.
// It uses TestExecutor as executor. If lateBind is set, the remote API port is not
// bound. If rootCA is set, this root is used to bootstrap the node's TLS certs.
func newTestNode(joinAddr, joinToken string, lateBind bool, rootCA *ca.RootCA) (*testNode, error) {
tmpDir, err := ioutil.TempDir("", "swarmkit-integration-")
if err != nil {
return nil, err
}
cAddr := filepath.Join(tmpDir, "control.sock")
cfg := &node.Config{
ListenControlAPI: cAddr,
JoinAddr: joinAddr,
StateDir: tmpDir,
Executor: &TestExecutor{},
JoinToken: joinToken,
}
if !lateBind {
cfg.ListenRemoteAPI = "127.0.0.1:0"
}
if rootCA != nil {
certDir := filepath.Join(tmpDir, "certificates")
if err := os.MkdirAll(certDir, 0700); err != nil {
return nil, err
}
certPaths := ca.NewConfigPaths(certDir)
if err := ioutil.WriteFile(certPaths.RootCA.Cert, rootCA.Cert, 0644); err != nil {
return nil, err
}
if err := ioutil.WriteFile(certPaths.RootCA.Key, rootCA.Key, 0600); err != nil {
return nil, err
}
// generate TLS certs for this manager for bootstrapping, else the node will generate its own CA
_, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(certPaths.Node, nil, nil),
identity.NewID(), ca.ManagerRole, identity.NewID())
if err != nil {
return nil, err
}
}
node, err := node.New(cfg)
if err != nil {
return nil, err
}
return &testNode{
config: cfg,
node: node,
stateDir: tmpDir,
}, nil
}
func getKRW(swarmdir, unlockKey string) (*ca.KeyReadWriter, error) {
var (
kek []byte
err error
)
if unlockKey != "" {
kek, err = encryption.ParseHumanReadableKey(unlockKey)
if err != nil {
return nil, err
}
}
krw := ca.NewKeyReadWriter(certPaths(swarmdir).Node, kek, manager.RaftDEKData{})
_, _, err = krw.Read() // loads all the key data into the KRW object
if err != nil {
return nil, err
}
return krw, nil
}
func TestCreateSecurityConfigEmptyDir(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Remove all the contents from the temp dir and try again with a new node
os.RemoveAll(tc.TempDir)
krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
}
func main() {
// Create root material within the current directory.
rootPaths := ca.CertPaths{
Cert: filepath.Join("ca", "root.crt"),
Key: filepath.Join("ca", "root.key"),
}
// Initialize the Root CA.
rootCA, err := ca.CreateRootCA("external-ca-example", rootPaths)
if err != nil {
logrus.Fatalf("unable to initialize Root CA: %s", err)
}
// Create the initial manager node credentials.
nodeConfigPaths := ca.NewConfigPaths("certificates")
clusterID := identity.NewID()
nodeID := identity.NewID()
kw := ca.NewKeyReadWriter(nodeConfigPaths.Node, nil, nil)
if _, err := rootCA.IssueAndSaveNewCertificates(kw, nodeID, ca.ManagerRole, clusterID); err != nil {
logrus.Fatalf("unable to create initial manager node credentials: %s", err)
}
// And copy the Root CA certificate into the node config path for its
// CA.
ioutil.WriteFile(nodeConfigPaths.RootCA.Cert, rootCA.Cert, os.FileMode(0644))
server, err := testutils.NewExternalSigningServer(rootCA, "ca")
if err != nil {
logrus.Fatalf("unable to start server: %s", err)
}
defer server.Stop()
logrus.Infof("Now run: swarmd -d . --listen-control-api ./swarmd.sock --external-ca protocol=cfssl,url=%s", server.URL)
sigC := make(chan os.Signal, 1)
signal.Notify(sigC, syscall.SIGTERM, syscall.SIGINT)
<-sigC
}
func TestNewRootCABundle(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// make one rootCA
secondRootCA, err := ca.CreateRootCA("rootCN2", paths.RootCA)
assert.NoError(t, err)
// make a second root CA
firstRootCA, err := ca.CreateRootCA("rootCN1", paths.RootCA)
assert.NoError(t, err)
// Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle
bundle := append(firstRootCA.Cert, secondRootCA.Cert...)
err = ioutil.WriteFile(paths.RootCA.Cert, bundle, 0644)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(bundle, firstRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, bundle, newRootCA.Cert)
assert.Equal(t, 2, len(newRootCA.Pool.Subjects()))
// If I use newRootCA's IssueAndSaveNewCertificates to sign certs, I'll get the correct CA in the chain
kw := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, err = newRootCA.IssueAndSaveNewCertificates(kw, "CN", "OU", "ORG")
assert.NoError(t, err)
certBytes, err := ioutil.ReadFile(paths.Node.Cert)
assert.NoError(t, err)
certs, err := helpers.ParseCertificatesPEM(certBytes)
assert.NoError(t, err)
assert.Len(t, certs, 2)
assert.Equal(t, "CN", certs[0].Subject.CommonName)
assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0])
assert.Equal(t, "ORG", certs[0].Subject.Organization[0])
assert.Equal(t, "rootCN1", certs[1].Subject.CommonName)
}
// NewRaftDEKManager creates a key if one doesn't exist
func TestNewRaftDEKManager(t *testing.T) {
tempDir, err := ioutil.TempDir("", "manager-new-dek-manager-")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
paths := ca.NewConfigPaths(tempDir)
cert, key, err := cautils.CreateRootCertAndKey("cn")
require.NoError(t, err)
krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
require.NoError(t, krw.Write(cert, key, nil))
keyBytes, err := ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
require.NotContains(t, string(keyBytes), pemHeaderRaftDEK) // headers are not written
dekManager, err := NewRaftDEKManager(krw) // this should create a new DEK and write it to the file
require.NoError(t, err)
keyBytes, err = ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
require.Contains(t, string(keyBytes), pemHeaderRaftDEK) // header is written now
keys := dekManager.GetKeys()
require.NotNil(t, keys.CurrentDEK)
require.Nil(t, keys.PendingDEK)
require.False(t, dekManager.NeedsRotation())
// If one exists, nothing is updated
dekManager, err = NewRaftDEKManager(krw) // this should create a new DEK and write it to the file
require.NoError(t, err)
keyBytes2, err := ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
require.Equal(t, keyBytes, keyBytes2)
require.Equal(t, keys, dekManager.GetKeys())
require.False(t, dekManager.NeedsRotation())
}
func TestLoadSecurityConfigInvalidKey(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Write some garbage to the Key
ioutil.WriteFile(tc.Paths.Node.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n
some random garbage\n
-----END EC PRIVATE KEY-----`), 0644)
krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
_, err := ca.LoadSecurityConfig(tc.Context, tc.RootCA, krw, false)
assert.Error(t, err)
nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
ca.CertificateRequestConfig{
ConnBroker: tc.ConnBroker,
})
assert.NoError(t, err)
assert.NotNil(t, nodeConfig)
assert.NotNil(t, nodeConfig.ClientTLSCreds)
assert.NotNil(t, nodeConfig.ServerTLSCreds)
assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
}
func TestRequestAndSaveNewCertificates(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Copy the current RootCA without the signer
rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool}
cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.ManagerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, cert)
perms, err := permbits.Stat(tc.Paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
// there was no encryption config in the remote, so the key should be unencrypted
unencryptedKeyReader := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
// the worker token is also unencrypted
cert, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, cert)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
// If there is a different kek in the remote store, when TLS certs are renewed the new key will
// be encrypted with that kek
assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
cluster := store.GetCluster(tx, tc.Organization)
cluster.Spec.EncryptionConfig.AutoLockManagers = true
cluster.UnlockKeys = []*api.EncryptionKey{{
Subsystem: ca.ManagerRole,
Key: []byte("kek!"),
}}
return store.UpdateCluster(tx, cluster)
}))
assert.NoError(t, os.RemoveAll(tc.Paths.Node.Cert))
assert.NoError(t, os.RemoveAll(tc.Paths.Node.Key))
_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.ManagerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
// key can no longer be read without a kek
_, _, err = unencryptedKeyReader.Read()
require.Error(t, err)
_, _, err = ca.NewKeyReadWriter(tc.Paths.Node, []byte("kek!"), nil).Read()
require.NoError(t, err)
// if it's a worker though, the key is always unencrypted, even though the manager key is encrypted
_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
}
// If there is no CA, and a join addr is provided, one is downloaded from the
// join server. If there is a CA, it is just loaded from disk. The TLS key and
// cert are also downloaded.
func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) {
tempdir, err := ioutil.TempDir("", "test-join-node")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
// join addr is invalid
node, err := New(&Config{
StateDir: tempdir,
JoinAddr: "127.0.0.1:12",
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.Error(t, err)
tc := cautils.NewTestCA(t)
defer tc.Stop()
peer, err := tc.ConnBroker.Remotes().Select()
require.NoError(t, err)
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.NoError(t, err)
// the TLS key and cert were written to disk unencrypted
_, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read()
require.NoError(t, err)
// remove the TLS cert and key, and mark the root CA cert so that we will
// know if it gets replaced
require.NoError(t, os.Remove(paths.Node.Cert))
require.NoError(t, os.Remove(paths.Node.Key))
certBytes, err := ioutil.ReadFile(paths.RootCA.Cert)
require.NoError(t, err)
pemBlock, _ := pem.Decode(certBytes)
require.NotNil(t, pemBlock)
pemBlock.Headers["marked"] = "true"
certBytes = pem.EncodeToMemory(pemBlock)
require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, certBytes, 0644))
// also make sure the new set gets downloaded and written to disk with a passphrase
// by updating the memory store with manager autolock on and an unlock key
require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
clusters, err := store.FindClusters(tx, store.All)
require.NoError(t, err)
require.Len(t, clusters, 1)
newCluster := clusters[0].Copy()
newCluster.Spec.EncryptionConfig.AutoLockManagers = true
newCluster.UnlockKeys = []*api.EncryptionKey{{
Subsystem: ca.ManagerRole,
Key: []byte("passphrase"),
}}
return store.UpdateCluster(tx, newCluster)
}))
// Join with without any passphrase - this should be fine, because the TLS
// key is downloaded and then loaded just fine. However, it *is* written
// to disk encrypted.
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.NoError(t, err)
// make sure the CA cert has not been replaced
readCertBytes, err := ioutil.ReadFile(paths.RootCA.Cert)
require.NoError(t, err)
require.Equal(t, certBytes, readCertBytes)
// the TLS node cert and key were saved to disk encrypted, though
_, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read()
require.Error(t, err)
_, _, err = ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil).Read()
require.NoError(t, err)
}
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
var securityConfig *ca.SecurityConfig
krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
if err := krw.Migrate(); err != nil {
return nil, err
}
// Check if we already have a valid certificates on disk.
rootCA, err := ca.GetLocalRootCA(paths.RootCA)
if err != nil && err != ca.ErrNoLocalRootCA {
return nil, err
}
if err == nil {
securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
if err != nil {
_, isInvalidKEK := errors.Cause(err).(ca.ErrInvalidKEK)
if isInvalidKEK {
return nil, ErrInvalidUnlockKey
} else if !os.IsNotExist(err) {
return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
}
}
}
if securityConfig == nil {
if n.config.JoinAddr == "" {
// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
n.unlockKey = nil
if n.config.AutoLockManagers {
n.unlockKey = encryption.GenerateSecretKey()
}
krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
if err != nil {
return nil, err
}
log.G(ctx).Debug("generated CA key and certificate")
} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
if err != nil {
return nil, err
}
log.G(ctx).Debug("downloaded CA certificate")
}
// Obtain new certs and setup TLS certificates renewal for this node:
// - If certificates weren't present on disk, we call CreateSecurityConfig, which blocks
// until a valid certificate has been issued.
// - We wait for CreateSecurityConfig to finish since we need a certificate to operate.
// Attempt to load certificate from disk
securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
if err == nil {
log.G(ctx).WithFields(logrus.Fields{
"node.id": securityConfig.ClientTLSCreds.NodeID(),
}).Debugf("loaded TLS certificate")
} else {
if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
return nil, ErrInvalidUnlockKey
}
log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target())
securityConfig, err = rootCA.CreateSecurityConfig(ctx, krw, ca.CertificateRequestConfig{
Token: n.config.JoinToken,
Availability: n.config.Availability,
Remotes: n.remotes,
})
if err != nil {
return nil, err
}
}
}
n.Lock()
n.role = securityConfig.ClientTLSCreds.Role()
n.nodeID = securityConfig.ClientTLSCreds.NodeID()
n.roleCond.Broadcast()
n.Unlock()
return securityConfig, nil
}
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
var securityConfig *ca.SecurityConfig
krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
if err := krw.Migrate(); err != nil {
return nil, err
}
// Check if we already have a valid certificates on disk.
rootCA, err := ca.GetLocalRootCA(paths.RootCA)
if err != nil && err != ca.ErrNoLocalRootCA {
return nil, err
}
if err == nil {
clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw)
_, ok := errors.Cause(err).(ca.ErrInvalidKEK)
switch {
case err == nil:
securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds)
log.G(ctx).Debug("loaded CA and TLS certificates")
case ok:
return nil, ErrInvalidUnlockKey
case os.IsNotExist(err):
break
default:
return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
}
}
if securityConfig == nil {
if n.config.JoinAddr == "" {
// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
n.unlockKey = nil
if n.config.AutoLockManagers {
n.unlockKey = encryption.GenerateSecretKey()
}
krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
if err != nil {
return nil, err
}
log.G(ctx).Debug("generated CA key and certificate")
} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
if err != nil {
return nil, err
}
log.G(ctx).Debug("downloaded CA certificate")
}
// Obtain new certs and setup TLS certificates renewal for this node:
// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
// us to display the ID before the certificate gets issued (for potential approval).
// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
// up to date.
issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
go func() {
select {
case <-ctx.Done():
case resp := <-issueResponseChan:
log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
"node.id": resp.NodeID,
}).Debugf("loaded TLS certificate")
n.Lock()
n.nodeID = resp.NodeID
n.nodeMembership = resp.NodeMembership
n.Unlock()
close(n.certificateRequested)
}
}()
// LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS
// certificates and write them to disk
securityConfig, err = ca.LoadOrCreateSecurityConfig(
ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw)
if err != nil {
if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
return nil, ErrInvalidUnlockKey
}
return nil, err
}
}
n.Lock()
n.role = securityConfig.ClientTLSCreds.Role()
n.nodeID = securityConfig.ClientTLSCreds.NodeID()
n.nodeMembership = api.NodeMembershipAccepted
n.roleCond.Broadcast()
n.Unlock()
return securityConfig, nil
}
func TestGetRemoteCA(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
shaHash := sha256.New()
shaHash.Write(tc.RootCA.Cert)
md := shaHash.Sum(nil)
mdStr := hex.EncodeToString(md)
d, err := digest.Parse("sha256:" + mdStr)
require.NoError(t, err)
downloadedRootCA, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
require.NoError(t, err)
require.Equal(t, downloadedRootCA.Cert, tc.RootCA.Cert)
// update the test CA to include a multi-certificate bundle as the root - the digest
// we use to verify with must be the digest of the whole bundle
tmpDir, err := ioutil.TempDir("", "GetRemoteCA")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
paths := ca.NewConfigPaths(tmpDir)
otherRootCA, err := ca.CreateRootCA("other", paths.RootCA)
require.NoError(t, err)
comboCertBundle := append(tc.RootCA.Cert, otherRootCA.Cert...)
require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
cluster := store.GetCluster(tx, tc.Organization)
cluster.RootCA.CACert = comboCertBundle
cluster.RootCA.CAKey = tc.RootCA.Key
return store.UpdateCluster(tx, cluster)
}))
require.NoError(t, raftutils.PollFunc(nil, func() error {
_, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
if err == nil {
return fmt.Errorf("testca's rootca hasn't updated yet")
}
require.Contains(t, err.Error(), "remote CA does not match fingerprint")
return nil
}))
// If we provide the right digest, the root CA is updated and we can validate
// certs signed by either one
d = digest.FromBytes(comboCertBundle)
downloadedRootCA, err = ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
require.NoError(t, err)
require.Equal(t, comboCertBundle, downloadedRootCA.Cert)
require.Equal(t, 2, len(downloadedRootCA.Pool.Subjects()))
for _, rootCA := range []ca.RootCA{tc.RootCA, otherRootCA} {
krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, err := rootCA.IssueAndSaveNewCertificates(krw, "cn", "ou", "org")
require.NoError(t, err)
certPEM, _, err := krw.Read()
require.NoError(t, err)
cert, err := helpers.ParseCertificatesPEM(certPEM)
require.NoError(t, err)
chains, err := cert[0].Verify(x509.VerifyOptions{
Roots: downloadedRootCA.Pool,
})
require.NoError(t, err)
require.Len(t, chains, 1)
}
}
func TestManager(t *testing.T) {
ctx := context.Background()
temp, err := ioutil.TempFile("", "test-socket")
assert.NoError(t, err)
assert.NoError(t, temp.Close())
assert.NoError(t, os.Remove(temp.Name()))
defer os.RemoveAll(temp.Name())
stateDir, err := ioutil.TempDir("", "test-raft")
assert.NoError(t, err)
defer os.RemoveAll(stateDir)
tc := testutils.NewTestCA(t, func(p ca.CertPaths) *ca.KeyReadWriter {
return ca.NewKeyReadWriter(p, []byte("kek"), nil)
})
defer tc.Stop()
agentSecurityConfig, err := tc.NewNodeConfig(ca.WorkerRole)
assert.NoError(t, err)
agentDiffOrgSecurityConfig, err := tc.NewNodeConfigOrg(ca.WorkerRole, "another-org")
assert.NoError(t, err)
managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
m, err := New(&Config{
RemoteAPI: RemoteAddrs{ListenAddr: "127.0.0.1:0"},
ControlAPI: temp.Name(),
StateDir: stateDir,
SecurityConfig: managerSecurityConfig,
AutoLockManagers: true,
UnlockKey: []byte("kek"),
})
assert.NoError(t, err)
assert.NotNil(t, m)
tcpAddr := m.Addr()
done := make(chan error)
defer close(done)
go func() {
done <- m.Run(ctx)
}()
opts := []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(agentSecurityConfig.ClientTLSCreds),
}
conn, err := grpc.Dial(tcpAddr, opts...)
assert.NoError(t, err)
defer func() {
assert.NoError(t, conn.Close())
}()
// We have to send a dummy request to verify if the connection is actually up.
client := api.NewDispatcherClient(conn)
_, err = client.Heartbeat(ctx, &api.HeartbeatRequest{})
assert.Equal(t, dispatcher.ErrNodeNotRegistered.Error(), grpc.ErrorDesc(err))
_, err = client.Session(ctx, &api.SessionRequest{})
assert.NoError(t, err)
// Try to have a client in a different org access this manager
opts = []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(agentDiffOrgSecurityConfig.ClientTLSCreds),
}
conn2, err := grpc.Dial(tcpAddr, opts...)
assert.NoError(t, err)
defer func() {
assert.NoError(t, conn2.Close())
}()
client = api.NewDispatcherClient(conn2)
_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
assert.Contains(t, grpc.ErrorDesc(err), "Permission denied: unauthorized peer role: rpc error: code = 7 desc = Permission denied: remote certificate not part of organization")
// Verify that requests to the various GRPC services running on TCP
// are rejected if they don't have certs.
opts = []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})),
}
noCertConn, err := grpc.Dial(tcpAddr, opts...)
assert.NoError(t, err)
defer func() {
assert.NoError(t, noCertConn.Close())
}()
client = api.NewDispatcherClient(noCertConn)
_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")
controlClient := api.NewControlClient(noCertConn)
_, err = controlClient.ListNodes(context.Background(), &api.ListNodesRequest{})
assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")
raftClient := api.NewRaftMembershipClient(noCertConn)
_, err = raftClient.Join(context.Background(), &api.JoinRequest{})
assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")
opts = []grpc.DialOption{
grpc.WithTimeout(10 * time.Second),
grpc.WithTransportCredentials(managerSecurityConfig.ClientTLSCreds),
}
controlConn, err := grpc.Dial(tcpAddr, opts...)
assert.NoError(t, err)
defer func() {
assert.NoError(t, controlConn.Close())
}()
// check that the kek is added to the config
var cluster api.Cluster
m.raftNode.MemoryStore().View(func(tx store.ReadTx) {
clusters, err := store.FindClusters(tx, store.All)
require.NoError(t, err)
require.Len(t, clusters, 1)
cluster = *clusters[0]
})
require.NotNil(t, cluster)
require.Len(t, cluster.UnlockKeys, 1)
require.Equal(t, &api.EncryptionKey{
Subsystem: ca.ManagerRole,
Key: []byte("kek"),
}, cluster.UnlockKeys[0])
// Test removal of the agent node
agentID := agentSecurityConfig.ClientTLSCreds.NodeID()
assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
return store.CreateNode(tx,
&api.Node{
ID: agentID,
Certificate: api.Certificate{
Role: api.NodeRoleWorker,
CN: agentID,
},
},
)
}))
controlClient = api.NewControlClient(controlConn)
_, err = controlClient.RemoveNode(context.Background(),
&api.RemoveNodeRequest{
NodeID: agentID,
Force: true,
},
)
assert.NoError(t, err)
client = api.NewDispatcherClient(conn)
_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
assert.Contains(t, grpc.ErrorDesc(err), "removed from swarm")
m.Stop(ctx)
// After stopping we should MAY receive an error from ListenAndServe if
// all this happened before WaitForLeader completed, so don't check the
// error.
<-done
}
func TestKeyReadWriterViewAndUpdateHeaders(t *testing.T) {
cert, key, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir))
// write a key with headers to the key to make sure it gets passed when reading/writing headers
keyBlock, _ := pem.Decode(key)
require.NotNil(t, keyBlock)
keyBlock.Headers = map[string]string{"hello": "world"}
key = pem.EncodeToMemory(keyBlock)
require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644))
require.NoError(t, ioutil.WriteFile(path.Node.Key, key, 0600))
// if the update headers callback function fails, updating headers fails
k := ca.NewKeyReadWriter(path.Node, nil, nil)
err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
require.Nil(t, h)
return nil, fmt.Errorf("nope")
})
require.Error(t, err)
require.Equal(t, "nope", err.Error())
// updating headers succeed and is called with the latest kek data
err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
require.Nil(t, h)
return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
return map[string]string{"updated": "headers"}, nil
}}, nil
})
require.NoError(t, err)
k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
require.Equal(t, map[string]string{"updated": "headers"}, h)
require.Equal(t, ca.KEKData{}, k)
return testHeaders{}, nil
}})
_, _, err = k.Read()
require.NoError(t, err)
// we can also update headers on an encrypted key
k = ca.NewKeyReadWriter(path.Node, []byte("kek"), nil)
require.NoError(t, k.Write(cert, key, nil))
err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
require.Nil(t, h)
return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
require.Equal(t, ca.KEKData{KEK: []byte("kek")}, kek)
return map[string]string{"updated": "headers"}, nil
}}, nil
})
require.NoError(t, err)
k = ca.NewKeyReadWriter(path.Node, []byte("kek"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
require.Equal(t, map[string]string{"updated": "headers"}, h)
require.Equal(t, ca.KEKData{KEK: []byte("kek")}, k)
return testHeaders{}, nil
}})
_, _, err = k.Read()
require.NoError(t, err)
}
// can read and write tls keys that aren't encrypted, and that are encrypted. without
// a pem header manager, the headers are all preserved and not overwritten
func TestKeyReadWriter(t *testing.T) {
cert, key, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
expectedKey := key
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created
checkCanReadWithKEK := func(kek []byte) *ca.KeyReadWriter {
k := ca.NewKeyReadWriter(path.Node, kek, nil)
readCert, readKey, err := k.Read()
require.NoError(t, err)
require.Equal(t, cert, readCert)
require.Equal(t, expectedKey, readKey, "Expected %s, Got %s", string(expectedKey), string(readKey))
return k
}
k := ca.NewKeyReadWriter(path.Node, nil, nil)
// can't read things that don't exist
_, _, err = k.Read()
require.Error(t, err)
// can write an unencrypted key with no updates
require.NoError(t, k.Write(cert, expectedKey, nil))
// can read unencrypted
k = checkCanReadWithKEK(nil)
_, kekData := k.GetCurrentState()
require.EqualValues(t, 0, kekData.Version) // the first version was 0
// write a key with headers to the key to make sure they're cleaned
keyBlock, _ := pem.Decode(expectedKey)
require.NotNil(t, keyBlock)
keyBlock.Headers = map[string]string{"hello": "world"}
expectedKey = pem.EncodeToMemory(keyBlock)
// write a version, but that's not what we'd expect back once we read
keyBlock.Headers["kek-version"] = "8"
require.NoError(t, ioutil.WriteFile(path.Node.Key, pem.EncodeToMemory(keyBlock), 0600))
// if a kek is provided, we can still read unencrypted keys, and read
// the provided version
k = checkCanReadWithKEK([]byte("original kek"))
_, kekData = k.GetCurrentState()
require.EqualValues(t, 8, kekData.Version)
// we can update the kek and write at the same time
require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("new kek!"), Version: 3}))
// the same kek can still read, and will continue to write with this key if
// no further kek updates are provided
_, _, err = k.Read()
require.NoError(t, err)
require.NoError(t, k.Write(cert, expectedKey, nil))
expectedKey = key
// without the right kek, we can't read
k = ca.NewKeyReadWriter(path.Node, []byte("original kek"), nil)
_, _, err = k.Read()
require.Error(t, err)
// same new key, just for sanity
k = checkCanReadWithKEK([]byte("new kek!"))
_, kekData = k.GetCurrentState()
require.EqualValues(t, 3, kekData.Version)
// we can also change the kek back to nil, which means the key is unencrypted
require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: nil}))
k = checkCanReadWithKEK(nil)
_, kekData = k.GetCurrentState()
require.EqualValues(t, 0, kekData.Version)
}
// KeyReaderWriter makes a call to a get headers updater, if write is called,
// and set headers, if read is called. The KEK version header is always preserved
// no matter what.
func TestKeyReadWriterWithPemHeaderManager(t *testing.T) {
cert, key, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
// write a key with headers to the key to make sure it gets overwritten
keyBlock, _ := pem.Decode(key)
require.NotNil(t, keyBlock)
keyBlock.Headers = map[string]string{"hello": "world"}
key = pem.EncodeToMemory(keyBlock)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created
// if if getting new headers fail, writing a key fails, and the key does not rotate
var count int
badKEKData := ca.KEKData{KEK: []byte("failed kek"), Version: 3}
k := ca.NewKeyReadWriter(path.Node, nil, testHeaders{newHeaders: func(k ca.KEKData) (map[string]string, error) {
if count == 0 {
count++
require.Equal(t, badKEKData, k)
return nil, fmt.Errorf("fail")
}
require.Equal(t, ca.KEKData{}, k)
return nil, nil
}})
// first write will fail
require.Error(t, k.Write(cert, key, &badKEKData))
// the stored kek data will be not be updated because the write failed
_, kekData := k.GetCurrentState()
require.Equal(t, ca.KEKData{}, kekData)
// second write will succeed, using the original kek (nil)
require.NoError(t, k.Write(cert, key, nil))
var (
headers map[string]string
kek ca.KEKData
)
// if setting headers fail, reading fails
k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(map[string]string, ca.KEKData) (ca.PEMKeyHeaders, error) {
return nil, fmt.Errorf("nope")
}})
_, _, err = k.Read()
require.Error(t, err)
k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
headers = h
kek = k
return testHeaders{}, nil
}})
_, _, err = k.Read()
require.NoError(t, err)
require.Equal(t, ca.KEKData{}, kek)
require.Equal(t, keyBlock.Headers, headers)
// writing new headers is called with existing headers, and will write a key that has the headers
// returned by the header update function
k = ca.NewKeyReadWriter(path.Node, []byte("oldKek"), testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
require.Equal(t, []byte("newKEK"), kek.KEK)
return map[string]string{"updated": "headers"}, nil
}})
require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("newKEK"), Version: 2}))
// make sure headers were correctly set
k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
headers = h
kek = k
return testHeaders{}, nil
}})
_, _, err = k.Read()
require.NoError(t, err)
require.Equal(t, ca.KEKData{KEK: []byte("newKEK"), Version: 2}, kek)
_, kekData = k.GetCurrentState()
require.Equal(t, kek, kekData)
require.Equal(t, map[string]string{"updated": "headers"}, headers)
}
func TestIsStateDirty(t *testing.T) {
ctx := context.Background()
temp, err := ioutil.TempFile("", "test-socket")
assert.NoError(t, err)
assert.NoError(t, temp.Close())
assert.NoError(t, os.Remove(temp.Name()))
defer os.RemoveAll(temp.Name())
stateDir, err := ioutil.TempDir("", "test-raft")
assert.NoError(t, err)
defer os.RemoveAll(stateDir)
tc := testutils.NewTestCA(t, func(p ca.CertPaths) *ca.KeyReadWriter {
return ca.NewKeyReadWriter(p, []byte("kek"), nil)
})
defer tc.Stop()
managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
assert.NoError(t, err)
m, err := New(&Config{
RemoteAPI: &RemoteAddrs{ListenAddr: "127.0.0.1:0"},
ControlAPI: temp.Name(),
StateDir: stateDir,
SecurityConfig: managerSecurityConfig,
AutoLockManagers: true,
UnlockKey: []byte("kek"),
})
assert.NoError(t, err)
assert.NotNil(t, m)
go m.Run(ctx)
defer m.Stop(ctx, false)
// State should never be dirty just after creating the manager
isDirty, err := m.IsStateDirty()
assert.NoError(t, err)
assert.False(t, isDirty)
// Wait for cluster and node to be created.
watch, cancel := state.Watch(m.raftNode.MemoryStore().WatchQueue())
defer cancel()
<-watch
<-watch
// Updating the node should not cause the state to become dirty
assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
node := store.GetNode(tx, m.config.SecurityConfig.ClientTLSCreds.NodeID())
require.NotNil(t, node)
node.Spec.Availability = api.NodeAvailabilityPause
return store.UpdateNode(tx, node)
}))
isDirty, err = m.IsStateDirty()
assert.NoError(t, err)
assert.False(t, isDirty)
// Adding a service should cause the state to become dirty
assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
return store.CreateService(tx, &api.Service{ID: "foo"})
}))
isDirty, err = m.IsStateDirty()
assert.NoError(t, err)
assert.True(t, isDirty)
}
// NewTestCA is a helper method that creates a TestCA and a bunch of default
// connections and security configs.
func NewTestCA(t *testing.T, krwGenerators ...func(ca.CertPaths) *ca.KeyReadWriter) *TestCA {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
s := store.NewMemoryStore(nil)
paths := ca.NewConfigPaths(tempBaseDir)
organization := identity.NewID()
rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
var (
externalSigningServer *ExternalSigningServer
externalCAs []*api.ExternalCA
)
if External {
// Start the CA API server.
externalSigningServer, err = NewExternalSigningServer(rootCA, tempBaseDir)
assert.NoError(t, err)
externalCAs = []*api.ExternalCA{
{
Protocol: api.ExternalCA_CAProtocolCFSSL,
URL: externalSigningServer.URL,
},
}
}
krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
if len(krwGenerators) > 0 {
krw = krwGenerators[0](paths.Node)
}
managerConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, organization, "", External)
assert.NoError(t, err)
managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, "swarm-test-org-2", "", External)
assert.NoError(t, err)
workerConfig, err := genSecurityConfig(s, rootCA, krw, ca.WorkerRole, organization, "", External)
assert.NoError(t, err)
l, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
clientOpts := append(baseOpts, grpc.WithTransportCredentials(workerConfig.ClientTLSCreds))
managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds))
managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds))
conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...)
assert.NoError(t, err)
conn2, err := grpc.Dial(l.Addr().String(), clientOpts...)
assert.NoError(t, err)
conn3, err := grpc.Dial(l.Addr().String(), managerOpts...)
assert.NoError(t, err)
conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...)
assert.NoError(t, err)
serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)}
grpcServer := grpc.NewServer(serverOpts...)
managerToken := ca.GenerateJoinToken(&rootCA)
workerToken := ca.GenerateJoinToken(&rootCA)
createClusterObject(t, s, organization, workerToken, managerToken, externalCAs...)
caServer := ca.NewServer(s, managerConfig)
caServer.SetReconciliationRetryInterval(50 * time.Millisecond)
api.RegisterCAServer(grpcServer, caServer)
api.RegisterNodeCAServer(grpcServer, caServer)
ctx := context.Background()
go grpcServer.Serve(l)
go caServer.Run(ctx)
// Wait for caServer to be ready to serve
<-caServer.Ready()
remotes := remotes.NewRemotes(api.Peer{Addr: l.Addr().String()})
caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)}
nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)}
conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4}
return &TestCA{
RootCA: rootCA,
ExternalSigningServer: externalSigningServer,
MemoryStore: s,
TempDir: tempBaseDir,
Organization: organization,
Paths: paths,
Context: ctx,
CAClients: caClients,
NodeCAClients: nodeCAClients,
Conns: conns,
Server: grpcServer,
CAServer: caServer,
WorkerToken: workerToken,
ManagerToken: managerToken,
ConnBroker: connectionbroker.New(remotes),
KeyReadWriter: krw,
}
}