func TestBootstrapCluster(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
err = ca.BootstrapCluster(tempBaseDir)
assert.NoError(t, err)
perms, err := permbits.Stat(paths.RootCA.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
perms, err = permbits.Stat(paths.RootCA.Key)
assert.NoError(t, err)
assert.False(t, perms.GroupRead())
assert.False(t, perms.OtherRead())
perms, err = permbits.Stat(paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
perms, err = permbits.Stat(paths.Node.Key)
assert.NoError(t, err)
assert.False(t, perms.GroupRead())
assert.False(t, perms.OtherRead())
}
func (n *Node) loadCertificates() error {
certDir := filepath.Join(n.config.StateDir, "certificates")
rootCA, err := ca.GetLocalRootCA(certDir)
if err != nil {
if err == ca.ErrNoLocalRootCA {
return nil
}
return err
}
configPaths := ca.NewConfigPaths(certDir)
clientTLSCreds, _, err := ca.LoadTLSCreds(rootCA, configPaths.Node)
if err != nil {
if os.IsNotExist(err) {
return nil
}
return errors.Wrapf(err, "error while loading TLS Certificate in %s", configPaths.Node.Cert)
}
// todo: try csr if no cert or store nodeID/role in some other way
n.Lock()
n.role = clientTLSCreds.Role()
n.nodeID = clientTLSCreds.NodeID()
n.nodeMembership = api.NodeMembershipAccepted
n.roleCond.Broadcast()
n.Unlock()
return nil
}
// If there is nothing on disk and no join addr, we create a new CA and a new set of TLS certs.
// If AutoLockManagers is enabled, the TLS key is encrypted with a randomly generated lock key.
func TestLoadSecurityConfigNewNode(t *testing.T) {
for _, autoLockManagers := range []bool{true, false} {
tempdir, err := ioutil.TempDir("", "test-new-node")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
node, err := New(&Config{
StateDir: tempdir,
AutoLockManagers: autoLockManagers,
})
require.NoError(t, err)
securityConfig, err := node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
unencryptedReader := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, _, err = unencryptedReader.Read()
if !autoLockManagers {
require.NoError(t, err)
} else {
require.IsType(t, ca.ErrInvalidKEK{}, err)
}
}
}
func TestParseValidateAndSignCSR(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
csr, _, err := ca.GenerateAndWriteNewKey(paths.Node)
assert.NoError(t, err)
signedCert, err := rootCA.ParseValidateAndSignCSR(csr, "CN", "OU", "ORG")
assert.NoError(t, err)
assert.NotNil(t, signedCert)
parsedCert, err := helpers.ParseCertificatesPEM(signedCert)
assert.NoError(t, err)
assert.Equal(t, 2, len(parsedCert))
assert.Equal(t, "CN", parsedCert[0].Subject.CommonName)
assert.Equal(t, 1, len(parsedCert[0].Subject.OrganizationalUnit))
assert.Equal(t, "OU", parsedCert[0].Subject.OrganizationalUnit[0])
assert.Equal(t, 3, len(parsedCert[0].Subject.Names))
assert.Equal(t, "ORG", parsedCert[0].Subject.Organization[0])
assert.Equal(t, "rootCN", parsedCert[1].Subject.CommonName)
}
func TestKeyReadWriterMigrate(t *testing.T) {
cert, key, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir))
// if the key exists in an old location, migrate it from there.
tempKeyPath := filepath.Join(filepath.Dir(path.Node.Key), "."+filepath.Base(path.Node.Key))
require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644))
require.NoError(t, ioutil.WriteFile(tempKeyPath, key, 0600))
krw := ca.NewKeyReadWriter(path.Node, nil, nil)
require.NoError(t, krw.Migrate())
_, err = os.Stat(tempKeyPath)
require.True(t, os.IsNotExist(err)) // it's been moved to the right place
_, _, err = krw.Read()
require.NoError(t, err)
// migrate does not affect any existing files
dirList, err := ioutil.ReadDir(filepath.Dir(path.Node.Key))
require.NoError(t, err)
require.NoError(t, krw.Migrate())
dirList2, err := ioutil.ReadDir(filepath.Dir(path.Node.Key))
require.NoError(t, err)
require.Equal(t, dirList, dirList2)
_, _, err = krw.Read()
require.NoError(t, err)
}
func TestGenerateAndSignNewTLSCert(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
_, err = ca.GenerateAndSignNewTLSCert(rootCA, "CN", "OU", "ORG", paths.Node)
assert.NoError(t, err)
perms, err := permbits.Stat(paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
perms, err = permbits.Stat(paths.Node.Key)
assert.NoError(t, err)
assert.False(t, perms.GroupRead())
assert.False(t, perms.OtherRead())
certBytes, err := ioutil.ReadFile(paths.Node.Cert)
assert.NoError(t, err)
certs, err := helpers.ParseCertificatesPEM(certBytes)
assert.NoError(t, err)
assert.Len(t, certs, 2)
assert.Equal(t, "CN", certs[0].Subject.CommonName)
assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0])
assert.Equal(t, "ORG", certs[0].Subject.Organization[0])
assert.Equal(t, "rootCN", certs[1].Subject.CommonName)
}
// The TLS KEK and the KEK for the headers should be in sync, and so failing
// to decrypt the TLS key should be mean we won't be able to decrypt the headers.
// However, the TLS Key encryption uses AES-256-CBC (golang as of 1.7.x does not seem
// to support GCM, so no cipher modes with digests) so sometimes decrypting with
// the wrong passphrase will not result in an error. This means we will ultimately
// have to rely on the header encryption mechanism, which does include a digest, to
// determine if the KEK is valid.
func TestDecryptTLSKeyFalsePositive(t *testing.T) {
badKey := []byte(`
-----BEGIN EC PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-256-CBC,e7927e79e748233776c03c2eb7275f09
kek-version: 392
raft-dek: CAESMBrzZ0gNVPe3FRs42743q8RtkUBrK1ICQpHWX2vdQ8iqSKt1WoKdFDFD2r28LYAVLxoYQguwHbijMx9k+BALUNBAI3s199S5tvnr
JfGenNvzm++AvsOh+UmcBY+JgI6lnfzaCB68agmlmEZYLYi5tqtAU7gif6VIJpCW
+Pj23Fzkw8sKKOOBeapSC5lp+Cjx9OsCci/R9xrdx+uxnnzKJNxOB/qzqcQfZDMh
id2LxdliFcPEk/Yj5gNGpT0UMFJ4G52enbOwOru46f0=
-----END EC PRIVATE KEY-----
`)
// not actually a real swarm cert - generated a cert corresponding to the key that expires in 20 years
matchingCert := []byte(`
-----BEGIN CERTIFICATE-----
MIIB9jCCAZygAwIBAgIRAIdzF3Z9VT2OXbRvEw5cR68wCgYIKoZIzj0EAwIwYDEi
MCAGA1UEChMZbWRwMXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dh
cm0tbWFuYWdlcjEiMCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTAg
GA8wMDAxMDEwMTAwMDAwMFoXDTM2MTEwODA2MjMwMlowYDEiMCAGA1UEChMZbWRw
MXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dhcm0tbWFuYWdlcjEi
MCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTBZMBMGByqGSM49AgEG
CCqGSM49AwEHA0IABGOivD25E/zcupRFQdKOKbPHS9Mx7JlUhlWnl0iR0K5VhVIU
XjUHt98GuX6gDjs4yUzEKSGxYPsSYlnG9zQqbQSjNTAzMA4GA1UdDwEB/wQEAwIF
oDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMAoGCCqGSM49BAMC
A0gAMEUCIQDWtjg1ITGznQILipaEe70G/NgZAOtFfuPXTVkUl3el+wIgSVOVKB/Q
O0T3aXuZGYNyh//KqAoA3erCmh6HauMz84Y=
-----END CERTIFICATE-----
`)
var wrongKEK []byte // empty passphrase doesn't decrypt without errors
falsePositiveKEK, err := base64.RawStdEncoding.DecodeString("bIQgLAAMoGCrHdjMLVhEVqnYTAM7ZNF2xWMiwtw7AiQ")
require.NoError(t, err)
realKEK, err := base64.RawStdEncoding.DecodeString("fDg9YejLnMjU+FpulWR62oJLzVpkD2j7VQuP5xiK9QA")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter-false-positive-decryption")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(tempdir)
require.NoError(t, ioutil.WriteFile(path.Node.Key, badKey, 0600))
require.NoError(t, ioutil.WriteFile(path.Node.Cert, matchingCert, 0644))
krw := ca.NewKeyReadWriter(path.Node, wrongKEK, RaftDEKData{})
_, _, err = krw.Read()
require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err))
krw = ca.NewKeyReadWriter(path.Node, falsePositiveKEK, RaftDEKData{})
_, _, err = krw.Read()
require.Error(t, err)
require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err))
krw = ca.NewKeyReadWriter(path.Node, realKEK, RaftDEKData{})
_, _, err = krw.Read()
require.NoError(t, err)
}
// If we abort in the middle of writing the key and cert, such that only the key is written
// to the final location, when we read we can still read the cert from the temporary
// location.
func TestTwoPhaseReadWrite(t *testing.T) {
cert1, _, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
cert2, key2, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir))
krw := ca.NewKeyReadWriter(path.Node, nil, nil)
// put a directory in the location where the cert goes, so we can't actually move
// the cert from the temporary location to the final location.
require.NoError(t, os.Mkdir(filepath.Join(path.Node.Cert), 0755))
require.Error(t, krw.Write(cert2, key2, nil))
// the temp cert file should exist
tempCertPath := filepath.Join(filepath.Dir(path.Node.Cert), "."+filepath.Base(path.Node.Cert))
readCert, err := ioutil.ReadFile(tempCertPath)
require.NoError(t, err)
require.Equal(t, cert2, readCert)
// remove the directory, to simulate it failing to write the first time
os.RemoveAll(path.Node.Cert)
readCert, readKey, err := krw.Read()
require.NoError(t, err)
require.Equal(t, cert2, readCert)
require.Equal(t, key2, readKey)
// the cert should have been moved to its proper location
_, err = os.Stat(tempCertPath)
require.True(t, os.IsNotExist(err))
// If the cert in the proper location doesn't match the key, the temp location is checked
require.NoError(t, ioutil.WriteFile(tempCertPath, cert2, 0644))
require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert1, 0644))
readCert, readKey, err = krw.Read()
require.NoError(t, err)
require.Equal(t, cert2, readCert)
require.Equal(t, key2, readKey)
// the cert should have been moved to its proper location
_, err = os.Stat(tempCertPath)
require.True(t, os.IsNotExist(err))
// If the cert in the temp location also doesn't match, the failure matching the
// correctly-located cert is returned
require.NoError(t, os.Remove(path.Node.Cert))
require.NoError(t, ioutil.WriteFile(tempCertPath, cert1, 0644)) // mismatching cert
_, _, err = krw.Read()
require.True(t, os.IsNotExist(err))
// the cert should have been removed
_, err = os.Stat(tempCertPath)
require.True(t, os.IsNotExist(err))
}
// If there are CAs and TLS certs on disk, it tries to load and fails if there
// are any errors, even if a join token is provided.
func TestLoadSecurityConfigLoadFromDisk(t *testing.T) {
tempdir, err := ioutil.TempDir("", "test-load-node-tls")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
tc := cautils.NewTestCA(t)
defer tc.Stop()
peer, err := tc.ConnBroker.Remotes().Select()
require.NoError(t, err)
// Load successfully with valid passphrase
rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
krw := ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil)
require.NoError(t, err)
_, err = rootCA.IssueAndSaveNewCertificates(krw, identity.NewID(), ca.WorkerRole, identity.NewID())
require.NoError(t, err)
node, err := New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
UnlockKey: []byte("passphrase"),
})
require.NoError(t, err)
securityConfig, err := node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
// Invalid passphrase
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.Equal(t, ErrInvalidUnlockKey, err)
// Invalid CA
rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
UnlockKey: []byte("passphrase"),
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.IsType(t, x509.UnknownAuthorityError{}, errors.Cause(err))
}
func TestLoadSecurityConfigIncorrectPassphrase(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
paths := ca.NewConfigPaths(tc.TempDir)
_, err := tc.RootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(paths.Node, []byte("kek"), nil),
"nodeID", ca.WorkerRole, tc.Organization)
require.NoError(t, err)
_, err = ca.LoadSecurityConfig(tc.Context, tc.RootCA, ca.NewKeyReadWriter(paths.Node, nil, nil))
require.IsType(t, ca.ErrInvalidKEK{}, err)
}
func TestNewRootCAWithPassphrase(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
defer os.Setenv(ca.PassphraseENVVar, "")
defer os.Setenv(ca.PassphraseENVVarPrev, "")
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
// Ensure that we're encrypting the Key bytes out of NewRoot if there
// is a passphrase set as an env Var
os.Setenv(ca.PassphraseENVVar, "password1")
newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.NotEqual(t, rootCA.Key, newRootCA.Key)
assert.Equal(t, rootCA.Cert, newRootCA.Cert)
assert.NotContains(t, string(rootCA.Key), string(newRootCA.Key))
assert.Contains(t, string(newRootCA.Key), "Proc-Type: 4,ENCRYPTED")
// Ensure that we're decrypting the Key bytes out of NewRoot if there
// is a passphrase set as an env Var
anotherNewRootCA, err := ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, newRootCA, anotherNewRootCA)
assert.NotContains(t, string(rootCA.Key), string(anotherNewRootCA.Key))
assert.Contains(t, string(anotherNewRootCA.Key), "Proc-Type: 4,ENCRYPTED")
// Ensure that we cant decrypt the Key bytes out of NewRoot if there
// is a wrong passphrase set as an env Var
os.Setenv(ca.PassphraseENVVar, "password2")
anotherNewRootCA, err = ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.Error(t, err)
// Ensure that we cant decrypt the Key bytes out of NewRoot if there
// is a wrong passphrase set as an env Var
os.Setenv(ca.PassphraseENVVarPrev, "password2")
anotherNewRootCA, err = ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.Error(t, err)
// Ensure that we can decrypt the Key bytes out of NewRoot if there
// is a wrong passphrase set as an env Var, but a valid as Prev
os.Setenv(ca.PassphraseENVVarPrev, "password1")
anotherNewRootCA, err = ca.NewRootCA(newRootCA.Cert, newRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, newRootCA, anotherNewRootCA)
assert.NotContains(t, string(rootCA.Key), string(anotherNewRootCA.Key))
assert.Contains(t, string(anotherNewRootCA.Key), "Proc-Type: 4,ENCRYPTED")
}
func TestKeyReadWriterViewAndRotateKEK(t *testing.T) {
cert, key, err := testutils.CreateRootCertAndKey("cn")
require.NoError(t, err)
tempdir, err := ioutil.TempDir("", "KeyReadWriter")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
path := ca.NewConfigPaths(filepath.Join(tempdir))
// write a key with headers to the key to make sure it gets passed when reading/writing headers
keyBlock, _ := pem.Decode(key)
require.NotNil(t, keyBlock)
keyBlock.Headers = map[string]string{"hello": "world"}
key = pem.EncodeToMemory(keyBlock)
require.NoError(t, ca.NewKeyReadWriter(path.Node, nil, nil).Write(cert, key, nil))
// if if getting new kek and headers fail, rotating a KEK fails, and the kek does not rotate
k := ca.NewKeyReadWriter(path.Node, nil, nil)
require.Error(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
require.Equal(t, ca.KEKData{}, k)
require.Nil(t, h)
return ca.KEKData{}, nil, fmt.Errorf("Nope")
}))
// writing new headers will write a key that has the headers returned by the header update function
k = ca.NewKeyReadWriter(path.Node, []byte("oldKEK"), nil)
require.NoError(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
require.Equal(t, ca.KEKData{KEK: []byte("oldKEK")}, k)
require.Nil(t, h)
return ca.KEKData{KEK: []byte("newKEK"), Version: uint64(2)},
testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
require.Equal(t, []byte("newKEK"), kek.KEK)
return map[string]string{"updated": "headers"}, nil
}}, nil
}))
// ensure the key has been re-encrypted and we can read it
k = ca.NewKeyReadWriter(path.Node, nil, nil)
_, _, err = k.Read()
require.Error(t, err)
var headers map[string]string
k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, _ ca.KEKData) (ca.PEMKeyHeaders, error) {
headers = h
return testHeaders{}, nil
}})
_, _, err = k.Read()
require.NoError(t, err)
require.Equal(t, map[string]string{"updated": "headers"}, headers)
}
func TestRaftDEKManagerUpdateKeys(t *testing.T) {
tempDir, err := ioutil.TempDir("", "manager-update-keys-")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
paths := ca.NewConfigPaths(tempDir)
cert, key, err := cautils.CreateRootCertAndKey("cn")
require.NoError(t, err)
keys := raft.EncryptionKeys{
CurrentDEK: []byte("key1"),
PendingDEK: []byte("key2"),
}
krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{
EncryptionKeys: keys,
NeedsRotation: true,
})
require.NoError(t, krw.Write(cert, key, nil))
dekManager, err := NewRaftDEKManager(krw)
require.NoError(t, err)
newKeys := raft.EncryptionKeys{
CurrentDEK: []byte("new current"),
}
require.NoError(t, dekManager.UpdateKeys(newKeys))
// don't run GetKeys, because NeedsRotation is true and it'd just generate a new one
h, _ := krw.GetCurrentState()
dekData, ok := h.(RaftDEKData)
require.True(t, ok)
require.True(t, dekData.NeedsRotation)
// UpdateKeys so there is no CurrentDEK: all the headers should be wiped out
require.NoError(t, dekManager.UpdateKeys(raft.EncryptionKeys{}))
require.Equal(t, raft.EncryptionKeys{}, dekManager.GetKeys())
require.False(t, dekManager.NeedsRotation())
h, _ = krw.GetCurrentState()
require.Nil(t, h)
keyBytes, err := ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
keyBlock, _ := pem.Decode(keyBytes)
require.NotNil(t, keyBlock)
// the only header remaining should be the kek version
require.Len(t, keyBlock.Headers, 1)
require.Contains(t, keyBlock.Headers, "kek-version")
}
func TestNewRootCA(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, rootCA, newRootCA)
}
func TestGetLocalRootCAInvalidCert(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// Write some garbage to the CA cert
require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, []byte(`-----BEGIN CERTIFICATE-----\n
some random garbage\n
-----END CERTIFICATE-----`), 0644))
_, err = ca.GetLocalRootCA(paths.RootCA)
require.Error(t, err)
}
// newNode creates new node with specific role(manager or agent) and joins to
// existing cluster. if joinAddr is empty string, then new cluster will be initialized.
// It uses TestExecutor as executor. If lateBind is set, the remote API port is not
// bound. If rootCA is set, this root is used to bootstrap the node's TLS certs.
func newTestNode(joinAddr, joinToken string, lateBind bool, rootCA *ca.RootCA) (*testNode, error) {
tmpDir, err := ioutil.TempDir("", "swarmkit-integration-")
if err != nil {
return nil, err
}
cAddr := filepath.Join(tmpDir, "control.sock")
cfg := &node.Config{
ListenControlAPI: cAddr,
JoinAddr: joinAddr,
StateDir: tmpDir,
Executor: &TestExecutor{},
JoinToken: joinToken,
}
if !lateBind {
cfg.ListenRemoteAPI = "127.0.0.1:0"
}
if rootCA != nil {
certDir := filepath.Join(tmpDir, "certificates")
if err := os.MkdirAll(certDir, 0700); err != nil {
return nil, err
}
certPaths := ca.NewConfigPaths(certDir)
if err := ioutil.WriteFile(certPaths.RootCA.Cert, rootCA.Cert, 0644); err != nil {
return nil, err
}
if err := ioutil.WriteFile(certPaths.RootCA.Key, rootCA.Key, 0600); err != nil {
return nil, err
}
// generate TLS certs for this manager for bootstrapping, else the node will generate its own CA
_, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(certPaths.Node, nil, nil),
identity.NewID(), ca.ManagerRole, identity.NewID())
if err != nil {
return nil, err
}
}
node, err := node.New(cfg)
if err != nil {
return nil, err
}
return &testNode{
config: cfg,
node: node,
stateDir: tmpDir,
}, nil
}
func TestNewRootCABundle(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// Write one root CA to disk, keep the bytes
secondRootCA, err := ca.CreateAndWriteRootCA("rootCN2", paths.RootCA)
assert.NoError(t, err)
// Overwrite the first root CA on disk
firstRootCA, err := ca.CreateAndWriteRootCA("rootCN1", paths.RootCA)
assert.NoError(t, err)
// Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle
bundle := append(firstRootCA.Cert, secondRootCA.Cert...)
err = ioutil.WriteFile(paths.RootCA.Cert, bundle, 0644)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(bundle, firstRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, bundle, newRootCA.Cert)
assert.Equal(t, 2, len(newRootCA.Pool.Subjects()))
// Now load the bundle from disk
diskRootCA, err := ca.GetLocalRootCA(tempBaseDir)
assert.NoError(t, err)
assert.Equal(t, bundle, diskRootCA.Cert)
assert.Equal(t, 2, len(diskRootCA.Pool.Subjects()))
// If I use GenerateAndSignNewTLSCert to sign certs, I'll get the correct CA in the chain
_, err = ca.GenerateAndSignNewTLSCert(diskRootCA, "CN", "OU", "ORG", paths.Node)
assert.NoError(t, err)
certBytes, err := ioutil.ReadFile(paths.Node.Cert)
assert.NoError(t, err)
certs, err := helpers.ParseCertificatesPEM(certBytes)
assert.NoError(t, err)
assert.Len(t, certs, 2)
assert.Equal(t, "CN", certs[0].Subject.CommonName)
assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0])
assert.Equal(t, "ORG", certs[0].Subject.Organization[0])
assert.Equal(t, "rootCN1", certs[1].Subject.CommonName)
}
func TestEncryptECPrivateKey(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
_, key, err := ca.GenerateAndWriteNewKey(paths.Node)
assert.NoError(t, err)
encryptedKey, err := ca.EncryptECPrivateKey(key, "passphrase")
assert.NoError(t, err)
keyBlock, _ := pem.Decode(encryptedKey)
assert.NotNil(t, keyBlock)
assert.Equal(t, keyBlock.Headers["Proc-Type"], "4,ENCRYPTED")
assert.Contains(t, keyBlock.Headers["DEK-Info"], "AES-256-CBC")
}
func TestCreateRootCA(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
_, err = ca.CreateRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
perms, err := permbits.Stat(paths.RootCA.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
_, err = permbits.Stat(paths.RootCA.Key)
assert.True(t, os.IsNotExist(err))
}
func TestGetLocalRootCAInvalidKey(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// Create the local Root CA to ensure that we can reload it correctly.
_, err = ca.CreateRootCA("rootCN", paths.RootCA)
require.NoError(t, err)
// Write some garbage to the root key - this will cause the loading to fail
require.NoError(t, ioutil.WriteFile(paths.RootCA.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n
some random garbage\n
-----END EC PRIVATE KEY-----`), 0600))
_, err = ca.GetLocalRootCA(paths.RootCA)
require.Error(t, err)
}
func TestCreateAndWriteRootCAExpiry(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
// Convert the certificate into an object to create a RootCA
parsedCert, err := helpers.ParseCertificatePEM(rootCA.Cert)
assert.NoError(t, err)
duration, err := time.ParseDuration(ca.RootCAExpiration)
assert.NoError(t, err)
assert.True(t, time.Now().Add(duration).AddDate(0, -1, 0).Before(parsedCert.NotAfter))
}
func TestGetLocalRootCA(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// First, try to load the local Root CA with the certificate missing.
_, err = ca.GetLocalRootCA(tempBaseDir)
assert.Equal(t, ca.ErrNoLocalRootCA, err)
// Create the local Root CA to ensure that we can reload it correctly.
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.True(t, rootCA.CanSign())
assert.NoError(t, err)
rootCA2, err := ca.GetLocalRootCA(tempBaseDir)
assert.NoError(t, err)
assert.True(t, rootCA2.CanSign())
assert.Equal(t, rootCA, rootCA2)
// Try again, this time without a private key.
assert.NoError(t, os.Remove(paths.RootCA.Key))
rootCA3, err := ca.GetLocalRootCA(tempBaseDir)
assert.NoError(t, err)
assert.False(t, rootCA3.CanSign())
assert.Equal(t, rootCA.Cert, rootCA3.Cert)
// Try with a private key that does not match the CA cert public key.
privKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
assert.NoError(t, err)
privKeyBytes, err := x509.MarshalECPrivateKey(privKey)
assert.NoError(t, err)
privKeyPem := pem.EncodeToMemory(&pem.Block{
Type: "EC PRIVATE KEY",
Bytes: privKeyBytes,
})
assert.NoError(t, ioutil.WriteFile(paths.RootCA.Key, privKeyPem, os.FileMode(0600)))
_, err = ca.GetLocalRootCA(tempBaseDir)
assert.EqualError(t, err, "certificate key mismatch")
}
func main() {
// Create root material within the current directory.
rootPaths := ca.CertPaths{
Cert: filepath.Join("ca", "root.crt"),
Key: filepath.Join("ca", "root.key"),
}
// Initialize the Root CA.
rootCA, err := ca.CreateRootCA("external-ca-example", rootPaths)
if err != nil {
logrus.Fatalf("unable to initialize Root CA: %s", err)
}
// Create the initial manager node credentials.
nodeConfigPaths := ca.NewConfigPaths("certificates")
clusterID := identity.NewID()
nodeID := identity.NewID()
kw := ca.NewKeyReadWriter(nodeConfigPaths.Node, nil, nil)
if _, err := rootCA.IssueAndSaveNewCertificates(kw, nodeID, ca.ManagerRole, clusterID); err != nil {
logrus.Fatalf("unable to create initial manager node credentials: %s", err)
}
// And copy the Root CA certificate into the node config path for its
// CA.
ioutil.WriteFile(nodeConfigPaths.RootCA.Cert, rootCA.Cert, os.FileMode(0644))
server, err := testutils.NewExternalSigningServer(rootCA, "ca")
if err != nil {
logrus.Fatalf("unable to start server: %s", err)
}
defer server.Stop()
logrus.Infof("Now run: swarmd -d . --listen-control-api ./swarmd.sock --external-ca protocol=cfssl,url=%s", server.URL)
sigC := make(chan os.Signal, 1)
signal.Notify(sigC, syscall.SIGTERM, syscall.SIGINT)
<-sigC
}
func TestGenerateAndWriteNewKey(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
csr, key, err := ca.GenerateAndWriteNewKey(paths.Node)
assert.NoError(t, err)
assert.NotNil(t, csr)
assert.NotNil(t, key)
perms, err := permbits.Stat(paths.Node.Key)
assert.NoError(t, err)
assert.False(t, perms.GroupRead())
assert.False(t, perms.OtherRead())
_, err = helpers.ParseCSRPEM(csr)
assert.NoError(t, err)
}
func TestParseValidateAndSignMaliciousCSR(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
req := &cfcsr.CertificateRequest{
Names: []cfcsr.Name{
{
O: "maliciousOrg",
OU: "maliciousOU",
L: "maliciousLocality",
},
},
CN: "maliciousCN",
Hosts: []string{"docker.com"},
KeyRequest: &cfcsr.BasicKeyRequest{A: "ecdsa", S: 256},
}
csr, _, err := cfcsr.ParseRequest(req)
assert.NoError(t, err)
signedCert, err := rootCA.ParseValidateAndSignCSR(csr, "CN", "OU", "ORG")
assert.NoError(t, err)
assert.NotNil(t, signedCert)
parsedCert, err := helpers.ParseCertificatesPEM(signedCert)
assert.NoError(t, err)
assert.Equal(t, 2, len(parsedCert))
assert.Equal(t, "CN", parsedCert[0].Subject.CommonName)
assert.Equal(t, 1, len(parsedCert[0].Subject.OrganizationalUnit))
assert.Equal(t, "OU", parsedCert[0].Subject.OrganizationalUnit[0])
assert.Equal(t, 3, len(parsedCert[0].Subject.Names))
assert.Empty(t, parsedCert[0].Subject.Locality)
assert.Equal(t, "ORG", parsedCert[0].Subject.Organization[0])
assert.Equal(t, "rootCN", parsedCert[1].Subject.CommonName)
}
func TestNewRootCABundle(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
// make one rootCA
secondRootCA, err := ca.CreateRootCA("rootCN2", paths.RootCA)
assert.NoError(t, err)
// make a second root CA
firstRootCA, err := ca.CreateRootCA("rootCN1", paths.RootCA)
assert.NoError(t, err)
// Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle
bundle := append(firstRootCA.Cert, secondRootCA.Cert...)
err = ioutil.WriteFile(paths.RootCA.Cert, bundle, 0644)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(bundle, firstRootCA.Key, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
assert.Equal(t, bundle, newRootCA.Cert)
assert.Equal(t, 2, len(newRootCA.Pool.Subjects()))
// If I use newRootCA's IssueAndSaveNewCertificates to sign certs, I'll get the correct CA in the chain
kw := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, err = newRootCA.IssueAndSaveNewCertificates(kw, "CN", "OU", "ORG")
assert.NoError(t, err)
certBytes, err := ioutil.ReadFile(paths.Node.Cert)
assert.NoError(t, err)
certs, err := helpers.ParseCertificatesPEM(certBytes)
assert.NoError(t, err)
assert.Len(t, certs, 2)
assert.Equal(t, "CN", certs[0].Subject.CommonName)
assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0])
assert.Equal(t, "ORG", certs[0].Subject.Organization[0])
assert.Equal(t, "rootCN1", certs[1].Subject.CommonName)
}
// If there's only a root CA on disk (no TLS certs), and no join addr, we create a new CA
// and a new set of TLS certs. Similarly if there's only a TLS cert and key, and no CA.
func TestLoadSecurityConfigPartialCertsOnDisk(t *testing.T) {
tempdir, err := ioutil.TempDir("", "test-new-node")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
require.NoError(t, err)
node, err := New(&Config{
StateDir: tempdir,
})
require.NoError(t, err)
securityConfig, err := node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
cert, key, err := securityConfig.KeyReader().Read()
require.NoError(t, err)
// a new CA was generated because no existing TLS certs were present
require.NotEqual(t, rootCA.Cert, securityConfig.RootCA().Cert)
// if the TLS key and cert are on disk, but there's no CA, a new CA and TLS
// key+cert are generated
require.NoError(t, os.RemoveAll(paths.RootCA.Cert))
node, err = New(&Config{
StateDir: tempdir,
})
require.NoError(t, err)
securityConfig, err = node.loadSecurityConfig(context.Background())
require.NoError(t, err)
require.NotNil(t, securityConfig)
newCert, newKey, err := securityConfig.KeyReader().Read()
require.NoError(t, err)
require.NotEqual(t, cert, newCert)
require.NotEqual(t, key, newKey)
}
func TestNewRootCANonDefaultExpiry(t *testing.T) {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
defer os.RemoveAll(tempBaseDir)
paths := ca.NewConfigPaths(tempBaseDir)
rootCA, err := ca.CreateAndWriteRootCA("rootCN", paths.RootCA)
assert.NoError(t, err)
newRootCA, err := ca.NewRootCA(rootCA.Cert, rootCA.Key, 1*time.Hour)
assert.NoError(t, err)
// Create and sign a new CSR
csr, _, err := ca.GenerateAndWriteNewKey(paths.Node)
assert.NoError(t, err)
cert, err := newRootCA.ParseValidateAndSignCSR(csr, "CN", ca.ManagerRole, "ORG")
assert.NoError(t, err)
parsedCerts, err := helpers.ParseCertificatesPEM(cert)
assert.NoError(t, err)
assert.Len(t, parsedCerts, 2)
assert.True(t, time.Now().Add(time.Minute*59).Before(parsedCerts[0].NotAfter))
assert.True(t, time.Now().Add(time.Hour).Add(time.Minute).After(parsedCerts[0].NotAfter))
// Sign the same CSR again, this time with a 59 Minute expiration RootCA (under the 60 minute minimum).
// This should use the default of 3 months
newRootCA, err = ca.NewRootCA(rootCA.Cert, rootCA.Key, 59*time.Minute)
assert.NoError(t, err)
cert, err = newRootCA.ParseValidateAndSignCSR(csr, "CN", ca.ManagerRole, "ORG")
assert.NoError(t, err)
parsedCerts, err = helpers.ParseCertificatesPEM(cert)
assert.NoError(t, err)
assert.Len(t, parsedCerts, 2)
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, -1).Before(parsedCerts[0].NotAfter))
assert.True(t, time.Now().Add(ca.DefaultNodeCertExpiration).AddDate(0, 0, 1).After(parsedCerts[0].NotAfter))
}
// NewRaftDEKManager creates a key if one doesn't exist
func TestNewRaftDEKManager(t *testing.T) {
tempDir, err := ioutil.TempDir("", "manager-new-dek-manager-")
require.NoError(t, err)
defer os.RemoveAll(tempDir)
paths := ca.NewConfigPaths(tempDir)
cert, key, err := cautils.CreateRootCertAndKey("cn")
require.NoError(t, err)
krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
require.NoError(t, krw.Write(cert, key, nil))
keyBytes, err := ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
require.NotContains(t, string(keyBytes), pemHeaderRaftDEK) // headers are not written
dekManager, err := NewRaftDEKManager(krw) // this should create a new DEK and write it to the file
require.NoError(t, err)
keyBytes, err = ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
require.Contains(t, string(keyBytes), pemHeaderRaftDEK) // header is written now
keys := dekManager.GetKeys()
require.NotNil(t, keys.CurrentDEK)
require.Nil(t, keys.PendingDEK)
require.False(t, dekManager.NeedsRotation())
// If one exists, nothing is updated
dekManager, err = NewRaftDEKManager(krw) // this should create a new DEK and write it to the file
require.NoError(t, err)
keyBytes2, err := ioutil.ReadFile(paths.Node.Key)
require.NoError(t, err)
require.Equal(t, keyBytes, keyBytes2)
require.Equal(t, keys, dekManager.GetKeys())
require.False(t, dekManager.NeedsRotation())
}
// NewTestCA is a helper method that creates a TestCA and a bunch of default
// connections and security configs.
func NewTestCA(t *testing.T) *TestCA {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
s := store.NewMemoryStore(nil)
paths := ca.NewConfigPaths(tempBaseDir)
organization := identity.NewID()
rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
var (
externalSigningServer *ExternalSigningServer
externalCAs []*api.ExternalCA
)
if External {
// Start the CA API server.
externalSigningServer, err = NewExternalSigningServer(rootCA, tempBaseDir)
assert.NoError(t, err)
externalCAs = []*api.ExternalCA{
{
Protocol: api.ExternalCA_CAProtocolCFSSL,
URL: externalSigningServer.URL,
},
}
}
managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "", External)
assert.NoError(t, err)
managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "", External)
assert.NoError(t, err)
agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "", External)
assert.NoError(t, err)
l, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds))
managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds))
managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds))
conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...)
assert.NoError(t, err)
conn2, err := grpc.Dial(l.Addr().String(), clientOpts...)
assert.NoError(t, err)
conn3, err := grpc.Dial(l.Addr().String(), managerOpts...)
assert.NoError(t, err)
conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...)
assert.NoError(t, err)
serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)}
grpcServer := grpc.NewServer(serverOpts...)
managerToken := ca.GenerateJoinToken(&rootCA)
workerToken := ca.GenerateJoinToken(&rootCA)
createClusterObject(t, s, organization, workerToken, managerToken, externalCAs...)
caServer := ca.NewServer(s, managerConfig)
api.RegisterCAServer(grpcServer, caServer)
api.RegisterNodeCAServer(grpcServer, caServer)
ctx := context.Background()
go grpcServer.Serve(l)
go caServer.Run(ctx)
// Wait for caServer to be ready to serve
<-caServer.Ready()
remotes := remotes.NewRemotes(api.Peer{Addr: l.Addr().String()})
caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)}
nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)}
conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4}
return &TestCA{
RootCA: rootCA,
ExternalSigningServer: externalSigningServer,
MemoryStore: s,
TempDir: tempBaseDir,
Organization: organization,
Paths: paths,
Context: ctx,
CAClients: caClients,
NodeCAClients: nodeCAClients,
Conns: conns,
Server: grpcServer,
CAServer: caServer,
WorkerToken: workerToken,
ManagerToken: managerToken,
Remotes: remotes,
}
}
