// UpdateCluster updates a Cluster referenced by ClusterID with the given ClusterSpec.
// - Returns `NotFound` if the Cluster is not found.
// - Returns `InvalidArgument` if the ClusterSpec is malformed.
// - Returns `Unimplemented` if the ClusterSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) {
if request.ClusterID == "" || request.ClusterVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateClusterSpec(request.Spec); err != nil {
return nil, err
}
var cluster *api.Cluster
err := s.store.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, request.ClusterID)
if cluster == nil {
return nil
}
cluster.Meta.Version = *request.ClusterVersion
cluster.Spec = *request.Spec.Copy()
return store.UpdateCluster(tx, cluster)
})
if err != nil {
return nil, err
}
if cluster == nil {
return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
}
redactedClusters := redactClusters([]*api.Cluster{cluster})
// WARN: we should never return cluster here. We need to redact the private fields first.
return &api.UpdateClusterResponse{
Cluster: redactedClusters[0],
}, nil
}
func (k *KeyManager) updateKey(cluster *api.Cluster) error {
return k.store.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, cluster.ID)
if cluster == nil {
return nil
}
cluster.EncryptionKeyLamportClock = k.keyRing.lClock
cluster.NetworkBootstrapKeys = k.keyRing.keys
return store.UpdateCluster(tx, cluster)
})
}
func TestGetUnlockKey(t *testing.T) {
t.Parallel()
tc := testutils.NewTestCA(t)
defer tc.Stop()
var cluster *api.Cluster
tc.MemoryStore.View(func(tx store.ReadTx) {
clusters, err := store.FindClusters(tx, store.ByName(store.DefaultClusterName))
require.NoError(t, err)
cluster = clusters[0]
})
resp, err := tc.CAClients[0].GetUnlockKey(context.Background(), &api.GetUnlockKeyRequest{})
require.NoError(t, err)
require.Nil(t, resp.UnlockKey)
require.Equal(t, cluster.Meta.Version, resp.Version)
// Update the unlock key
require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, cluster.ID)
cluster.Spec.EncryptionConfig.AutoLockManagers = true
cluster.UnlockKeys = []*api.EncryptionKey{{
Subsystem: ca.ManagerRole,
Key: []byte("secret"),
}}
return store.UpdateCluster(tx, cluster)
}))
tc.MemoryStore.View(func(tx store.ReadTx) {
cluster = store.GetCluster(tx, cluster.ID)
})
require.NoError(t, raftutils.PollFuncWithTimeout(nil, func() error {
resp, err = tc.CAClients[0].GetUnlockKey(context.Background(), &api.GetUnlockKeyRequest{})
if err != nil {
return fmt.Errorf("get unlock key: %v", err)
}
if !bytes.Equal(resp.UnlockKey, []byte("secret")) {
return fmt.Errorf("secret hasn't rotated yet")
}
if cluster.Meta.Version.Index > resp.Version.Index {
return fmt.Errorf("hasn't updated to the right version yet")
}
return nil
}, 250*time.Millisecond))
}
// rotateRootCAKEK will attempt to rotate the key-encryption-key for root CA key-material in raft.
// If there is no passphrase set in ENV, it returns.
// If there is plain-text root key-material, and a passphrase set, it encrypts it.
// If there is encrypted root key-material and it is using the current passphrase, it returns.
// If there is encrypted root key-material, and it is using the previous passphrase, it
// re-encrypts it with the current passphrase.
func (m *Manager) rotateRootCAKEK(ctx context.Context, clusterID string) error {
// If we don't have a KEK, we won't ever be rotating anything
strPassphrase := os.Getenv(ca.PassphraseENVVar)
if strPassphrase == "" {
return nil
}
strPassphrasePrev := os.Getenv(ca.PassphraseENVVarPrev)
passphrase := []byte(strPassphrase)
passphrasePrev := []byte(strPassphrasePrev)
s := m.RaftNode.MemoryStore()
var (
cluster *api.Cluster
err error
finalKey []byte
)
// Retrieve the cluster identified by ClusterID
s.View(func(readTx store.ReadTx) {
cluster = store.GetCluster(readTx, clusterID)
})
if cluster == nil {
return fmt.Errorf("cluster not found: %s", clusterID)
}
// Try to get the private key from the cluster
privKeyPEM := cluster.RootCA.CAKey
if privKeyPEM == nil || len(privKeyPEM) == 0 {
// We have no PEM root private key in this cluster.
log.G(ctx).Warnf("cluster %s does not have private key material", clusterID)
return nil
}
// Decode the PEM private key
keyBlock, _ := pem.Decode(privKeyPEM)
if keyBlock == nil {
return fmt.Errorf("invalid PEM-encoded private key inside of cluster %s", clusterID)
}
// If this key is not encrypted, then we have to encrypt it
if !x509.IsEncryptedPEMBlock(keyBlock) {
finalKey, err = ca.EncryptECPrivateKey(privKeyPEM, strPassphrase)
if err != nil {
return err
}
} else {
// This key is already encrypted, let's try to decrypt with the current main passphrase
_, err = x509.DecryptPEMBlock(keyBlock, []byte(passphrase))
if err == nil {
// The main key is the correct KEK, nothing to do here
return nil
}
// This key is already encrypted, but failed with current main passphrase.
// Let's try to decrypt with the previous passphrase
unencryptedKey, err := x509.DecryptPEMBlock(keyBlock, []byte(passphrasePrev))
if err != nil {
// We were not able to decrypt either with the main or backup passphrase, error
return err
}
unencryptedKeyBlock := &pem.Block{
Type: keyBlock.Type,
Bytes: unencryptedKey,
Headers: keyBlock.Headers,
}
// We were able to decrypt the key, but with the previous passphrase. Let's encrypt
// with the new one and store it in raft
finalKey, err = ca.EncryptECPrivateKey(pem.EncodeToMemory(unencryptedKeyBlock), strPassphrase)
if err != nil {
log.G(ctx).Debugf("failed to rotate the key-encrypting-key for the root key material of cluster %s", clusterID)
return err
}
}
log.G(ctx).Infof("Re-encrypting the root key material of cluster %s", clusterID)
// Let's update the key in the cluster object
return s.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, clusterID)
if cluster == nil {
return fmt.Errorf("cluster not found: %s", clusterID)
}
cluster.RootCA.CAKey = finalKey
return store.UpdateCluster(tx, cluster)
})
}
func TestNewNodeCertificateRequiresToken(t *testing.T) {
t.Parallel()
tc := testutils.NewTestCA(t)
defer tc.Stop()
csr, _, err := ca.GenerateNewCSR()
assert.NoError(t, err)
// Issuance fails if no secret is provided
role := api.NodeRoleManager
issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance fails if wrong secret is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance succeeds if correct token is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
// Rotate manager and worker tokens
var (
newManagerToken string
newWorkerToken string
)
assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
clusters, _ := store.FindClusters(tx, store.ByName(store.DefaultClusterName))
newWorkerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Worker = newWorkerToken
newManagerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Manager = newManagerToken
return store.UpdateCluster(tx, clusters[0])
}))
// updating the join token may take a little bit in order to register on the CA server, so poll
assert.NoError(t, raftutils.PollFunc(nil, func() error {
// Old token should fail
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
if err == nil {
return fmt.Errorf("join token not updated yet")
}
return nil
}))
// Old token should fail
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// New token should succeed
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newWorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
}
func TestRequestAndSaveNewCertificates(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
// Copy the current RootCA without the signer
rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool}
cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.ManagerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, cert)
perms, err := permbits.Stat(tc.Paths.Node.Cert)
assert.NoError(t, err)
assert.False(t, perms.GroupWrite())
assert.False(t, perms.OtherWrite())
// there was no encryption config in the remote, so the key should be unencrypted
unencryptedKeyReader := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
// the worker token is also unencrypted
cert, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
assert.NotNil(t, cert)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
// If there is a different kek in the remote store, when TLS certs are renewed the new key will
// be encrypted with that kek
assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
cluster := store.GetCluster(tx, tc.Organization)
cluster.Spec.EncryptionConfig.AutoLockManagers = true
cluster.UnlockKeys = []*api.EncryptionKey{{
Subsystem: ca.ManagerRole,
Key: []byte("kek!"),
}}
return store.UpdateCluster(tx, cluster)
}))
assert.NoError(t, os.RemoveAll(tc.Paths.Node.Cert))
assert.NoError(t, os.RemoveAll(tc.Paths.Node.Key))
_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.ManagerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
// key can no longer be read without a kek
_, _, err = unencryptedKeyReader.Read()
require.Error(t, err)
_, _, err = ca.NewKeyReadWriter(tc.Paths.Node, []byte("kek!"), nil).Read()
require.NoError(t, err)
// if it's a worker though, the key is always unencrypted, even though the manager key is encrypted
_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
ca.CertificateRequestConfig{
Token: tc.WorkerToken,
Remotes: tc.Remotes,
})
assert.NoError(t, err)
_, _, err = unencryptedKeyReader.Read()
require.NoError(t, err)
}
// UpdateCluster updates a Cluster referenced by ClusterID with the given ClusterSpec.
// - Returns `NotFound` if the Cluster is not found.
// - Returns `InvalidArgument` if the ClusterSpec is malformed.
// - Returns `Unimplemented` if the ClusterSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) {
if request.ClusterID == "" || request.ClusterVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateClusterSpec(request.Spec); err != nil {
return nil, err
}
var cluster *api.Cluster
err := s.store.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, request.ClusterID)
if cluster == nil {
return nil
}
cluster.Meta.Version = *request.ClusterVersion
cluster.Spec = *request.Spec.Copy()
expireBlacklistedCerts(cluster)
if request.Rotation.WorkerJoinToken {
cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA)
}
if request.Rotation.ManagerJoinToken {
cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA)
}
var unlockKeys []*api.EncryptionKey
var managerKey *api.EncryptionKey
for _, eKey := range cluster.UnlockKeys {
if eKey.Subsystem == ca.ManagerRole {
if !cluster.Spec.EncryptionConfig.AutoLockManagers {
continue
}
managerKey = eKey
}
unlockKeys = append(unlockKeys, eKey)
}
switch {
case !cluster.Spec.EncryptionConfig.AutoLockManagers:
break
case managerKey == nil:
unlockKeys = append(unlockKeys, &api.EncryptionKey{
Subsystem: ca.ManagerRole,
Key: encryption.GenerateSecretKey(),
})
case request.Rotation.ManagerUnlockKey:
managerKey.Key = encryption.GenerateSecretKey()
}
cluster.UnlockKeys = unlockKeys
return store.UpdateCluster(tx, cluster)
})
if err != nil {
return nil, err
}
if cluster == nil {
return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
}
redactedClusters := redactClusters([]*api.Cluster{cluster})
// WARN: we should never return cluster here. We need to redact the private fields first.
return &api.UpdateClusterResponse{
Cluster: redactedClusters[0],
}, nil
}
func TestNewNodeCertificateRequiresToken(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
// Issuance fails if no secret is provided
role := api.NodeRoleManager
issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance fails if wrong secret is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance succeeds if correct token is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
// Rotate manager and worker tokens
var (
newManagerToken string
newWorkerToken string
)
assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
clusters, _ := store.FindClusters(tx, store.ByName(store.DefaultClusterName))
newWorkerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Worker = newWorkerToken
newManagerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Manager = newManagerToken
return store.UpdateCluster(tx, clusters[0])
}))
time.Sleep(500 * time.Millisecond)
// Old token should fail
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// New token should succeed
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newWorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
}
func TestGetRemoteCA(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
shaHash := sha256.New()
shaHash.Write(tc.RootCA.Cert)
md := shaHash.Sum(nil)
mdStr := hex.EncodeToString(md)
d, err := digest.Parse("sha256:" + mdStr)
require.NoError(t, err)
downloadedRootCA, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
require.NoError(t, err)
require.Equal(t, downloadedRootCA.Cert, tc.RootCA.Cert)
// update the test CA to include a multi-certificate bundle as the root - the digest
// we use to verify with must be the digest of the whole bundle
tmpDir, err := ioutil.TempDir("", "GetRemoteCA")
require.NoError(t, err)
defer os.RemoveAll(tmpDir)
paths := ca.NewConfigPaths(tmpDir)
otherRootCA, err := ca.CreateRootCA("other", paths.RootCA)
require.NoError(t, err)
comboCertBundle := append(tc.RootCA.Cert, otherRootCA.Cert...)
require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
cluster := store.GetCluster(tx, tc.Organization)
cluster.RootCA.CACert = comboCertBundle
cluster.RootCA.CAKey = tc.RootCA.Key
return store.UpdateCluster(tx, cluster)
}))
require.NoError(t, raftutils.PollFunc(nil, func() error {
_, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
if err == nil {
return fmt.Errorf("testca's rootca hasn't updated yet")
}
require.Contains(t, err.Error(), "remote CA does not match fingerprint")
return nil
}))
// If we provide the right digest, the root CA is updated and we can validate
// certs signed by either one
d = digest.FromBytes(comboCertBundle)
downloadedRootCA, err = ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
require.NoError(t, err)
require.Equal(t, comboCertBundle, downloadedRootCA.Cert)
require.Equal(t, 2, len(downloadedRootCA.Pool.Subjects()))
for _, rootCA := range []ca.RootCA{tc.RootCA, otherRootCA} {
krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
_, err := rootCA.IssueAndSaveNewCertificates(krw, "cn", "ou", "org")
require.NoError(t, err)
certPEM, _, err := krw.Read()
require.NoError(t, err)
cert, err := helpers.ParseCertificatesPEM(certPEM)
require.NoError(t, err)
chains, err := cert[0].Verify(x509.VerifyOptions{
Roots: downloadedRootCA.Pool,
})
require.NoError(t, err)
require.Len(t, chains, 1)
}
}
// If there is no CA, and a join addr is provided, one is downloaded from the
// join server. If there is a CA, it is just loaded from disk. The TLS key and
// cert are also downloaded.
func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) {
tempdir, err := ioutil.TempDir("", "test-join-node")
require.NoError(t, err)
defer os.RemoveAll(tempdir)
paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))
// join addr is invalid
node, err := New(&Config{
StateDir: tempdir,
JoinAddr: "127.0.0.1:12",
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.Error(t, err)
tc := cautils.NewTestCA(t)
defer tc.Stop()
peer, err := tc.ConnBroker.Remotes().Select()
require.NoError(t, err)
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.NoError(t, err)
// the TLS key and cert were written to disk unencrypted
_, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read()
require.NoError(t, err)
// remove the TLS cert and key, and mark the root CA cert so that we will
// know if it gets replaced
require.NoError(t, os.Remove(paths.Node.Cert))
require.NoError(t, os.Remove(paths.Node.Key))
certBytes, err := ioutil.ReadFile(paths.RootCA.Cert)
require.NoError(t, err)
pemBlock, _ := pem.Decode(certBytes)
require.NotNil(t, pemBlock)
pemBlock.Headers["marked"] = "true"
certBytes = pem.EncodeToMemory(pemBlock)
require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, certBytes, 0644))
// also make sure the new set gets downloaded and written to disk with a passphrase
// by updating the memory store with manager autolock on and an unlock key
require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
clusters, err := store.FindClusters(tx, store.All)
require.NoError(t, err)
require.Len(t, clusters, 1)
newCluster := clusters[0].Copy()
newCluster.Spec.EncryptionConfig.AutoLockManagers = true
newCluster.UnlockKeys = []*api.EncryptionKey{{
Subsystem: ca.ManagerRole,
Key: []byte("passphrase"),
}}
return store.UpdateCluster(tx, newCluster)
}))
// Join with without any passphrase - this should be fine, because the TLS
// key is downloaded and then loaded just fine. However, it *is* written
// to disk encrypted.
node, err = New(&Config{
StateDir: tempdir,
JoinAddr: peer.Addr,
JoinToken: tc.ManagerToken,
})
require.NoError(t, err)
_, err = node.loadSecurityConfig(context.Background())
require.NoError(t, err)
// make sure the CA cert has not been replaced
readCertBytes, err := ioutil.ReadFile(paths.RootCA.Cert)
require.NoError(t, err)
require.Equal(t, certBytes, readCertBytes)
// the TLS node cert and key were saved to disk encrypted, though
_, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read()
require.Error(t, err)
_, _, err = ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil).Read()
require.NoError(t, err)
}
// RemoveNode removes a Node referenced by NodeID with the given NodeSpec.
// - Returns NotFound if the Node is not found.
// - Returns FailedPrecondition if the Node has manager role (and is part of the memberlist) or is not shut down.
// - Returns InvalidArgument if NodeID or NodeVersion is not valid.
// - Returns an error if the delete fails.
func (s *Server) RemoveNode(ctx context.Context, request *api.RemoveNodeRequest) (*api.RemoveNodeResponse, error) {
if request.NodeID == "" {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
err := s.store.Update(func(tx store.Tx) error {
node := store.GetNode(tx, request.NodeID)
if node == nil {
return grpc.Errorf(codes.NotFound, "node %s not found", request.NodeID)
}
if node.Spec.Role == api.NodeRoleManager {
if s.raft == nil {
return grpc.Errorf(codes.FailedPrecondition, "node %s is a manager but cannot access node information from the raft memberlist", request.NodeID)
}
if member := s.raft.GetMemberByNodeID(request.NodeID); member != nil {
return grpc.Errorf(codes.FailedPrecondition, "node %s is a cluster manager and is a member of the raft cluster. It must be demoted to worker before removal", request.NodeID)
}
}
if !request.Force && node.Status.State == api.NodeStatus_READY {
return grpc.Errorf(codes.FailedPrecondition, "node %s is not down and can't be removed", request.NodeID)
}
// lookup the cluster
clusters, err := store.FindClusters(tx, store.ByName("default"))
if err != nil {
return err
}
if len(clusters) != 1 {
return grpc.Errorf(codes.Internal, "could not fetch cluster object")
}
cluster := clusters[0]
removedNode := &api.RemovedNode{ID: node.ID}
// Set an expiry time for this RemovedNode if a certificate
// exists and can be parsed.
if len(node.Certificate.Certificate) != 0 {
certBlock, _ := pem.Decode(node.Certificate.Certificate)
if certBlock != nil {
X509Cert, err := x509.ParseCertificate(certBlock.Bytes)
if err == nil && !X509Cert.NotAfter.IsZero() {
expiry, err := ptypes.TimestampProto(X509Cert.NotAfter)
if err == nil {
removedNode.Expiry = expiry
}
}
}
}
cluster.RemovedNodes = append(cluster.RemovedNodes, removedNode)
if err := store.UpdateCluster(tx, cluster); err != nil {
return err
}
return store.DeleteNode(tx, request.NodeID)
})
if err != nil {
return nil, err
}
return &api.RemoveNodeResponse{}, nil
}