예제 #1
0
// The TLS KEK and the KEK for the headers should be in sync, and so failing
// to decrypt the TLS key should be mean we won't be able to decrypt the headers.
// However, the TLS Key encryption uses AES-256-CBC (golang as of 1.7.x does not seem
// to support GCM, so no cipher modes with digests) so sometimes decrypting with
// the wrong passphrase will not result in an error.  This means we will ultimately
// have to rely on the header encryption mechanism, which does include a digest, to
// determine if the KEK is valid.
func TestDecryptTLSKeyFalsePositive(t *testing.T) {
	badKey := []byte(`
-----BEGIN EC PRIVATE KEY-----
Proc-Type: 4,ENCRYPTED
DEK-Info: AES-256-CBC,e7927e79e748233776c03c2eb7275f09
kek-version: 392
raft-dek: CAESMBrzZ0gNVPe3FRs42743q8RtkUBrK1ICQpHWX2vdQ8iqSKt1WoKdFDFD2r28LYAVLxoYQguwHbijMx9k+BALUNBAI3s199S5tvnr

JfGenNvzm++AvsOh+UmcBY+JgI6lnfzaCB68agmlmEZYLYi5tqtAU7gif6VIJpCW
+Pj23Fzkw8sKKOOBeapSC5lp+Cjx9OsCci/R9xrdx+uxnnzKJNxOB/qzqcQfZDMh
id2LxdliFcPEk/Yj5gNGpT0UMFJ4G52enbOwOru46f0=
-----END EC PRIVATE KEY-----
`)

	// not actually a real swarm cert - generated a cert corresponding to the key that expires in 20 years
	matchingCert := []byte(`
-----BEGIN CERTIFICATE-----
MIIB9jCCAZygAwIBAgIRAIdzF3Z9VT2OXbRvEw5cR68wCgYIKoZIzj0EAwIwYDEi
MCAGA1UEChMZbWRwMXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dh
cm0tbWFuYWdlcjEiMCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTAg
GA8wMDAxMDEwMTAwMDAwMFoXDTM2MTEwODA2MjMwMlowYDEiMCAGA1UEChMZbWRw
MXU5Z3FoOTV1NXN2MmNodDRrcDB1cTEWMBQGA1UECxMNc3dhcm0tbWFuYWdlcjEi
MCAGA1UEAxMZcXJzYmwza2FqOWhiZWprM2R5aWFlc3FiYTBZMBMGByqGSM49AgEG
CCqGSM49AwEHA0IABGOivD25E/zcupRFQdKOKbPHS9Mx7JlUhlWnl0iR0K5VhVIU
XjUHt98GuX6gDjs4yUzEKSGxYPsSYlnG9zQqbQSjNTAzMA4GA1UdDwEB/wQEAwIF
oDATBgNVHSUEDDAKBggrBgEFBQcDATAMBgNVHRMBAf8EAjAAMAoGCCqGSM49BAMC
A0gAMEUCIQDWtjg1ITGznQILipaEe70G/NgZAOtFfuPXTVkUl3el+wIgSVOVKB/Q
O0T3aXuZGYNyh//KqAoA3erCmh6HauMz84Y=
-----END CERTIFICATE-----
	`)

	var wrongKEK []byte // empty passphrase doesn't decrypt without errors
	falsePositiveKEK, err := base64.RawStdEncoding.DecodeString("bIQgLAAMoGCrHdjMLVhEVqnYTAM7ZNF2xWMiwtw7AiQ")
	require.NoError(t, err)
	realKEK, err := base64.RawStdEncoding.DecodeString("fDg9YejLnMjU+FpulWR62oJLzVpkD2j7VQuP5xiK9QA")
	require.NoError(t, err)

	tempdir, err := ioutil.TempDir("", "KeyReadWriter-false-positive-decryption")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	path := ca.NewConfigPaths(tempdir)
	require.NoError(t, ioutil.WriteFile(path.Node.Key, badKey, 0600))
	require.NoError(t, ioutil.WriteFile(path.Node.Cert, matchingCert, 0644))

	krw := ca.NewKeyReadWriter(path.Node, wrongKEK, RaftDEKData{})
	_, _, err = krw.Read()
	require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err))

	krw = ca.NewKeyReadWriter(path.Node, falsePositiveKEK, RaftDEKData{})
	_, _, err = krw.Read()
	require.Error(t, err)
	require.IsType(t, ca.ErrInvalidKEK{}, errors.Cause(err))

	krw = ca.NewKeyReadWriter(path.Node, realKEK, RaftDEKData{})
	_, _, err = krw.Read()
	require.NoError(t, err)
}
예제 #2
0
func TestLoadSecurityConfigIncorrectPassphrase(t *testing.T) {
	tc := testutils.NewTestCA(t)
	defer tc.Stop()

	paths := ca.NewConfigPaths(tc.TempDir)
	_, err := tc.RootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(paths.Node, []byte("kek"), nil),
		"nodeID", ca.WorkerRole, tc.Organization)
	require.NoError(t, err)

	_, err = ca.LoadSecurityConfig(tc.Context, tc.RootCA, ca.NewKeyReadWriter(paths.Node, nil, nil))
	require.IsType(t, ca.ErrInvalidKEK{}, err)
}
예제 #3
0
func TestKeyReadWriterViewAndRotateKEK(t *testing.T) {
	cert, key, err := testutils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	tempdir, err := ioutil.TempDir("", "KeyReadWriter")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	path := ca.NewConfigPaths(filepath.Join(tempdir))

	// write a key with headers to the key to make sure it gets passed when reading/writing headers
	keyBlock, _ := pem.Decode(key)
	require.NotNil(t, keyBlock)
	keyBlock.Headers = map[string]string{"hello": "world"}
	key = pem.EncodeToMemory(keyBlock)
	require.NoError(t, ca.NewKeyReadWriter(path.Node, nil, nil).Write(cert, key, nil))

	// if if getting new kek and headers fail, rotating a KEK fails, and the kek does not rotate
	k := ca.NewKeyReadWriter(path.Node, nil, nil)
	require.Error(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
		require.Equal(t, ca.KEKData{}, k)
		require.Nil(t, h)
		return ca.KEKData{}, nil, fmt.Errorf("Nope")
	}))

	// writing new headers will write a key that has the headers returned by the header update function
	k = ca.NewKeyReadWriter(path.Node, []byte("oldKEK"), nil)
	require.NoError(t, k.ViewAndRotateKEK(func(k ca.KEKData, h ca.PEMKeyHeaders) (ca.KEKData, ca.PEMKeyHeaders, error) {
		require.Equal(t, ca.KEKData{KEK: []byte("oldKEK")}, k)
		require.Nil(t, h)
		return ca.KEKData{KEK: []byte("newKEK"), Version: uint64(2)},
			testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
				require.Equal(t, []byte("newKEK"), kek.KEK)
				return map[string]string{"updated": "headers"}, nil
			}}, nil
	}))

	// ensure the key has been re-encrypted and we can read it
	k = ca.NewKeyReadWriter(path.Node, nil, nil)
	_, _, err = k.Read()
	require.Error(t, err)

	var headers map[string]string

	k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, _ ca.KEKData) (ca.PEMKeyHeaders, error) {
		headers = h
		return testHeaders{}, nil
	}})
	_, _, err = k.Read()
	require.NoError(t, err)
	require.Equal(t, map[string]string{"updated": "headers"}, headers)
}
예제 #4
0
func TestKeyReadWriterMigrate(t *testing.T) {
	cert, key, err := testutils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	tempdir, err := ioutil.TempDir("", "KeyReadWriter")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	path := ca.NewConfigPaths(filepath.Join(tempdir))

	// if the key exists in an old location, migrate it from there.
	tempKeyPath := filepath.Join(filepath.Dir(path.Node.Key), "."+filepath.Base(path.Node.Key))
	require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644))
	require.NoError(t, ioutil.WriteFile(tempKeyPath, key, 0600))

	krw := ca.NewKeyReadWriter(path.Node, nil, nil)
	require.NoError(t, krw.Migrate())
	_, err = os.Stat(tempKeyPath)
	require.True(t, os.IsNotExist(err)) // it's been moved to the right place
	_, _, err = krw.Read()
	require.NoError(t, err)

	// migrate does not affect any existing files
	dirList, err := ioutil.ReadDir(filepath.Dir(path.Node.Key))
	require.NoError(t, err)
	require.NoError(t, krw.Migrate())
	dirList2, err := ioutil.ReadDir(filepath.Dir(path.Node.Key))
	require.NoError(t, err)
	require.Equal(t, dirList, dirList2)
	_, _, err = krw.Read()
	require.NoError(t, err)
}
예제 #5
0
// If there is nothing on disk and no join addr, we create a new CA and a new set of TLS certs.
// If AutoLockManagers is enabled, the TLS key is encrypted with a randomly generated lock key.
func TestLoadSecurityConfigNewNode(t *testing.T) {
	for _, autoLockManagers := range []bool{true, false} {
		tempdir, err := ioutil.TempDir("", "test-new-node")
		require.NoError(t, err)
		defer os.RemoveAll(tempdir)

		paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))

		node, err := New(&Config{
			StateDir:         tempdir,
			AutoLockManagers: autoLockManagers,
		})
		require.NoError(t, err)
		securityConfig, err := node.loadSecurityConfig(context.Background())
		require.NoError(t, err)
		require.NotNil(t, securityConfig)

		unencryptedReader := ca.NewKeyReadWriter(paths.Node, nil, nil)
		_, _, err = unencryptedReader.Read()
		if !autoLockManagers {
			require.NoError(t, err)
		} else {
			require.IsType(t, ca.ErrInvalidKEK{}, err)
		}
	}
}
예제 #6
0
func TestLoadSecurityConfigInvalidCert(t *testing.T) {
	tc := testutils.NewTestCA(t)
	defer tc.Stop()

	// Write some garbage to the cert
	ioutil.WriteFile(tc.Paths.Node.Cert, []byte(`-----BEGIN CERTIFICATE-----\n
some random garbage\n
-----END CERTIFICATE-----`), 0644)

	krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)

	_, err := ca.LoadSecurityConfig(tc.Context, tc.RootCA, krw)
	assert.Error(t, err)

	nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
		ca.CertificateRequestConfig{
			Remotes: tc.Remotes,
		})

	assert.NoError(t, err)
	assert.NotNil(t, nodeConfig)
	assert.NotNil(t, nodeConfig.ClientTLSCreds)
	assert.NotNil(t, nodeConfig.ServerTLSCreds)
	assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
}
예제 #7
0
func TestCreateSecurityConfigNoCerts(t *testing.T) {
	tc := testutils.NewTestCA(t)
	defer tc.Stop()

	// Remove only the node certificates form the directory, and attest that we get
	// new certificates that are locally signed
	os.RemoveAll(tc.Paths.Node.Cert)
	krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
	nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
		ca.CertificateRequestConfig{
			Token:   tc.WorkerToken,
			Remotes: tc.Remotes,
		})
	assert.NoError(t, err)
	assert.NotNil(t, nodeConfig)
	assert.NotNil(t, nodeConfig.ClientTLSCreds)
	assert.NotNil(t, nodeConfig.ServerTLSCreds)
	assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())

	// Remove only the node certificates form the directory, get a new rootCA, and attest that we get
	// new certificates that are issued by the remote CA
	os.RemoveAll(tc.Paths.Node.Cert)
	rootCA, err := ca.GetLocalRootCA(tc.Paths.RootCA)
	assert.NoError(t, err)
	nodeConfig, err = rootCA.CreateSecurityConfig(tc.Context, krw,
		ca.CertificateRequestConfig{
			Token:   tc.WorkerToken,
			Remotes: tc.Remotes,
		})
	assert.NoError(t, err)
	assert.NotNil(t, nodeConfig)
	assert.NotNil(t, nodeConfig.ClientTLSCreds)
	assert.NotNil(t, nodeConfig.ServerTLSCreds)
	assert.Equal(t, rootCA, *nodeConfig.RootCA())
}
예제 #8
0
// If we abort in the middle of writing the key and cert, such that only the key is written
// to the final location, when we read we can still read the cert from the temporary
// location.
func TestTwoPhaseReadWrite(t *testing.T) {
	cert1, _, err := testutils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	cert2, key2, err := testutils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	tempdir, err := ioutil.TempDir("", "KeyReadWriter")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	path := ca.NewConfigPaths(filepath.Join(tempdir))
	krw := ca.NewKeyReadWriter(path.Node, nil, nil)

	// put a directory in the location where the cert goes, so we can't actually move
	// the cert from the temporary location to the final location.
	require.NoError(t, os.Mkdir(filepath.Join(path.Node.Cert), 0755))
	require.Error(t, krw.Write(cert2, key2, nil))

	// the temp cert file should exist
	tempCertPath := filepath.Join(filepath.Dir(path.Node.Cert), "."+filepath.Base(path.Node.Cert))
	readCert, err := ioutil.ReadFile(tempCertPath)
	require.NoError(t, err)
	require.Equal(t, cert2, readCert)

	// remove the directory, to simulate it failing to write the first time
	os.RemoveAll(path.Node.Cert)
	readCert, readKey, err := krw.Read()
	require.NoError(t, err)
	require.Equal(t, cert2, readCert)
	require.Equal(t, key2, readKey)
	// the cert should have been moved to its proper location
	_, err = os.Stat(tempCertPath)
	require.True(t, os.IsNotExist(err))

	// If the cert in the proper location doesn't match the key, the temp location is checked
	require.NoError(t, ioutil.WriteFile(tempCertPath, cert2, 0644))
	require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert1, 0644))
	readCert, readKey, err = krw.Read()
	require.NoError(t, err)
	require.Equal(t, cert2, readCert)
	require.Equal(t, key2, readKey)
	// the cert should have been moved to its proper location
	_, err = os.Stat(tempCertPath)
	require.True(t, os.IsNotExist(err))

	// If the cert in the temp location also doesn't match, the failure matching the
	// correctly-located cert is returned
	require.NoError(t, os.Remove(path.Node.Cert))
	require.NoError(t, ioutil.WriteFile(tempCertPath, cert1, 0644)) // mismatching cert
	_, _, err = krw.Read()
	require.True(t, os.IsNotExist(err))
	// the cert should have been removed
	_, err = os.Stat(tempCertPath)
	require.True(t, os.IsNotExist(err))
}
예제 #9
0
// If there are CAs and TLS certs on disk, it tries to load and fails if there
// are any errors, even if a join token is provided.
func TestLoadSecurityConfigLoadFromDisk(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "test-load-node-tls")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))

	tc := cautils.NewTestCA(t)
	defer tc.Stop()
	peer, err := tc.ConnBroker.Remotes().Select()
	require.NoError(t, err)

	// Load successfully with valid passphrase
	rootCA, err := ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
	require.NoError(t, err)
	krw := ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil)
	require.NoError(t, err)
	_, err = rootCA.IssueAndSaveNewCertificates(krw, identity.NewID(), ca.WorkerRole, identity.NewID())
	require.NoError(t, err)

	node, err := New(&Config{
		StateDir:  tempdir,
		JoinAddr:  peer.Addr,
		JoinToken: tc.ManagerToken,
		UnlockKey: []byte("passphrase"),
	})
	require.NoError(t, err)
	securityConfig, err := node.loadSecurityConfig(context.Background())
	require.NoError(t, err)
	require.NotNil(t, securityConfig)

	// Invalid passphrase
	node, err = New(&Config{
		StateDir:  tempdir,
		JoinAddr:  peer.Addr,
		JoinToken: tc.ManagerToken,
	})
	require.NoError(t, err)
	_, err = node.loadSecurityConfig(context.Background())
	require.Equal(t, ErrInvalidUnlockKey, err)

	// Invalid CA
	rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
	require.NoError(t, err)
	node, err = New(&Config{
		StateDir:  tempdir,
		JoinAddr:  peer.Addr,
		JoinToken: tc.ManagerToken,
		UnlockKey: []byte("passphrase"),
	})
	require.NoError(t, err)
	_, err = node.loadSecurityConfig(context.Background())
	require.IsType(t, x509.UnknownAuthorityError{}, errors.Cause(err))
}
예제 #10
0
// NewExternalSigningServer creates and runs a new ExternalSigningServer which
// uses the given rootCA to sign node certificates. A server key and cert are
// generated and saved into the given basedir and then a TLS listener is
// started on a random available port. On success, an HTTPS server will be
// running in a separate goroutine. The URL of the singing endpoint is
// available in the returned *ExternalSignerServer value. Calling the Close()
// method will stop the server.
func NewExternalSigningServer(rootCA ca.RootCA, basedir string) (*ExternalSigningServer, error) {
	serverCN := "external-ca-example-server"
	serverOU := "localhost" // Make a valid server cert for localhost.

	// Create TLS credentials for the external CA server which we will run.
	serverPaths := ca.CertPaths{
		Cert: filepath.Join(basedir, "server.crt"),
		Key:  filepath.Join(basedir, "server.key"),
	}
	serverCert, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(serverPaths, nil, nil), serverCN, serverOU, "")
	if err != nil {
		return nil, errors.Wrap(err, "unable to get TLS server certificate")
	}

	serverTLSConfig := &tls.Config{
		Certificates: []tls.Certificate{*serverCert},
		ClientAuth:   tls.RequireAndVerifyClientCert,
		ClientCAs:    rootCA.Pool,
	}

	tlsListener, err := tls.Listen("tcp", "localhost:0", serverTLSConfig)
	if err != nil {
		return nil, errors.Wrap(err, "unable to create TLS connection listener")
	}

	assignedPort := tlsListener.Addr().(*net.TCPAddr).Port

	signURL := url.URL{
		Scheme: "https",
		Host:   net.JoinHostPort("localhost", strconv.Itoa(assignedPort)),
		Path:   "/sign",
	}

	ess := &ExternalSigningServer{
		listener: tlsListener,
		URL:      signURL.String(),
	}

	mux := http.NewServeMux()
	handler := &signHandler{
		numIssued: &ess.NumIssued,
		rootCA:    rootCA,
		flaky:     &ess.flaky,
	}
	mux.Handle(signURL.Path, handler)

	server := &http.Server{
		Handler: mux,
	}

	go server.Serve(tlsListener)

	return ess, nil
}
예제 #11
0
func TestDecrypt(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "rafttool")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	kek := []byte("kek")
	dek := []byte("dek")
	unlockKey := encryption.HumanReadableKey(kek)

	// write a key to disk, else we won't be able to decrypt anything
	paths := certPaths(tempdir)
	krw := ca.NewKeyReadWriter(paths.Node, kek,
		manager.RaftDEKData{EncryptionKeys: raft.EncryptionKeys{CurrentDEK: dek}})
	cert, key, err := testutils.CreateRootCertAndKey("not really a root, just need cert and key")
	require.NoError(t, err)
	require.NoError(t, krw.Write(cert, key, nil))

	// create the encrypted v3 directory
	origSnapshot := raftpb.Snapshot{
		Data: []byte("snapshot"),
		Metadata: raftpb.SnapshotMetadata{
			Index: 1,
			Term:  1,
		},
	}
	e, d := encryption.Defaults(dek)
	writeFakeRaftData(t, tempdir, &origSnapshot, storage.NewWALFactory(e, d), storage.NewSnapFactory(e, d))

	outdir := filepath.Join(tempdir, "outdir")
	// if we use the wrong unlock key, we can't actually decrypt anything.  The output directory won't get created.
	err = decryptRaftData(tempdir, outdir, "")
	require.IsType(t, ca.ErrInvalidKEK{}, err)
	require.False(t, fileutil.Exist(outdir))

	// Using the right unlock key, we produce data that is unencrypted
	require.NoError(t, decryptRaftData(tempdir, outdir, unlockKey))
	require.True(t, fileutil.Exist(outdir))

	// The snapshot directory is readable by the regular snapshotter
	snapshot, err := storage.OriginalSnap.New(filepath.Join(outdir, "snap-decrypted")).Load()
	require.NoError(t, err)
	require.NotNil(t, snapshot)
	require.Equal(t, origSnapshot, *snapshot)

	// The wals are readable by the regular wal
	walreader, err := storage.OriginalWAL.Open(filepath.Join(outdir, "wal-decrypted"), walpb.Snapshot{Index: 1, Term: 1})
	require.NoError(t, err)
	metadata, _, entries, err := walreader.ReadAll()
	require.NoError(t, err)
	require.Equal(t, []byte("v3metadata"), metadata)
	require.Len(t, entries, 5)
}
예제 #12
0
func TestRaftDEKManagerUpdateKeys(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "manager-update-keys-")
	require.NoError(t, err)
	defer os.RemoveAll(tempDir)

	paths := ca.NewConfigPaths(tempDir)
	cert, key, err := cautils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	keys := raft.EncryptionKeys{
		CurrentDEK: []byte("key1"),
		PendingDEK: []byte("key2"),
	}
	krw := ca.NewKeyReadWriter(paths.Node, nil, RaftDEKData{
		EncryptionKeys: keys,
		NeedsRotation:  true,
	})
	require.NoError(t, krw.Write(cert, key, nil))

	dekManager, err := NewRaftDEKManager(krw)
	require.NoError(t, err)

	newKeys := raft.EncryptionKeys{
		CurrentDEK: []byte("new current"),
	}
	require.NoError(t, dekManager.UpdateKeys(newKeys))
	// don't run GetKeys, because NeedsRotation is true and it'd just generate a new one

	h, _ := krw.GetCurrentState()
	dekData, ok := h.(RaftDEKData)
	require.True(t, ok)
	require.True(t, dekData.NeedsRotation)

	// UpdateKeys so there is no CurrentDEK: all the headers should be wiped out
	require.NoError(t, dekManager.UpdateKeys(raft.EncryptionKeys{}))
	require.Equal(t, raft.EncryptionKeys{}, dekManager.GetKeys())
	require.False(t, dekManager.NeedsRotation())

	h, _ = krw.GetCurrentState()
	require.Nil(t, h)

	keyBytes, err := ioutil.ReadFile(paths.Node.Key)
	require.NoError(t, err)
	keyBlock, _ := pem.Decode(keyBytes)
	require.NotNil(t, keyBlock)

	// the only header remaining should be the kek version
	require.Len(t, keyBlock.Headers, 1)
	require.Contains(t, keyBlock.Headers, "kek-version")
}
예제 #13
0
파일: node.go 프로젝트: yongtang/swarmkit
// newNode creates new node with specific role(manager or agent) and joins to
// existing cluster. if joinAddr is empty string, then new cluster will be initialized.
// It uses TestExecutor as executor. If lateBind is set, the remote API port is not
// bound.  If rootCA is set, this root is used to bootstrap the node's TLS certs.
func newTestNode(joinAddr, joinToken string, lateBind bool, rootCA *ca.RootCA) (*testNode, error) {
	tmpDir, err := ioutil.TempDir("", "swarmkit-integration-")
	if err != nil {
		return nil, err
	}

	cAddr := filepath.Join(tmpDir, "control.sock")
	cfg := &node.Config{
		ListenControlAPI: cAddr,
		JoinAddr:         joinAddr,
		StateDir:         tmpDir,
		Executor:         &TestExecutor{},
		JoinToken:        joinToken,
	}
	if !lateBind {
		cfg.ListenRemoteAPI = "127.0.0.1:0"
	}
	if rootCA != nil {
		certDir := filepath.Join(tmpDir, "certificates")
		if err := os.MkdirAll(certDir, 0700); err != nil {
			return nil, err
		}
		certPaths := ca.NewConfigPaths(certDir)
		if err := ioutil.WriteFile(certPaths.RootCA.Cert, rootCA.Cert, 0644); err != nil {
			return nil, err
		}
		if err := ioutil.WriteFile(certPaths.RootCA.Key, rootCA.Key, 0600); err != nil {
			return nil, err
		}
		// generate TLS certs for this manager for bootstrapping, else the node will generate its own CA
		_, err := rootCA.IssueAndSaveNewCertificates(ca.NewKeyReadWriter(certPaths.Node, nil, nil),
			identity.NewID(), ca.ManagerRole, identity.NewID())
		if err != nil {
			return nil, err
		}
	}

	node, err := node.New(cfg)
	if err != nil {
		return nil, err
	}
	return &testNode{
		config:   cfg,
		node:     node,
		stateDir: tmpDir,
	}, nil
}
예제 #14
0
파일: common.go 프로젝트: yongtang/swarmkit
func getKRW(swarmdir, unlockKey string) (*ca.KeyReadWriter, error) {
	var (
		kek []byte
		err error
	)
	if unlockKey != "" {
		kek, err = encryption.ParseHumanReadableKey(unlockKey)
		if err != nil {
			return nil, err
		}
	}
	krw := ca.NewKeyReadWriter(certPaths(swarmdir).Node, kek, manager.RaftDEKData{})
	_, _, err = krw.Read() // loads all the key data into the KRW object
	if err != nil {
		return nil, err
	}
	return krw, nil
}
예제 #15
0
func TestCreateSecurityConfigEmptyDir(t *testing.T) {
	tc := testutils.NewTestCA(t)
	defer tc.Stop()

	// Remove all the contents from the temp dir and try again with a new node
	os.RemoveAll(tc.TempDir)
	krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
	nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
		ca.CertificateRequestConfig{
			Token:   tc.WorkerToken,
			Remotes: tc.Remotes,
		})
	assert.NoError(t, err)
	assert.NotNil(t, nodeConfig)
	assert.NotNil(t, nodeConfig.ClientTLSCreds)
	assert.NotNil(t, nodeConfig.ServerTLSCreds)
	assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
}
예제 #16
0
파일: main.go 프로젝트: yongtang/swarmkit
func main() {
	// Create root material within the current directory.
	rootPaths := ca.CertPaths{
		Cert: filepath.Join("ca", "root.crt"),
		Key:  filepath.Join("ca", "root.key"),
	}

	// Initialize the Root CA.
	rootCA, err := ca.CreateRootCA("external-ca-example", rootPaths)
	if err != nil {
		logrus.Fatalf("unable to initialize Root CA: %s", err)
	}

	// Create the initial manager node credentials.
	nodeConfigPaths := ca.NewConfigPaths("certificates")

	clusterID := identity.NewID()
	nodeID := identity.NewID()

	kw := ca.NewKeyReadWriter(nodeConfigPaths.Node, nil, nil)
	if _, err := rootCA.IssueAndSaveNewCertificates(kw, nodeID, ca.ManagerRole, clusterID); err != nil {
		logrus.Fatalf("unable to create initial manager node credentials: %s", err)
	}

	// And copy the Root CA certificate into the node config path for its
	// CA.
	ioutil.WriteFile(nodeConfigPaths.RootCA.Cert, rootCA.Cert, os.FileMode(0644))

	server, err := testutils.NewExternalSigningServer(rootCA, "ca")
	if err != nil {
		logrus.Fatalf("unable to start server: %s", err)
	}

	defer server.Stop()

	logrus.Infof("Now run: swarmd -d . --listen-control-api ./swarmd.sock --external-ca protocol=cfssl,url=%s", server.URL)

	sigC := make(chan os.Signal, 1)
	signal.Notify(sigC, syscall.SIGTERM, syscall.SIGINT)

	<-sigC
}
예제 #17
0
func TestNewRootCABundle(t *testing.T) {
	tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
	assert.NoError(t, err)
	defer os.RemoveAll(tempBaseDir)

	paths := ca.NewConfigPaths(tempBaseDir)

	// make one rootCA
	secondRootCA, err := ca.CreateRootCA("rootCN2", paths.RootCA)
	assert.NoError(t, err)

	// make a second root CA
	firstRootCA, err := ca.CreateRootCA("rootCN1", paths.RootCA)
	assert.NoError(t, err)

	// Overwrite the bytes of the second Root CA with the bundle, creating a valid 2 cert bundle
	bundle := append(firstRootCA.Cert, secondRootCA.Cert...)
	err = ioutil.WriteFile(paths.RootCA.Cert, bundle, 0644)
	assert.NoError(t, err)

	newRootCA, err := ca.NewRootCA(bundle, firstRootCA.Key, ca.DefaultNodeCertExpiration)
	assert.NoError(t, err)
	assert.Equal(t, bundle, newRootCA.Cert)
	assert.Equal(t, 2, len(newRootCA.Pool.Subjects()))

	// If I use newRootCA's IssueAndSaveNewCertificates to sign certs, I'll get the correct CA in the chain
	kw := ca.NewKeyReadWriter(paths.Node, nil, nil)
	_, err = newRootCA.IssueAndSaveNewCertificates(kw, "CN", "OU", "ORG")
	assert.NoError(t, err)

	certBytes, err := ioutil.ReadFile(paths.Node.Cert)
	assert.NoError(t, err)
	certs, err := helpers.ParseCertificatesPEM(certBytes)
	assert.NoError(t, err)
	assert.Len(t, certs, 2)
	assert.Equal(t, "CN", certs[0].Subject.CommonName)
	assert.Equal(t, "OU", certs[0].Subject.OrganizationalUnit[0])
	assert.Equal(t, "ORG", certs[0].Subject.Organization[0])
	assert.Equal(t, "rootCN1", certs[1].Subject.CommonName)

}
예제 #18
0
// NewRaftDEKManager creates a key if one doesn't exist
func TestNewRaftDEKManager(t *testing.T) {
	tempDir, err := ioutil.TempDir("", "manager-new-dek-manager-")
	require.NoError(t, err)
	defer os.RemoveAll(tempDir)

	paths := ca.NewConfigPaths(tempDir)
	cert, key, err := cautils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
	require.NoError(t, krw.Write(cert, key, nil))

	keyBytes, err := ioutil.ReadFile(paths.Node.Key)
	require.NoError(t, err)
	require.NotContains(t, string(keyBytes), pemHeaderRaftDEK) // headers are not written

	dekManager, err := NewRaftDEKManager(krw) // this should create a new DEK and write it to the file
	require.NoError(t, err)

	keyBytes, err = ioutil.ReadFile(paths.Node.Key)
	require.NoError(t, err)
	require.Contains(t, string(keyBytes), pemHeaderRaftDEK) // header is written now

	keys := dekManager.GetKeys()
	require.NotNil(t, keys.CurrentDEK)
	require.Nil(t, keys.PendingDEK)
	require.False(t, dekManager.NeedsRotation())

	// If one exists, nothing is updated
	dekManager, err = NewRaftDEKManager(krw) // this should create a new DEK and write it to the file
	require.NoError(t, err)

	keyBytes2, err := ioutil.ReadFile(paths.Node.Key)
	require.NoError(t, err)
	require.Equal(t, keyBytes, keyBytes2)

	require.Equal(t, keys, dekManager.GetKeys())
	require.False(t, dekManager.NeedsRotation())
}
예제 #19
0
func TestLoadSecurityConfigInvalidKey(t *testing.T) {
	tc := testutils.NewTestCA(t)
	defer tc.Stop()

	// Write some garbage to the Key
	ioutil.WriteFile(tc.Paths.Node.Key, []byte(`-----BEGIN EC PRIVATE KEY-----\n
some random garbage\n
-----END EC PRIVATE KEY-----`), 0644)

	krw := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)

	_, err := ca.LoadSecurityConfig(tc.Context, tc.RootCA, krw, false)
	assert.Error(t, err)

	nodeConfig, err := tc.RootCA.CreateSecurityConfig(tc.Context, krw,
		ca.CertificateRequestConfig{
			ConnBroker: tc.ConnBroker,
		})
	assert.NoError(t, err)
	assert.NotNil(t, nodeConfig)
	assert.NotNil(t, nodeConfig.ClientTLSCreds)
	assert.NotNil(t, nodeConfig.ServerTLSCreds)
	assert.Equal(t, tc.RootCA, *nodeConfig.RootCA())
}
예제 #20
0
func TestRequestAndSaveNewCertificates(t *testing.T) {
	tc := testutils.NewTestCA(t)
	defer tc.Stop()

	// Copy the current RootCA without the signer
	rca := ca.RootCA{Cert: tc.RootCA.Cert, Pool: tc.RootCA.Pool}
	cert, err := rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
		ca.CertificateRequestConfig{
			Token:   tc.ManagerToken,
			Remotes: tc.Remotes,
		})
	assert.NoError(t, err)
	assert.NotNil(t, cert)
	perms, err := permbits.Stat(tc.Paths.Node.Cert)
	assert.NoError(t, err)
	assert.False(t, perms.GroupWrite())
	assert.False(t, perms.OtherWrite())

	// there was no encryption config in the remote, so the key should be unencrypted
	unencryptedKeyReader := ca.NewKeyReadWriter(tc.Paths.Node, nil, nil)
	_, _, err = unencryptedKeyReader.Read()
	require.NoError(t, err)

	// the worker token is also unencrypted
	cert, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
		ca.CertificateRequestConfig{
			Token:   tc.WorkerToken,
			Remotes: tc.Remotes,
		})
	assert.NoError(t, err)
	assert.NotNil(t, cert)
	_, _, err = unencryptedKeyReader.Read()
	require.NoError(t, err)

	// If there is a different kek in the remote store, when TLS certs are renewed the new key will
	// be encrypted with that kek
	assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
		cluster := store.GetCluster(tx, tc.Organization)
		cluster.Spec.EncryptionConfig.AutoLockManagers = true
		cluster.UnlockKeys = []*api.EncryptionKey{{
			Subsystem: ca.ManagerRole,
			Key:       []byte("kek!"),
		}}
		return store.UpdateCluster(tx, cluster)
	}))
	assert.NoError(t, os.RemoveAll(tc.Paths.Node.Cert))
	assert.NoError(t, os.RemoveAll(tc.Paths.Node.Key))

	_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
		ca.CertificateRequestConfig{
			Token:   tc.ManagerToken,
			Remotes: tc.Remotes,
		})
	assert.NoError(t, err)

	// key can no longer be read without a kek
	_, _, err = unencryptedKeyReader.Read()
	require.Error(t, err)

	_, _, err = ca.NewKeyReadWriter(tc.Paths.Node, []byte("kek!"), nil).Read()
	require.NoError(t, err)

	// if it's a worker though, the key is always unencrypted, even though the manager key is encrypted
	_, err = rca.RequestAndSaveNewCertificates(tc.Context, tc.KeyReadWriter,
		ca.CertificateRequestConfig{
			Token:   tc.WorkerToken,
			Remotes: tc.Remotes,
		})
	assert.NoError(t, err)
	_, _, err = unencryptedKeyReader.Read()
	require.NoError(t, err)
}
예제 #21
0
// If there is no CA, and a join addr is provided, one is downloaded from the
// join server. If there is a CA, it is just loaded from disk.  The TLS key and
// cert are also downloaded.
func TestLoadSecurityConfigDownloadAllCerts(t *testing.T) {
	tempdir, err := ioutil.TempDir("", "test-join-node")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	paths := ca.NewConfigPaths(filepath.Join(tempdir, "certificates"))

	// join addr is invalid
	node, err := New(&Config{
		StateDir: tempdir,
		JoinAddr: "127.0.0.1:12",
	})
	require.NoError(t, err)
	_, err = node.loadSecurityConfig(context.Background())
	require.Error(t, err)

	tc := cautils.NewTestCA(t)
	defer tc.Stop()

	peer, err := tc.ConnBroker.Remotes().Select()
	require.NoError(t, err)

	node, err = New(&Config{
		StateDir:  tempdir,
		JoinAddr:  peer.Addr,
		JoinToken: tc.ManagerToken,
	})
	require.NoError(t, err)
	_, err = node.loadSecurityConfig(context.Background())
	require.NoError(t, err)

	// the TLS key and cert were written to disk unencrypted
	_, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read()
	require.NoError(t, err)

	// remove the TLS cert and key, and mark the root CA cert so that we will
	// know if it gets replaced
	require.NoError(t, os.Remove(paths.Node.Cert))
	require.NoError(t, os.Remove(paths.Node.Key))
	certBytes, err := ioutil.ReadFile(paths.RootCA.Cert)
	require.NoError(t, err)
	pemBlock, _ := pem.Decode(certBytes)
	require.NotNil(t, pemBlock)
	pemBlock.Headers["marked"] = "true"
	certBytes = pem.EncodeToMemory(pemBlock)
	require.NoError(t, ioutil.WriteFile(paths.RootCA.Cert, certBytes, 0644))

	// also make sure the new set gets downloaded and written to disk with a passphrase
	// by updating the memory store with manager autolock on and an unlock key
	require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
		clusters, err := store.FindClusters(tx, store.All)
		require.NoError(t, err)
		require.Len(t, clusters, 1)

		newCluster := clusters[0].Copy()
		newCluster.Spec.EncryptionConfig.AutoLockManagers = true
		newCluster.UnlockKeys = []*api.EncryptionKey{{
			Subsystem: ca.ManagerRole,
			Key:       []byte("passphrase"),
		}}
		return store.UpdateCluster(tx, newCluster)
	}))

	// Join with without any passphrase - this should be fine, because the TLS
	// key is downloaded and then loaded just fine.  However, it *is* written
	// to disk encrypted.
	node, err = New(&Config{
		StateDir:  tempdir,
		JoinAddr:  peer.Addr,
		JoinToken: tc.ManagerToken,
	})
	require.NoError(t, err)
	_, err = node.loadSecurityConfig(context.Background())
	require.NoError(t, err)

	// make sure the CA cert has not been replaced
	readCertBytes, err := ioutil.ReadFile(paths.RootCA.Cert)
	require.NoError(t, err)
	require.Equal(t, certBytes, readCertBytes)

	// the TLS node cert and key were saved to disk encrypted, though
	_, _, err = ca.NewKeyReadWriter(paths.Node, nil, nil).Read()
	require.Error(t, err)
	_, _, err = ca.NewKeyReadWriter(paths.Node, []byte("passphrase"), nil).Read()
	require.NoError(t, err)
}
예제 #22
0
파일: node.go 프로젝트: docker/swarmkit
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
	paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
	var securityConfig *ca.SecurityConfig

	krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
	if err := krw.Migrate(); err != nil {
		return nil, err
	}

	// Check if we already have a valid certificates on disk.
	rootCA, err := ca.GetLocalRootCA(paths.RootCA)
	if err != nil && err != ca.ErrNoLocalRootCA {
		return nil, err
	}
	if err == nil {
		securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
		if err != nil {
			_, isInvalidKEK := errors.Cause(err).(ca.ErrInvalidKEK)
			if isInvalidKEK {
				return nil, ErrInvalidUnlockKey
			} else if !os.IsNotExist(err) {
				return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
			}
		}
	}

	if securityConfig == nil {
		if n.config.JoinAddr == "" {
			// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
			n.unlockKey = nil
			if n.config.AutoLockManagers {
				n.unlockKey = encryption.GenerateSecretKey()
			}
			krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
			rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
			if err != nil {
				return nil, err
			}
			log.G(ctx).Debug("generated CA key and certificate")
		} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
			rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
			if err != nil {
				return nil, err
			}
			log.G(ctx).Debug("downloaded CA certificate")
		}

		// Obtain new certs and setup TLS certificates renewal for this node:
		// - If certificates weren't present on disk, we call CreateSecurityConfig, which blocks
		//   until a valid certificate has been issued.
		// - We wait for CreateSecurityConfig to finish since we need a certificate to operate.

		// Attempt to load certificate from disk
		securityConfig, err = ca.LoadSecurityConfig(ctx, rootCA, krw)
		if err == nil {
			log.G(ctx).WithFields(logrus.Fields{
				"node.id": securityConfig.ClientTLSCreds.NodeID(),
			}).Debugf("loaded TLS certificate")
		} else {
			if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
				return nil, ErrInvalidUnlockKey
			}
			log.G(ctx).WithError(err).Debugf("no node credentials found in: %s", krw.Target())

			securityConfig, err = rootCA.CreateSecurityConfig(ctx, krw, ca.CertificateRequestConfig{
				Token:        n.config.JoinToken,
				Availability: n.config.Availability,
				Remotes:      n.remotes,
			})

			if err != nil {
				return nil, err
			}
		}
	}

	n.Lock()
	n.role = securityConfig.ClientTLSCreds.Role()
	n.nodeID = securityConfig.ClientTLSCreds.NodeID()
	n.roleCond.Broadcast()
	n.Unlock()

	return securityConfig, nil
}
예제 #23
0
파일: node.go 프로젝트: ollie314/docker
func (n *Node) loadSecurityConfig(ctx context.Context) (*ca.SecurityConfig, error) {
	paths := ca.NewConfigPaths(filepath.Join(n.config.StateDir, certDirectory))
	var securityConfig *ca.SecurityConfig

	krw := ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
	if err := krw.Migrate(); err != nil {
		return nil, err
	}

	// Check if we already have a valid certificates on disk.
	rootCA, err := ca.GetLocalRootCA(paths.RootCA)
	if err != nil && err != ca.ErrNoLocalRootCA {
		return nil, err
	}
	if err == nil {
		clientTLSCreds, serverTLSCreds, err := ca.LoadTLSCreds(rootCA, krw)
		_, ok := errors.Cause(err).(ca.ErrInvalidKEK)
		switch {
		case err == nil:
			securityConfig = ca.NewSecurityConfig(&rootCA, krw, clientTLSCreds, serverTLSCreds)
			log.G(ctx).Debug("loaded CA and TLS certificates")
		case ok:
			return nil, ErrInvalidUnlockKey
		case os.IsNotExist(err):
			break
		default:
			return nil, errors.Wrapf(err, "error while loading TLS certificate in %s", paths.Node.Cert)
		}
	}

	if securityConfig == nil {
		if n.config.JoinAddr == "" {
			// if we're not joining a cluster, bootstrap a new one - and we have to set the unlock key
			n.unlockKey = nil
			if n.config.AutoLockManagers {
				n.unlockKey = encryption.GenerateSecretKey()
			}
			krw = ca.NewKeyReadWriter(paths.Node, n.unlockKey, &manager.RaftDEKData{})
			rootCA, err = ca.CreateRootCA(ca.DefaultRootCN, paths.RootCA)
			if err != nil {
				return nil, err
			}
			log.G(ctx).Debug("generated CA key and certificate")
		} else if err == ca.ErrNoLocalRootCA { // from previous error loading the root CA from disk
			rootCA, err = ca.DownloadRootCA(ctx, paths.RootCA, n.config.JoinToken, n.remotes)
			if err != nil {
				return nil, err
			}
			log.G(ctx).Debug("downloaded CA certificate")
		}

		// Obtain new certs and setup TLS certificates renewal for this node:
		// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
		// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
		// us to display the ID before the certificate gets issued (for potential approval).
		// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
		// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
		// up to date.
		issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
		go func() {
			select {
			case <-ctx.Done():
			case resp := <-issueResponseChan:
				log.G(log.WithModule(ctx, "tls")).WithFields(logrus.Fields{
					"node.id": resp.NodeID,
				}).Debugf("loaded TLS certificate")
				n.Lock()
				n.nodeID = resp.NodeID
				n.nodeMembership = resp.NodeMembership
				n.Unlock()
				close(n.certificateRequested)
			}
		}()

		// LoadOrCreateSecurityConfig is the point at which a new node joining a cluster will retrieve TLS
		// certificates and write them to disk
		securityConfig, err = ca.LoadOrCreateSecurityConfig(
			ctx, rootCA, n.config.JoinToken, ca.ManagerRole, n.remotes, issueResponseChan, krw)
		if err != nil {
			if _, ok := errors.Cause(err).(ca.ErrInvalidKEK); ok {
				return nil, ErrInvalidUnlockKey
			}
			return nil, err
		}
	}

	n.Lock()
	n.role = securityConfig.ClientTLSCreds.Role()
	n.nodeID = securityConfig.ClientTLSCreds.NodeID()
	n.nodeMembership = api.NodeMembershipAccepted
	n.roleCond.Broadcast()
	n.Unlock()

	return securityConfig, nil
}
예제 #24
0
func TestGetRemoteCA(t *testing.T) {
	tc := testutils.NewTestCA(t)
	defer tc.Stop()

	shaHash := sha256.New()
	shaHash.Write(tc.RootCA.Cert)
	md := shaHash.Sum(nil)
	mdStr := hex.EncodeToString(md)

	d, err := digest.Parse("sha256:" + mdStr)
	require.NoError(t, err)

	downloadedRootCA, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
	require.NoError(t, err)
	require.Equal(t, downloadedRootCA.Cert, tc.RootCA.Cert)

	// update the test CA to include a multi-certificate bundle as the root - the digest
	// we use to verify with must be the digest of the whole bundle
	tmpDir, err := ioutil.TempDir("", "GetRemoteCA")
	require.NoError(t, err)
	defer os.RemoveAll(tmpDir)
	paths := ca.NewConfigPaths(tmpDir)

	otherRootCA, err := ca.CreateRootCA("other", paths.RootCA)
	require.NoError(t, err)

	comboCertBundle := append(tc.RootCA.Cert, otherRootCA.Cert...)
	require.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
		cluster := store.GetCluster(tx, tc.Organization)
		cluster.RootCA.CACert = comboCertBundle
		cluster.RootCA.CAKey = tc.RootCA.Key
		return store.UpdateCluster(tx, cluster)
	}))
	require.NoError(t, raftutils.PollFunc(nil, func() error {
		_, err := ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
		if err == nil {
			return fmt.Errorf("testca's rootca hasn't updated yet")
		}
		require.Contains(t, err.Error(), "remote CA does not match fingerprint")
		return nil
	}))

	// If we provide the right digest, the root CA is updated and we can validate
	// certs signed by either one
	d = digest.FromBytes(comboCertBundle)
	downloadedRootCA, err = ca.GetRemoteCA(tc.Context, d, tc.ConnBroker)
	require.NoError(t, err)
	require.Equal(t, comboCertBundle, downloadedRootCA.Cert)
	require.Equal(t, 2, len(downloadedRootCA.Pool.Subjects()))

	for _, rootCA := range []ca.RootCA{tc.RootCA, otherRootCA} {
		krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
		_, err := rootCA.IssueAndSaveNewCertificates(krw, "cn", "ou", "org")
		require.NoError(t, err)

		certPEM, _, err := krw.Read()
		require.NoError(t, err)

		cert, err := helpers.ParseCertificatesPEM(certPEM)
		require.NoError(t, err)

		chains, err := cert[0].Verify(x509.VerifyOptions{
			Roots: downloadedRootCA.Pool,
		})
		require.NoError(t, err)
		require.Len(t, chains, 1)
	}
}
예제 #25
0
func TestManager(t *testing.T) {
	ctx := context.Background()

	temp, err := ioutil.TempFile("", "test-socket")
	assert.NoError(t, err)
	assert.NoError(t, temp.Close())
	assert.NoError(t, os.Remove(temp.Name()))

	defer os.RemoveAll(temp.Name())

	stateDir, err := ioutil.TempDir("", "test-raft")
	assert.NoError(t, err)
	defer os.RemoveAll(stateDir)

	tc := testutils.NewTestCA(t, func(p ca.CertPaths) *ca.KeyReadWriter {
		return ca.NewKeyReadWriter(p, []byte("kek"), nil)
	})
	defer tc.Stop()

	agentSecurityConfig, err := tc.NewNodeConfig(ca.WorkerRole)
	assert.NoError(t, err)
	agentDiffOrgSecurityConfig, err := tc.NewNodeConfigOrg(ca.WorkerRole, "another-org")
	assert.NoError(t, err)
	managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
	assert.NoError(t, err)

	m, err := New(&Config{
		RemoteAPI:        RemoteAddrs{ListenAddr: "127.0.0.1:0"},
		ControlAPI:       temp.Name(),
		StateDir:         stateDir,
		SecurityConfig:   managerSecurityConfig,
		AutoLockManagers: true,
		UnlockKey:        []byte("kek"),
	})
	assert.NoError(t, err)
	assert.NotNil(t, m)

	tcpAddr := m.Addr()

	done := make(chan error)
	defer close(done)
	go func() {
		done <- m.Run(ctx)
	}()

	opts := []grpc.DialOption{
		grpc.WithTimeout(10 * time.Second),
		grpc.WithTransportCredentials(agentSecurityConfig.ClientTLSCreds),
	}

	conn, err := grpc.Dial(tcpAddr, opts...)
	assert.NoError(t, err)
	defer func() {
		assert.NoError(t, conn.Close())
	}()

	// We have to send a dummy request to verify if the connection is actually up.
	client := api.NewDispatcherClient(conn)
	_, err = client.Heartbeat(ctx, &api.HeartbeatRequest{})
	assert.Equal(t, dispatcher.ErrNodeNotRegistered.Error(), grpc.ErrorDesc(err))
	_, err = client.Session(ctx, &api.SessionRequest{})
	assert.NoError(t, err)

	// Try to have a client in a different org access this manager
	opts = []grpc.DialOption{
		grpc.WithTimeout(10 * time.Second),
		grpc.WithTransportCredentials(agentDiffOrgSecurityConfig.ClientTLSCreds),
	}

	conn2, err := grpc.Dial(tcpAddr, opts...)
	assert.NoError(t, err)
	defer func() {
		assert.NoError(t, conn2.Close())
	}()

	client = api.NewDispatcherClient(conn2)
	_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
	assert.Contains(t, grpc.ErrorDesc(err), "Permission denied: unauthorized peer role: rpc error: code = 7 desc = Permission denied: remote certificate not part of organization")

	// Verify that requests to the various GRPC services running on TCP
	// are rejected if they don't have certs.
	opts = []grpc.DialOption{
		grpc.WithTimeout(10 * time.Second),
		grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})),
	}

	noCertConn, err := grpc.Dial(tcpAddr, opts...)
	assert.NoError(t, err)
	defer func() {
		assert.NoError(t, noCertConn.Close())
	}()

	client = api.NewDispatcherClient(noCertConn)
	_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
	assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")

	controlClient := api.NewControlClient(noCertConn)
	_, err = controlClient.ListNodes(context.Background(), &api.ListNodesRequest{})
	assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")

	raftClient := api.NewRaftMembershipClient(noCertConn)
	_, err = raftClient.Join(context.Background(), &api.JoinRequest{})
	assert.EqualError(t, err, "rpc error: code = 7 desc = Permission denied: unauthorized peer role: rpc error: code = 7 desc = no client certificates in request")

	opts = []grpc.DialOption{
		grpc.WithTimeout(10 * time.Second),
		grpc.WithTransportCredentials(managerSecurityConfig.ClientTLSCreds),
	}

	controlConn, err := grpc.Dial(tcpAddr, opts...)
	assert.NoError(t, err)
	defer func() {
		assert.NoError(t, controlConn.Close())
	}()

	// check that the kek is added to the config
	var cluster api.Cluster
	m.raftNode.MemoryStore().View(func(tx store.ReadTx) {
		clusters, err := store.FindClusters(tx, store.All)
		require.NoError(t, err)
		require.Len(t, clusters, 1)
		cluster = *clusters[0]
	})
	require.NotNil(t, cluster)
	require.Len(t, cluster.UnlockKeys, 1)
	require.Equal(t, &api.EncryptionKey{
		Subsystem: ca.ManagerRole,
		Key:       []byte("kek"),
	}, cluster.UnlockKeys[0])

	// Test removal of the agent node
	agentID := agentSecurityConfig.ClientTLSCreds.NodeID()
	assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
		return store.CreateNode(tx,
			&api.Node{
				ID: agentID,
				Certificate: api.Certificate{
					Role: api.NodeRoleWorker,
					CN:   agentID,
				},
			},
		)
	}))
	controlClient = api.NewControlClient(controlConn)
	_, err = controlClient.RemoveNode(context.Background(),
		&api.RemoveNodeRequest{
			NodeID: agentID,
			Force:  true,
		},
	)
	assert.NoError(t, err)

	client = api.NewDispatcherClient(conn)
	_, err = client.Heartbeat(context.Background(), &api.HeartbeatRequest{})
	assert.Contains(t, grpc.ErrorDesc(err), "removed from swarm")

	m.Stop(ctx)

	// After stopping we should MAY receive an error from ListenAndServe if
	// all this happened before WaitForLeader completed, so don't check the
	// error.
	<-done
}
예제 #26
0
func TestKeyReadWriterViewAndUpdateHeaders(t *testing.T) {
	cert, key, err := testutils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	tempdir, err := ioutil.TempDir("", "KeyReadWriter")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	path := ca.NewConfigPaths(filepath.Join(tempdir))

	// write a key with headers to the key to make sure it gets passed when reading/writing headers
	keyBlock, _ := pem.Decode(key)
	require.NotNil(t, keyBlock)
	keyBlock.Headers = map[string]string{"hello": "world"}
	key = pem.EncodeToMemory(keyBlock)
	require.NoError(t, ioutil.WriteFile(path.Node.Cert, cert, 0644))
	require.NoError(t, ioutil.WriteFile(path.Node.Key, key, 0600))

	// if the update headers callback function fails, updating headers fails
	k := ca.NewKeyReadWriter(path.Node, nil, nil)
	err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
		require.Nil(t, h)
		return nil, fmt.Errorf("nope")
	})
	require.Error(t, err)
	require.Equal(t, "nope", err.Error())

	// updating headers succeed and is called with the latest kek data
	err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
		require.Nil(t, h)
		return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
			return map[string]string{"updated": "headers"}, nil
		}}, nil
	})
	require.NoError(t, err)

	k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
		require.Equal(t, map[string]string{"updated": "headers"}, h)
		require.Equal(t, ca.KEKData{}, k)
		return testHeaders{}, nil
	}})
	_, _, err = k.Read()
	require.NoError(t, err)

	// we can also update headers on an encrypted key
	k = ca.NewKeyReadWriter(path.Node, []byte("kek"), nil)
	require.NoError(t, k.Write(cert, key, nil))

	err = k.ViewAndUpdateHeaders(func(h ca.PEMKeyHeaders) (ca.PEMKeyHeaders, error) {
		require.Nil(t, h)
		return testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
			require.Equal(t, ca.KEKData{KEK: []byte("kek")}, kek)
			return map[string]string{"updated": "headers"}, nil
		}}, nil
	})
	require.NoError(t, err)

	k = ca.NewKeyReadWriter(path.Node, []byte("kek"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
		require.Equal(t, map[string]string{"updated": "headers"}, h)
		require.Equal(t, ca.KEKData{KEK: []byte("kek")}, k)
		return testHeaders{}, nil
	}})
	_, _, err = k.Read()
	require.NoError(t, err)
}
예제 #27
0
// can read and write tls keys that aren't encrypted, and that are encrypted.  without
// a pem header manager, the headers are all preserved and not overwritten
func TestKeyReadWriter(t *testing.T) {
	cert, key, err := testutils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	expectedKey := key

	tempdir, err := ioutil.TempDir("", "KeyReadWriter")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created

	checkCanReadWithKEK := func(kek []byte) *ca.KeyReadWriter {
		k := ca.NewKeyReadWriter(path.Node, kek, nil)
		readCert, readKey, err := k.Read()
		require.NoError(t, err)
		require.Equal(t, cert, readCert)
		require.Equal(t, expectedKey, readKey, "Expected %s, Got %s", string(expectedKey), string(readKey))
		return k
	}

	k := ca.NewKeyReadWriter(path.Node, nil, nil)

	// can't read things that don't exist
	_, _, err = k.Read()
	require.Error(t, err)

	// can write an unencrypted key with no updates
	require.NoError(t, k.Write(cert, expectedKey, nil))

	// can read unencrypted
	k = checkCanReadWithKEK(nil)
	_, kekData := k.GetCurrentState()
	require.EqualValues(t, 0, kekData.Version) // the first version was 0

	// write a key with headers to the key to make sure they're cleaned
	keyBlock, _ := pem.Decode(expectedKey)
	require.NotNil(t, keyBlock)
	keyBlock.Headers = map[string]string{"hello": "world"}
	expectedKey = pem.EncodeToMemory(keyBlock)
	// write a version, but that's not what we'd expect back once we read
	keyBlock.Headers["kek-version"] = "8"
	require.NoError(t, ioutil.WriteFile(path.Node.Key, pem.EncodeToMemory(keyBlock), 0600))

	// if a kek is provided, we can still read unencrypted keys, and read
	// the provided version
	k = checkCanReadWithKEK([]byte("original kek"))
	_, kekData = k.GetCurrentState()
	require.EqualValues(t, 8, kekData.Version)

	// we can update the kek and write at the same time
	require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("new kek!"), Version: 3}))

	// the same kek can still read, and will continue to write with this key if
	// no further kek updates are provided
	_, _, err = k.Read()
	require.NoError(t, err)
	require.NoError(t, k.Write(cert, expectedKey, nil))

	expectedKey = key

	// without the right kek, we can't read
	k = ca.NewKeyReadWriter(path.Node, []byte("original kek"), nil)
	_, _, err = k.Read()
	require.Error(t, err)

	// same new key, just for sanity
	k = checkCanReadWithKEK([]byte("new kek!"))
	_, kekData = k.GetCurrentState()
	require.EqualValues(t, 3, kekData.Version)

	// we can also change the kek back to nil, which means the key is unencrypted
	require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: nil}))
	k = checkCanReadWithKEK(nil)
	_, kekData = k.GetCurrentState()
	require.EqualValues(t, 0, kekData.Version)
}
예제 #28
0
// KeyReaderWriter makes a call to a get headers updater, if write is called,
// and set headers, if read is called.  The KEK version header is always preserved
// no matter what.
func TestKeyReadWriterWithPemHeaderManager(t *testing.T) {
	cert, key, err := testutils.CreateRootCertAndKey("cn")
	require.NoError(t, err)

	// write a key with headers to the key to make sure it gets overwritten
	keyBlock, _ := pem.Decode(key)
	require.NotNil(t, keyBlock)
	keyBlock.Headers = map[string]string{"hello": "world"}
	key = pem.EncodeToMemory(keyBlock)

	tempdir, err := ioutil.TempDir("", "KeyReadWriter")
	require.NoError(t, err)
	defer os.RemoveAll(tempdir)

	path := ca.NewConfigPaths(filepath.Join(tempdir, "subdir")) // to make sure subdirectories are created

	// if if getting new headers fail, writing a key fails, and the key does not rotate
	var count int
	badKEKData := ca.KEKData{KEK: []byte("failed kek"), Version: 3}
	k := ca.NewKeyReadWriter(path.Node, nil, testHeaders{newHeaders: func(k ca.KEKData) (map[string]string, error) {
		if count == 0 {
			count++
			require.Equal(t, badKEKData, k)
			return nil, fmt.Errorf("fail")
		}
		require.Equal(t, ca.KEKData{}, k)
		return nil, nil
	}})
	// first write will fail
	require.Error(t, k.Write(cert, key, &badKEKData))
	// the stored kek data will be not be updated because the write failed
	_, kekData := k.GetCurrentState()
	require.Equal(t, ca.KEKData{}, kekData)
	// second write will succeed, using the original kek (nil)
	require.NoError(t, k.Write(cert, key, nil))

	var (
		headers map[string]string
		kek     ca.KEKData
	)

	// if setting headers fail, reading fails
	k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(map[string]string, ca.KEKData) (ca.PEMKeyHeaders, error) {
		return nil, fmt.Errorf("nope")
	}})
	_, _, err = k.Read()
	require.Error(t, err)

	k = ca.NewKeyReadWriter(path.Node, nil, testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
		headers = h
		kek = k
		return testHeaders{}, nil
	}})

	_, _, err = k.Read()
	require.NoError(t, err)
	require.Equal(t, ca.KEKData{}, kek)
	require.Equal(t, keyBlock.Headers, headers)

	// writing new headers is called with existing headers, and will write a key that has the headers
	// returned by the header update function
	k = ca.NewKeyReadWriter(path.Node, []byte("oldKek"), testHeaders{newHeaders: func(kek ca.KEKData) (map[string]string, error) {
		require.Equal(t, []byte("newKEK"), kek.KEK)
		return map[string]string{"updated": "headers"}, nil
	}})
	require.NoError(t, k.Write(cert, key, &ca.KEKData{KEK: []byte("newKEK"), Version: 2}))

	// make sure headers were correctly set
	k = ca.NewKeyReadWriter(path.Node, []byte("newKEK"), testHeaders{setHeaders: func(h map[string]string, k ca.KEKData) (ca.PEMKeyHeaders, error) {
		headers = h
		kek = k
		return testHeaders{}, nil
	}})
	_, _, err = k.Read()
	require.NoError(t, err)
	require.Equal(t, ca.KEKData{KEK: []byte("newKEK"), Version: 2}, kek)

	_, kekData = k.GetCurrentState()
	require.Equal(t, kek, kekData)
	require.Equal(t, map[string]string{"updated": "headers"}, headers)
}
예제 #29
0
func TestIsStateDirty(t *testing.T) {
	ctx := context.Background()

	temp, err := ioutil.TempFile("", "test-socket")
	assert.NoError(t, err)
	assert.NoError(t, temp.Close())
	assert.NoError(t, os.Remove(temp.Name()))

	defer os.RemoveAll(temp.Name())

	stateDir, err := ioutil.TempDir("", "test-raft")
	assert.NoError(t, err)
	defer os.RemoveAll(stateDir)

	tc := testutils.NewTestCA(t, func(p ca.CertPaths) *ca.KeyReadWriter {
		return ca.NewKeyReadWriter(p, []byte("kek"), nil)
	})
	defer tc.Stop()

	managerSecurityConfig, err := tc.NewNodeConfig(ca.ManagerRole)
	assert.NoError(t, err)

	m, err := New(&Config{
		RemoteAPI:        &RemoteAddrs{ListenAddr: "127.0.0.1:0"},
		ControlAPI:       temp.Name(),
		StateDir:         stateDir,
		SecurityConfig:   managerSecurityConfig,
		AutoLockManagers: true,
		UnlockKey:        []byte("kek"),
	})
	assert.NoError(t, err)
	assert.NotNil(t, m)

	go m.Run(ctx)
	defer m.Stop(ctx, false)

	// State should never be dirty just after creating the manager
	isDirty, err := m.IsStateDirty()
	assert.NoError(t, err)
	assert.False(t, isDirty)

	// Wait for cluster and node to be created.
	watch, cancel := state.Watch(m.raftNode.MemoryStore().WatchQueue())
	defer cancel()
	<-watch
	<-watch

	// Updating the node should not cause the state to become dirty
	assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
		node := store.GetNode(tx, m.config.SecurityConfig.ClientTLSCreds.NodeID())
		require.NotNil(t, node)
		node.Spec.Availability = api.NodeAvailabilityPause
		return store.UpdateNode(tx, node)
	}))
	isDirty, err = m.IsStateDirty()
	assert.NoError(t, err)
	assert.False(t, isDirty)

	// Adding a service should cause the state to become dirty
	assert.NoError(t, m.raftNode.MemoryStore().Update(func(tx store.Tx) error {
		return store.CreateService(tx, &api.Service{ID: "foo"})
	}))
	isDirty, err = m.IsStateDirty()
	assert.NoError(t, err)
	assert.True(t, isDirty)
}
예제 #30
0
// NewTestCA is a helper method that creates a TestCA and a bunch of default
// connections and security configs.
func NewTestCA(t *testing.T, krwGenerators ...func(ca.CertPaths) *ca.KeyReadWriter) *TestCA {
	tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
	assert.NoError(t, err)

	s := store.NewMemoryStore(nil)

	paths := ca.NewConfigPaths(tempBaseDir)
	organization := identity.NewID()

	rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration)
	assert.NoError(t, err)

	var (
		externalSigningServer *ExternalSigningServer
		externalCAs           []*api.ExternalCA
	)

	if External {
		// Start the CA API server.
		externalSigningServer, err = NewExternalSigningServer(rootCA, tempBaseDir)
		assert.NoError(t, err)
		externalCAs = []*api.ExternalCA{
			{
				Protocol: api.ExternalCA_CAProtocolCFSSL,
				URL:      externalSigningServer.URL,
			},
		}
	}

	krw := ca.NewKeyReadWriter(paths.Node, nil, nil)
	if len(krwGenerators) > 0 {
		krw = krwGenerators[0](paths.Node)
	}

	managerConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, organization, "", External)
	assert.NoError(t, err)

	managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, krw, ca.ManagerRole, "swarm-test-org-2", "", External)
	assert.NoError(t, err)

	workerConfig, err := genSecurityConfig(s, rootCA, krw, ca.WorkerRole, organization, "", External)
	assert.NoError(t, err)

	l, err := net.Listen("tcp", "127.0.0.1:0")
	assert.NoError(t, err)

	baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
	insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
	clientOpts := append(baseOpts, grpc.WithTransportCredentials(workerConfig.ClientTLSCreds))
	managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds))
	managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds))

	conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...)
	assert.NoError(t, err)

	conn2, err := grpc.Dial(l.Addr().String(), clientOpts...)
	assert.NoError(t, err)

	conn3, err := grpc.Dial(l.Addr().String(), managerOpts...)
	assert.NoError(t, err)

	conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...)
	assert.NoError(t, err)

	serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)}
	grpcServer := grpc.NewServer(serverOpts...)

	managerToken := ca.GenerateJoinToken(&rootCA)
	workerToken := ca.GenerateJoinToken(&rootCA)

	createClusterObject(t, s, organization, workerToken, managerToken, externalCAs...)
	caServer := ca.NewServer(s, managerConfig)
	caServer.SetReconciliationRetryInterval(50 * time.Millisecond)
	api.RegisterCAServer(grpcServer, caServer)
	api.RegisterNodeCAServer(grpcServer, caServer)

	ctx := context.Background()

	go grpcServer.Serve(l)
	go caServer.Run(ctx)

	// Wait for caServer to be ready to serve
	<-caServer.Ready()
	remotes := remotes.NewRemotes(api.Peer{Addr: l.Addr().String()})

	caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)}
	nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)}
	conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4}

	return &TestCA{
		RootCA:                rootCA,
		ExternalSigningServer: externalSigningServer,
		MemoryStore:           s,
		TempDir:               tempBaseDir,
		Organization:          organization,
		Paths:                 paths,
		Context:               ctx,
		CAClients:             caClients,
		NodeCAClients:         nodeCAClients,
		Conns:                 conns,
		Server:                grpcServer,
		CAServer:              caServer,
		WorkerToken:           workerToken,
		ManagerToken:          managerToken,
		ConnBroker:            connectionbroker.New(remotes),
		KeyReadWriter:         krw,
	}
}