// defaultClusterObject creates a default cluster.
func defaultClusterObject(clusterID string, initialCAConfig api.CAConfig, raftCfg api.RaftConfig, rootCA *ca.RootCA) *api.Cluster {
return &api.Cluster{
ID: clusterID,
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: store.DefaultClusterName,
},
Orchestration: api.OrchestrationConfig{
TaskHistoryRetentionLimit: defaultTaskHistoryRetentionLimit,
},
Dispatcher: api.DispatcherConfig{
HeartbeatPeriod: ptypes.DurationProto(dispatcher.DefaultHeartBeatPeriod),
},
Raft: raftCfg,
CAConfig: initialCAConfig,
},
RootCA: api.RootCA{
CAKey: rootCA.Key,
CACert: rootCA.Cert,
CACertHash: rootCA.Digest.String(),
JoinTokens: api.JoinTokens{
Worker: ca.GenerateJoinToken(rootCA),
Manager: ca.GenerateJoinToken(rootCA),
},
},
}
}
func createClusterObj(id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster {
spec := createClusterSpec(name)
spec.AcceptancePolicy = policy
return &api.Cluster{
ID: id,
Spec: *spec,
RootCA: api.RootCA{
CACert: []byte("-----BEGIN CERTIFICATE-----AwEHoUQDQgAEZ4vGYkSt/kjoHbUjDx9eyO1xBVJEH2F+AwM9lACIZ414cD1qYy8u-----BEGIN CERTIFICATE-----"),
CAKey: []byte("-----BEGIN EC PRIVATE KEY-----AwEHoUQDQgAEZ4vGYkSt/kjoHbUjDx9eyO1xBVJEH2F+AwM9lACIZ414cD1qYy8u-----END EC PRIVATE KEY-----"),
CACertHash: "hash",
JoinTokens: api.JoinTokens{
Worker: ca.GenerateJoinToken(rootCA),
Manager: ca.GenerateJoinToken(rootCA),
},
},
}
}
// UpdateCluster updates a Cluster referenced by ClusterID with the given ClusterSpec.
// - Returns `NotFound` if the Cluster is not found.
// - Returns `InvalidArgument` if the ClusterSpec is malformed.
// - Returns `Unimplemented` if the ClusterSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) {
if request.ClusterID == "" || request.ClusterVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateClusterSpec(request.Spec); err != nil {
return nil, err
}
var cluster *api.Cluster
err := s.store.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, request.ClusterID)
if cluster == nil {
return nil
}
cluster.Meta.Version = *request.ClusterVersion
cluster.Spec = *request.Spec.Copy()
expireBlacklistedCerts(cluster)
if request.Rotation.RotateWorkerToken {
cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA)
}
if request.Rotation.RotateManagerToken {
cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA)
}
return store.UpdateCluster(tx, cluster)
})
if err != nil {
return nil, err
}
if cluster == nil {
return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
}
redactedClusters := redactClusters([]*api.Cluster{cluster})
// WARN: we should never return cluster here. We need to redact the private fields first.
return &api.UpdateClusterResponse{
Cluster: redactedClusters[0],
}, nil
}
func createCluster(t *testing.T, ts *testServer, id, name string, policy api.AcceptancePolicy, rootCA *ca.RootCA) *api.Cluster {
spec := createClusterSpec(name)
spec.AcceptancePolicy = policy
cluster := &api.Cluster{
ID: id,
Spec: *spec,
RootCA: api.RootCA{
CACert: []byte("-----BEGIN CERTIFICATE-----AwEHoUQDQgAEZ4vGYkSt/kjoHbUjDx9eyO1xBVJEH2F+AwM9lACIZ414cD1qYy8u-----BEGIN CERTIFICATE-----"),
CAKey: []byte("-----BEGIN EC PRIVATE KEY-----AwEHoUQDQgAEZ4vGYkSt/kjoHbUjDx9eyO1xBVJEH2F+AwM9lACIZ414cD1qYy8u-----END EC PRIVATE KEY-----"),
CACertHash: "hash",
JoinTokens: api.JoinTokens{
Worker: ca.GenerateJoinToken(rootCA),
Manager: ca.GenerateJoinToken(rootCA),
},
},
}
assert.NoError(t, ts.Store.Update(func(tx store.Tx) error {
return store.CreateCluster(tx, cluster)
}))
return cluster
}
// Run starts all manager sub-systems and the gRPC server at the configured
// address.
// The call never returns unless an error occurs or `Stop()` is called.
//
// TODO(aluzzardi): /!\ This function is *way* too complex. /!\
// It needs to be split into smaller manageable functions.
func (m *Manager) Run(parent context.Context) error {
ctx, ctxCancel := context.WithCancel(parent)
defer ctxCancel()
// Harakiri.
go func() {
select {
case <-ctx.Done():
case <-m.stopped:
ctxCancel()
}
}()
leadershipCh, cancel := m.RaftNode.SubscribeLeadership()
defer cancel()
go func() {
for leadershipEvent := range leadershipCh {
// read out and discard all of the messages when we've stopped
// don't acquire the mutex yet. if stopped is closed, we don't need
// this stops this loop from starving Run()'s attempt to Lock
select {
case <-m.stopped:
continue
default:
// do nothing, we're not stopped
}
// we're not stopping so NOW acquire the mutex
m.mu.Lock()
newState := leadershipEvent.(raft.LeadershipState)
if newState == raft.IsLeader {
s := m.RaftNode.MemoryStore()
rootCA := m.config.SecurityConfig.RootCA()
nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID()
raftCfg := raft.DefaultRaftConfig()
raftCfg.ElectionTick = uint32(m.RaftNode.Config.ElectionTick)
raftCfg.HeartbeatTick = uint32(m.RaftNode.Config.HeartbeatTick)
clusterID := m.config.SecurityConfig.ClientTLSCreds.Organization()
initialCAConfig := ca.DefaultCAConfig()
initialCAConfig.ExternalCAs = m.config.ExternalCAs
s.Update(func(tx store.Tx) error {
// Add a default cluster object to the
// store. Don't check the error because
// we expect this to fail unless this
// is a brand new cluster.
store.CreateCluster(tx, &api.Cluster{
ID: clusterID,
Spec: api.ClusterSpec{
Annotations: api.Annotations{
Name: store.DefaultClusterName,
},
Orchestration: api.OrchestrationConfig{
TaskHistoryRetentionLimit: defaultTaskHistoryRetentionLimit,
},
Dispatcher: api.DispatcherConfig{
HeartbeatPeriod: ptypes.DurationProto(dispatcher.DefaultHeartBeatPeriod),
},
Raft: raftCfg,
CAConfig: initialCAConfig,
},
RootCA: api.RootCA{
CAKey: rootCA.Key,
CACert: rootCA.Cert,
CACertHash: rootCA.Digest.String(),
JoinTokens: api.JoinTokens{
Worker: ca.GenerateJoinToken(rootCA),
Manager: ca.GenerateJoinToken(rootCA),
},
},
})
// Add Node entry for ourself, if one
// doesn't exist already.
store.CreateNode(tx, &api.Node{
ID: nodeID,
Certificate: api.Certificate{
CN: nodeID,
Role: api.NodeRoleManager,
Status: api.IssuanceStatus{
State: api.IssuanceStateIssued,
},
},
Spec: api.NodeSpec{
Role: api.NodeRoleManager,
Membership: api.NodeMembershipAccepted,
},
})
return nil
})
// Attempt to rotate the key-encrypting-key of the root CA key-material
err := m.rotateRootCAKEK(ctx, clusterID)
if err != nil {
log.G(ctx).WithError(err).Error("root key-encrypting-key rotation failed")
}
m.replicatedOrchestrator = orchestrator.NewReplicatedOrchestrator(s)
m.globalOrchestrator = orchestrator.NewGlobalOrchestrator(s)
m.taskReaper = orchestrator.NewTaskReaper(s)
m.scheduler = scheduler.New(s)
m.keyManager = keymanager.New(m.RaftNode.MemoryStore(), keymanager.DefaultConfig())
// TODO(stevvooe): Allocate a context that can be used to
// shutdown underlying manager processes when leadership is
// lost.
m.allocator, err = allocator.New(s)
if err != nil {
log.G(ctx).WithError(err).Error("failed to create allocator")
// TODO(stevvooe): It doesn't seem correct here to fail
// creating the allocator but then use it anyway.
}
if m.keyManager != nil {
go func(keyManager *keymanager.KeyManager) {
if err := keyManager.Run(ctx); err != nil {
log.G(ctx).WithError(err).Error("keymanager failed with an error")
}
}(m.keyManager)
}
go func(d *dispatcher.Dispatcher) {
if err := d.Run(ctx); err != nil {
log.G(ctx).WithError(err).Error("Dispatcher exited with an error")
}
}(m.Dispatcher)
go func(server *ca.Server) {
if err := server.Run(ctx); err != nil {
log.G(ctx).WithError(err).Error("CA signer exited with an error")
}
}(m.caserver)
// Start all sub-components in separate goroutines.
// TODO(aluzzardi): This should have some kind of error handling so that
// any component that goes down would bring the entire manager down.
if m.allocator != nil {
go func(allocator *allocator.Allocator) {
if err := allocator.Run(ctx); err != nil {
log.G(ctx).WithError(err).Error("allocator exited with an error")
}
}(m.allocator)
}
go func(scheduler *scheduler.Scheduler) {
if err := scheduler.Run(ctx); err != nil {
log.G(ctx).WithError(err).Error("scheduler exited with an error")
}
}(m.scheduler)
go func(taskReaper *orchestrator.TaskReaper) {
taskReaper.Run()
}(m.taskReaper)
go func(orchestrator *orchestrator.ReplicatedOrchestrator) {
if err := orchestrator.Run(ctx); err != nil {
log.G(ctx).WithError(err).Error("replicated orchestrator exited with an error")
}
}(m.replicatedOrchestrator)
go func(globalOrchestrator *orchestrator.GlobalOrchestrator) {
if err := globalOrchestrator.Run(ctx); err != nil {
log.G(ctx).WithError(err).Error("global orchestrator exited with an error")
}
}(m.globalOrchestrator)
} else if newState == raft.IsFollower {
m.Dispatcher.Stop()
m.caserver.Stop()
if m.allocator != nil {
m.allocator.Stop()
m.allocator = nil
}
m.replicatedOrchestrator.Stop()
m.replicatedOrchestrator = nil
m.globalOrchestrator.Stop()
m.globalOrchestrator = nil
m.taskReaper.Stop()
m.taskReaper = nil
m.scheduler.Stop()
m.scheduler = nil
if m.keyManager != nil {
m.keyManager.Stop()
m.keyManager = nil
}
}
m.mu.Unlock()
}
}()
proxyOpts := []grpc.DialOption{
grpc.WithTimeout(5 * time.Second),
grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds),
}
cs := raftpicker.NewConnSelector(m.RaftNode, proxyOpts...)
m.connSelector = cs
// We need special connSelector for controlapi because it provides automatic
// leader tracking.
// Other APIs are using connSelector which errors out on leader change, but
// allows to react quickly to reelections.
controlAPIProxyOpts := []grpc.DialOption{
grpc.WithBackoffMaxDelay(time.Second),
grpc.WithTransportCredentials(m.config.SecurityConfig.ClientTLSCreds),
}
controlAPIConnSelector := hackpicker.NewConnSelector(m.RaftNode, controlAPIProxyOpts...)
authorize := func(ctx context.Context, roles []string) error {
// Authorize the remote roles, ensure they can only be forwarded by managers
_, err := ca.AuthorizeForwardedRoleAndOrg(ctx, roles, []string{ca.ManagerRole}, m.config.SecurityConfig.ClientTLSCreds.Organization())
return err
}
baseControlAPI := controlapi.NewServer(m.RaftNode.MemoryStore(), m.RaftNode, m.config.SecurityConfig.RootCA())
healthServer := health.NewHealthServer()
authenticatedControlAPI := api.NewAuthenticatedWrapperControlServer(baseControlAPI, authorize)
authenticatedDispatcherAPI := api.NewAuthenticatedWrapperDispatcherServer(m.Dispatcher, authorize)
authenticatedCAAPI := api.NewAuthenticatedWrapperCAServer(m.caserver, authorize)
authenticatedNodeCAAPI := api.NewAuthenticatedWrapperNodeCAServer(m.caserver, authorize)
authenticatedRaftAPI := api.NewAuthenticatedWrapperRaftServer(m.RaftNode, authorize)
authenticatedHealthAPI := api.NewAuthenticatedWrapperHealthServer(healthServer, authorize)
authenticatedRaftMembershipAPI := api.NewAuthenticatedWrapperRaftMembershipServer(m.RaftNode, authorize)
proxyDispatcherAPI := api.NewRaftProxyDispatcherServer(authenticatedDispatcherAPI, cs, m.RaftNode, ca.WithMetadataForwardTLSInfo)
proxyCAAPI := api.NewRaftProxyCAServer(authenticatedCAAPI, cs, m.RaftNode, ca.WithMetadataForwardTLSInfo)
proxyNodeCAAPI := api.NewRaftProxyNodeCAServer(authenticatedNodeCAAPI, cs, m.RaftNode, ca.WithMetadataForwardTLSInfo)
proxyRaftMembershipAPI := api.NewRaftProxyRaftMembershipServer(authenticatedRaftMembershipAPI, cs, m.RaftNode, ca.WithMetadataForwardTLSInfo)
// localProxyControlAPI is a special kind of proxy. It is only wired up
// to receive requests from a trusted local socket, and these requests
// don't use TLS, therefore the requests it handles locally should
// bypass authorization. When it proxies, it sends them as requests from
// this manager rather than forwarded requests (it has no TLS
// information to put in the metadata map).
forwardAsOwnRequest := func(ctx context.Context) (context.Context, error) { return ctx, nil }
localProxyControlAPI := api.NewRaftProxyControlServer(baseControlAPI, controlAPIConnSelector, m.RaftNode, forwardAsOwnRequest)
// Everything registered on m.server should be an authenticated
// wrapper, or a proxy wrapping an authenticated wrapper!
api.RegisterCAServer(m.server, proxyCAAPI)
api.RegisterNodeCAServer(m.server, proxyNodeCAAPI)
api.RegisterRaftServer(m.server, authenticatedRaftAPI)
api.RegisterHealthServer(m.server, authenticatedHealthAPI)
api.RegisterRaftMembershipServer(m.server, proxyRaftMembershipAPI)
api.RegisterControlServer(m.localserver, localProxyControlAPI)
api.RegisterControlServer(m.server, authenticatedControlAPI)
api.RegisterDispatcherServer(m.server, proxyDispatcherAPI)
errServe := make(chan error, 2)
for proto, l := range m.listeners {
go func(proto string, lis net.Listener) {
ctx := log.WithLogger(ctx, log.G(ctx).WithFields(
logrus.Fields{
"proto": lis.Addr().Network(),
"addr": lis.Addr().String()}))
if proto == "unix" {
log.G(ctx).Info("Listening for local connections")
// we need to disallow double closes because UnixListener.Close
// can delete unix-socket file of newer listener. grpc calls
// Close twice indeed: in Serve and in Stop.
errServe <- m.localserver.Serve(&closeOnceListener{Listener: lis})
} else {
log.G(ctx).Info("Listening for connections")
errServe <- m.server.Serve(lis)
}
}(proto, l)
}
// Set the raft server as serving for the health server
healthServer.SetServingStatus("Raft", api.HealthCheckResponse_SERVING)
if err := m.RaftNode.JoinAndStart(); err != nil {
for _, lis := range m.listeners {
lis.Close()
}
return fmt.Errorf("can't initialize raft node: %v", err)
}
close(m.started)
go func() {
err := m.RaftNode.Run(ctx)
if err != nil {
log.G(ctx).Error(err)
m.Stop(ctx)
}
}()
if err := raft.WaitForLeader(ctx, m.RaftNode); err != nil {
m.server.Stop()
return err
}
c, err := raft.WaitForCluster(ctx, m.RaftNode)
if err != nil {
m.server.Stop()
return err
}
raftConfig := c.Spec.Raft
if int(raftConfig.ElectionTick) != m.RaftNode.Config.ElectionTick {
log.G(ctx).Warningf("election tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.RaftNode.Config.ElectionTick, raftConfig.ElectionTick)
}
if int(raftConfig.HeartbeatTick) != m.RaftNode.Config.HeartbeatTick {
log.G(ctx).Warningf("heartbeat tick value (%ds) is different from the one defined in the cluster config (%vs), the cluster may be unstable", m.RaftNode.Config.HeartbeatTick, raftConfig.HeartbeatTick)
}
// wait for an error in serving.
err = <-errServe
select {
// check to see if stopped was posted to. if so, we're in the process of
// stopping, or done and that's why we got the error. if stopping is
// deliberate, stopped will ALWAYS be closed before the error is trigger,
// so this path will ALWAYS be taken if the stop was deliberate
case <-m.stopped:
// shutdown was requested, do not return an error
// but first, we wait to acquire a mutex to guarantee that stopping is
// finished. as long as we acquire the mutex BEFORE we return, we know
// that stopping is stopped.
m.mu.Lock()
m.mu.Unlock()
return nil
// otherwise, we'll get something from errServe, which indicates that an
// error in serving has actually occurred and this isn't a planned shutdown
default:
return err
}
}
func TestNewNodeCertificateRequiresToken(t *testing.T) {
t.Parallel()
tc := testutils.NewTestCA(t)
defer tc.Stop()
csr, _, err := ca.GenerateNewCSR()
assert.NoError(t, err)
// Issuance fails if no secret is provided
role := api.NodeRoleManager
issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance fails if wrong secret is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance succeeds if correct token is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
// Rotate manager and worker tokens
var (
newManagerToken string
newWorkerToken string
)
assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
clusters, _ := store.FindClusters(tx, store.ByName(store.DefaultClusterName))
newWorkerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Worker = newWorkerToken
newManagerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Manager = newManagerToken
return store.UpdateCluster(tx, clusters[0])
}))
// updating the join token may take a little bit in order to register on the CA server, so poll
assert.NoError(t, raftutils.PollFunc(nil, func() error {
// Old token should fail
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
if err == nil {
return fmt.Errorf("join token not updated yet")
}
return nil
}))
// Old token should fail
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// New token should succeed
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newWorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
}
// NewTestCA is a helper method that creates a TestCA and a bunch of default
// connections and security configs.
func NewTestCA(t *testing.T) *TestCA {
tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
assert.NoError(t, err)
s := store.NewMemoryStore(nil)
paths := ca.NewConfigPaths(tempBaseDir)
organization := identity.NewID()
rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration)
assert.NoError(t, err)
var (
externalSigningServer *ExternalSigningServer
externalCAs []*api.ExternalCA
)
if External {
// Start the CA API server.
externalSigningServer, err = NewExternalSigningServer(rootCA, tempBaseDir)
assert.NoError(t, err)
externalCAs = []*api.ExternalCA{
{
Protocol: api.ExternalCA_CAProtocolCFSSL,
URL: externalSigningServer.URL,
},
}
}
managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "", External)
assert.NoError(t, err)
managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "", External)
assert.NoError(t, err)
agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "", External)
assert.NoError(t, err)
l, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds))
managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds))
managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds))
conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...)
assert.NoError(t, err)
conn2, err := grpc.Dial(l.Addr().String(), clientOpts...)
assert.NoError(t, err)
conn3, err := grpc.Dial(l.Addr().String(), managerOpts...)
assert.NoError(t, err)
conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...)
assert.NoError(t, err)
serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)}
grpcServer := grpc.NewServer(serverOpts...)
managerToken := ca.GenerateJoinToken(&rootCA)
workerToken := ca.GenerateJoinToken(&rootCA)
createClusterObject(t, s, organization, workerToken, managerToken, externalCAs...)
caServer := ca.NewServer(s, managerConfig)
api.RegisterCAServer(grpcServer, caServer)
api.RegisterNodeCAServer(grpcServer, caServer)
ctx := context.Background()
go grpcServer.Serve(l)
go caServer.Run(ctx)
// Wait for caServer to be ready to serve
<-caServer.Ready()
remotes := remotes.NewRemotes(api.Peer{Addr: l.Addr().String()})
caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)}
nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)}
conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4}
return &TestCA{
RootCA: rootCA,
ExternalSigningServer: externalSigningServer,
MemoryStore: s,
TempDir: tempBaseDir,
Organization: organization,
Paths: paths,
Context: ctx,
CAClients: caClients,
NodeCAClients: nodeCAClients,
Conns: conns,
Server: grpcServer,
CAServer: caServer,
WorkerToken: workerToken,
ManagerToken: managerToken,
Remotes: remotes,
}
}
// UpdateCluster updates a Cluster referenced by ClusterID with the given ClusterSpec.
// - Returns `NotFound` if the Cluster is not found.
// - Returns `InvalidArgument` if the ClusterSpec is malformed.
// - Returns `Unimplemented` if the ClusterSpec references unimplemented features.
// - Returns an error if the update fails.
func (s *Server) UpdateCluster(ctx context.Context, request *api.UpdateClusterRequest) (*api.UpdateClusterResponse, error) {
if request.ClusterID == "" || request.ClusterVersion == nil {
return nil, grpc.Errorf(codes.InvalidArgument, errInvalidArgument.Error())
}
if err := validateClusterSpec(request.Spec); err != nil {
return nil, err
}
var cluster *api.Cluster
err := s.store.Update(func(tx store.Tx) error {
cluster = store.GetCluster(tx, request.ClusterID)
if cluster == nil {
return nil
}
cluster.Meta.Version = *request.ClusterVersion
cluster.Spec = *request.Spec.Copy()
expireBlacklistedCerts(cluster)
if request.Rotation.WorkerJoinToken {
cluster.RootCA.JoinTokens.Worker = ca.GenerateJoinToken(s.rootCA)
}
if request.Rotation.ManagerJoinToken {
cluster.RootCA.JoinTokens.Manager = ca.GenerateJoinToken(s.rootCA)
}
var unlockKeys []*api.EncryptionKey
var managerKey *api.EncryptionKey
for _, eKey := range cluster.UnlockKeys {
if eKey.Subsystem == ca.ManagerRole {
if !cluster.Spec.EncryptionConfig.AutoLockManagers {
continue
}
managerKey = eKey
}
unlockKeys = append(unlockKeys, eKey)
}
switch {
case !cluster.Spec.EncryptionConfig.AutoLockManagers:
break
case managerKey == nil:
unlockKeys = append(unlockKeys, &api.EncryptionKey{
Subsystem: ca.ManagerRole,
Key: encryption.GenerateSecretKey(),
})
case request.Rotation.ManagerUnlockKey:
managerKey.Key = encryption.GenerateSecretKey()
}
cluster.UnlockKeys = unlockKeys
return store.UpdateCluster(tx, cluster)
})
if err != nil {
return nil, err
}
if cluster == nil {
return nil, grpc.Errorf(codes.NotFound, "cluster %s not found", request.ClusterID)
}
redactedClusters := redactClusters([]*api.Cluster{cluster})
// WARN: we should never return cluster here. We need to redact the private fields first.
return &api.UpdateClusterResponse{
Cluster: redactedClusters[0],
}, nil
}
func TestNewNodeCertificateRequiresToken(t *testing.T) {
tc := testutils.NewTestCA(t)
defer tc.Stop()
csr, _, err := ca.GenerateAndWriteNewKey(tc.Paths.Node)
assert.NoError(t, err)
// Issuance fails if no secret is provided
role := api.NodeRoleManager
issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance fails if wrong secret is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: "invalid-secret"}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// Issuance succeeds if correct token is provided
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
// Rotate manager and worker tokens
var (
newManagerToken string
newWorkerToken string
)
assert.NoError(t, tc.MemoryStore.Update(func(tx store.Tx) error {
clusters, _ := store.FindClusters(tx, store.ByName(store.DefaultClusterName))
newWorkerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Worker = newWorkerToken
newManagerToken = ca.GenerateJoinToken(&tc.RootCA)
clusters[0].RootCA.JoinTokens.Manager = newManagerToken
return store.UpdateCluster(tx, clusters[0])
}))
time.Sleep(500 * time.Millisecond)
// Old token should fail
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.ManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: tc.WorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.EqualError(t, err, "rpc error: code = 3 desc = A valid join token is necessary to join this cluster")
// New token should succeed
role = api.NodeRoleManager
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newManagerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
role = api.NodeRoleWorker
issueRequest = &api.IssueNodeCertificateRequest{CSR: csr, Role: role, Token: newWorkerToken}
_, err = tc.NodeCAClients[0].IssueNodeCertificate(context.Background(), issueRequest)
assert.NoError(t, err)
}
