// GetRemoteSignedCertificate submits a CSR together with the intended role to a remote CA server address // available through a picker, and that is part of a CA identified by a specific certificate pool. func GetRemoteSignedCertificate(ctx context.Context, csr []byte, role, secret string, rootCAPool *x509.CertPool, picker *picker.Picker, creds credentials.TransportAuthenticator, nodeInfo chan<- api.IssueNodeCertificateResponse) ([]byte, error) { if rootCAPool == nil { return nil, fmt.Errorf("valid root CA pool required") } if picker == nil { return nil, fmt.Errorf("valid remote address picker required") } if creds == nil { // This is our only non-MTLS request, and it happens when we are boostraping our TLS certs // We're using CARole as server name, so an external CA doesn't also have to have ManagerRole in the cert SANs creds = credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rootCAPool}) } opts := []grpc.DialOption{ grpc.WithTransportCredentials(creds), grpc.WithBackoffMaxDelay(10 * time.Second), grpc.WithPicker(picker)} firstAddr, err := picker.PickAddr() if err != nil { return nil, err } conn, err := grpc.Dial(firstAddr, opts...) if err != nil { return nil, err } defer conn.Close() // Create a CAClient to retrieve a new Certificate caClient := api.NewNodeCAClient(conn) // Convert our internal string roles into an API role apiRole, err := FormatRole(role) if err != nil { return nil, err } // Send the Request and retrieve the request token issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Role: apiRole, Secret: secret} issueResponse, err := caClient.IssueNodeCertificate(ctx, issueRequest) if err != nil { return nil, err } // Send back the NodeID on the nodeInfo, so the caller can know what ID was assigned by the CA if nodeInfo != nil { nodeInfo <- *issueResponse } statusRequest := &api.NodeCertificateStatusRequest{NodeID: issueResponse.NodeID} expBackoff := events.NewExponentialBackoff(events.ExponentialBackoffConfig{ Base: time.Second, Factor: time.Second, Max: 30 * time.Second, }) log.Infof("Waiting for TLS certificate to be issued...") // Exponential backoff with Max of 30 seconds to wait for a new retry for { // Send the Request and retrieve the certificate statusResponse, err := caClient.NodeCertificateStatus(ctx, statusRequest) if err != nil { return nil, err } // If the certificate was issued, return if statusResponse.Status.State == api.IssuanceStateIssued { if statusResponse.Certificate == nil { return nil, fmt.Errorf("no certificate in CertificateStatus response") } // The certificate in the response must match the CSR // we submitted. If we are getting a response for a // certificate that was previously issued, we need to // retry until the certificate gets updated per our // current request. if bytes.Equal(statusResponse.Certificate.CSR, csr) { return statusResponse.Certificate.Certificate, nil } } // If we're still pending, the issuance failed, or the state is unknown // let's continue trying. expBackoff.Failure(nil, nil) time.Sleep(expBackoff.Proceed(nil)) } }
// NewTestCA is a helper method that creates a TestCA and a bunch of default // connections and security configs. func NewTestCA(t *testing.T) *TestCA { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) s := store.NewMemoryStore(nil) paths := ca.NewConfigPaths(tempBaseDir) organization := identity.NewID() rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration) assert.NoError(t, err) var ( externalSigningServer *ExternalSigningServer externalCAs []*api.ExternalCA ) if External { // Start the CA API server. externalSigningServer, err = NewExternalSigningServer(rootCA, tempBaseDir) assert.NoError(t, err) externalCAs = []*api.ExternalCA{ { Protocol: api.ExternalCA_CAProtocolCFSSL, URL: externalSigningServer.URL, }, } } managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "", External) assert.NoError(t, err) managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "", External) assert.NoError(t, err) agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "", External) assert.NoError(t, err) l, err := net.Listen("tcp", "127.0.0.1:0") assert.NoError(t, err) baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)} insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}))) clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds)) managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds)) managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds)) conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...) assert.NoError(t, err) conn2, err := grpc.Dial(l.Addr().String(), clientOpts...) assert.NoError(t, err) conn3, err := grpc.Dial(l.Addr().String(), managerOpts...) assert.NoError(t, err) conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...) assert.NoError(t, err) serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)} grpcServer := grpc.NewServer(serverOpts...) managerToken := ca.GenerateJoinToken(&rootCA) workerToken := ca.GenerateJoinToken(&rootCA) createClusterObject(t, s, organization, workerToken, managerToken, externalCAs...) caServer := ca.NewServer(s, managerConfig) api.RegisterCAServer(grpcServer, caServer) api.RegisterNodeCAServer(grpcServer, caServer) ctx := context.Background() go grpcServer.Serve(l) go caServer.Run(ctx) // Wait for caServer to be ready to serve <-caServer.Ready() remotes := remotes.NewRemotes(api.Peer{Addr: l.Addr().String()}) caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)} nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)} conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4} return &TestCA{ RootCA: rootCA, ExternalSigningServer: externalSigningServer, MemoryStore: s, TempDir: tempBaseDir, Organization: organization, Paths: paths, Context: ctx, CAClients: caClients, NodeCAClients: nodeCAClients, Conns: conns, Server: grpcServer, CAServer: caServer, WorkerToken: workerToken, ManagerToken: managerToken, Remotes: remotes, } }
// GetRemoteSignedCertificate submits a CSR to a remote CA server address, // and that is part of a CA identified by a specific certificate pool. func GetRemoteSignedCertificate(ctx context.Context, csr []byte, rootCAPool *x509.CertPool, config CertificateRequestConfig) ([]byte, error) { if rootCAPool == nil { return nil, errors.New("valid root CA pool required") } creds := config.Credentials if creds == nil { // This is our only non-MTLS request, and it happens when we are boostraping our TLS certs // We're using CARole as server name, so an external CA doesn't also have to have ManagerRole in the cert SANs creds = credentials.NewTLS(&tls.Config{ServerName: CARole, RootCAs: rootCAPool}) } conn, peer, err := getGRPCConnection(creds, config.Remotes) if err != nil { return nil, err } defer conn.Close() // Create a CAClient to retrieve a new Certificate caClient := api.NewNodeCAClient(conn) // Send the Request and retrieve the request token issueRequest := &api.IssueNodeCertificateRequest{CSR: csr, Token: config.Token} issueResponse, err := caClient.IssueNodeCertificate(ctx, issueRequest) if err != nil { config.Remotes.Observe(peer, -remotes.DefaultObservationWeight) return nil, err } statusRequest := &api.NodeCertificateStatusRequest{NodeID: issueResponse.NodeID} expBackoff := events.NewExponentialBackoff(events.ExponentialBackoffConfig{ Base: time.Second, Factor: time.Second, Max: 30 * time.Second, }) // Exponential backoff with Max of 30 seconds to wait for a new retry for { // Send the Request and retrieve the certificate ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() statusResponse, err := caClient.NodeCertificateStatus(ctx, statusRequest) if err != nil { config.Remotes.Observe(peer, -remotes.DefaultObservationWeight) return nil, err } // If the certificate was issued, return if statusResponse.Status.State == api.IssuanceStateIssued { if statusResponse.Certificate == nil { return nil, errors.New("no certificate in CertificateStatus response") } // The certificate in the response must match the CSR // we submitted. If we are getting a response for a // certificate that was previously issued, we need to // retry until the certificate gets updated per our // current request. if bytes.Equal(statusResponse.Certificate.CSR, csr) { config.Remotes.Observe(peer, remotes.DefaultObservationWeight) return statusResponse.Certificate.Certificate, nil } } // If we're still pending, the issuance failed, or the state is unknown // let's continue trying. expBackoff.Failure(nil, nil) time.Sleep(expBackoff.Proceed(nil)) } }
// NewTestCA is a helper method that creates a TestCA and a bunch of default // connections and security configs func NewTestCA(t *testing.T, policy api.AcceptancePolicy) *TestCA { tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-") assert.NoError(t, err) s := store.NewMemoryStore(nil) paths := ca.NewConfigPaths(tempBaseDir) organization := identity.NewID() rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration) assert.NoError(t, err) managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "") assert.NoError(t, err) managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "") assert.NoError(t, err) agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "") assert.NoError(t, err) l, err := net.Listen("tcp", "127.0.0.1:0") assert.NoError(t, err) baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)} insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true}))) clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds)) managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds)) managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds)) conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...) assert.NoError(t, err) conn2, err := grpc.Dial(l.Addr().String(), clientOpts...) assert.NoError(t, err) conn3, err := grpc.Dial(l.Addr().String(), managerOpts...) assert.NoError(t, err) conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...) assert.NoError(t, err) serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)} grpcServer := grpc.NewServer(serverOpts...) createClusterObject(t, s, policy) caServer := ca.NewServer(s, managerConfig) api.RegisterCAServer(grpcServer, caServer) api.RegisterNodeCAServer(grpcServer, caServer) ctx := context.Background() go grpcServer.Serve(l) go caServer.Run(ctx) // Wait for caServer to be ready to serve <-caServer.Ready() remotes := picker.NewRemotes(api.Peer{Addr: l.Addr().String()}) picker := picker.NewPicker(remotes, l.Addr().String()) caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)} nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)} conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4} return &TestCA{ RootCA: rootCA, MemoryStore: s, Picker: picker, TempDir: tempBaseDir, Organization: organization, Paths: paths, Context: ctx, CAClients: caClients, NodeCAClients: nodeCAClients, Conns: conns, CAServer: caServer, } }