Example #1
0
func agentTestEnv(t *testing.T) (*Agent, func()) {
	var cleanup []func()
	tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
	cleanup = append(cleanup, func() { tc.Stop() })

	agentSecurityConfig, err := tc.NewNodeConfig(ca.AgentRole)
	assert.NoError(t, err)

	addr := "localhost:4949"
	remotes := picker.NewRemotes(api.Peer{Addr: addr})

	conn, err := grpc.Dial(addr,
		grpc.WithPicker(picker.NewPicker(remotes, addr)),
		grpc.WithTransportCredentials(agentSecurityConfig.ClientTLSCreds))
	assert.NoError(t, err)

	db, cleanupStorage := storageTestEnv(t)
	cleanup = append(cleanup, func() { cleanupStorage() })

	agent, err := New(&Config{
		Executor: &NoopExecutor{},
		Managers: remotes,
		Conn:     conn,
		DB:       db,
	})
	return agent, func() {
		for i := len(cleanup) - 1; i > 0; i-- {
			cleanup[i]()
		}
	}
}
Example #2
0
func (n *Node) runAgent(ctx context.Context, db *bolt.DB, creds credentials.TransportAuthenticator, ready chan<- struct{}) error {
	var manager api.Peer
	select {
	case <-ctx.Done():
	case manager = <-n.remotes.WaitSelect(ctx):
	}
	if ctx.Err() != nil {
		return ctx.Err()
	}
	picker := picker.NewPicker(n.remotes, manager.Addr)
	conn, err := grpc.Dial(manager.Addr,
		grpc.WithPicker(picker),
		grpc.WithTransportCredentials(creds),
		grpc.WithBackoffMaxDelay(maxSessionFailureBackoff))
	if err != nil {
		return err
	}

	agent, err := New(&Config{
		Hostname:         n.config.Hostname,
		Managers:         n.remotes,
		Executor:         n.config.Executor,
		DB:               db,
		Conn:             conn,
		Picker:           picker,
		NotifyRoleChange: n.roleChangeReq,
	})
	if err != nil {
		return err
	}
	if err := agent.Start(ctx); err != nil {
		return err
	}

	n.Lock()
	n.agent = agent
	n.Unlock()

	defer func() {
		n.Lock()
		n.agent = nil
		n.Unlock()
	}()

	go func() {
		<-agent.Ready()
		close(ready)
	}()

	// todo: manually call stop on context cancellation?

	return agent.Err(context.Background())
}
Example #3
0
func (mc *managersCluster) addAgents(count int) error {
	var addrs []api.Peer
	for _, m := range mc.ms {
		addrs = append(addrs, api.Peer{Addr: m.addr})
	}
	for i := 0; i < count; i++ {
		asConfig, err := mc.tc.NewNodeConfig(ca.AgentRole)
		if err != nil {
			return err
		}

		managers := picker.NewRemotes(addrs...)
		peer, err := managers.Select()
		if err != nil {
			return err
		}
		conn, err := grpc.Dial(peer.Addr,
			grpc.WithPicker(picker.NewPicker(managers)),
			grpc.WithTransportCredentials(asConfig.ClientTLSCreds))
		if err != nil {
			return err
		}

		id := strconv.Itoa(rand.Int())
		a, err := agent.New(&agent.Config{
			Hostname: "hostname_" + id,
			Managers: managers,
			Executor: &NoopExecutor{},
			Conn:     conn,
		})
		if err != nil {
			return err
		}
		if err := a.Start(context.Background()); err != nil {
			return err
		}
		mc.agents = append(mc.agents, a)
	}
	return nil
}
Example #4
0
func TestAgentStartStop(t *testing.T) {
	tc := testutils.NewTestCA(t, testutils.AcceptancePolicy(true, true, ""))
	defer tc.Stop()

	agentSecurityConfig, err := tc.NewNodeConfig(ca.AgentRole)
	assert.NoError(t, err)

	addr := "localhost:4949"
	remotes := picker.NewRemotes(api.Peer{Addr: addr})

	conn, err := grpc.Dial(addr,
		grpc.WithPicker(picker.NewPicker(remotes, addr)),
		grpc.WithTransportCredentials(agentSecurityConfig.ClientTLSCreds))
	assert.NoError(t, err)

	db, cleanup := storageTestEnv(t)
	defer cleanup()

	agent, err := New(&Config{
		Executor: &NoopExecutor{},
		Managers: remotes,
		Conn:     conn,
		DB:       db,
	})
	assert.NoError(t, err)
	assert.NotNil(t, agent)

	ctx, _ := context.WithTimeout(context.Background(), 5000*time.Millisecond)

	assert.Equal(t, errAgentNotStarted, agent.Stop(ctx))
	assert.NoError(t, agent.Start(ctx))

	if err := agent.Start(ctx); err != errAgentStarted {
		t.Fatalf("expected agent started error: %v", err)
	}

	assert.NoError(t, agent.Stop(ctx))
}
Example #5
0
func (n *Node) run(ctx context.Context) (err error) {
	defer func() {
		n.err = err
		close(n.closed)
	}()
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	ctx = log.WithLogger(ctx, log.G(ctx).WithField("module", "node"))

	go func() {
		select {
		case <-ctx.Done():
		case <-n.stopped:
			cancel()
		}
	}()

	// NOTE: When this node is created by NewNode(), our nodeID is set if
	// n.loadCertificates() succeeded in loading TLS credentials.
	if n.config.JoinAddr == "" && n.nodeID == "" {
		if err := n.bootstrapCA(); err != nil {
			return err
		}
	}

	if n.config.JoinAddr != "" || n.config.ForceNewCluster {
		n.remotes = newPersistentRemotes(filepath.Join(n.config.StateDir, stateFilename))
		if n.config.JoinAddr != "" {
			n.remotes.Observe(api.Peer{Addr: n.config.JoinAddr}, 1)
		}
	}

	// Obtain new certs and setup TLS certificates renewal for this node:
	// - We call LoadOrCreateSecurityConfig which blocks until a valid certificate has been issued
	// - We retrieve the nodeID from LoadOrCreateSecurityConfig through the info channel. This allows
	// us to display the ID before the certificate gets issued (for potential approval).
	// - We wait for LoadOrCreateSecurityConfig to finish since we need a certificate to operate.
	// - Given a valid certificate, spin a renewal go-routine that will ensure that certificates stay
	// up to date.
	issueResponseChan := make(chan api.IssueNodeCertificateResponse, 1)
	go func() {
		select {
		case <-ctx.Done():
		case resp := <-issueResponseChan:
			logrus.Debugf("Requesting certificate for NodeID: %v", resp.NodeID)
			n.Lock()
			n.nodeID = resp.NodeID
			n.nodeMembership = resp.NodeMembership
			n.Unlock()
			close(n.certificateRequested)
		}
	}()

	certDir := filepath.Join(n.config.StateDir, "certificates")
	securityConfig, err := ca.LoadOrCreateSecurityConfig(ctx, certDir, n.config.JoinToken, ca.ManagerRole, picker.NewPicker(n.remotes), issueResponseChan)
	if err != nil {
		return err
	}

	taskDBPath := filepath.Join(n.config.StateDir, "worker/tasks.db")
	if err := os.MkdirAll(filepath.Dir(taskDBPath), 0777); err != nil {
		return err
	}

	db, err := bolt.Open(taskDBPath, 0666, nil)
	if err != nil {
		return err
	}
	defer db.Close()

	if err := n.loadCertificates(); err != nil {
		return err
	}

	forceCertRenewal := make(chan struct{})
	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			case apirole := <-n.roleChangeReq:
				n.Lock()
				lastRole := n.role
				role := ca.AgentRole
				if apirole == api.NodeRoleManager {
					role = ca.ManagerRole
				}
				if lastRole == role {
					n.Unlock()
					continue
				}
				// switch role to agent immediately to shutdown manager early
				if role == ca.AgentRole {
					n.role = role
					n.roleCond.Broadcast()
				}
				n.Unlock()
				select {
				case forceCertRenewal <- struct{}{}:
				case <-ctx.Done():
					return
				}
			}
		}
	}()

	updates := ca.RenewTLSConfig(ctx, securityConfig, certDir, picker.NewPicker(n.remotes), forceCertRenewal)
	go func() {
		for {
			select {
			case certUpdate := <-updates:
				if certUpdate.Err != nil {
					logrus.Warnf("error renewing TLS certificate: %v", certUpdate.Err)
					continue
				}
				n.Lock()
				n.role = certUpdate.Role
				n.roleCond.Broadcast()
				n.Unlock()
			case <-ctx.Done():
				return
			}
		}
	}()

	role := n.role

	managerReady := make(chan struct{})
	agentReady := make(chan struct{})
	var managerErr error
	var agentErr error
	var wg sync.WaitGroup
	wg.Add(2)
	go func() {
		managerErr = n.runManager(ctx, securityConfig, managerReady) // store err and loop
		wg.Done()
		cancel()
	}()
	go func() {
		agentErr = n.runAgent(ctx, db, securityConfig.ClientTLSCreds, agentReady)
		wg.Done()
		cancel()
	}()

	go func() {
		<-agentReady
		if role == ca.ManagerRole {
			<-managerReady
		}
		close(n.ready)
	}()

	wg.Wait()
	if managerErr != nil && managerErr != context.Canceled {
		return managerErr
	}
	if agentErr != nil && agentErr != context.Canceled {
		return agentErr
	}
	return err
}
Example #6
0
// NewTestCA is a helper method that creates a TestCA and a bunch of default
// connections and security configs
func NewTestCA(t *testing.T, policy api.AcceptancePolicy) *TestCA {
	tempBaseDir, err := ioutil.TempDir("", "swarm-ca-test-")
	assert.NoError(t, err)

	s := store.NewMemoryStore(nil)

	paths := ca.NewConfigPaths(tempBaseDir)
	organization := identity.NewID()

	rootCA, err := createAndWriteRootCA("swarm-test-CA", paths.RootCA, ca.DefaultNodeCertExpiration)
	assert.NoError(t, err)

	managerConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, organization, "")
	assert.NoError(t, err)

	managerDiffOrgConfig, err := genSecurityConfig(s, rootCA, ca.ManagerRole, "swarm-test-org-2", "")
	assert.NoError(t, err)

	agentConfig, err := genSecurityConfig(s, rootCA, ca.AgentRole, organization, "")
	assert.NoError(t, err)

	l, err := net.Listen("tcp", "127.0.0.1:0")
	assert.NoError(t, err)

	baseOpts := []grpc.DialOption{grpc.WithTimeout(10 * time.Second)}
	insecureClientOpts := append(baseOpts, grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})))
	clientOpts := append(baseOpts, grpc.WithTransportCredentials(agentConfig.ClientTLSCreds))
	managerOpts := append(baseOpts, grpc.WithTransportCredentials(managerConfig.ClientTLSCreds))
	managerDiffOrgOpts := append(baseOpts, grpc.WithTransportCredentials(managerDiffOrgConfig.ClientTLSCreds))

	conn1, err := grpc.Dial(l.Addr().String(), insecureClientOpts...)
	assert.NoError(t, err)

	conn2, err := grpc.Dial(l.Addr().String(), clientOpts...)
	assert.NoError(t, err)

	conn3, err := grpc.Dial(l.Addr().String(), managerOpts...)
	assert.NoError(t, err)

	conn4, err := grpc.Dial(l.Addr().String(), managerDiffOrgOpts...)
	assert.NoError(t, err)

	serverOpts := []grpc.ServerOption{grpc.Creds(managerConfig.ServerTLSCreds)}
	grpcServer := grpc.NewServer(serverOpts...)

	createClusterObject(t, s, policy)
	caServer := ca.NewServer(s, managerConfig)
	api.RegisterCAServer(grpcServer, caServer)
	api.RegisterNodeCAServer(grpcServer, caServer)

	ctx := context.Background()

	go grpcServer.Serve(l)
	go caServer.Run(ctx)

	// Wait for caServer to be ready to serve
	<-caServer.Ready()

	remotes := picker.NewRemotes(api.Peer{Addr: l.Addr().String()})
	picker := picker.NewPicker(remotes, l.Addr().String())

	caClients := []api.CAClient{api.NewCAClient(conn1), api.NewCAClient(conn2), api.NewCAClient(conn3)}
	nodeCAClients := []api.NodeCAClient{api.NewNodeCAClient(conn1), api.NewNodeCAClient(conn2), api.NewNodeCAClient(conn3), api.NewNodeCAClient(conn4)}
	conns := []*grpc.ClientConn{conn1, conn2, conn3, conn4}

	return &TestCA{
		RootCA:        rootCA,
		MemoryStore:   s,
		Picker:        picker,
		TempDir:       tempBaseDir,
		Organization:  organization,
		Paths:         paths,
		Context:       ctx,
		CAClients:     caClients,
		NodeCAClients: nodeCAClients,
		Conns:         conns,
		CAServer:      caServer,
	}
}