// StartWithStopper is the same as Start, but allows passing a stopper // explicitly. func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error { if ts.Ctx == nil { ts.Ctx = NewTestContext() } if stopper == nil { stopper = stop.NewStopper() } // Change the replication requirements so we don't get log spam // about ranges not being replicated enough. // TODO(marc): set this in the zones table when we have an entry // for the default cluster-wide zone config and remove these // shenanigans about mutating the global default. oldDefaultZC := proto.Clone(config.DefaultZoneConfig).(*config.ZoneConfig) config.DefaultZoneConfig.ReplicaAttrs = []roachpb.Attributes{{}} stopper.AddCloser(stop.CloserFn(func() { config.DefaultZoneConfig = oldDefaultZC })) var err error ts.Server, err = NewServer(ts.Ctx, stopper) if err != nil { return err } // Ensure we have the correct number of engines. Add in in-memory ones where // needed. There must be at least one store/engine. if ts.StoresPerNode < 1 { ts.StoresPerNode = 1 } for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ { ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper)) } if !ts.SkipBootstrap { stopper := stop.NewStopper() _, err := BootstrapCluster("cluster-1", ts.Ctx.Engines, stopper) if err != nil { return util.Errorf("could not bootstrap cluster: %s", err) } stopper.Stop() } if err := ts.Server.Start(true); err != nil { return err } // If enabled, wait for initial splits to complete before returning control. // If initial splits do not complete, the server is stopped before // returning. if config.TestingTableSplitsDisabled() { return nil } if err := ts.WaitForInitialSplits(); err != nil { ts.Stop() return err } return nil }
// TestingSetupZoneConfigHook initializes the zone config hook // to 'testingZoneConfigHook' which uses 'testingZoneConfig'. // Settings go back to their previous values when the stopper runs our closer. func TestingSetupZoneConfigHook(stopper *stop.Stopper) { testingLock.Lock() defer testingLock.Unlock() if testingHasHook { panic("TestingSetupZoneConfigHook called without restoring state") } testingHasHook = true testingZoneConfig = map[uint32]*ZoneConfig{} testingPreviousHook = ZoneConfigHook ZoneConfigHook = testingZoneConfigHook testingLargestIDHook = func(maxID uint32) (max uint32) { testingLock.Lock() defer testingLock.Unlock() for id := range testingZoneConfig { if maxID > 0 && id > maxID { continue } if id > max { max = id } } return } stopper.AddCloser(stop.CloserFn(testingResetZoneConfigHook)) }
// start starts the node by registering the storage instance for the // RPC service "Node" and initializing stores for each specified // engine. Launches periodic store gossiping in a goroutine. func (n *Node) start(rpcServer *rpc.Server, engines []engine.Engine, attrs proto.Attributes, stopper *stop.Stopper) error { n.initDescriptor(rpcServer.Addr(), attrs) if err := rpcServer.RegisterName("Node", (*nodeServer)(n)); err != nil { log.Fatalf("unable to register node service with RPC server: %s", err) } // Start status monitor. n.status.StartMonitorFeed(n.ctx.EventFeed) stopper.AddCloser(n.ctx.EventFeed) // Initialize stores, including bootstrapping new ones. if err := n.initStores(engines, stopper); err != nil { return err } n.startedAt = n.ctx.Clock.Now().WallTime // Initialize publisher for Node Events. This requires the NodeID, which is // initialized by initStores(); because of this, some Store initialization // events will precede the StartNodeEvent on the feed. n.feed = status.NewNodeEventFeed(n.Descriptor.NodeID, n.ctx.EventFeed) n.feed.StartNode(n.Descriptor, n.startedAt) n.startPublishStatuses(stopper) n.startGossip(stopper) log.Infoc(n.context(), "Started node with %v engine(s) and attributes %v", engines, attrs.Attrs) return nil }
// StartWithStopper is the same as Start, but allows passing a stopper // explicitly. func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error { if ts.Ctx == nil { ctx := MakeTestContext() ts.Ctx = &ctx } if stopper == nil { stopper = stop.NewStopper() } // Change the replication requirements so we don't get log spam about ranges // not being replicated enough. cfg := config.DefaultZoneConfig() cfg.ReplicaAttrs = []roachpb.Attributes{{}} fn := config.TestingSetDefaultZoneConfig(cfg) stopper.AddCloser(stop.CloserFn(fn)) // Needs to be called before NewServer to ensure resolvers are initialized. if err := ts.Ctx.InitNode(); err != nil { return err } // Ensure we have the correct number of engines. Add in-memory ones where // needed. There must be at least one store/engine. if ts.StoresPerNode < 1 { ts.StoresPerNode = 1 } for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ { ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, stopper)) } var err error ts.Server, err = NewServer(*ts.Ctx, stopper) if err != nil { return err } // Our context must be shared with our server. ts.Ctx = &ts.Server.ctx if err := ts.Server.Start(); err != nil { return err } // If enabled, wait for initial splits to complete before returning control. // If initial splits do not complete, the server is stopped before // returning. if config.TestingTableSplitsDisabled() { return nil } if err := ts.WaitForInitialSplits(); err != nil { ts.Stop() return err } return nil }
// NewMultiRaft creates a MultiRaft object. func NewMultiRaft(nodeID roachpb.NodeID, storeID roachpb.StoreID, config *Config, stopper *stop.Stopper) (*MultiRaft, error) { if nodeID <= 0 { return nil, util.Errorf("invalid NodeID: %s", nodeID) } if storeID <= 0 { return nil, util.Errorf("invalid StoreID: %s", storeID) } if err := config.validate(); err != nil { return nil, err } if config.Ticker == nil { config.Ticker = newTicker(config.TickInterval) stopper.AddCloser(config.Ticker) } if config.EntryFormatter != nil { // Wrap the EntryFormatter to strip off the command id. ef := config.EntryFormatter config.EntryFormatter = func(data []byte) string { if len(data) == 0 { return "[empty]" } id, cmd := decodeCommand(data) formatted := ef(cmd) return fmt.Sprintf("%x: %s", id, formatted) } } m := &MultiRaft{ Config: *config, stopper: stopper, nodeID: nodeID, storeID: storeID, // Output channel. Events: make(chan []interface{}), // Input channels. reqChan: make(chan *RaftMessageRequest, reqBufferSize), createGroupChan: make(chan createGroupOp), removeGroupChan: make(chan removeGroupOp), proposalChan: make(chan *proposal), campaignChan: make(chan roachpb.RangeID), callbackChan: make(chan func()), statusChan: make(chan statusOp), } if err := m.Transport.Listen(storeID, (*multiraftServer)(m)); err != nil { return nil, err } return m, nil }
// start starts the node by registering the storage instance for the // RPC service "Node" and initializing stores for each specified // engine. Launches periodic store gossiping in a goroutine. func (n *Node) start(rpcServer *rpc.Server, engines []engine.Engine, attrs proto.Attributes, stopper *stop.Stopper) error { n.initDescriptor(rpcServer.Addr(), attrs) requests := []proto.Request{ &proto.GetRequest{}, &proto.PutRequest{}, &proto.ConditionalPutRequest{}, &proto.IncrementRequest{}, &proto.DeleteRequest{}, &proto.DeleteRangeRequest{}, &proto.ScanRequest{}, &proto.EndTransactionRequest{}, &proto.AdminSplitRequest{}, &proto.AdminMergeRequest{}, &proto.InternalRangeLookupRequest{}, &proto.InternalHeartbeatTxnRequest{}, &proto.InternalGCRequest{}, &proto.InternalPushTxnRequest{}, &proto.InternalResolveIntentRequest{}, &proto.InternalResolveIntentRangeRequest{}, &proto.InternalMergeRequest{}, &proto.InternalTruncateLogRequest{}, &proto.InternalLeaderLeaseRequest{}, } for _, r := range requests { if err := rpcServer.Register("Node."+r.Method().String(), n.executeCmd, r); err != nil { log.Fatalf("unable to register node service with RPC server: %s", err) } } // Start status monitor. n.status.StartMonitorFeed(n.ctx.EventFeed) stopper.AddCloser(n.ctx.EventFeed) // Initialize stores, including bootstrapping new ones. if err := n.initStores(engines, stopper); err != nil { return err } n.startedAt = n.ctx.Clock.Now().WallTime // Initialize publisher for Node Events. This requires the NodeID, which is // initialized by initStores(); because of this, some Store initialization // events will precede the StartNodeEvent on the feed. n.feed = status.NewNodeEventFeed(n.Descriptor.NodeID, n.ctx.EventFeed) n.feed.StartNode(n.Descriptor, n.startedAt) n.startPublishStatuses(stopper) n.startGossip(stopper) log.Infoc(n.context(), "Started node with %v engine(s) and attributes %v", engines, attrs.Attrs) return nil }
// NewMultiRaft creates a MultiRaft object. func NewMultiRaft(nodeID proto.RaftNodeID, config *Config, stopper *stop.Stopper) (*MultiRaft, error) { if nodeID == 0 { return nil, util.Error("Invalid RaftNodeID") } if err := config.validate(); err != nil { return nil, err } if config.Ticker == nil { config.Ticker = newTicker(config.TickInterval) stopper.AddCloser(config.Ticker) } if config.EntryFormatter != nil { // Wrap the EntryFormatter to strip off the command id. ef := config.EntryFormatter config.EntryFormatter = func(data []byte) string { if len(data) == 0 { return "[empty]" } id, cmd := decodeCommand(data) formatted := ef(cmd) return fmt.Sprintf("%x: %s", id, formatted) } } m := &MultiRaft{ Config: *config, stopper: stopper, multiNode: raft.StartMultiNode(uint64(nodeID)), nodeID: nodeID, // Output channel. Events: make(chan interface{}, config.EventBufferSize), // Input channels. reqChan: make(chan *RaftMessageRequest, reqBufferSize), createGroupChan: make(chan *createGroupOp), removeGroupChan: make(chan *removeGroupOp), proposalChan: make(chan *proposal), callbackChan: make(chan func()), } if err := m.Transport.Listen(nodeID, (*multiraftServer)(m)); err != nil { return nil, err } return m, nil }
// New creates an instance of a gossip node. func New( ctx context.Context, rpcContext *rpc.Context, grpcServer *grpc.Server, resolvers []resolver.Resolver, stopper *stop.Stopper, registry *metric.Registry, ) *Gossip { ctx = log.WithEventLog(ctx, "gossip", "gossip") g := &Gossip{ ctx: ctx, Connected: make(chan struct{}), rpcContext: rpcContext, server: newServer(ctx, stopper, registry), outgoing: makeNodeSet(minPeers, metric.NewGauge(MetaConnectionsOutgoingGauge)), bootstrapping: map[string]struct{}{}, disconnected: make(chan *client, 10), stalledCh: make(chan struct{}, 1), stallInterval: defaultStallInterval, bootstrapInterval: defaultBootstrapInterval, cullInterval: defaultCullInterval, nodeDescs: map[roachpb.NodeID]*roachpb.NodeDescriptor{}, resolverAddrs: map[util.UnresolvedAddr]resolver.Resolver{}, bootstrapAddrs: map[util.UnresolvedAddr]struct{}{}, } stopper.AddCloser(stop.CloserFn(func() { log.FinishEventLog(ctx) })) registry.AddMetric(g.outgoing.gauge) g.clientsMu.breakers = map[string]*circuit.Breaker{} log.Infof(g.ctx, "initial resolvers: %s", resolvers) g.SetResolvers(resolvers) g.mu.Lock() // Add ourselves as a SystemConfig watcher. g.mu.is.registerCallback(KeySystemConfig, g.updateSystemConfig) // Add ourselves as a node descriptor watcher. g.mu.is.registerCallback(MakePrefixPattern(KeyNodeIDPrefix), g.updateNodeAddress) g.mu.Unlock() RegisterGossipServer(grpcServer, g.server) return g }
func newTestCluster(transport Transport, size int, stopper *stop.Stopper, t *testing.T) *testCluster { if transport == nil { transport = NewLocalRPCTransport(stopper) } stopper.AddCloser(transport) cluster := &testCluster{ t: t, transport: transport, groups: map[proto.RangeID][]int{}, } for i := 0; i < size; i++ { ticker := newManualTicker() storage := &BlockableStorage{storage: NewMemoryStorage()} config := &Config{ Transport: transport, Storage: storage, Ticker: ticker, ElectionTimeoutTicks: 2, HeartbeatIntervalTicks: 1, TickInterval: time.Hour, // not in use } mr, err := NewMultiRaft(proto.RaftNodeID(i+1), config, stopper) if err != nil { t.Fatal(err) } state := newState(mr) demux := newEventDemux(state.Events) demux.start(stopper) cluster.nodes = append(cluster.nodes, state) cluster.tickers = append(cluster.tickers, ticker) cluster.events = append(cluster.events, demux) cluster.storages = append(cluster.storages, storage) } cluster.start() return cluster }