// StartWithStopper is the same as Start, but allows passing a stopper
// explicitly.
func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error {
if ts.Ctx == nil {
ts.Ctx = NewTestContext()
}
if stopper == nil {
stopper = stop.NewStopper()
}
// Change the replication requirements so we don't get log spam
// about ranges not being replicated enough.
// TODO(marc): set this in the zones table when we have an entry
// for the default cluster-wide zone config and remove these
// shenanigans about mutating the global default.
oldDefaultZC := proto.Clone(config.DefaultZoneConfig).(*config.ZoneConfig)
config.DefaultZoneConfig.ReplicaAttrs = []roachpb.Attributes{{}}
stopper.AddCloser(stop.CloserFn(func() {
config.DefaultZoneConfig = oldDefaultZC
}))
var err error
ts.Server, err = NewServer(ts.Ctx, stopper)
if err != nil {
return err
}
// Ensure we have the correct number of engines. Add in in-memory ones where
// needed. There must be at least one store/engine.
if ts.StoresPerNode < 1 {
ts.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper))
}
if !ts.SkipBootstrap {
stopper := stop.NewStopper()
_, err := BootstrapCluster("cluster-1", ts.Ctx.Engines, stopper)
if err != nil {
return util.Errorf("could not bootstrap cluster: %s", err)
}
stopper.Stop()
}
if err := ts.Server.Start(true); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if config.TestingTableSplitsDisabled() {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
// TestingSetupZoneConfigHook initializes the zone config hook
// to 'testingZoneConfigHook' which uses 'testingZoneConfig'.
// Settings go back to their previous values when the stopper runs our closer.
func TestingSetupZoneConfigHook(stopper *stop.Stopper) {
testingLock.Lock()
defer testingLock.Unlock()
if testingHasHook {
panic("TestingSetupZoneConfigHook called without restoring state")
}
testingHasHook = true
testingZoneConfig = map[uint32]*ZoneConfig{}
testingPreviousHook = ZoneConfigHook
ZoneConfigHook = testingZoneConfigHook
testingLargestIDHook = func(maxID uint32) (max uint32) {
testingLock.Lock()
defer testingLock.Unlock()
for id := range testingZoneConfig {
if maxID > 0 && id > maxID {
continue
}
if id > max {
max = id
}
}
return
}
stopper.AddCloser(stop.CloserFn(testingResetZoneConfigHook))
}
// start starts the node by registering the storage instance for the
// RPC service "Node" and initializing stores for each specified
// engine. Launches periodic store gossiping in a goroutine.
func (n *Node) start(rpcServer *rpc.Server, engines []engine.Engine,
attrs proto.Attributes, stopper *stop.Stopper) error {
n.initDescriptor(rpcServer.Addr(), attrs)
if err := rpcServer.RegisterName("Node", (*nodeServer)(n)); err != nil {
log.Fatalf("unable to register node service with RPC server: %s", err)
}
// Start status monitor.
n.status.StartMonitorFeed(n.ctx.EventFeed)
stopper.AddCloser(n.ctx.EventFeed)
// Initialize stores, including bootstrapping new ones.
if err := n.initStores(engines, stopper); err != nil {
return err
}
n.startedAt = n.ctx.Clock.Now().WallTime
// Initialize publisher for Node Events. This requires the NodeID, which is
// initialized by initStores(); because of this, some Store initialization
// events will precede the StartNodeEvent on the feed.
n.feed = status.NewNodeEventFeed(n.Descriptor.NodeID, n.ctx.EventFeed)
n.feed.StartNode(n.Descriptor, n.startedAt)
n.startPublishStatuses(stopper)
n.startGossip(stopper)
log.Infoc(n.context(), "Started node with %v engine(s) and attributes %v", engines, attrs.Attrs)
return nil
}
// StartWithStopper is the same as Start, but allows passing a stopper
// explicitly.
func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error {
if ts.Ctx == nil {
ctx := MakeTestContext()
ts.Ctx = &ctx
}
if stopper == nil {
stopper = stop.NewStopper()
}
// Change the replication requirements so we don't get log spam about ranges
// not being replicated enough.
cfg := config.DefaultZoneConfig()
cfg.ReplicaAttrs = []roachpb.Attributes{{}}
fn := config.TestingSetDefaultZoneConfig(cfg)
stopper.AddCloser(stop.CloserFn(fn))
// Needs to be called before NewServer to ensure resolvers are initialized.
if err := ts.Ctx.InitNode(); err != nil {
return err
}
// Ensure we have the correct number of engines. Add in-memory ones where
// needed. There must be at least one store/engine.
if ts.StoresPerNode < 1 {
ts.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, stopper))
}
var err error
ts.Server, err = NewServer(*ts.Ctx, stopper)
if err != nil {
return err
}
// Our context must be shared with our server.
ts.Ctx = &ts.Server.ctx
if err := ts.Server.Start(); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if config.TestingTableSplitsDisabled() {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
// NewMultiRaft creates a MultiRaft object.
func NewMultiRaft(nodeID roachpb.NodeID, storeID roachpb.StoreID, config *Config, stopper *stop.Stopper) (*MultiRaft, error) {
if nodeID <= 0 {
return nil, util.Errorf("invalid NodeID: %s", nodeID)
}
if storeID <= 0 {
return nil, util.Errorf("invalid StoreID: %s", storeID)
}
if err := config.validate(); err != nil {
return nil, err
}
if config.Ticker == nil {
config.Ticker = newTicker(config.TickInterval)
stopper.AddCloser(config.Ticker)
}
if config.EntryFormatter != nil {
// Wrap the EntryFormatter to strip off the command id.
ef := config.EntryFormatter
config.EntryFormatter = func(data []byte) string {
if len(data) == 0 {
return "[empty]"
}
id, cmd := decodeCommand(data)
formatted := ef(cmd)
return fmt.Sprintf("%x: %s", id, formatted)
}
}
m := &MultiRaft{
Config: *config,
stopper: stopper,
nodeID: nodeID,
storeID: storeID,
// Output channel.
Events: make(chan []interface{}),
// Input channels.
reqChan: make(chan *RaftMessageRequest, reqBufferSize),
createGroupChan: make(chan createGroupOp),
removeGroupChan: make(chan removeGroupOp),
proposalChan: make(chan *proposal),
campaignChan: make(chan roachpb.RangeID),
callbackChan: make(chan func()),
statusChan: make(chan statusOp),
}
if err := m.Transport.Listen(storeID, (*multiraftServer)(m)); err != nil {
return nil, err
}
return m, nil
}
// start starts the node by registering the storage instance for the
// RPC service "Node" and initializing stores for each specified
// engine. Launches periodic store gossiping in a goroutine.
func (n *Node) start(rpcServer *rpc.Server, engines []engine.Engine,
attrs proto.Attributes, stopper *stop.Stopper) error {
n.initDescriptor(rpcServer.Addr(), attrs)
requests := []proto.Request{
&proto.GetRequest{},
&proto.PutRequest{},
&proto.ConditionalPutRequest{},
&proto.IncrementRequest{},
&proto.DeleteRequest{},
&proto.DeleteRangeRequest{},
&proto.ScanRequest{},
&proto.EndTransactionRequest{},
&proto.AdminSplitRequest{},
&proto.AdminMergeRequest{},
&proto.InternalRangeLookupRequest{},
&proto.InternalHeartbeatTxnRequest{},
&proto.InternalGCRequest{},
&proto.InternalPushTxnRequest{},
&proto.InternalResolveIntentRequest{},
&proto.InternalResolveIntentRangeRequest{},
&proto.InternalMergeRequest{},
&proto.InternalTruncateLogRequest{},
&proto.InternalLeaderLeaseRequest{},
}
for _, r := range requests {
if err := rpcServer.Register("Node."+r.Method().String(), n.executeCmd, r); err != nil {
log.Fatalf("unable to register node service with RPC server: %s", err)
}
}
// Start status monitor.
n.status.StartMonitorFeed(n.ctx.EventFeed)
stopper.AddCloser(n.ctx.EventFeed)
// Initialize stores, including bootstrapping new ones.
if err := n.initStores(engines, stopper); err != nil {
return err
}
n.startedAt = n.ctx.Clock.Now().WallTime
// Initialize publisher for Node Events. This requires the NodeID, which is
// initialized by initStores(); because of this, some Store initialization
// events will precede the StartNodeEvent on the feed.
n.feed = status.NewNodeEventFeed(n.Descriptor.NodeID, n.ctx.EventFeed)
n.feed.StartNode(n.Descriptor, n.startedAt)
n.startPublishStatuses(stopper)
n.startGossip(stopper)
log.Infoc(n.context(), "Started node with %v engine(s) and attributes %v", engines, attrs.Attrs)
return nil
}
// NewMultiRaft creates a MultiRaft object.
func NewMultiRaft(nodeID proto.RaftNodeID, config *Config, stopper *stop.Stopper) (*MultiRaft, error) {
if nodeID == 0 {
return nil, util.Error("Invalid RaftNodeID")
}
if err := config.validate(); err != nil {
return nil, err
}
if config.Ticker == nil {
config.Ticker = newTicker(config.TickInterval)
stopper.AddCloser(config.Ticker)
}
if config.EntryFormatter != nil {
// Wrap the EntryFormatter to strip off the command id.
ef := config.EntryFormatter
config.EntryFormatter = func(data []byte) string {
if len(data) == 0 {
return "[empty]"
}
id, cmd := decodeCommand(data)
formatted := ef(cmd)
return fmt.Sprintf("%x: %s", id, formatted)
}
}
m := &MultiRaft{
Config: *config,
stopper: stopper,
multiNode: raft.StartMultiNode(uint64(nodeID)),
nodeID: nodeID,
// Output channel.
Events: make(chan interface{}, config.EventBufferSize),
// Input channels.
reqChan: make(chan *RaftMessageRequest, reqBufferSize),
createGroupChan: make(chan *createGroupOp),
removeGroupChan: make(chan *removeGroupOp),
proposalChan: make(chan *proposal),
callbackChan: make(chan func()),
}
if err := m.Transport.Listen(nodeID, (*multiraftServer)(m)); err != nil {
return nil, err
}
return m, nil
}
// New creates an instance of a gossip node.
func New(
ctx context.Context,
rpcContext *rpc.Context,
grpcServer *grpc.Server,
resolvers []resolver.Resolver,
stopper *stop.Stopper,
registry *metric.Registry,
) *Gossip {
ctx = log.WithEventLog(ctx, "gossip", "gossip")
g := &Gossip{
ctx: ctx,
Connected: make(chan struct{}),
rpcContext: rpcContext,
server: newServer(ctx, stopper, registry),
outgoing: makeNodeSet(minPeers, metric.NewGauge(MetaConnectionsOutgoingGauge)),
bootstrapping: map[string]struct{}{},
disconnected: make(chan *client, 10),
stalledCh: make(chan struct{}, 1),
stallInterval: defaultStallInterval,
bootstrapInterval: defaultBootstrapInterval,
cullInterval: defaultCullInterval,
nodeDescs: map[roachpb.NodeID]*roachpb.NodeDescriptor{},
resolverAddrs: map[util.UnresolvedAddr]resolver.Resolver{},
bootstrapAddrs: map[util.UnresolvedAddr]struct{}{},
}
stopper.AddCloser(stop.CloserFn(func() {
log.FinishEventLog(ctx)
}))
registry.AddMetric(g.outgoing.gauge)
g.clientsMu.breakers = map[string]*circuit.Breaker{}
log.Infof(g.ctx, "initial resolvers: %s", resolvers)
g.SetResolvers(resolvers)
g.mu.Lock()
// Add ourselves as a SystemConfig watcher.
g.mu.is.registerCallback(KeySystemConfig, g.updateSystemConfig)
// Add ourselves as a node descriptor watcher.
g.mu.is.registerCallback(MakePrefixPattern(KeyNodeIDPrefix), g.updateNodeAddress)
g.mu.Unlock()
RegisterGossipServer(grpcServer, g.server)
return g
}
func newTestCluster(transport Transport, size int, stopper *stop.Stopper, t *testing.T) *testCluster {
if transport == nil {
transport = NewLocalRPCTransport(stopper)
}
stopper.AddCloser(transport)
cluster := &testCluster{
t: t,
transport: transport,
groups: map[proto.RangeID][]int{},
}
for i := 0; i < size; i++ {
ticker := newManualTicker()
storage := &BlockableStorage{storage: NewMemoryStorage()}
config := &Config{
Transport: transport,
Storage: storage,
Ticker: ticker,
ElectionTimeoutTicks: 2,
HeartbeatIntervalTicks: 1,
TickInterval: time.Hour, // not in use
}
mr, err := NewMultiRaft(proto.RaftNodeID(i+1), config, stopper)
if err != nil {
t.Fatal(err)
}
state := newState(mr)
demux := newEventDemux(state.Events)
demux.start(stopper)
cluster.nodes = append(cluster.nodes, state)
cluster.tickers = append(cluster.tickers, ticker)
cluster.events = append(cluster.events, demux)
cluster.storages = append(cluster.storages, storage)
}
cluster.start()
return cluster
}
