// NewContext creates an rpc Context with the supplied values.
func NewContext(baseCtx *base.Context, clock *hlc.Clock, stopper *stop.Stopper) *Context {
ctx := &Context{
Context: baseCtx,
}
if clock != nil {
ctx.localClock = clock
} else {
ctx.localClock = hlc.NewClock(hlc.UnixNano)
}
ctx.Stopper = stopper
ctx.RemoteClocks = newRemoteClockMonitor(clock, 10*defaultHeartbeatInterval)
ctx.HeartbeatInterval = defaultHeartbeatInterval
ctx.HeartbeatTimeout = 2 * defaultHeartbeatInterval
stopper.RunWorker(func() {
<-stopper.ShouldQuiesce()
ctx.conns.Lock()
for key, meta := range ctx.conns.cache {
ctx.removeConn(key, meta.conn)
}
ctx.conns.Unlock()
})
return ctx
}
// NewExecutor creates an Executor and registers a callback on the
// system config.
func NewExecutor(db client.DB, gossip *gossip.Gossip, leaseMgr *LeaseManager, metaRegistry *metric.Registry, stopper *stop.Stopper) *Executor {
exec := &Executor{
db: db,
reCache: parser.NewRegexpCache(512),
leaseMgr: leaseMgr,
latency: metaRegistry.Latency("sql.latency"),
}
exec.systemConfigCond = sync.NewCond(&exec.systemConfigMu)
gossipUpdateC := gossip.RegisterSystemConfigChannel()
stopper.RunWorker(func() {
for {
select {
case <-gossipUpdateC:
cfg := gossip.GetSystemConfig()
exec.updateSystemConfig(cfg)
case <-stopper.ShouldStop():
return
}
}
})
return exec
}
// RefreshLeases starts a goroutine that refreshes the lease manager
// leases for tables received in the latest system configuration via gossip.
func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gossip.Gossip) {
s.RunWorker(func() {
descKeyPrefix := keys.MakeTablePrefix(uint32(sqlbase.DescriptorTable.ID))
gossipUpdateC := gossip.RegisterSystemConfigChannel()
for {
select {
case <-gossipUpdateC:
cfg, _ := gossip.GetSystemConfig()
if m.testingKnobs.GossipUpdateEvent != nil {
m.testingKnobs.GossipUpdateEvent(cfg)
}
// Read all tables and their versions
if log.V(2) {
log.Info("received a new config; will refresh leases")
}
// Loop through the configuration to find all the tables.
for _, kv := range cfg.Values {
if !bytes.HasPrefix(kv.Key, descKeyPrefix) {
continue
}
// Attempt to unmarshal config into a table/database descriptor.
var descriptor sqlbase.Descriptor
if err := kv.Value.GetProto(&descriptor); err != nil {
log.Warningf("%s: unable to unmarshal descriptor %v", kv.Key, kv.Value)
continue
}
switch union := descriptor.Union.(type) {
case *sqlbase.Descriptor_Table:
table := union.Table
if err := table.Validate(); err != nil {
log.Errorf("%s: received invalid table descriptor: %v", kv.Key, table)
continue
}
if log.V(2) {
log.Infof("%s: refreshing lease table: %d (%s), version: %d",
kv.Key, table.ID, table.Name, table.Version)
}
// Try to refresh the table lease to one >= this version.
if t := m.findTableState(table.ID, false /* create */, nil); t != nil {
if err := t.purgeOldLeases(
db, table.Deleted(), table.Version, m.LeaseStore); err != nil {
log.Warningf("error purging leases for table %d(%s): %s",
table.ID, table.Name, err)
}
}
case *sqlbase.Descriptor_Database:
// Ignore.
}
}
if m.testingKnobs.TestingLeasesRefreshedEvent != nil {
m.testingKnobs.TestingLeasesRefreshedEvent(cfg)
}
case <-s.ShouldStop():
return
}
}
})
}
// waitAndProcess waits for the pace interval and processes the range
// if rng is not nil. The method returns true when the scanner needs
// to be stopped. The method also removes a range from queues when it
// is signaled via the removed channel.
func (rs *rangeScanner) waitAndProcess(start time.Time, clock *hlc.Clock, stopper *stop.Stopper,
rng *Replica) bool {
waitInterval := rs.paceInterval(start, time.Now())
nextTime := time.After(waitInterval)
if log.V(6) {
log.Infof("Wait time interval set to %s", waitInterval)
}
for {
select {
case <-nextTime:
if rng == nil {
return false
}
return !stopper.RunTask(func() {
// Try adding range to all queues.
for _, q := range rs.queues {
q.MaybeAdd(rng, clock.Now())
}
})
case rng := <-rs.removed:
// Remove range from all queues as applicable.
for _, q := range rs.queues {
q.MaybeRemove(rng)
}
if log.V(6) {
log.Infof("removed range %s", rng)
}
case <-stopper.ShouldStop():
return true
}
}
}
// start initializes the infostore with the rpc server address and
// then begins processing connecting clients in an infinite select
// loop via goroutine. Periodically, clients connected and awaiting
// the next round of gossip are awoken via the conditional variable.
func (s *server) start(rpcServer *rpc.Server, stopper *stop.Stopper) {
addr := rpcServer.Addr()
s.is.NodeAddr = util.MakeUnresolvedAddr(addr.Network(), addr.String())
if err := rpcServer.Register("Gossip.Gossip", s.Gossip, &Request{}); err != nil {
log.Fatalf("unable to register gossip service with RPC server: %s", err)
}
rpcServer.AddCloseCallback(s.onClose)
updateCallback := func(_ string, _ roachpb.Value) {
// Wakeup all pending clients.
s.ready.Broadcast()
}
unregister := s.is.registerCallback(".*", updateCallback)
stopper.RunWorker(func() {
// Periodically wakeup blocked client gossip requests.
for {
select {
case <-stopper.ShouldStop():
s.stop(unregister)
return
}
}
})
}
// NewExecutor creates an Executor and registers a callback on the
// system config.
func NewExecutor(db client.DB, gossip *gossip.Gossip, leaseMgr *LeaseManager, stopper *stop.Stopper) *Executor {
registry := metric.NewRegistry()
exec := &Executor{
db: db,
reCache: parser.NewRegexpCache(512),
leaseMgr: leaseMgr,
registry: registry,
latency: registry.Latency("latency"),
txnBeginCount: registry.Counter("transaction.begincount"),
selectCount: registry.Counter("select.count"),
updateCount: registry.Counter("update.count"),
insertCount: registry.Counter("insert.count"),
deleteCount: registry.Counter("delete.count"),
ddlCount: registry.Counter("ddl.count"),
miscCount: registry.Counter("misc.count"),
}
exec.systemConfigCond = sync.NewCond(&exec.systemConfigMu)
gossipUpdateC := gossip.RegisterSystemConfigChannel()
stopper.RunWorker(func() {
for {
select {
case <-gossipUpdateC:
cfg := gossip.GetSystemConfig()
exec.updateSystemConfig(cfg)
case <-stopper.ShouldStop():
return
}
}
})
return exec
}
// startGossip loops on a periodic ticker to gossip node-related
// information. Starts a goroutine to loop until the node is closed.
func (n *Node) startGossip(ctx context.Context, stopper *stop.Stopper) {
stopper.RunWorker(func() {
gossipStoresInterval := envutil.EnvOrDefaultDuration("gossip_stores_interval",
gossip.DefaultGossipStoresInterval)
statusTicker := time.NewTicker(gossipStatusInterval)
storesTicker := time.NewTicker(gossipStoresInterval)
nodeTicker := time.NewTicker(gossipNodeDescriptorInterval)
defer storesTicker.Stop()
defer nodeTicker.Stop()
n.gossipStores(ctx) // one-off run before going to sleep
for {
select {
case <-statusTicker.C:
n.ctx.Gossip.LogStatus()
case <-storesTicker.C:
n.gossipStores(ctx)
case <-nodeTicker.C:
if err := n.ctx.Gossip.SetNodeDescriptor(&n.Descriptor); err != nil {
log.Warningf(ctx, "couldn't gossip descriptor for node %d: %s", n.Descriptor.NodeID, err)
}
case <-stopper.ShouldStop():
return
}
}
})
}
// waitAndProcess waits for the pace interval and processes the replica
// if repl is not nil. The method returns true when the scanner needs
// to be stopped. The method also removes a replica from queues when it
// is signaled via the removed channel.
func (rs *replicaScanner) waitAndProcess(start time.Time, clock *hlc.Clock, stopper *stop.Stopper,
repl *Replica) bool {
waitInterval := rs.paceInterval(start, timeutil.Now())
rs.waitTimer.Reset(waitInterval)
if log.V(6) {
log.Infof("Wait time interval set to %s", waitInterval)
}
for {
select {
case <-rs.waitTimer.C:
rs.waitTimer.Read = true
if repl == nil {
return false
}
return !stopper.RunTask(func() {
// Try adding replica to all queues.
for _, q := range rs.queues {
q.MaybeAdd(repl, clock.Now())
}
})
case repl := <-rs.removed:
// Remove replica from all queues as applicable.
for _, q := range rs.queues {
q.MaybeRemove(repl)
}
if log.V(6) {
log.Infof("removed replica %s", repl)
}
case <-stopper.ShouldStop():
return true
}
}
}
// TestingSetupZoneConfigHook initializes the zone config hook
// to 'testingZoneConfigHook' which uses 'testingZoneConfig'.
// Settings go back to their previous values when the stopper runs our closer.
func TestingSetupZoneConfigHook(stopper *stop.Stopper) {
testingLock.Lock()
defer testingLock.Unlock()
if testingHasHook {
panic("TestingSetupZoneConfigHook called without restoring state")
}
testingHasHook = true
testingZoneConfig = map[uint32]*ZoneConfig{}
testingPreviousHook = ZoneConfigHook
ZoneConfigHook = testingZoneConfigHook
testingLargestIDHook = func(maxID uint32) (max uint32) {
testingLock.Lock()
defer testingLock.Unlock()
for id := range testingZoneConfig {
if maxID > 0 && id > maxID {
continue
}
if id > max {
max = id
}
}
return
}
stopper.AddCloser(stop.CloserFn(testingResetZoneConfigHook))
}
func (e *eventDemux) start(stopper *stop.Stopper) {
stopper.RunWorker(func() {
for {
select {
case events := <-e.events:
for _, event := range events {
switch event := event.(type) {
case *EventLeaderElection:
e.LeaderElection <- event
case *EventCommandCommitted:
e.CommandCommitted <- event
case *EventMembershipChangeCommitted:
e.MembershipChangeCommitted <- event
default:
panic(fmt.Sprintf("got unknown event type %T", event))
}
}
case <-stopper.ShouldStop():
close(e.CommandCommitted)
close(e.MembershipChangeCommitted)
close(e.LeaderElection)
return
}
}
})
}
// waitAndProcess waits for the pace interval and processes the replica
// if repl is not nil. The method returns true when the scanner needs
// to be stopped. The method also removes a replica from queues when it
// is signaled via the removed channel.
func (rs *replicaScanner) waitAndProcess(
start time.Time, clock *hlc.Clock, stopper *stop.Stopper, repl *Replica,
) bool {
waitInterval := rs.paceInterval(start, timeutil.Now())
rs.waitTimer.Reset(waitInterval)
if log.V(6) {
log.Infof(context.TODO(), "wait timer interval set to %s", waitInterval)
}
for {
select {
case <-rs.waitTimer.C:
if log.V(6) {
log.Infof(context.TODO(), "wait timer fired")
}
rs.waitTimer.Read = true
if repl == nil {
return false
}
return nil != stopper.RunTask(func() {
// Try adding replica to all queues.
for _, q := range rs.queues {
q.MaybeAdd(repl, clock.Now())
}
})
case repl := <-rs.removed:
rs.removeReplica(repl)
case <-stopper.ShouldStop():
return true
}
}
}
// start dials the remote addr and commences gossip once connected. Upon exit,
// the client is sent on the disconnected channel. This method starts client
// processing in a goroutine and returns immediately.
func (c *client) start(g *Gossip, disconnected chan *client, ctx *rpc.Context, stopper *stop.Stopper) {
stopper.RunWorker(func() {
defer func() {
disconnected <- c
}()
conn, err := ctx.GRPCDial(c.addr.String(), grpc.WithBlock())
if err != nil {
log.Errorf("failed to dial: %v", err)
return
}
// Start gossiping.
if err := c.gossip(g, NewGossipClient(conn), stopper); err != nil {
if !grpcutil.IsClosedConnection(err) {
g.mu.Lock()
peerID := c.peerID
g.mu.Unlock()
if peerID != 0 {
log.Infof("closing client to node %d (%s): %s", peerID, c.addr, err)
} else {
log.Infof("closing client to %s: %s", c.addr, err)
}
}
}
})
}
// start starts the node by registering the storage instance for the
// RPC service "Node" and initializing stores for each specified
// engine. Launches periodic store gossiping in a goroutine.
func (n *Node) start(rpcServer *rpc.Server, engines []engine.Engine,
attrs proto.Attributes, stopper *stop.Stopper) error {
n.initDescriptor(rpcServer.Addr(), attrs)
if err := rpcServer.RegisterName("Node", (*nodeServer)(n)); err != nil {
log.Fatalf("unable to register node service with RPC server: %s", err)
}
// Start status monitor.
n.status.StartMonitorFeed(n.ctx.EventFeed)
stopper.AddCloser(n.ctx.EventFeed)
// Initialize stores, including bootstrapping new ones.
if err := n.initStores(engines, stopper); err != nil {
return err
}
n.startedAt = n.ctx.Clock.Now().WallTime
// Initialize publisher for Node Events. This requires the NodeID, which is
// initialized by initStores(); because of this, some Store initialization
// events will precede the StartNodeEvent on the feed.
n.feed = status.NewNodeEventFeed(n.Descriptor.NodeID, n.ctx.EventFeed)
n.feed.StartNode(n.Descriptor, n.startedAt)
n.startPublishStatuses(stopper)
n.startGossip(stopper)
log.Infoc(n.context(), "Started node with %v engine(s) and attributes %v", engines, attrs.Attrs)
return nil
}
func (bq *baseQueue) processOne(clock *hlc.Clock, stopper *stop.Stopper) {
stopper.RunTask(func() {
start := time.Now()
bq.Lock()
rng := bq.pop()
bq.Unlock()
if rng != nil {
now := clock.Now()
if log.V(1) {
log.Infof("processing range %s from %s queue...", rng, bq.name)
}
// If the queue requires the leader lease to process the
// range, check whether this replica has leader lease and
// renew or acquire if necessary.
if bq.impl.needsLeaderLease() {
// Create a "fake" get request in order to invoke redirectOnOrAcquireLease.
args := &proto.GetRequest{RequestHeader: proto.RequestHeader{Timestamp: now}}
if err := rng.redirectOnOrAcquireLeaderLease(nil /* Trace */, args.Header().Timestamp); err != nil {
if log.V(1) {
log.Infof("this replica of %s could not acquire leader lease; skipping...", rng)
}
return
}
}
if err := bq.impl.process(now, rng); err != nil {
log.Errorf("failure processing range %s from %s queue: %s", rng, bq.name, err)
}
if log.V(1) {
log.Infof("processed range %s from %s queue in %s", rng, bq.name, time.Now().Sub(start))
}
}
})
}
// start dials the remote addr and commences gossip once connected. Upon exit,
// the client is sent on the disconnected channel. This method starts client
// processing in a goroutine and returns immediately.
func (c *client) start(g *Gossip, disconnected chan *client, ctx *rpc.Context, stopper *stop.Stopper) {
stopper.RunWorker(func() {
defer func() {
disconnected <- c
}()
// Note: avoid using `grpc.WithBlock` here. This code is already
// asynchronous from the caller's perspective, so the only effect of
// `WithBlock` here is blocking shutdown - at the time of this writing,
// that ends ups up making `kv` tests take twice as long.
conn, err := ctx.GRPCDial(c.addr.String())
if err != nil {
log.Errorf("failed to dial: %v", err)
return
}
// Start gossiping.
if err := c.gossip(g, NewGossipClient(conn), stopper); err != nil {
if !grpcutil.IsClosedConnection(err) {
g.mu.Lock()
peerID := c.peerID
g.mu.Unlock()
if peerID != 0 {
log.Infof("closing client to node %d (%s): %s", peerID, c.addr, err)
} else {
log.Infof("closing client to %s: %s", c.addr, err)
}
}
}
})
}
// gossip loops, sending deltas of the infostore and receiving deltas
// in turn. If an alternate is proposed on response, the client addr
// is modified and method returns for forwarding by caller.
func (c *client) gossip(g *Gossip, gossipClient GossipClient, stopper *stop.Stopper) error {
// For un-bootstrapped node, g.is.NodeID is 0 when client start gossip,
// so it's better to get nodeID from g.is every time.
g.mu.Lock()
addr := g.is.NodeAddr
g.mu.Unlock()
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
stream, err := gossipClient.Gossip(ctx)
if err != nil {
return err
}
if err := c.requestGossip(g, addr, stream); err != nil {
return err
}
sendGossipChan := make(chan struct{}, 1)
// Register a callback for gossip updates.
updateCallback := func(_ string, _ roachpb.Value) {
select {
case sendGossipChan <- struct{}{}:
default:
}
}
// Defer calling "undoer" callback returned from registration.
defer g.RegisterCallback(".*", updateCallback)()
errCh := make(chan error, 1)
stopper.RunWorker(func() {
errCh <- func() error {
for {
reply, err := stream.Recv()
if err != nil {
return err
}
if err := c.handleResponse(g, reply); err != nil {
return err
}
}
}()
})
for {
select {
case <-c.closer:
return nil
case <-stopper.ShouldStop():
return nil
case err := <-errCh:
return err
case <-sendGossipChan:
if err := c.sendGossip(g, addr, stream); err != nil {
return err
}
}
}
}
// StartWithStopper is the same as Start, but allows passing a stopper
// explicitly.
func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error {
if ts.Ctx == nil {
ts.Ctx = NewTestContext()
}
if stopper == nil {
stopper = stop.NewStopper()
}
// Change the replication requirements so we don't get log spam
// about ranges not being replicated enough.
// TODO(marc): set this in the zones table when we have an entry
// for the default cluster-wide zone config and remove these
// shenanigans about mutating the global default.
oldDefaultZC := proto.Clone(config.DefaultZoneConfig).(*config.ZoneConfig)
config.DefaultZoneConfig.ReplicaAttrs = []roachpb.Attributes{{}}
stopper.AddCloser(stop.CloserFn(func() {
config.DefaultZoneConfig = oldDefaultZC
}))
var err error
ts.Server, err = NewServer(ts.Ctx, stopper)
if err != nil {
return err
}
// Ensure we have the correct number of engines. Add in in-memory ones where
// needed. There must be at least one store/engine.
if ts.StoresPerNode < 1 {
ts.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper))
}
if !ts.SkipBootstrap {
stopper := stop.NewStopper()
_, err := BootstrapCluster("cluster-1", ts.Ctx.Engines, stopper)
if err != nil {
return util.Errorf("could not bootstrap cluster: %s", err)
}
stopper.Stop()
}
if err := ts.Server.Start(true); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if config.TestingTableSplitsDisabled() {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
// NewContext creates an rpc Context with the supplied values.
func NewContext(baseCtx *base.Context, clock *hlc.Clock, stopper *stop.Stopper) *Context {
var ctx *Context
if baseCtx != nil {
// TODO(tamird): This form fools `go vet`; `baseCtx` contains several
// `sync.Mutex`s, and this deference copies them, which is bad. The problem
// predates this comment, so I'm kicking the can down the road for now, but
// we should fix this.
ctx = &Context{
Context: *baseCtx,
}
} else {
ctx = new(Context)
}
if clock != nil {
ctx.localClock = clock
} else {
ctx.localClock = hlc.NewClock(hlc.UnixNano)
}
ctx.Stopper = stopper
ctx.RemoteClocks = newRemoteClockMonitor(clock)
ctx.HeartbeatInterval = defaultHeartbeatInterval
ctx.HeartbeatTimeout = 2 * defaultHeartbeatInterval
stopper.RunWorker(func() {
<-stopper.ShouldDrain()
ctx.conns.Lock()
for key, conn := range ctx.conns.cache {
ctx.removeConn(key, conn)
}
ctx.conns.Unlock()
})
return ctx
}
// NewExecutor creates an Executor and registers a callback on the
// system config.
func NewExecutor(ctx ExecutorContext, stopper *stop.Stopper, registry *metric.Registry) *Executor {
exec := &Executor{
ctx: ctx,
reCache: parser.NewRegexpCache(512),
registry: registry,
latency: registry.Latency("latency"),
txnBeginCount: registry.Counter("txn.begin.count"),
txnCommitCount: registry.Counter("txn.commit.count"),
txnAbortCount: registry.Counter("txn.abort.count"),
txnRollbackCount: registry.Counter("txn.rollback.count"),
selectCount: registry.Counter("select.count"),
updateCount: registry.Counter("update.count"),
insertCount: registry.Counter("insert.count"),
deleteCount: registry.Counter("delete.count"),
ddlCount: registry.Counter("ddl.count"),
miscCount: registry.Counter("misc.count"),
}
exec.systemConfigCond = sync.NewCond(exec.systemConfigMu.RLocker())
gossipUpdateC := ctx.Gossip.RegisterSystemConfigChannel()
stopper.RunWorker(func() {
for {
select {
case <-gossipUpdateC:
cfg, _ := ctx.Gossip.GetSystemConfig()
exec.updateSystemConfig(cfg)
case <-stopper.ShouldStop():
return
}
}
})
return exec
}
// start will run continuously and mark stores as offline if they haven't been
// heard from in longer than timeUntilStoreDead.
func (sp *StorePool) start(stopper *stop.Stopper) {
stopper.RunWorker(func() {
for {
var timeout time.Duration
sp.mu.Lock()
detail := sp.queue.peek()
if detail == nil {
// No stores yet, wait the full timeout.
timeout = sp.timeUntilStoreDead
} else {
// Check to see if the store should be marked as dead.
deadAsOf := detail.lastUpdatedTime.GoTime().Add(sp.timeUntilStoreDead)
now := sp.clock.Now()
if now.GoTime().After(deadAsOf) {
deadDetail := sp.queue.dequeue()
deadDetail.markDead(now)
// The next store might be dead as well, set the timeout to
// 0 to process it immediately.
timeout = 0
} else {
// Store is still alive, schedule the next check for when
// it should timeout.
timeout = deadAsOf.Sub(now.GoTime())
}
}
sp.mu.Unlock()
select {
case <-time.After(timeout):
case <-stopper.ShouldStop():
return
}
}
})
}
// initStores initializes the Stores map from ID to Store. Stores are
// added to the local sender if already bootstrapped. A bootstrapped
// Store has a valid ident with cluster, node and Store IDs set. If
// the Store doesn't yet have a valid ident, it's added to the
// bootstraps list for initialization once the cluster and node IDs
// have been determined.
func (n *Node) initStores(engines []engine.Engine, stopper *stop.Stopper) error {
bootstraps := list.New()
if len(engines) == 0 {
return util.Errorf("no engines")
}
for _, e := range engines {
s := storage.NewStore(n.ctx, e, &n.Descriptor)
// Initialize each store in turn, handling un-bootstrapped errors by
// adding the store to the bootstraps list.
if err := s.Start(stopper); err != nil {
if _, ok := err.(*storage.NotBootstrappedError); ok {
log.Infof("store %s not bootstrapped", s)
bootstraps.PushBack(s)
continue
}
return util.Errorf("failed to start store: %s", err)
}
if s.Ident.ClusterID == "" || s.Ident.NodeID == 0 {
return util.Errorf("unidentified store: %s", s)
}
capacity, err := s.Capacity()
if err != nil {
return util.Errorf("could not query store capacity: %s", err)
}
log.Infof("initialized store %s: %+v", s, capacity)
n.stores.AddStore(s)
}
// Verify all initialized stores agree on cluster and node IDs.
if err := n.validateStores(); err != nil {
return err
}
// Set the stores map as the gossip persistent storage, so that
// gossip can bootstrap using the most recently persisted set of
// node addresses.
if err := n.ctx.Gossip.SetStorage(n.stores); err != nil {
return fmt.Errorf("failed to initialize the gossip interface: %s", err)
}
// Connect gossip before starting bootstrap. For new nodes, connecting
// to the gossip network is necessary to get the cluster ID.
n.connectGossip()
// If no NodeID has been assigned yet, allocate a new node ID by
// supplying 0 to initNodeID.
if n.Descriptor.NodeID == 0 {
n.initNodeID(0)
}
// Bootstrap any uninitialized stores asynchronously.
if bootstraps.Len() > 0 {
stopper.RunAsyncTask(func() {
n.bootstrapStores(bootstraps, stopper)
})
}
return nil
}
// ServeWith accepts connections on ln and serves them using serveConn.
func (s *Server) ServeWith(stopper *stop.Stopper, l net.Listener, serveConn func(net.Conn)) error {
// Inspired by net/http.(*Server).Serve
var tempDelay time.Duration // how long to sleep on accept failure
for {
rw, e := l.Accept()
if e != nil {
if ne, ok := e.(net.Error); ok && ne.Temporary() {
if tempDelay == 0 {
tempDelay = 5 * time.Millisecond
} else {
tempDelay *= 2
}
if max := 1 * time.Second; tempDelay > max {
tempDelay = max
}
httpLogger.Printf("http: Accept error: %v; retrying in %v", e, tempDelay)
time.Sleep(tempDelay)
continue
}
return e
}
tempDelay = 0
go func() {
defer stopper.Recover()
s.Server.ConnState(rw, http.StateNew) // before Serve can return
serveConn(rw)
s.Server.ConnState(rw, http.StateClosed)
}()
}
}
// maybeWarnAboutInit looks for signs indicating a cluster which
// hasn't been initialized and warns. There's no absolutely sure way
// to determine whether the current node is simply waiting to be
// bootstrapped to an existing cluster vs. the operator having failed
// to initialize the cluster via the "cockroach init" command, so
// we can only warn.
//
// This method checks whether all gossip bootstrap hosts are
// connected, and whether the node itself is a bootstrap host, but
// there is still no sentinel gossip.
func (g *Gossip) maybeWarnAboutInit(stopper *stop.Stopper) {
stopper.RunWorker(func() {
// Wait 5s before first check.
select {
case <-stopper.ShouldStop():
return
case <-time.After(5 * time.Second):
}
retryOptions := retry.Options{
InitialBackoff: 5 * time.Second, // first backoff at 5s
MaxBackoff: 60 * time.Second, // max backoff is 60s
Multiplier: 2, // doubles
Stopper: stopper, // stop no matter what on stopper
}
// This will never error because of infinite retries.
for r := retry.Start(retryOptions); r.Next(); {
g.mu.Lock()
hasSentinel := g.is.getInfo(KeySentinel) != nil
triedAll := g.triedAll
g.mu.Unlock()
// If we have the sentinel, exit the retry loop.
if hasSentinel {
break
}
// Otherwise, if all bootstrap hosts are connected, warn.
if triedAll {
log.Warningf("connected to gossip but missing sentinel. Has the cluster been initialized? " +
"Use \"cockroach init\" to initialize.")
}
}
})
}
// start dials the remote addr and commences gossip once connected.
// Upon exit, signals client is done by pushing it onto the done
// channel. If the client experienced an error, its err field will
// be set. This method starts client processing in a goroutine and
// returns immediately.
func (c *client) start(g *Gossip, done chan *client, context *rpc.Context, stopper *stop.Stopper) {
stopper.RunWorker(func() {
var err error
c.rpcClient = rpc.NewClient(c.addr, context)
select {
case <-c.rpcClient.Healthy():
// Start gossiping and wait for disconnect or error.
err = c.gossip(g, stopper)
if context.DisableCache {
c.rpcClient.Close()
}
case <-c.rpcClient.Closed:
err = util.Errorf("client closed")
}
done <- c
if err != nil {
if c.peerID != 0 {
log.Infof("closing client to node %d (%s): %s", c.peerID, c.addr, err)
} else {
log.Infof("closing client to %s: %s", c.addr, err)
}
}
})
}
// bootstrap connects the node to the gossip network. Bootstrapping
// commences in the event there are no connected clients or the
// sentinel gossip info is not available. After a successful bootstrap
// connection, this method will block on the stalled condvar, which
// receives notifications that gossip network connectivity has been
// lost and requires re-bootstrapping.
func (g *Gossip) bootstrap(stopper *stop.Stopper) {
stopper.RunWorker(func() {
for {
g.mu.Lock()
if g.closed {
g.mu.Unlock()
return
}
// Check whether or not we need bootstrap.
haveClients := g.outgoing.len() > 0
haveSentinel := g.is.getInfo(KeySentinel) != nil
if !haveClients || !haveSentinel {
// Try to get another bootstrap address from the resolvers.
if addr := g.getNextBootstrapAddress(); addr != nil {
g.startClient(addr, g.bsRPCContext, stopper)
}
}
g.mu.Unlock()
// Block until we need bootstrapping again.
select {
case <-g.stalled:
// continue
case <-stopper.ShouldStop():
return
}
}
})
}
// waitForStopper stops the supplied stop.Stopper and waits up to five seconds
// for it to complete.
func waitForStopper(t testing.TB, stopper *stop.Stopper) {
stopper.Stop()
select {
case <-stopper.IsStopped():
case <-time.After(5 * time.Second):
t.Fatalf("Stopper failed to stop after 5 seconds")
}
}
// gossip loops, sending deltas of the infostore and receiving deltas
// in turn. If an alternate is proposed on response, the client addr
// is modified and method returns for forwarding by caller.
func (c *client) gossip(g *Gossip, stopper *stop.Stopper) error {
// For un-bootstrapped node, g.is.NodeID is 0 when client start gossip,
// so it's better to get nodeID from g.is every time.
g.mu.Lock()
addr := util.MakeUnresolvedAddr(g.is.NodeAddr.Network(), g.is.NodeAddr.String())
g.mu.Unlock()
lAddr := util.MakeUnresolvedAddr(c.rpcClient.LocalAddr().Network(), c.rpcClient.LocalAddr().String())
done := make(chan *netrpc.Call, 10)
c.getGossip(g, addr, lAddr, done)
// Register a callback for gossip updates.
updateCallback := func(_ string, _ roachpb.Value) {
c.sendGossip(g, addr, lAddr, done)
}
// Defer calling "undoer" callback returned from registration.
defer g.RegisterCallback(".*", updateCallback)()
// Loop until stopper is signalled, or until either the gossip or
// RPC clients are closed. getGossip is a hanging get, returning
// results only when the remote server has new gossip information to
// share. sendGossip is sent to the remote server when this node has
// new gossip information to share with the server.
//
// Nodes "pull" gossip in order to guarantee that they're connected
// to the sentinel and not too distant from other nodes in the
// network. The also "push" their own gossip which guarantees that
// the sentinel node will contain their info, and therefore every
// node connected to the sentinel. Just pushing or just pulling
// wouldn't guarantee a fully connected network.
for {
select {
case call := <-done:
if err := c.handleGossip(g, call); err != nil {
return err
}
req := call.Args.(*Request)
// If this was from a gossip pull request, fetch again.
if req.Delta == nil {
c.getGossip(g, addr, lAddr, done)
} else {
// Otherwise, it's a gossip push request; set sendingGossip
// flag false and maybe send more gossip if there have been
// additional updates.
g.mu.Lock()
c.sendingGossip = false
g.mu.Unlock()
c.sendGossip(g, addr, lAddr, done)
}
case <-c.rpcClient.Closed:
return util.Errorf("client closed")
case <-c.closer:
return nil
case <-stopper.ShouldStop():
return nil
}
}
}
// StartWithStopper is the same as Start, but allows passing a stopper
// explicitly.
func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error {
if ts.Ctx == nil {
ctx := MakeTestContext()
ts.Ctx = &ctx
}
if stopper == nil {
stopper = stop.NewStopper()
}
// Change the replication requirements so we don't get log spam about ranges
// not being replicated enough.
cfg := config.DefaultZoneConfig()
cfg.ReplicaAttrs = []roachpb.Attributes{{}}
fn := config.TestingSetDefaultZoneConfig(cfg)
stopper.AddCloser(stop.CloserFn(fn))
// Needs to be called before NewServer to ensure resolvers are initialized.
if err := ts.Ctx.InitNode(); err != nil {
return err
}
// Ensure we have the correct number of engines. Add in-memory ones where
// needed. There must be at least one store/engine.
if ts.StoresPerNode < 1 {
ts.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, stopper))
}
var err error
ts.Server, err = NewServer(*ts.Ctx, stopper)
if err != nil {
return err
}
// Our context must be shared with our server.
ts.Ctx = &ts.Server.ctx
if err := ts.Server.Start(); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if config.TestingTableSplitsDisabled() {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
// RefreshLeases starts a goroutine that refreshes the lease manager
// leases for tables received in the latest system configuration via gossip.
func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gossip.Gossip) {
s.RunWorker(func() {
descKeyPrefix := keys.MakeTablePrefix(uint32(DescriptorTable.ID))
gossip.RegisterSystemConfigCallback(m.updateSystemConfig)
for {
select {
case <-m.newConfig:
// Read all tables and their versions
cfg := m.getSystemConfig()
if log.V(2) {
log.Info("received a new config %v", cfg)
}
// Loop through the configuration to find all the tables.
for _, kv := range cfg.Values {
if kv.Value.Tag != roachpb.ValueType_BYTES {
continue
}
if !bytes.HasPrefix(kv.Key, descKeyPrefix) {
continue
}
// Attempt to unmarshal config into a table/database descriptor.
var descriptor Descriptor
if err := kv.Value.GetProto(&descriptor); err != nil {
log.Warningf("unable to unmarshal descriptor %v", kv.Value)
continue
}
switch union := descriptor.Union.(type) {
case *Descriptor_Table:
table := union.Table
if err := table.Validate(); err != nil {
log.Errorf("received invalid table descriptor: %v", table)
continue
}
if log.V(2) {
log.Infof("refreshing lease table: %d, version: %d", table.ID, table.Version)
}
// Try to refresh the table lease to one >= this version.
if err := m.refreshLease(db, table.ID, table.Version); err != nil {
log.Warning(err)
}
case *Descriptor_Database:
// Ignore.
}
}
case <-s.ShouldStop():
return
}
}
})
}
// NewMultiRaft creates a MultiRaft object.
func NewMultiRaft(nodeID roachpb.NodeID, storeID roachpb.StoreID, config *Config, stopper *stop.Stopper) (*MultiRaft, error) {
if nodeID <= 0 {
return nil, util.Errorf("invalid NodeID: %s", nodeID)
}
if storeID <= 0 {
return nil, util.Errorf("invalid StoreID: %s", storeID)
}
if err := config.validate(); err != nil {
return nil, err
}
if config.Ticker == nil {
config.Ticker = newTicker(config.TickInterval)
stopper.AddCloser(config.Ticker)
}
if config.EntryFormatter != nil {
// Wrap the EntryFormatter to strip off the command id.
ef := config.EntryFormatter
config.EntryFormatter = func(data []byte) string {
if len(data) == 0 {
return "[empty]"
}
id, cmd := decodeCommand(data)
formatted := ef(cmd)
return fmt.Sprintf("%x: %s", id, formatted)
}
}
m := &MultiRaft{
Config: *config,
stopper: stopper,
nodeID: nodeID,
storeID: storeID,
// Output channel.
Events: make(chan []interface{}),
// Input channels.
reqChan: make(chan *RaftMessageRequest, reqBufferSize),
createGroupChan: make(chan createGroupOp),
removeGroupChan: make(chan removeGroupOp),
proposalChan: make(chan *proposal),
campaignChan: make(chan roachpb.RangeID),
callbackChan: make(chan func()),
statusChan: make(chan statusOp),
}
if err := m.Transport.Listen(storeID, (*multiraftServer)(m)); err != nil {
return nil, err
}
return m, nil
}