// NewExecutor creates an Executor and registers a callback on the // system config. func NewExecutor(db client.DB, gossip *gossip.Gossip, leaseMgr *LeaseManager, metaRegistry *metric.Registry, stopper *stop.Stopper) *Executor { exec := &Executor{ db: db, reCache: parser.NewRegexpCache(512), leaseMgr: leaseMgr, latency: metaRegistry.Latency("sql.latency"), } exec.systemConfigCond = sync.NewCond(&exec.systemConfigMu) gossipUpdateC := gossip.RegisterSystemConfigChannel() stopper.RunWorker(func() { for { select { case <-gossipUpdateC: cfg := gossip.GetSystemConfig() exec.updateSystemConfig(cfg) case <-stopper.ShouldStop(): return } } }) return exec }
// storeDescFromGossip retrieves a StoreDescriptor from the specified // store gossip key. Returns an error if the gossip doesn't exist // or is not a StoreDescriptor. func storeDescFromGossip(key string, g *gossip.Gossip) (*proto.StoreDescriptor, error) { storeDesc := &proto.StoreDescriptor{} if err := g.GetInfoProto(key, storeDesc); err != nil { return nil, err } return storeDesc, nil }
// newReplicateQueue returns a new instance of replicateQueue. func newReplicateQueue(store *Store, g *gossip.Gossip, allocator Allocator, clock *hlc.Clock, options AllocatorOptions) *replicateQueue { rq := &replicateQueue{ allocator: allocator, clock: clock, updateChan: make(chan struct{}, 1), } rq.baseQueue = makeBaseQueue("replicate", rq, store, g, queueConfig{ maxSize: replicateQueueMaxSize, needsLease: true, acceptsUnsplitRanges: false, }) if g != nil { // gossip is nil for some unittests // Register a gossip callback to signal queue that replicas in // purgatory might be retried due to new store gossip. g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) { select { case rq.updateChan <- struct{}{}: default: } }) } return rq }
// computeSplitKeys returns an array of keys at which the supplied // range should be split, as computed by intersecting the range with // accounting and zone config map boundaries. func computeSplitKeys(g *gossip.Gossip, rng *Range) []proto.Key { // Now split the range into pieces by intersecting it with the // boundaries of the config map. splitKeys := proto.KeySlice{} for _, configKey := range []string{gossip.KeyConfigAccounting, gossip.KeyConfigZone} { info, err := g.GetInfo(configKey) if err != nil { log.Errorf("unable to fetch %s config from gossip: %s", configKey, err) continue } configMap := info.(PrefixConfigMap) splits, err := configMap.SplitRangeByPrefixes(rng.Desc().StartKey, rng.Desc().EndKey) if err != nil { log.Errorf("unable to split %s by prefix map %s", rng, configMap) continue } // Gather new splits. for _, split := range splits { if split.end.Less(rng.Desc().EndKey) { splitKeys = append(splitKeys, split.end) } } } // Sort and unique the combined split keys from intersections with // both the accounting and zone config maps. sort.Sort(splitKeys) var unique []proto.Key for i, key := range splitKeys { if i == 0 || !key.Equal(splitKeys[i-1]) { unique = append(unique, key) } } return unique }
// RefreshLeases starts a goroutine that refreshes the lease manager // leases for tables received in the latest system configuration via gossip. func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gossip.Gossip) { s.RunWorker(func() { descKeyPrefix := keys.MakeTablePrefix(uint32(sqlbase.DescriptorTable.ID)) gossipUpdateC := gossip.RegisterSystemConfigChannel() for { select { case <-gossipUpdateC: cfg, _ := gossip.GetSystemConfig() if m.testingKnobs.GossipUpdateEvent != nil { m.testingKnobs.GossipUpdateEvent(cfg) } // Read all tables and their versions if log.V(2) { log.Info("received a new config; will refresh leases") } // Loop through the configuration to find all the tables. for _, kv := range cfg.Values { if !bytes.HasPrefix(kv.Key, descKeyPrefix) { continue } // Attempt to unmarshal config into a table/database descriptor. var descriptor sqlbase.Descriptor if err := kv.Value.GetProto(&descriptor); err != nil { log.Warningf("%s: unable to unmarshal descriptor %v", kv.Key, kv.Value) continue } switch union := descriptor.Union.(type) { case *sqlbase.Descriptor_Table: table := union.Table if err := table.Validate(); err != nil { log.Errorf("%s: received invalid table descriptor: %v", kv.Key, table) continue } if log.V(2) { log.Infof("%s: refreshing lease table: %d (%s), version: %d", kv.Key, table.ID, table.Name, table.Version) } // Try to refresh the table lease to one >= this version. if t := m.findTableState(table.ID, false /* create */, nil); t != nil { if err := t.purgeOldLeases( db, table.Deleted(), table.Version, m.LeaseStore); err != nil { log.Warningf("error purging leases for table %d(%s): %s", table.ID, table.Name, err) } } case *sqlbase.Descriptor_Database: // Ignore. } } if m.testingKnobs.TestingLeasesRefreshedEvent != nil { m.testingKnobs.TestingLeasesRefreshedEvent(cfg) } case <-s.ShouldStop(): return } } }) }
// NewStorePool creates a StorePool and registers the store updating callback // with gossip. func NewStorePool( g *gossip.Gossip, clock *hlc.Clock, rpcContext *rpc.Context, reservationsEnabled bool, timeUntilStoreDead time.Duration, stopper *stop.Stopper, ) *StorePool { sp := &StorePool{ clock: clock, timeUntilStoreDead: timeUntilStoreDead, rpcContext: rpcContext, reservationsEnabled: reservationsEnabled, failedReservationsTimeout: envutil.EnvOrDefaultDuration("failed_reservation_timeout", defaultFailedReservationsTimeout), declinedReservationsTimeout: envutil.EnvOrDefaultDuration("declined_reservation_timeout", defaultDeclinedReservationsTimeout), reserveRPCTimeout: envutil.EnvOrDefaultDuration("reserve_rpc_timeout", defaultReserveRPCTimeout), } sp.mu.stores = make(map[roachpb.StoreID]*storeDetail) heap.Init(&sp.mu.queue) storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix) g.RegisterCallback(storeRegex, sp.storeGossipUpdate) sp.start(stopper) return sp }
// NewExecutor creates an Executor and registers a callback on the // system config. func NewExecutor(db client.DB, gossip *gossip.Gossip, leaseMgr *LeaseManager, stopper *stop.Stopper) *Executor { registry := metric.NewRegistry() exec := &Executor{ db: db, reCache: parser.NewRegexpCache(512), leaseMgr: leaseMgr, registry: registry, latency: registry.Latency("latency"), txnBeginCount: registry.Counter("transaction.begincount"), selectCount: registry.Counter("select.count"), updateCount: registry.Counter("update.count"), insertCount: registry.Counter("insert.count"), deleteCount: registry.Counter("delete.count"), ddlCount: registry.Counter("ddl.count"), miscCount: registry.Counter("misc.count"), } exec.systemConfigCond = sync.NewCond(&exec.systemConfigMu) gossipUpdateC := gossip.RegisterSystemConfigChannel() stopper.RunWorker(func() { for { select { case <-gossipUpdateC: cfg := gossip.GetSystemConfig() exec.updateSystemConfig(cfg) case <-stopper.ShouldStop(): return } } }) return exec }
// NewStorePool creates a StorePool and registers the store updating callback // with gossip. func NewStorePool( g *gossip.Gossip, clock *hlc.Clock, rpcContext *rpc.Context, reservationsEnabled bool, timeUntilStoreDead time.Duration, stopper *stop.Stopper, ) *StorePool { sp := &StorePool{ clock: clock, timeUntilStoreDead: timeUntilStoreDead, rpcContext: rpcContext, reservationsEnabled: reservationsEnabled, failedReservationsTimeout: envutil.EnvOrDefaultDuration("COCKROACH_FAILED_RESERVATION_TIMEOUT", defaultFailedReservationsTimeout), declinedReservationsTimeout: envutil.EnvOrDefaultDuration("COCKROACH_DECLINED_RESERVATION_TIMEOUT", defaultDeclinedReservationsTimeout), reserveRPCTimeout: envutil.EnvOrDefaultDuration("COCKROACH_RESERVE_RPC_TIMEOUT", defaultReserveRPCTimeout), resolver: GossipAddressResolver(g), } sp.mu.stores = make(map[roachpb.StoreID]*storeDetail) heap.Init(&sp.mu.queue) storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix) g.RegisterCallback(storeRegex, sp.storeGossipUpdate) deadReplicasRegex := gossip.MakePrefixPattern(gossip.KeyDeadReplicasPrefix) g.RegisterCallback(deadReplicasRegex, sp.deadReplicasGossipUpdate) sp.start(stopper) return sp }
func newStoreGossiper(g *gossip.Gossip) *storeGossiper { sg := &storeGossiper{ g: g, } g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ []byte) { sg.wg.Done() }) return sg }
// lookupZoneConfig returns the zone config matching the range. func lookupZoneConfig(g *gossip.Gossip, repl *Replica) (config.ZoneConfig, error) { zoneMap, err := g.GetZoneConfig() if err != nil { return config.ZoneConfig{}, util.Errorf("unable to lookup zone config for range %s: %s", repl, err) } prefixConfig := zoneMap.MatchByPrefix(repl.Desc().StartKey) return *prefixConfig.Config.GetValue().(*config.ZoneConfig), nil }
// lookupZoneConfig returns the zone config matching the range. func lookupZoneConfig(g *gossip.Gossip, rng *Range) (proto.ZoneConfig, error) { zoneMap, err := g.GetInfo(gossip.KeyConfigZone) if err != nil || zoneMap == nil { return proto.ZoneConfig{}, util.Errorf("unable to lookup zone config for range %s: %s", rng, err) } prefixConfig := zoneMap.(PrefixConfigMap).MatchByPrefix(rng.Desc().StartKey) return *prefixConfig.Config.(*proto.ZoneConfig), nil }
// RefreshLeases starts a goroutine that refreshes the lease manager // leases for tables received in the latest system configuration via gossip. func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gossip.Gossip) { s.RunWorker(func() { descKeyPrefix := keys.MakeTablePrefix(uint32(DescriptorTable.ID)) gossipUpdateC := gossip.RegisterSystemConfigChannel() for { select { case <-gossipUpdateC: cfg := *gossip.GetSystemConfig() m.updateSystemConfig(cfg) // Read all tables and their versions if log.V(2) { log.Info("received a new config %v", cfg) } // Loop through the configuration to find all the tables. for _, kv := range cfg.Values { if !bytes.HasPrefix(kv.Key, descKeyPrefix) { continue } // Attempt to unmarshal config into a table/database descriptor. var descriptor Descriptor if err := kv.Value.GetProto(&descriptor); err != nil { log.Warningf("%s: unable to unmarshal descriptor %v", kv.Key, kv.Value) continue } switch union := descriptor.Union.(type) { case *Descriptor_Table: table := union.Table if err := table.Validate(); err != nil { log.Errorf("%s: received invalid table descriptor: %v", kv.Key, table) continue } if log.V(2) { log.Infof("%s: refreshing lease table: %d, version: %d", kv.Key, table.ID, table.Version) } // Try to refresh the table lease to one >= this version. if err := m.refreshLease(db, table.ID, table.Version); err != nil { log.Warningf("%s: %v", kv.Key, err) } case *Descriptor_Database: // Ignore. } } case <-s.ShouldStop(): return } } }) }
// storeDescFromGossip retrieves a StoreDescriptor from the specified // capacity gossip key. Returns an error if the gossip doesn't exist // or is not a StoreDescriptor. func storeDescFromGossip(key string, g *gossip.Gossip) (*proto.StoreDescriptor, error) { info, err := g.GetInfo(key) if err != nil { return nil, err } storeDesc, ok := info.(proto.StoreDescriptor) if !ok { return nil, fmt.Errorf("gossiped info is not a StoreDescriptor: %+v", info) } return &storeDesc, nil }
// newStoreGossiper creates a store gossiper for use by tests. It adds the // callback to gossip. func newStoreGossiper(g *gossip.Gossip) *storeGossiper { sg := &storeGossiper{ g: g, storeKeyMap: make(map[string]struct{}), } g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(key string, _ []byte) { sg.mu.Lock() defer sg.mu.Unlock() if _, ok := sg.storeKeyMap[key]; ok { sg.wg.Done() } }) return sg }
// NewStoreGossiper creates a store gossiper for use by tests. It adds the // callback to gossip. func NewStoreGossiper(g *gossip.Gossip) *StoreGossiper { sg := &StoreGossiper{ g: g, storeKeyMap: make(map[string]struct{}), } sg.cond = sync.NewCond(&sg.mu) g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(key string, _ roachpb.Value) { sg.mu.Lock() defer sg.mu.Unlock() delete(sg.storeKeyMap, key) sg.cond.Broadcast() }) return sg }
// NewStorePool creates a StorePool and registers the store updating callback // with gossip. func NewStorePool(g *gossip.Gossip, timeUntilStoreDead time.Duration, stopper *stop.Stopper) *StorePool { sp := &StorePool{ timeUntilStoreDead: timeUntilStoreDead, stores: make(map[roachpb.StoreID]*storeDetail), } heap.Init(&sp.queue) storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix) g.RegisterCallback(storeRegex, sp.storeGossipUpdate) sp.start(stopper) return sp }
// newExecutor creates an Executor and registers a callback on the // system config. func newExecutor(db client.DB, gossip *gossip.Gossip, clock *hlc.Clock) *Executor { exec := &Executor{ db: db, reCache: parser.NewRegexpCache(512), } gossip.RegisterSystemConfigCallback(exec.updateSystemConfig) return exec }
func gossipStores(g *gossip.Gossip, stores []*proto.StoreDescriptor, t *testing.T) { var wg sync.WaitGroup wg.Add(len(stores)) g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyCapacityPrefix), func(_ string, _ bool) { wg.Done() }) for _, s := range stores { keyMaxCapacity := gossip.MakeCapacityKey(s.Node.NodeID, s.StoreID) // Gossip store descriptor. err := g.AddInfo(keyMaxCapacity, *s, 0) if err != nil { t.Fatal(err) } } // Wait for all gossip callbacks to be invoked. wg.Wait() }
// newExecutor creates an Executor and registers a callback on the // system config. func newExecutor(db client.DB, gossip *gossip.Gossip, leaseMgr *LeaseManager) *Executor { exec := &Executor{ db: db, reCache: parser.NewRegexpCache(512), leaseMgr: leaseMgr, } exec.systemConfigCond = sync.NewCond(&exec.systemConfigMu) gossip.RegisterSystemConfigCallback(exec.updateSystemConfig) return exec }
// newReplicaSlice creates a replicaSlice from the replicas listed in the range // descriptor and using gossip to lookup node descriptors. Replicas on nodes // that are not gossipped are omitted from the result. func newReplicaSlice(gossip *gossip.Gossip, desc *proto.RangeDescriptor) replicaSlice { replicas := make(replicaSlice, 0, len(desc.Replicas)) for _, r := range desc.Replicas { nd, err := gossip.GetNodeDescriptor(r.NodeID) if err != nil { if log.V(1) { log.Infof("node %d is not gossiped: %v", r.NodeID, err) } continue } replicas = append(replicas, replicaInfo{ Replica: r, NodeDesc: nd, }) } return replicas }
// newReplicaSlice creates a ReplicaSlice from the replicas listed in the range // descriptor and using gossip to lookup node descriptors. Replicas on nodes // that are not gossipped are omitted from the result. func newReplicaSlice(gossip *gossip.Gossip, desc *roachpb.RangeDescriptor) ReplicaSlice { if gossip == nil { return nil } replicas := make(ReplicaSlice, 0, len(desc.Replicas)) for _, r := range desc.Replicas { nd, err := gossip.GetNodeDescriptor(r.NodeID) if err != nil { if log.V(1) { log.Infof(context.TODO(), "node %d is not gossiped: %v", r.NodeID, err) } continue } replicas = append(replicas, ReplicaInfo{ ReplicaDescriptor: r, NodeDesc: nd, }) } return replicas }
// NewDistSender returns a batch.Sender instance which connects to the // Cockroach cluster via the supplied gossip instance. Supplying a // DistSenderContext or the fields within is optional. For omitted values, sane // defaults will be used. func NewDistSender(cfg *DistSenderConfig, g *gossip.Gossip) *DistSender { if cfg == nil { cfg = &DistSenderConfig{} } ds := &DistSender{gossip: g} ds.Ctx = cfg.Ctx if ds.Ctx == nil { ds.Ctx = context.Background() } if ds.Ctx.Done() != nil { panic("context with cancel or deadline") } if tracing.TracerFromCtx(ds.Ctx) == nil { ds.Ctx = tracing.WithTracer(ds.Ctx, tracing.NewTracer()) } ds.clock = cfg.Clock if ds.clock == nil { ds.clock = hlc.NewClock(hlc.UnixNano) } if cfg.nodeDescriptor != nil { atomic.StorePointer(&ds.nodeDescriptor, unsafe.Pointer(cfg.nodeDescriptor)) } rcSize := cfg.RangeDescriptorCacheSize if rcSize <= 0 { rcSize = defaultRangeDescriptorCacheSize } rdb := cfg.RangeDescriptorDB if rdb == nil { rdb = ds } ds.rangeCache = newRangeDescriptorCache(rdb, int(rcSize)) lcSize := cfg.LeaseHolderCacheSize if lcSize <= 0 { lcSize = defaultLeaseHolderCacheSize } ds.leaseHolderCache = newLeaseHolderCache(int(lcSize)) if cfg.RangeLookupMaxRanges <= 0 { ds.rangeLookupMaxRanges = defaultRangeLookupMaxRanges } if cfg.TransportFactory != nil { ds.transportFactory = cfg.TransportFactory } ds.rpcRetryOptions = base.DefaultRetryOptions() if cfg.RPCRetryOptions != nil { ds.rpcRetryOptions = *cfg.RPCRetryOptions } if cfg.RPCContext != nil { ds.rpcContext = cfg.RPCContext if ds.rpcRetryOptions.Closer == nil { ds.rpcRetryOptions.Closer = ds.rpcContext.Stopper.ShouldQuiesce() } } if cfg.SendNextTimeout != 0 { ds.sendNextTimeout = cfg.SendNextTimeout } else { ds.sendNextTimeout = defaultSendNextTimeout } if g != nil { g.RegisterCallback(gossip.KeyFirstRangeDescriptor, func(_ string, value roachpb.Value) { if log.V(1) { var desc roachpb.RangeDescriptor if err := value.GetProto(&desc); err != nil { log.Errorf(ds.Ctx, "unable to parse gossipped first range descriptor: %s", err) } else { log.Infof(ds.Ctx, "gossipped first range descriptor: %+v", desc.Replicas) } } err := ds.rangeCache.EvictCachedRangeDescriptor(roachpb.RKeyMin, nil, false) if err != nil { log.Warningf(ds.Ctx, "failed to evict first range descriptor: %s", err) } }) } return ds }
// NewExecutor creates an Executor and registers a callback on the // system config. func NewExecutor(db client.DB, gossip *gossip.Gossip) *Executor { exec := &Executor{db: db} gossip.RegisterSystemConfigCallback(exec.updateSystemConfig) return exec }
// GossipAddressResolver is a thin wrapper around gossip's GetNodeIDAddress // that allows its return value to be used as the net.Addr interface. func GossipAddressResolver(gossip *gossip.Gossip) NodeAddressResolver { return func(nodeID roachpb.NodeID) (net.Addr, error) { return gossip.GetNodeIDAddress(nodeID) } }