// newReplicateQueue returns a new instance of replicateQueue.
func newReplicateQueue(
store *Store, g *gossip.Gossip, allocator Allocator, clock *hlc.Clock, options AllocatorOptions,
) *replicateQueue {
rq := &replicateQueue{
allocator: allocator,
clock: clock,
updateChan: make(chan struct{}, 1),
}
rq.baseQueue = newBaseQueue(
"replicate", rq, store, g,
queueConfig{
maxSize: replicateQueueMaxSize,
needsLease: true,
acceptsUnsplitRanges: store.TestingKnobs().ReplicateQueueAcceptsUnsplit,
successes: store.metrics.ReplicateQueueSuccesses,
failures: store.metrics.ReplicateQueueFailures,
pending: store.metrics.ReplicateQueuePending,
processingNanos: store.metrics.ReplicateQueueProcessingNanos,
purgatory: store.metrics.ReplicateQueuePurgatory,
},
)
if g != nil { // gossip is nil for some unittests
// Register a gossip callback to signal queue that replicas in
// purgatory might be retried due to new store gossip.
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) {
select {
case rq.updateChan <- struct{}{}:
default:
}
})
}
return rq
}
// NewStorePool creates a StorePool and registers the store updating callback
// with gossip.
func NewStorePool(
ambient log.AmbientContext,
g *gossip.Gossip,
clock *hlc.Clock,
rpcContext *rpc.Context,
timeUntilStoreDead time.Duration,
stopper *stop.Stopper,
deterministic bool,
) *StorePool {
sp := &StorePool{
AmbientContext: ambient,
clock: clock,
timeUntilStoreDead: timeUntilStoreDead,
rpcContext: rpcContext,
failedReservationsTimeout: envutil.EnvOrDefaultDuration("COCKROACH_FAILED_RESERVATION_TIMEOUT",
defaultFailedReservationsTimeout),
declinedReservationsTimeout: envutil.EnvOrDefaultDuration("COCKROACH_DECLINED_RESERVATION_TIMEOUT",
defaultDeclinedReservationsTimeout),
resolver: GossipAddressResolver(g),
deterministic: deterministic,
}
sp.mu.storeDetails = make(map[roachpb.StoreID]*storeDetail)
heap.Init(&sp.mu.queue)
sp.mu.nodeLocalities = make(map[roachpb.NodeID]roachpb.Locality)
storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
g.RegisterCallback(storeRegex, sp.storeGossipUpdate)
deadReplicasRegex := gossip.MakePrefixPattern(gossip.KeyDeadReplicasPrefix)
g.RegisterCallback(deadReplicasRegex, sp.deadReplicasGossipUpdate)
sp.start(stopper)
return sp
}
// WaitForStores waits for all of the store descriptors to be gossiped. Servers
// other than the first "bootstrap" their stores asynchronously, but we'd like
// to wait for all of the stores to be initialized before returning the
// TestCluster.
func (tc *TestCluster) WaitForStores(t testing.TB, g *gossip.Gossip) {
// Register a gossip callback for the store descriptors.
var storesMu syncutil.Mutex
stores := map[roachpb.StoreID]struct{}{}
storesDone := make(chan error)
storesDoneOnce := storesDone
unregister := g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix),
func(_ string, content roachpb.Value) {
storesMu.Lock()
defer storesMu.Unlock()
if storesDoneOnce == nil {
return
}
var desc roachpb.StoreDescriptor
if err := content.GetProto(&desc); err != nil {
storesDoneOnce <- err
return
}
stores[desc.StoreID] = struct{}{}
if len(stores) == len(tc.Servers) {
close(storesDoneOnce)
storesDoneOnce = nil
}
})
defer unregister()
// Wait for the store descriptors to be gossiped.
for err := range storesDone {
if err != nil {
t.Fatal(err)
}
}
}
// NewStoreGossiper creates a store gossiper for use by tests. It adds the
// callback to gossip.
func NewStoreGossiper(g *gossip.Gossip) *StoreGossiper {
sg := &StoreGossiper{
g: g,
storeKeyMap: make(map[string]struct{}),
}
sg.cond = sync.NewCond(&sg.mu)
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(key string, _ roachpb.Value) {
sg.mu.Lock()
defer sg.mu.Unlock()
delete(sg.storeKeyMap, key)
sg.cond.Broadcast()
})
return sg
}
// newReplicateQueue returns a new instance of replicateQueue.
func newReplicateQueue(
store *Store, g *gossip.Gossip, allocator Allocator, clock *hlc.Clock, options AllocatorOptions,
) *replicateQueue {
rq := &replicateQueue{
metrics: makeReplicateQueueMetrics(),
allocator: allocator,
clock: clock,
updateChan: make(chan struct{}, 1),
}
store.metrics.registry.AddMetricStruct(&rq.metrics)
rq.baseQueue = newBaseQueue(
"replicate", rq, store, g,
queueConfig{
maxSize: defaultQueueMaxSize,
needsLease: true,
acceptsUnsplitRanges: store.TestingKnobs().ReplicateQueueAcceptsUnsplit,
successes: store.metrics.ReplicateQueueSuccesses,
failures: store.metrics.ReplicateQueueFailures,
pending: store.metrics.ReplicateQueuePending,
processingNanos: store.metrics.ReplicateQueueProcessingNanos,
purgatory: store.metrics.ReplicateQueuePurgatory,
},
)
updateFn := func() {
select {
case rq.updateChan <- struct{}{}:
default:
}
}
// Register a gossip and node liveness callbacks to signal queue
// that replicas in purgatory might be retried.
if g != nil { // gossip is nil for some unittests
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) {
updateFn()
})
}
if nl := store.cfg.NodeLiveness; nl != nil { // node liveness is nil for some unittests
nl.RegisterCallback(func(_ roachpb.NodeID) {
updateFn()
})
}
return rq
}
// newReplicaSlice creates a ReplicaSlice from the replicas listed in the range
// descriptor and using gossip to lookup node descriptors. Replicas on nodes
// that are not gossiped are omitted from the result.
func newReplicaSlice(gossip *gossip.Gossip, desc *roachpb.RangeDescriptor) ReplicaSlice {
if gossip == nil {
return nil
}
replicas := make(ReplicaSlice, 0, len(desc.Replicas))
for _, r := range desc.Replicas {
nd, err := gossip.GetNodeDescriptor(r.NodeID)
if err != nil {
if log.V(1) {
log.Infof(context.TODO(), "node %d is not gossiped: %v", r.NodeID, err)
}
continue
}
replicas = append(replicas, ReplicaInfo{
ReplicaDescriptor: r,
NodeDesc: nd,
})
}
return replicas
}
// NewDistSender returns a batch.Sender instance which connects to the
// Cockroach cluster via the supplied gossip instance. Supplying a
// DistSenderContext or the fields within is optional. For omitted values, sane
// defaults will be used.
func NewDistSender(cfg DistSenderConfig, g *gossip.Gossip) *DistSender {
ds := &DistSender{gossip: g}
ds.AmbientContext = cfg.AmbientCtx
if ds.AmbientContext.Tracer == nil {
ds.AmbientContext.Tracer = tracing.NewTracer()
}
ds.clock = cfg.Clock
if ds.clock == nil {
ds.clock = hlc.NewClock(hlc.UnixNano)
}
if cfg.nodeDescriptor != nil {
atomic.StorePointer(&ds.nodeDescriptor, unsafe.Pointer(cfg.nodeDescriptor))
}
rcSize := cfg.RangeDescriptorCacheSize
if rcSize <= 0 {
rcSize = defaultRangeDescriptorCacheSize
}
rdb := cfg.RangeDescriptorDB
if rdb == nil {
rdb = ds
}
ds.rangeCache = newRangeDescriptorCache(ds.AnnotateCtx(context.TODO()), rdb, int(rcSize))
lcSize := cfg.LeaseHolderCacheSize
if lcSize <= 0 {
lcSize = defaultLeaseHolderCacheSize
}
ds.leaseHolderCache = newLeaseHolderCache(int(lcSize))
if cfg.RangeLookupMaxRanges <= 0 {
ds.rangeLookupMaxRanges = defaultRangeLookupMaxRanges
}
if cfg.TransportFactory != nil {
ds.transportFactory = cfg.TransportFactory
}
ds.rpcRetryOptions = base.DefaultRetryOptions()
if cfg.RPCRetryOptions != nil {
ds.rpcRetryOptions = *cfg.RPCRetryOptions
}
if cfg.RPCContext != nil {
ds.rpcContext = cfg.RPCContext
if ds.rpcRetryOptions.Closer == nil {
ds.rpcRetryOptions.Closer = ds.rpcContext.Stopper.ShouldQuiesce()
}
}
if cfg.SendNextTimeout != 0 {
ds.sendNextTimeout = cfg.SendNextTimeout
} else {
ds.sendNextTimeout = defaultSendNextTimeout
}
if cfg.SenderConcurrency != 0 {
ds.asyncSenderSem = make(chan struct{}, cfg.SenderConcurrency)
} else {
ds.asyncSenderSem = make(chan struct{}, defaultSenderConcurrency)
}
if g != nil {
ctx := ds.AnnotateCtx(context.Background())
g.RegisterCallback(gossip.KeyFirstRangeDescriptor,
func(_ string, value roachpb.Value) {
if log.V(1) {
var desc roachpb.RangeDescriptor
if err := value.GetProto(&desc); err != nil {
log.Errorf(ctx, "unable to parse gossiped first range descriptor: %s", err)
} else {
log.Infof(ctx, "gossiped first range descriptor: %+v", desc.Replicas)
}
}
err := ds.rangeCache.EvictCachedRangeDescriptor(roachpb.RKeyMin, nil, false)
if err != nil {
log.Warningf(ctx, "failed to evict first range descriptor: %s", err)
}
})
}
return ds
}
// RefreshLeases starts a goroutine that refreshes the lease manager
// leases for tables received in the latest system configuration via gossip.
func (m *LeaseManager) RefreshLeases(s *stop.Stopper, db *client.DB, gossip *gossip.Gossip) {
s.RunWorker(func() {
descKeyPrefix := keys.MakeTablePrefix(uint32(sqlbase.DescriptorTable.ID))
gossipUpdateC := gossip.RegisterSystemConfigChannel()
for {
select {
case <-gossipUpdateC:
cfg, _ := gossip.GetSystemConfig()
if m.testingKnobs.GossipUpdateEvent != nil {
m.testingKnobs.GossipUpdateEvent(cfg)
}
// Read all tables and their versions
if log.V(2) {
log.Info(context.TODO(), "received a new config; will refresh leases")
}
// Loop through the configuration to find all the tables.
for _, kv := range cfg.Values {
if !bytes.HasPrefix(kv.Key, descKeyPrefix) {
continue
}
// Attempt to unmarshal config into a table/database descriptor.
var descriptor sqlbase.Descriptor
if err := kv.Value.GetProto(&descriptor); err != nil {
log.Warningf(context.TODO(), "%s: unable to unmarshal descriptor %v", kv.Key, kv.Value)
continue
}
switch union := descriptor.Union.(type) {
case *sqlbase.Descriptor_Table:
table := union.Table
table.MaybeUpgradeFormatVersion()
if err := table.ValidateTable(); err != nil {
log.Errorf(context.TODO(), "%s: received invalid table descriptor: %v", kv.Key, table)
continue
}
if log.V(2) {
log.Infof(context.TODO(), "%s: refreshing lease table: %d (%s), version: %d, deleted: %t",
kv.Key, table.ID, table.Name, table.Version, table.Dropped())
}
// Try to refresh the table lease to one >= this version.
if t := m.findTableState(table.ID, false /* create */); t != nil {
if err := t.purgeOldLeases(
db, table.Dropped(), table.Version, m.LeaseStore); err != nil {
log.Warningf(context.TODO(), "error purging leases for table %d(%s): %s",
table.ID, table.Name, err)
}
}
case *sqlbase.Descriptor_Database:
// Ignore.
}
}
if m.testingKnobs.TestingLeasesRefreshedEvent != nil {
m.testingKnobs.TestingLeasesRefreshedEvent(cfg)
}
case <-s.ShouldStop():
return
}
}
})
}
// GossipAddressResolver is a thin wrapper around gossip's GetNodeIDAddress
// that allows its return value to be used as the net.Addr interface.
func GossipAddressResolver(gossip *gossip.Gossip) NodeAddressResolver {
return func(nodeID roachpb.NodeID) (net.Addr, error) {
return gossip.GetNodeIDAddress(nodeID)
}
}