func newStoreGossiper(g *gossip.Gossip) *storeGossiper {
sg := &storeGossiper{
g: g,
}
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ []byte) { sg.wg.Done() })
return sg
}
// newReplicateQueue returns a new instance of replicateQueue.
func newReplicateQueue(store *Store, g *gossip.Gossip, allocator Allocator, clock *hlc.Clock,
options AllocatorOptions) *replicateQueue {
rq := &replicateQueue{
allocator: allocator,
clock: clock,
updateChan: make(chan struct{}, 1),
}
rq.baseQueue = makeBaseQueue("replicate", rq, store, g, queueConfig{
maxSize: replicateQueueMaxSize,
needsLease: true,
acceptsUnsplitRanges: false,
})
if g != nil { // gossip is nil for some unittests
// Register a gossip callback to signal queue that replicas in
// purgatory might be retried due to new store gossip.
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) {
select {
case rq.updateChan <- struct{}{}:
default:
}
})
}
return rq
}
// NewStorePool creates a StorePool and registers the store updating callback
// with gossip.
func NewStorePool(
g *gossip.Gossip,
clock *hlc.Clock,
rpcContext *rpc.Context,
reservationsEnabled bool,
timeUntilStoreDead time.Duration,
stopper *stop.Stopper,
) *StorePool {
sp := &StorePool{
clock: clock,
timeUntilStoreDead: timeUntilStoreDead,
rpcContext: rpcContext,
reservationsEnabled: reservationsEnabled,
failedReservationsTimeout: envutil.EnvOrDefaultDuration("failed_reservation_timeout",
defaultFailedReservationsTimeout),
declinedReservationsTimeout: envutil.EnvOrDefaultDuration("declined_reservation_timeout",
defaultDeclinedReservationsTimeout),
reserveRPCTimeout: envutil.EnvOrDefaultDuration("reserve_rpc_timeout",
defaultReserveRPCTimeout),
}
sp.mu.stores = make(map[roachpb.StoreID]*storeDetail)
heap.Init(&sp.mu.queue)
storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
g.RegisterCallback(storeRegex, sp.storeGossipUpdate)
sp.start(stopper)
return sp
}
// NewStorePool creates a StorePool and registers the store updating callback
// with gossip.
func NewStorePool(
g *gossip.Gossip,
clock *hlc.Clock,
rpcContext *rpc.Context,
reservationsEnabled bool,
timeUntilStoreDead time.Duration,
stopper *stop.Stopper,
) *StorePool {
sp := &StorePool{
clock: clock,
timeUntilStoreDead: timeUntilStoreDead,
rpcContext: rpcContext,
reservationsEnabled: reservationsEnabled,
failedReservationsTimeout: envutil.EnvOrDefaultDuration("COCKROACH_FAILED_RESERVATION_TIMEOUT",
defaultFailedReservationsTimeout),
declinedReservationsTimeout: envutil.EnvOrDefaultDuration("COCKROACH_DECLINED_RESERVATION_TIMEOUT",
defaultDeclinedReservationsTimeout),
reserveRPCTimeout: envutil.EnvOrDefaultDuration("COCKROACH_RESERVE_RPC_TIMEOUT",
defaultReserveRPCTimeout),
resolver: GossipAddressResolver(g),
}
sp.mu.stores = make(map[roachpb.StoreID]*storeDetail)
heap.Init(&sp.mu.queue)
storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
g.RegisterCallback(storeRegex, sp.storeGossipUpdate)
deadReplicasRegex := gossip.MakePrefixPattern(gossip.KeyDeadReplicasPrefix)
g.RegisterCallback(deadReplicasRegex, sp.deadReplicasGossipUpdate)
sp.start(stopper)
return sp
}
// newStoreGossiper creates a store gossiper for use by tests. It adds the
// callback to gossip.
func newStoreGossiper(g *gossip.Gossip) *storeGossiper {
sg := &storeGossiper{
g: g,
storeKeyMap: make(map[string]struct{}),
}
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(key string, _ []byte) {
sg.mu.Lock()
defer sg.mu.Unlock()
if _, ok := sg.storeKeyMap[key]; ok {
sg.wg.Done()
}
})
return sg
}
// NewStorePool creates a StorePool and registers the store updating callback
// with gossip.
func NewStorePool(g *gossip.Gossip, timeUntilStoreDead time.Duration, stopper *stop.Stopper) *StorePool {
sp := &StorePool{
timeUntilStoreDead: timeUntilStoreDead,
stores: make(map[roachpb.StoreID]*storeDetail),
}
heap.Init(&sp.queue)
storeRegex := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
g.RegisterCallback(storeRegex, sp.storeGossipUpdate)
sp.start(stopper)
return sp
}
// NewStoreGossiper creates a store gossiper for use by tests. It adds the
// callback to gossip.
func NewStoreGossiper(g *gossip.Gossip) *StoreGossiper {
sg := &StoreGossiper{
g: g,
storeKeyMap: make(map[string]struct{}),
}
sg.cond = sync.NewCond(&sg.mu)
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(key string, _ roachpb.Value) {
sg.mu.Lock()
defer sg.mu.Unlock()
delete(sg.storeKeyMap, key)
sg.cond.Broadcast()
})
return sg
}
func gossipStores(g *gossip.Gossip, stores []*proto.StoreDescriptor, t *testing.T) {
var wg sync.WaitGroup
wg.Add(len(stores))
g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyCapacityPrefix), func(_ string, _ bool) { wg.Done() })
for _, s := range stores {
keyMaxCapacity := gossip.MakeCapacityKey(s.Node.NodeID, s.StoreID)
// Gossip store descriptor.
err := g.AddInfo(keyMaxCapacity, *s, 0)
if err != nil {
t.Fatal(err)
}
}
// Wait for all gossip callbacks to be invoked.
wg.Wait()
}
// NewDistSender returns a batch.Sender instance which connects to the
// Cockroach cluster via the supplied gossip instance. Supplying a
// DistSenderContext or the fields within is optional. For omitted values, sane
// defaults will be used.
func NewDistSender(cfg *DistSenderConfig, g *gossip.Gossip) *DistSender {
if cfg == nil {
cfg = &DistSenderConfig{}
}
ds := &DistSender{gossip: g}
ds.Ctx = cfg.Ctx
if ds.Ctx == nil {
ds.Ctx = context.Background()
}
if ds.Ctx.Done() != nil {
panic("context with cancel or deadline")
}
if tracing.TracerFromCtx(ds.Ctx) == nil {
ds.Ctx = tracing.WithTracer(ds.Ctx, tracing.NewTracer())
}
ds.clock = cfg.Clock
if ds.clock == nil {
ds.clock = hlc.NewClock(hlc.UnixNano)
}
if cfg.nodeDescriptor != nil {
atomic.StorePointer(&ds.nodeDescriptor, unsafe.Pointer(cfg.nodeDescriptor))
}
rcSize := cfg.RangeDescriptorCacheSize
if rcSize <= 0 {
rcSize = defaultRangeDescriptorCacheSize
}
rdb := cfg.RangeDescriptorDB
if rdb == nil {
rdb = ds
}
ds.rangeCache = newRangeDescriptorCache(rdb, int(rcSize))
lcSize := cfg.LeaseHolderCacheSize
if lcSize <= 0 {
lcSize = defaultLeaseHolderCacheSize
}
ds.leaseHolderCache = newLeaseHolderCache(int(lcSize))
if cfg.RangeLookupMaxRanges <= 0 {
ds.rangeLookupMaxRanges = defaultRangeLookupMaxRanges
}
if cfg.TransportFactory != nil {
ds.transportFactory = cfg.TransportFactory
}
ds.rpcRetryOptions = base.DefaultRetryOptions()
if cfg.RPCRetryOptions != nil {
ds.rpcRetryOptions = *cfg.RPCRetryOptions
}
if cfg.RPCContext != nil {
ds.rpcContext = cfg.RPCContext
if ds.rpcRetryOptions.Closer == nil {
ds.rpcRetryOptions.Closer = ds.rpcContext.Stopper.ShouldQuiesce()
}
}
if cfg.SendNextTimeout != 0 {
ds.sendNextTimeout = cfg.SendNextTimeout
} else {
ds.sendNextTimeout = defaultSendNextTimeout
}
if g != nil {
g.RegisterCallback(gossip.KeyFirstRangeDescriptor,
func(_ string, value roachpb.Value) {
if log.V(1) {
var desc roachpb.RangeDescriptor
if err := value.GetProto(&desc); err != nil {
log.Errorf(ds.Ctx, "unable to parse gossipped first range descriptor: %s", err)
} else {
log.Infof(ds.Ctx,
"gossipped first range descriptor: %+v", desc.Replicas)
}
}
err := ds.rangeCache.EvictCachedRangeDescriptor(roachpb.RKeyMin, nil, false)
if err != nil {
log.Warningf(ds.Ctx, "failed to evict first range descriptor: %s", err)
}
})
}
return ds
}
