// lookupReplica looks up replica by key [range]. Lookups are done
// by consulting each store in turn via Store.LookupRange(key).
// Returns RangeID and replica on success; RangeKeyMismatch error
// if not found.
// This is only for testing usage; performance doesn't matter.
func (ls *Stores) lookupReplica(start, end roachpb.RKey) (rangeID roachpb.RangeID, replica *roachpb.ReplicaDescriptor, pErr *roachpb.Error) {
ls.mu.RLock()
defer ls.mu.RUnlock()
var rng *Replica
for _, store := range ls.storeMap {
rng = store.LookupReplica(start, end)
if rng == nil {
if tmpRng := store.LookupReplica(start, nil); tmpRng != nil {
log.Warningf(fmt.Sprintf("range not contained in one range: [%s,%s), but have [%s,%s)", start, end, tmpRng.Desc().StartKey, tmpRng.Desc().EndKey))
}
continue
}
if replica == nil {
rangeID = rng.RangeID
replica = rng.GetReplica()
continue
}
// Should never happen outside of tests.
return 0, nil, roachpb.NewErrorf(
"range %+v exists on additional store: %+v", rng, store)
}
if replica == nil {
pErr = roachpb.NewError(roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), nil))
}
return rangeID, replica, pErr
}
// RangeLookup implements the RangeDescriptorDB interface.
// RangeLookup dispatches a RangeLookup request for the given metadata
// key to the replicas of the given range. Note that we allow
// inconsistent reads when doing range lookups for efficiency. Getting
// stale data is not a correctness problem but instead may
// infrequently result in additional latency as additional range
// lookups may be required. Note also that rangeLookup bypasses the
// DistSender's Send() method, so there is no error inspection and
// retry logic here; this is not an issue since the lookup performs a
// single inconsistent read only.
func (ds *DistSender) RangeLookup(
key roachpb.RKey, desc *roachpb.RangeDescriptor, considerIntents, useReverseScan bool,
) ([]roachpb.RangeDescriptor, []roachpb.RangeDescriptor, *roachpb.Error) {
ba := roachpb.BatchRequest{}
ba.ReadConsistency = roachpb.INCONSISTENT
ba.Add(&roachpb.RangeLookupRequest{
Span: roachpb.Span{
// We can interpret the RKey as a Key here since it's a metadata
// lookup; those are never local.
Key: key.AsRawKey(),
},
MaxRanges: ds.rangeLookupMaxRanges,
ConsiderIntents: considerIntents,
Reverse: useReverseScan,
})
replicas := newReplicaSlice(ds.gossip, desc)
replicas.Shuffle()
// TODO(tschottdorf): Ideally we would use the trace of the request which
// caused this lookup.
_ = context.TODO()
br, err := ds.sendRPC(ds.Ctx, desc.RangeID, replicas, ba)
if err != nil {
return nil, nil, roachpb.NewError(err)
}
if br.Error != nil {
return nil, nil, br.Error
}
resp := br.Responses[0].GetInner().(*roachpb.RangeLookupResponse)
return resp.Ranges, resp.PrefetchedRanges, nil
}
// RangeLookup dispatches an RangeLookup request for the given
// metadata key to the replicas of the given range. Note that we allow
// inconsistent reads when doing range lookups for efficiency. Getting
// stale data is not a correctness problem but instead may
// infrequently result in additional latency as additional range
// lookups may be required. Note also that rangeLookup bypasses the
// DistSender's Send() method, so there is no error inspection and
// retry logic here; this is not an issue since the lookup performs a
// single inconsistent read only.
func (ds *DistSender) RangeLookup(key roachpb.RKey, desc *roachpb.RangeDescriptor, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
ba := roachpb.BatchRequest{}
ba.ReadConsistency = roachpb.INCONSISTENT
ba.Add(&roachpb.RangeLookupRequest{
Span: roachpb.Span{
// We can interpret the RKey as a Key here since it's a metadata
// lookup; those are never local.
Key: key.AsRawKey(),
},
MaxRanges: ds.rangeLookupMaxRanges,
ConsiderIntents: considerIntents,
Reverse: useReverseScan,
})
replicas := newReplicaSlice(ds.gossip, desc)
// TODO(tschottdorf) consider a Trace here, potentially that of the request
// that had the cache miss and waits for the result.
br, err := ds.sendRPC(nil /* Trace */, desc.RangeID, replicas, rpc.OrderRandom, ba)
if err != nil {
return nil, err
}
if br.Error != nil {
return nil, br.Error
}
return br.Responses[0].GetInner().(*roachpb.RangeLookupResponse).Ranges, nil
}
// lookupReplica looks up replica by key [range]. Lookups are done
// by consulting each store in turn via Store.LookupRange(key).
// Returns RangeID and replica on success; RangeKeyMismatch error
// if not found.
// This is only for testing usage; performance doesn't matter.
func (ls *Stores) lookupReplica(start, end roachpb.RKey) (rangeID roachpb.RangeID, replica *roachpb.ReplicaDescriptor, err error) {
ls.mu.RLock()
defer ls.mu.RUnlock()
var rng *Replica
var partialDesc *roachpb.RangeDescriptor
for _, store := range ls.storeMap {
rng = store.LookupReplica(start, end)
if rng == nil {
if tmpRng := store.LookupReplica(start, nil); tmpRng != nil {
log.Warningf("range not contained in one range: [%s,%s), but have [%s,%s)",
start, end, tmpRng.Desc().StartKey, tmpRng.Desc().EndKey)
partialDesc = tmpRng.Desc()
break
}
continue
}
if replica == nil {
rangeID = rng.RangeID
replica, err = rng.GetReplica()
if err != nil {
if _, ok := err.(*errReplicaNotInRange); !ok {
return 0, nil, err
}
}
continue
}
// Should never happen outside of tests.
return 0, nil, util.Errorf(
"range %+v exists on additional store: %+v", rng, store)
}
if replica == nil {
err = roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), partialDesc)
}
return rangeID, replica, err
}
// RangeLookup dispatches an RangeLookup request for the given
// metadata key to the replicas of the given range. Note that we allow
// inconsistent reads when doing range lookups for efficiency. Getting
// stale data is not a correctness problem but instead may
// infrequently result in additional latency as additional range
// lookups may be required. Note also that rangeLookup bypasses the
// DistSender's Send() method, so there is no error inspection and
// retry logic here; this is not an issue since the lookup performs a
// single inconsistent read only.
func (ds *DistSender) RangeLookup(key roachpb.RKey, desc *roachpb.RangeDescriptor, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
ba := roachpb.BatchRequest{}
ba.ReadConsistency = roachpb.INCONSISTENT
ba.Add(&roachpb.RangeLookupRequest{
Span: roachpb.Span{
// We can interpret the RKey as a Key here since it's a metadata
// lookup; those are never local.
Key: key.AsRawKey(),
},
MaxRanges: ds.rangeLookupMaxRanges,
ConsiderIntents: considerIntents,
Reverse: useReverseScan,
})
replicas := newReplicaSlice(ds.gossip, desc)
trace := ds.Tracer.StartSpan("range lookup")
defer trace.Finish()
// TODO(tschottdorf): Ideally we would use the trace of the request which
// caused this lookup instead of a new one.
br, err := ds.sendRPC(trace, desc.RangeID, replicas, orderRandom, ba)
if err != nil {
return nil, err
}
if br.Error != nil {
return nil, br.Error
}
return br.Responses[0].GetInner().(*roachpb.RangeLookupResponse).Ranges, nil
}
// LookupReplica looks up replica by key [range]. Lookups are done
// by consulting each store in turn via Store.LookupReplica(key).
// Returns RangeID and replica on success; RangeKeyMismatch error
// if not found.
// If end is nil, a replica containing start is looked up.
// This is only for testing usage; performance doesn't matter.
func (ls *Stores) LookupReplica(
start, end roachpb.RKey,
) (roachpb.RangeID, roachpb.ReplicaDescriptor, error) {
ls.mu.RLock()
defer ls.mu.RUnlock()
var rangeID roachpb.RangeID
var repDesc roachpb.ReplicaDescriptor
var repDescFound bool
for _, store := range ls.storeMap {
replica := store.LookupReplica(start, nil)
if replica == nil {
continue
}
// Verify that the descriptor contains the entire range.
if desc := replica.Desc(); !desc.ContainsKeyRange(start, end) {
log.Warningf(context.TODO(), "range not contained in one range: [%s,%s), but have [%s,%s)",
start, end, desc.StartKey, desc.EndKey)
err := roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), desc)
return 0, roachpb.ReplicaDescriptor{}, err
}
rangeID = replica.RangeID
var err error
repDesc, err = replica.GetReplicaDescriptor()
if err != nil {
if _, ok := err.(*roachpb.RangeNotFoundError); ok {
// We are not holding a lock across this block; the replica could have
// been removed from the range (via down-replication) between the
// LookupReplica and the GetReplicaDescriptor calls. In this case just
// ignore this replica.
continue
}
return 0, roachpb.ReplicaDescriptor{}, err
}
if repDescFound {
// We already found the range; this should never happen outside of tests.
err := errors.Errorf("range %+v exists on additional store: %+v", replica, store)
return 0, roachpb.ReplicaDescriptor{}, err
}
repDescFound = true
}
if !repDescFound {
return 0, roachpb.ReplicaDescriptor{}, roachpb.NewRangeNotFoundError(0)
}
return rangeID, repDesc, nil
}
// RangeLookup implements the RangeDescriptorDB interface. It looks up
// the descriptors for the given (meta) key.
func (ls *Stores) RangeLookup(key roachpb.RKey, _ *roachpb.RangeDescriptor, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
ba := roachpb.BatchRequest{}
ba.ReadConsistency = roachpb.INCONSISTENT
ba.Add(&roachpb.RangeLookupRequest{
Span: roachpb.Span{
// key is a meta key, so it's guaranteed not local-prefixed.
Key: key.AsRawKey(),
},
MaxRanges: 1,
ConsiderIntents: considerIntents,
Reverse: useReverseScan,
})
br, pErr := ls.Send(context.Background(), ba)
if pErr != nil {
return nil, pErr
}
return br.Responses[0].GetInner().(*roachpb.RangeLookupResponse).Ranges, nil
}
// LookupReplica looks up replica by key [range]. Lookups are done
// by consulting each store in turn via Store.LookupReplica(key).
// Returns RangeID and replica on success; RangeKeyMismatch error
// if not found.
// If end is nil, a replica containing start is looked up.
// This is only for testing usage; performance doesn't matter.
func (ls *Stores) LookupReplica(start, end roachpb.RKey) (rangeID roachpb.RangeID, repDesc roachpb.ReplicaDescriptor, err error) {
ls.mu.RLock()
defer ls.mu.RUnlock()
var rng *Replica
var partialDesc *roachpb.RangeDescriptor
var repDescFound bool
for _, store := range ls.storeMap {
rng = store.LookupReplica(start, end)
if rng == nil {
if tmpRng := store.LookupReplica(start, nil); tmpRng != nil {
partialDesc = tmpRng.Desc()
log.Warningf(context.TODO(), "range not contained in one range: [%s,%s), but have [%s,%s)",
start, end, partialDesc.StartKey, partialDesc.EndKey)
break
}
continue
}
if !repDescFound {
rangeID = rng.RangeID
repDesc, err = rng.GetReplicaDescriptor()
if err != nil {
if _, ok := err.(*roachpb.RangeNotFoundError); !ok {
return 0, roachpb.ReplicaDescriptor{}, err
}
} else {
repDescFound = true
}
continue
}
// Should never happen outside of tests.
return 0, roachpb.ReplicaDescriptor{}, errors.Errorf(
"range %+v exists on additional store: %+v", rng, store,
)
}
if !repDescFound {
err = roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), partialDesc)
}
return rangeID, repDesc, err
}
// truncate restricts all contained requests to the given key range.
// Even on error, the returned closure must be executed; it undoes any
// truncations performed.
// First, the boundaries of the truncation are obtained: This is the
// intersection between [from,to) and the descriptor's range.
// Secondly, all requests contained in the batch are "truncated" to
// the resulting range, inserting NoopRequest appropriately to
// replace requests which are left without a key range to operate on.
// The number of non-noop requests after truncation is returned along
// with a closure which must be executed to undo the truncation, even
// in case of an error.
// TODO(tschottdorf): Consider returning a new BatchRequest, which has more
// overhead in the common case of a batch which never needs truncation but is
// less magical.
func truncate(br *roachpb.BatchRequest, desc *roachpb.RangeDescriptor, from, to roachpb.RKey) (func(), int, error) {
if !desc.ContainsKey(from) {
from = desc.StartKey
}
if !desc.ContainsKeyRange(desc.StartKey, to) || to == nil {
to = desc.EndKey
}
truncateOne := func(args roachpb.Request) (bool, []func(), error) {
if _, ok := args.(*roachpb.NoopRequest); ok {
return true, nil, nil
}
header := args.Header()
if !roachpb.IsRange(args) {
// This is a point request.
if len(header.EndKey) > 0 {
return false, nil, util.Errorf("%T is not a range command, but EndKey is set", args)
}
if !desc.ContainsKey(keys.Addr(header.Key)) {
return true, nil, nil
}
return false, nil, nil
}
// We're dealing with a range-spanning request.
var undo []func()
keyAddr, endKeyAddr := keys.Addr(header.Key), keys.Addr(header.EndKey)
if l, r := !keyAddr.Equal(header.Key), !endKeyAddr.Equal(header.EndKey); l || r {
if !desc.ContainsKeyRange(keyAddr, endKeyAddr) {
return false, nil, util.Errorf("local key range must not span ranges")
}
if !l || !r {
return false, nil, util.Errorf("local key mixed with global key in range")
}
}
// Below, {end,}keyAddr equals header.{End,}Key, so nothing is local.
if keyAddr.Less(from) {
{
origKey := header.Key
undo = append(undo, func() { header.Key = origKey })
}
header.Key = from.AsRawKey() // "from" can't be local
keyAddr = from
}
if !endKeyAddr.Less(to) {
{
origEndKey := header.EndKey
undo = append(undo, func() { header.EndKey = origEndKey })
}
header.EndKey = to.AsRawKey() // "to" can't be local
endKeyAddr = to
}
// Check whether the truncation has left any keys in the range. If not,
// we need to cut it out of the request.
return !keyAddr.Less(endKeyAddr), undo, nil
}
var fns []func()
gUndo := func() {
for _, f := range fns {
f()
}
}
var numNoop int
for pos, arg := range br.Requests {
omit, undo, err := truncateOne(arg.GetInner())
if omit {
numNoop++
nReq := &roachpb.RequestUnion{}
if !nReq.SetValue(&roachpb.NoopRequest{}) {
panic("RequestUnion excludes NoopRequest")
}
oReq := br.Requests[pos]
br.Requests[pos] = *nReq
posCpy := pos // for closure
undo = append(undo, func() {
br.Requests[posCpy] = oReq
})
}
fns = append(fns, undo...)
if err != nil {
return gUndo, 0, err
}
}
return gUndo, len(br.Requests) - numNoop, nil
}