// lookupReplica looks up replica by key [range]. Lookups are done // by consulting each store in turn via Store.LookupRange(key). // Returns RangeID and replica on success; RangeKeyMismatch error // if not found. // This is only for testing usage; performance doesn't matter. func (ls *Stores) lookupReplica(start, end roachpb.RKey) (rangeID roachpb.RangeID, replica *roachpb.ReplicaDescriptor, pErr *roachpb.Error) { ls.mu.RLock() defer ls.mu.RUnlock() var rng *Replica for _, store := range ls.storeMap { rng = store.LookupReplica(start, end) if rng == nil { if tmpRng := store.LookupReplica(start, nil); tmpRng != nil { log.Warningf(fmt.Sprintf("range not contained in one range: [%s,%s), but have [%s,%s)", start, end, tmpRng.Desc().StartKey, tmpRng.Desc().EndKey)) } continue } if replica == nil { rangeID = rng.RangeID replica = rng.GetReplica() continue } // Should never happen outside of tests. return 0, nil, roachpb.NewErrorf( "range %+v exists on additional store: %+v", rng, store) } if replica == nil { pErr = roachpb.NewError(roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), nil)) } return rangeID, replica, pErr }
// RangeLookup implements the RangeDescriptorDB interface. // RangeLookup dispatches a RangeLookup request for the given metadata // key to the replicas of the given range. Note that we allow // inconsistent reads when doing range lookups for efficiency. Getting // stale data is not a correctness problem but instead may // infrequently result in additional latency as additional range // lookups may be required. Note also that rangeLookup bypasses the // DistSender's Send() method, so there is no error inspection and // retry logic here; this is not an issue since the lookup performs a // single inconsistent read only. func (ds *DistSender) RangeLookup( key roachpb.RKey, desc *roachpb.RangeDescriptor, considerIntents, useReverseScan bool, ) ([]roachpb.RangeDescriptor, []roachpb.RangeDescriptor, *roachpb.Error) { ba := roachpb.BatchRequest{} ba.ReadConsistency = roachpb.INCONSISTENT ba.Add(&roachpb.RangeLookupRequest{ Span: roachpb.Span{ // We can interpret the RKey as a Key here since it's a metadata // lookup; those are never local. Key: key.AsRawKey(), }, MaxRanges: ds.rangeLookupMaxRanges, ConsiderIntents: considerIntents, Reverse: useReverseScan, }) replicas := newReplicaSlice(ds.gossip, desc) replicas.Shuffle() // TODO(tschottdorf): Ideally we would use the trace of the request which // caused this lookup. _ = context.TODO() br, err := ds.sendRPC(ds.Ctx, desc.RangeID, replicas, ba) if err != nil { return nil, nil, roachpb.NewError(err) } if br.Error != nil { return nil, nil, br.Error } resp := br.Responses[0].GetInner().(*roachpb.RangeLookupResponse) return resp.Ranges, resp.PrefetchedRanges, nil }
// RangeLookup dispatches an RangeLookup request for the given // metadata key to the replicas of the given range. Note that we allow // inconsistent reads when doing range lookups for efficiency. Getting // stale data is not a correctness problem but instead may // infrequently result in additional latency as additional range // lookups may be required. Note also that rangeLookup bypasses the // DistSender's Send() method, so there is no error inspection and // retry logic here; this is not an issue since the lookup performs a // single inconsistent read only. func (ds *DistSender) RangeLookup(key roachpb.RKey, desc *roachpb.RangeDescriptor, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { ba := roachpb.BatchRequest{} ba.ReadConsistency = roachpb.INCONSISTENT ba.Add(&roachpb.RangeLookupRequest{ Span: roachpb.Span{ // We can interpret the RKey as a Key here since it's a metadata // lookup; those are never local. Key: key.AsRawKey(), }, MaxRanges: ds.rangeLookupMaxRanges, ConsiderIntents: considerIntents, Reverse: useReverseScan, }) replicas := newReplicaSlice(ds.gossip, desc) // TODO(tschottdorf) consider a Trace here, potentially that of the request // that had the cache miss and waits for the result. br, err := ds.sendRPC(nil /* Trace */, desc.RangeID, replicas, rpc.OrderRandom, ba) if err != nil { return nil, err } if br.Error != nil { return nil, br.Error } return br.Responses[0].GetInner().(*roachpb.RangeLookupResponse).Ranges, nil }
// lookupReplica looks up replica by key [range]. Lookups are done // by consulting each store in turn via Store.LookupRange(key). // Returns RangeID and replica on success; RangeKeyMismatch error // if not found. // This is only for testing usage; performance doesn't matter. func (ls *Stores) lookupReplica(start, end roachpb.RKey) (rangeID roachpb.RangeID, replica *roachpb.ReplicaDescriptor, err error) { ls.mu.RLock() defer ls.mu.RUnlock() var rng *Replica var partialDesc *roachpb.RangeDescriptor for _, store := range ls.storeMap { rng = store.LookupReplica(start, end) if rng == nil { if tmpRng := store.LookupReplica(start, nil); tmpRng != nil { log.Warningf("range not contained in one range: [%s,%s), but have [%s,%s)", start, end, tmpRng.Desc().StartKey, tmpRng.Desc().EndKey) partialDesc = tmpRng.Desc() break } continue } if replica == nil { rangeID = rng.RangeID replica, err = rng.GetReplica() if err != nil { if _, ok := err.(*errReplicaNotInRange); !ok { return 0, nil, err } } continue } // Should never happen outside of tests. return 0, nil, util.Errorf( "range %+v exists on additional store: %+v", rng, store) } if replica == nil { err = roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), partialDesc) } return rangeID, replica, err }
// RangeLookup dispatches an RangeLookup request for the given // metadata key to the replicas of the given range. Note that we allow // inconsistent reads when doing range lookups for efficiency. Getting // stale data is not a correctness problem but instead may // infrequently result in additional latency as additional range // lookups may be required. Note also that rangeLookup bypasses the // DistSender's Send() method, so there is no error inspection and // retry logic here; this is not an issue since the lookup performs a // single inconsistent read only. func (ds *DistSender) RangeLookup(key roachpb.RKey, desc *roachpb.RangeDescriptor, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { ba := roachpb.BatchRequest{} ba.ReadConsistency = roachpb.INCONSISTENT ba.Add(&roachpb.RangeLookupRequest{ Span: roachpb.Span{ // We can interpret the RKey as a Key here since it's a metadata // lookup; those are never local. Key: key.AsRawKey(), }, MaxRanges: ds.rangeLookupMaxRanges, ConsiderIntents: considerIntents, Reverse: useReverseScan, }) replicas := newReplicaSlice(ds.gossip, desc) trace := ds.Tracer.StartSpan("range lookup") defer trace.Finish() // TODO(tschottdorf): Ideally we would use the trace of the request which // caused this lookup instead of a new one. br, err := ds.sendRPC(trace, desc.RangeID, replicas, orderRandom, ba) if err != nil { return nil, err } if br.Error != nil { return nil, br.Error } return br.Responses[0].GetInner().(*roachpb.RangeLookupResponse).Ranges, nil }
// LookupReplica looks up replica by key [range]. Lookups are done // by consulting each store in turn via Store.LookupReplica(key). // Returns RangeID and replica on success; RangeKeyMismatch error // if not found. // If end is nil, a replica containing start is looked up. // This is only for testing usage; performance doesn't matter. func (ls *Stores) LookupReplica( start, end roachpb.RKey, ) (roachpb.RangeID, roachpb.ReplicaDescriptor, error) { ls.mu.RLock() defer ls.mu.RUnlock() var rangeID roachpb.RangeID var repDesc roachpb.ReplicaDescriptor var repDescFound bool for _, store := range ls.storeMap { replica := store.LookupReplica(start, nil) if replica == nil { continue } // Verify that the descriptor contains the entire range. if desc := replica.Desc(); !desc.ContainsKeyRange(start, end) { log.Warningf(context.TODO(), "range not contained in one range: [%s,%s), but have [%s,%s)", start, end, desc.StartKey, desc.EndKey) err := roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), desc) return 0, roachpb.ReplicaDescriptor{}, err } rangeID = replica.RangeID var err error repDesc, err = replica.GetReplicaDescriptor() if err != nil { if _, ok := err.(*roachpb.RangeNotFoundError); ok { // We are not holding a lock across this block; the replica could have // been removed from the range (via down-replication) between the // LookupReplica and the GetReplicaDescriptor calls. In this case just // ignore this replica. continue } return 0, roachpb.ReplicaDescriptor{}, err } if repDescFound { // We already found the range; this should never happen outside of tests. err := errors.Errorf("range %+v exists on additional store: %+v", replica, store) return 0, roachpb.ReplicaDescriptor{}, err } repDescFound = true } if !repDescFound { return 0, roachpb.ReplicaDescriptor{}, roachpb.NewRangeNotFoundError(0) } return rangeID, repDesc, nil }
// RangeLookup implements the RangeDescriptorDB interface. It looks up // the descriptors for the given (meta) key. func (ls *Stores) RangeLookup(key roachpb.RKey, _ *roachpb.RangeDescriptor, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) { ba := roachpb.BatchRequest{} ba.ReadConsistency = roachpb.INCONSISTENT ba.Add(&roachpb.RangeLookupRequest{ Span: roachpb.Span{ // key is a meta key, so it's guaranteed not local-prefixed. Key: key.AsRawKey(), }, MaxRanges: 1, ConsiderIntents: considerIntents, Reverse: useReverseScan, }) br, pErr := ls.Send(context.Background(), ba) if pErr != nil { return nil, pErr } return br.Responses[0].GetInner().(*roachpb.RangeLookupResponse).Ranges, nil }
// LookupReplica looks up replica by key [range]. Lookups are done // by consulting each store in turn via Store.LookupReplica(key). // Returns RangeID and replica on success; RangeKeyMismatch error // if not found. // If end is nil, a replica containing start is looked up. // This is only for testing usage; performance doesn't matter. func (ls *Stores) LookupReplica(start, end roachpb.RKey) (rangeID roachpb.RangeID, repDesc roachpb.ReplicaDescriptor, err error) { ls.mu.RLock() defer ls.mu.RUnlock() var rng *Replica var partialDesc *roachpb.RangeDescriptor var repDescFound bool for _, store := range ls.storeMap { rng = store.LookupReplica(start, end) if rng == nil { if tmpRng := store.LookupReplica(start, nil); tmpRng != nil { partialDesc = tmpRng.Desc() log.Warningf(context.TODO(), "range not contained in one range: [%s,%s), but have [%s,%s)", start, end, partialDesc.StartKey, partialDesc.EndKey) break } continue } if !repDescFound { rangeID = rng.RangeID repDesc, err = rng.GetReplicaDescriptor() if err != nil { if _, ok := err.(*roachpb.RangeNotFoundError); !ok { return 0, roachpb.ReplicaDescriptor{}, err } } else { repDescFound = true } continue } // Should never happen outside of tests. return 0, roachpb.ReplicaDescriptor{}, errors.Errorf( "range %+v exists on additional store: %+v", rng, store, ) } if !repDescFound { err = roachpb.NewRangeKeyMismatchError(start.AsRawKey(), end.AsRawKey(), partialDesc) } return rangeID, repDesc, err }
// truncate restricts all contained requests to the given key range. // Even on error, the returned closure must be executed; it undoes any // truncations performed. // First, the boundaries of the truncation are obtained: This is the // intersection between [from,to) and the descriptor's range. // Secondly, all requests contained in the batch are "truncated" to // the resulting range, inserting NoopRequest appropriately to // replace requests which are left without a key range to operate on. // The number of non-noop requests after truncation is returned along // with a closure which must be executed to undo the truncation, even // in case of an error. // TODO(tschottdorf): Consider returning a new BatchRequest, which has more // overhead in the common case of a batch which never needs truncation but is // less magical. func truncate(br *roachpb.BatchRequest, desc *roachpb.RangeDescriptor, from, to roachpb.RKey) (func(), int, error) { if !desc.ContainsKey(from) { from = desc.StartKey } if !desc.ContainsKeyRange(desc.StartKey, to) || to == nil { to = desc.EndKey } truncateOne := func(args roachpb.Request) (bool, []func(), error) { if _, ok := args.(*roachpb.NoopRequest); ok { return true, nil, nil } header := args.Header() if !roachpb.IsRange(args) { // This is a point request. if len(header.EndKey) > 0 { return false, nil, util.Errorf("%T is not a range command, but EndKey is set", args) } if !desc.ContainsKey(keys.Addr(header.Key)) { return true, nil, nil } return false, nil, nil } // We're dealing with a range-spanning request. var undo []func() keyAddr, endKeyAddr := keys.Addr(header.Key), keys.Addr(header.EndKey) if l, r := !keyAddr.Equal(header.Key), !endKeyAddr.Equal(header.EndKey); l || r { if !desc.ContainsKeyRange(keyAddr, endKeyAddr) { return false, nil, util.Errorf("local key range must not span ranges") } if !l || !r { return false, nil, util.Errorf("local key mixed with global key in range") } } // Below, {end,}keyAddr equals header.{End,}Key, so nothing is local. if keyAddr.Less(from) { { origKey := header.Key undo = append(undo, func() { header.Key = origKey }) } header.Key = from.AsRawKey() // "from" can't be local keyAddr = from } if !endKeyAddr.Less(to) { { origEndKey := header.EndKey undo = append(undo, func() { header.EndKey = origEndKey }) } header.EndKey = to.AsRawKey() // "to" can't be local endKeyAddr = to } // Check whether the truncation has left any keys in the range. If not, // we need to cut it out of the request. return !keyAddr.Less(endKeyAddr), undo, nil } var fns []func() gUndo := func() { for _, f := range fns { f() } } var numNoop int for pos, arg := range br.Requests { omit, undo, err := truncateOne(arg.GetInner()) if omit { numNoop++ nReq := &roachpb.RequestUnion{} if !nReq.SetValue(&roachpb.NoopRequest{}) { panic("RequestUnion excludes NoopRequest") } oReq := br.Requests[pos] br.Requests[pos] = *nReq posCpy := pos // for closure undo = append(undo, func() { br.Requests[posCpy] = oReq }) } fns = append(fns, undo...) if err != nil { return gUndo, 0, err } } return gUndo, len(br.Requests) - numNoop, nil }