// TestRangeCacheClearOverlappingMeta prevents regression of a bug which caused
// a panic when clearing overlapping descriptors for [KeyMin, Meta2Key). The
// issue was that when attempting to clear out descriptors which were subsumed
// by the above range, an iteration over the corresponding meta keys was
// performed, with the left endpoint excluded. This exclusion was incorrect: it
// first incremented the start key (KeyMin) and then formed the meta key; for
// KeyMin this leads to Meta2Prefix\x00. For the above EndKey, the meta key is
// a Meta1key which sorts before Meta2Prefix\x00, causing a panic. The fix was
// simply to increment the meta key for StartKey, not StartKey itself.
func TestRangeCacheClearOverlappingMeta(t *testing.T) {
defer leaktest.AfterTest(t)()
firstDesc := &roachpb.RangeDescriptor{
StartKey: roachpb.RKeyMin,
EndKey: roachpb.RKey("zzz"),
}
restDesc := &roachpb.RangeDescriptor{
StartKey: firstDesc.StartKey,
EndKey: roachpb.RKeyMax,
}
cache := newRangeDescriptorCache(nil, 2<<10)
cache.rangeCache.cache.Add(rangeCacheKey(keys.RangeMetaKey(firstDesc.EndKey)),
firstDesc)
cache.rangeCache.cache.Add(rangeCacheKey(keys.RangeMetaKey(restDesc.EndKey)),
restDesc)
// Add new range, corresponding to splitting the first range at a meta key.
metaSplitDesc := &roachpb.RangeDescriptor{
StartKey: roachpb.RKeyMin,
EndKey: mustMeta(roachpb.RKey("foo")),
}
func() {
defer func() {
if r := recover(); r != nil {
t.Fatalf("invocation of clearOverlappingCachedRangeDescriptors panicked: %v", r)
}
}()
if err := cache.clearOverlappingCachedRangeDescriptors(metaSplitDesc); err != nil {
t.Fatal(err)
}
}()
}
// TestRangeSplitMeta executes various splits (including at meta addressing)
// and checks that all created intents are resolved. This includes both intents
// which are resolved synchronously with EndTransaction and via RPC.
func TestRangeSplitMeta(t *testing.T) {
defer leaktest.AfterTest(t)
s := createTestDB(t)
defer s.Stop()
splitKeys := []roachpb.RKey{roachpb.RKey("G"), meta(roachpb.RKey("F")),
meta(roachpb.RKey("K")), meta(roachpb.RKey("H"))}
// Execute the consecutive splits.
for _, splitKey := range splitKeys {
log.Infof("starting split at key %q...", splitKey)
if err := s.DB.AdminSplit(roachpb.Key(splitKey)); err != nil {
t.Fatal(err)
}
log.Infof("split at key %q complete", splitKey)
}
if err := util.IsTrueWithin(func() bool {
if _, _, err := engine.MVCCScan(s.Eng, keys.LocalMax, roachpb.KeyMax, 0, roachpb.MaxTimestamp, true, nil); err != nil {
log.Infof("mvcc scan should be clean: %s", err)
return false
}
return true
}, 500*time.Millisecond); err != nil {
t.Error("failed to verify no dangling intents within 500ms")
}
}
// newTestRangeSet creates a new range set that has the count number of ranges.
func newTestRangeSet(count int, t *testing.T) *testRangeSet {
rs := &testRangeSet{replicasByKey: btree.New(64 /* degree */)}
for i := 0; i < count; i++ {
desc := &roachpb.RangeDescriptor{
RangeID: roachpb.RangeID(i),
StartKey: roachpb.RKey(fmt.Sprintf("%03d", i)),
EndKey: roachpb.RKey(fmt.Sprintf("%03d", i+1)),
}
// Initialize the range stat so the scanner can use it.
rng := &Replica{
RangeID: desc.RangeID,
}
rng.mu.state.Stats = enginepb.MVCCStats{
KeyBytes: 1,
ValBytes: 2,
KeyCount: 1,
LiveCount: 1,
}
if err := rng.setDesc(desc); err != nil {
t.Fatal(err)
}
if exRngItem := rs.replicasByKey.ReplaceOrInsert(rng); exRngItem != nil {
t.Fatalf("failed to insert range %s", rng)
}
}
return rs
}
// TestStoreRangeUpReplicate verifies that the replication queue will notice
// under-replicated ranges and replicate them.
func TestStoreRangeUpReplicate(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 3)
defer mtc.Stop()
// Initialize the gossip network.
var wg sync.WaitGroup
wg.Add(len(mtc.stores))
key := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
mtc.stores[0].Gossip().RegisterCallback(key, func(_ string, _ roachpb.Value) { wg.Done() })
for _, s := range mtc.stores {
s.GossipStore()
}
wg.Wait()
// Once we know our peers, trigger a scan.
mtc.stores[0].ForceReplicationScanAndProcess()
// The range should become available on every node.
if err := util.IsTrueWithin(func() bool {
for _, s := range mtc.stores {
r := s.LookupReplica(roachpb.RKey("a"), roachpb.RKey("b"))
if r == nil {
return false
}
}
return true
}, replicationTimeout); err != nil {
t.Fatal(err)
}
}
// Addr returns the address for the key, used to lookup the range containing
// the key. In the normal case, this is simply the key's value. However, for
// local keys, such as transaction records, range-spanning binary tree node
// pointers, the address is the inner encoded key, with the local key prefix
// and the suffix and optional detail removed. This address unwrapping is
// performed repeatedly in the case of doubly-local keys. In this way, local
// keys address to the same range as non-local keys, but are stored separately
// so that they don't collide with user-space or global system keys.
//
// However, not all local keys are addressable in the global map. Only range
// local keys incorporating a range key (start key or transaction key) are
// addressable (e.g. range metadata and txn records). Range local keys
// incorporating the Range ID are not (e.g. abort cache entries, and range
// stats).
func Addr(k roachpb.Key) (roachpb.RKey, error) {
if !bytes.HasPrefix(k, localPrefix) {
return roachpb.RKey(k), nil
}
for {
if bytes.HasPrefix(k, localStorePrefix) {
return nil, errors.Errorf("store-local key %q is not addressable", k)
}
if bytes.HasPrefix(k, LocalRangeIDPrefix) {
return nil, errors.Errorf("local range ID key %q is not addressable", k)
}
if !bytes.HasPrefix(k, LocalRangePrefix) {
return nil, errors.Errorf("local key %q malformed; should contain prefix %q",
k, LocalRangePrefix)
}
k = k[len(LocalRangePrefix):]
var err error
// Decode the encoded key, throw away the suffix and detail.
if _, k, err = encoding.DecodeBytesAscending(k, nil); err != nil {
return nil, err
}
if !bytes.HasPrefix(k, localPrefix) {
break
}
}
return roachpb.RKey(k), nil
}
// TestStoreRangeUpReplicate verifies that the replication queue will notice
// under-replicated ranges and replicate them.
func TestStoreRangeUpReplicate(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := startMultiTestContext(t, 3)
defer mtc.Stop()
// Initialize the gossip network.
var wg sync.WaitGroup
wg.Add(len(mtc.stores))
key := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
mtc.stores[0].Gossip().RegisterCallback(key, func(_ string, _ roachpb.Value) { wg.Done() })
for _, s := range mtc.stores {
s.GossipStore()
}
wg.Wait()
// Once we know our peers, trigger a scan.
mtc.stores[0].ForceReplicationScanAndProcess()
// The range should become available on every node.
util.SucceedsSoon(t, func() error {
for _, s := range mtc.stores {
r := s.LookupReplica(roachpb.RKey("a"), roachpb.RKey("b"))
if r == nil {
return util.Errorf("expected replica for keys \"a\" - \"b\"")
}
}
return nil
})
}
func TestComputeStatsForKeySpan(t *testing.T) {
defer leaktest.AfterTest(t)()
mtc := startMultiTestContext(t, 3)
defer mtc.Stop()
// Create a number of ranges using splits.
splitKeys := []string{"a", "c", "e", "g", "i"}
for _, k := range splitKeys {
key := []byte(k)
repl := mtc.stores[0].LookupReplica(key, roachpb.RKeyMin)
args := adminSplitArgs(key, key)
header := roachpb.Header{
RangeID: repl.RangeID,
}
if _, err := client.SendWrappedWith(mtc.stores[0], nil, header, &args); err != nil {
t.Fatal(err)
}
}
// Wait for splits to finish.
util.SucceedsSoon(t, func() error {
repl := mtc.stores[0].LookupReplica(roachpb.RKey("z"), nil)
if actualRSpan := repl.Desc().RSpan(); !actualRSpan.Key.Equal(roachpb.RKey("i")) {
return errors.Errorf("expected range %s to begin at key 'i'", repl)
}
return nil
})
// Create some keys across the ranges.
incKeys := []string{"b", "bb", "bbb", "d", "dd", "h"}
for _, k := range incKeys {
if _, err := mtc.dbs[0].Inc([]byte(k), 5); err != nil {
t.Fatal(err)
}
}
// Verify stats across different spans.
for _, tcase := range []struct {
startKey string
endKey string
expectedRanges int
expectedKeys int64
}{
{"a", "i", 4, 6},
{"a", "c", 1, 3},
{"b", "e", 2, 5},
{"e", "i", 2, 1},
} {
start, end := tcase.startKey, tcase.endKey
stats, count := mtc.stores[0].ComputeStatsForKeySpan(
roachpb.RKey(start), roachpb.RKey(end))
if a, e := count, tcase.expectedRanges; a != e {
t.Errorf("Expected %d ranges in span [%s - %s], found %d", e, start, end, a)
}
if a, e := stats.LiveCount, tcase.expectedKeys; a != e {
t.Errorf("Expected %d keys in span [%s - %s], found %d", e, start, end, a)
}
}
}
// TestSendRPCRetry verifies that sendRPC failed on first address but succeed on
// second address, the second reply should be successfully returned back.
func TestSendRPCRetry(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
g.SetNodeID(1)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
// Fill RangeDescriptor with 2 replicas
var descriptor = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKey("z"),
}
for i := 1; i <= 2; i++ {
addr := util.MakeUnresolvedAddr("tcp", fmt.Sprintf("node%d", i))
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(i),
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(i)), nd, time.Hour); err != nil {
t.Fatal(err)
}
descriptor.Replicas = append(descriptor.Replicas, roachpb.ReplicaDescriptor{
NodeID: roachpb.NodeID(i),
StoreID: roachpb.StoreID(i),
})
}
// Define our rpcSend stub which returns success on the second address.
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method == "Node.Batch" {
// reply from first address failed
_ = getReply()
// reply from second address succeed
batchReply := getReply().(*roachpb.BatchResponse)
reply := &roachpb.ScanResponse{}
batchReply.Add(reply)
reply.Rows = append([]roachpb.KeyValue{}, roachpb.KeyValue{Key: roachpb.Key("b"), Value: roachpb.Value{}})
return []proto.Message{batchReply}, nil
}
return nil, util.Errorf("unexpected method %v", method)
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{descriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 1)
sr, err := client.SendWrapped(ds, nil, scan)
if err != nil {
t.Fatal(err)
}
if l := len(sr.(*roachpb.ScanResponse).Rows); l != 1 {
t.Fatalf("expected 1 row; got %d", l)
}
}
func localRangeIDKeyParse(input string) (remainder string, key roachpb.Key) {
var rangeID int64
var err error
input = mustShiftSlash(input)
if endPos := strings.Index(input, "/"); endPos > 0 {
rangeID, err = strconv.ParseInt(input[:endPos], 10, 64)
if err != nil {
panic(err)
}
input = input[endPos:]
} else {
panic(errors.Errorf("illegal RangeID: %q", input))
}
input = mustShiftSlash(input)
var infix string
infix, input = mustShift(input)
var replicated bool
switch {
case bytes.Equal(localRangeIDUnreplicatedInfix, []byte(infix)):
case bytes.Equal(localRangeIDReplicatedInfix, []byte(infix)):
replicated = true
default:
panic(errors.Errorf("invalid infix: %q", infix))
}
input = mustShiftSlash(input)
// Get the suffix.
var suffix roachpb.RKey
for _, s := range rangeIDSuffixDict {
if strings.HasPrefix(input, s.name) {
input = input[len(s.name):]
if s.psFunc != nil {
remainder, key = s.psFunc(roachpb.RangeID(rangeID), input)
return
}
suffix = roachpb.RKey(s.suffix)
break
}
}
maker := MakeRangeIDUnreplicatedKey
if replicated {
maker = MakeRangeIDReplicatedKey
}
if suffix != nil {
if input != "" {
panic(&errUglifyUnsupported{errors.New("nontrivial detail")})
}
var detail roachpb.RKey
// TODO(tschottdorf): can't do this, init cycle:
// detail, err := UglyPrint(input)
// if err != nil {
// return "", nil, err
// }
remainder = ""
key = maker(roachpb.RangeID(rangeID), suffix, roachpb.RKey(detail))
return
}
panic(&errUglifyUnsupported{errors.New("unhandled general range key")})
}
// TestUpdateRangeAddressingSplitMeta1 verifies that it's an error to
// attempt to update range addressing records that would allow a split
// of meta1 records.
func TestUpdateRangeAddressingSplitMeta1(t *testing.T) {
defer leaktest.AfterTest(t)
left := &roachpb.RangeDescriptor{StartKey: roachpb.RKeyMin, EndKey: meta1Key(roachpb.RKey("a"))}
right := &roachpb.RangeDescriptor{StartKey: meta1Key(roachpb.RKey("a")), EndKey: roachpb.RKeyMax}
if err := splitRangeAddressing(&client.Batch{}, left, right); err == nil {
t.Error("expected failure trying to update addressing records for meta1 split")
}
}
// TestChangeReplicasDuplicateError tests that a replica change aborts if
// another change has been made to the RangeDescriptor since it was initiated.
func TestChangeReplicasDescriptorInvariant(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 3)
defer mtc.Stop()
repl, err := mtc.stores[0].GetReplica(1)
if err != nil {
t.Fatal(err)
}
addReplica := func(storeNum int, desc *roachpb.RangeDescriptor) error {
return repl.ChangeReplicas(roachpb.ADD_REPLICA,
roachpb.ReplicaDescriptor{
NodeID: mtc.stores[storeNum].Ident.NodeID,
StoreID: mtc.stores[storeNum].Ident.StoreID,
},
desc)
}
// Retain the descriptor for the range at this point.
origDesc := repl.Desc()
// Add replica to the second store, which should succeed.
if err := addReplica(1, origDesc); err != nil {
t.Fatal(err)
}
if err := util.IsTrueWithin(func() bool {
r := mtc.stores[1].LookupReplica(roachpb.RKey("a"), roachpb.RKey("b"))
if r == nil {
return false
}
return true
}, time.Second); err != nil {
t.Fatal(err)
}
// Attempt to add replica to the third store with the original descriptor.
// This should fail because the descriptor is stale.
if err := addReplica(2, origDesc); err == nil {
t.Fatal("Expected error calling ChangeReplicas with stale RangeDescriptor")
}
// Add to third store with fresh descriptor.
if err := addReplica(2, repl.Desc()); err != nil {
t.Fatal(err)
}
if err := util.IsTrueWithin(func() bool {
r := mtc.stores[2].LookupReplica(roachpb.RKey("a"), roachpb.RKey("b"))
if r == nil {
return false
}
return true
}, time.Second); err != nil {
t.Fatal(err)
}
}
func initTestDescriptorDB(t *testing.T) *testDescriptorDB {
db := newTestDescriptorDB()
for i, char := range "abcdefghijklmnopqrstuvwx" {
db.splitRange(t, roachpb.RKey(string(char)))
if i > 0 && i%6 == 0 {
db.splitRange(t, mustMeta(roachpb.RKey(string(char))))
}
}
db.cache = newRangeDescriptorCache(db, 2<<10)
return db
}
// TestReplicateAfterSplit verifies that a new replica whose start key
// is not KeyMin replicating to a fresh store can apply snapshots correctly.
func TestReplicateAfterSplit(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 2)
defer mtc.Stop()
rangeID := roachpb.RangeID(1)
splitKey := roachpb.Key("m")
key := roachpb.Key("z")
store0 := mtc.stores[0]
// Make the split
splitArgs := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, err := client.SendWrapped(rg1(store0), nil, &splitArgs); err != nil {
t.Fatal(err)
}
rangeID2 := store0.LookupReplica(roachpb.RKey(key), nil).RangeID
if rangeID2 == rangeID {
t.Errorf("got same range id after split")
}
// Issue an increment for later check.
incArgs := incrementArgs(key, 11)
if _, err := client.SendWrappedWith(rg1(store0), nil, roachpb.Header{
RangeID: rangeID2,
}, &incArgs); err != nil {
t.Fatal(err)
}
// Now add the second replica.
mtc.replicateRange(rangeID2, 0, 1)
if mtc.stores[1].LookupReplica(roachpb.RKey(key), nil).GetMaxBytes() == 0 {
t.Error("Range MaxBytes is not set after snapshot applied")
}
// Once it catches up, the effects of increment commands can be seen.
if err := util.IsTrueWithin(func() bool {
getArgs := getArgs(key)
// Reading on non-leader replica should use inconsistent read
reply, err := client.SendWrappedWith(rg1(mtc.stores[1]), nil, roachpb.Header{
RangeID: rangeID2,
ReadConsistency: roachpb.INCONSISTENT,
}, &getArgs)
if err != nil {
return false
}
getResp := reply.(*roachpb.GetResponse)
if log.V(1) {
log.Infof("read value %d", mustGetInt(getResp.Value))
}
return mustGetInt(getResp.Value) == 11
}, replicaReadTimeout); err != nil {
t.Fatal(err)
}
}
// TestBatchPrevNext tests batch.{Prev,Next}.
func TestBatchPrevNext(t *testing.T) {
defer leaktest.AfterTest(t)()
loc := func(s string) string {
return string(keys.RangeDescriptorKey(roachpb.RKey(s)))
}
span := func(strs ...string) []roachpb.Span {
var r []roachpb.Span
for i, str := range strs {
if i%2 == 0 {
r = append(r, roachpb.Span{Key: roachpb.Key(str)})
} else {
r[len(r)-1].EndKey = roachpb.Key(str)
}
}
return r
}
max, min := string(roachpb.RKeyMax), string(roachpb.RKeyMin)
abc := span("a", "", "b", "", "c", "")
testCases := []struct {
spans []roachpb.Span
key, expFW, expBW string
}{
{spans: span("a", "c", "b", ""), key: "b", expFW: "b", expBW: "b"},
{spans: span("a", "c", "b", ""), key: "a", expFW: "a", expBW: "a"},
{spans: span("a", "c", "d", ""), key: "c", expFW: "d", expBW: "c"},
{spans: span("a", "c\x00", "d", ""), key: "c", expFW: "c", expBW: "c"},
{spans: abc, key: "b", expFW: "b", expBW: "b"},
{spans: abc, key: "b\x00", expFW: "c", expBW: "b\x00"},
{spans: abc, key: "bb", expFW: "c", expBW: "b"},
{spans: span(), key: "whatevs", expFW: max, expBW: min},
{spans: span(loc("a"), loc("c")), key: "c", expFW: "c", expBW: "c"},
{spans: span(loc("a"), loc("c")), key: "c\x00", expFW: max, expBW: "c\x00"},
}
for i, test := range testCases {
var ba roachpb.BatchRequest
for _, span := range test.spans {
args := &roachpb.ScanRequest{}
args.Key, args.EndKey = span.Key, span.EndKey
ba.Add(args)
}
if next, err := next(ba, roachpb.RKey(test.key)); err != nil {
t.Errorf("%d: %v", i, err)
} else if !bytes.Equal(next, roachpb.Key(test.expFW)) {
t.Errorf("%d: next: expected %q, got %q", i, test.expFW, next)
}
if prev, err := prev(ba, roachpb.RKey(test.key)); err != nil {
t.Errorf("%d: %v", i, err)
} else if !bytes.Equal(prev, roachpb.Key(test.expBW)) {
t.Errorf("%d: prev: expected %q, got %q", i, test.expBW, prev)
}
}
}
// TestSendRPCRetry verifies that sendRPC failed on first address but succeed on
// second address, the second reply should be successfully returned back.
func TestSendRPCRetry(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
g.SetNodeID(1)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
// Fill RangeDescriptor with 2 replicas
var descriptor = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKey("z"),
}
for i := 1; i <= 2; i++ {
addr := util.MakeUnresolvedAddr("tcp", fmt.Sprintf("node%d", i))
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(i),
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(i)), nd, time.Hour); err != nil {
t.Fatal(err)
}
descriptor.Replicas = append(descriptor.Replicas, roachpb.ReplicaDescriptor{
NodeID: roachpb.NodeID(i),
StoreID: roachpb.StoreID(i),
})
}
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (proto.Message, error) {
batchReply := &roachpb.BatchResponse{}
reply := &roachpb.ScanResponse{}
batchReply.Add(reply)
reply.Rows = append([]roachpb.KeyValue{}, roachpb.KeyValue{Key: roachpb.Key("b"), Value: roachpb.Value{}})
return batchReply, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{descriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 1)
sr, err := client.SendWrapped(ds, nil, scan)
if err != nil {
t.Fatal(err)
}
if l := len(sr.(*roachpb.ScanResponse).Rows); l != 1 {
t.Fatalf("expected 1 row; got %d", l)
}
}
// TestRangeCacheDetectSplit verifies that when the cache detects a split
// it will properly coalesce all requests to the right half of the split and
// will prefetch the left half of the split.
func TestRangeCacheDetectSplit(t *testing.T) {
defer leaktest.AfterTest(t)()
db := initTestDescriptorDB(t)
pauseLookupResumeAndAssert := func(key string, expected int64, evictToken *evictionToken) {
var wg, waitJoin sync.WaitGroup
db.pauseRangeLookups()
for i := 0; i < 3; i++ {
wg.Add(1)
waitJoin.Add(1)
go func(id int) {
// Each request goes to a different key.
doLookupWithToken(t, db.cache, fmt.Sprintf("%s%d", key, id), evictToken, false, false, &waitJoin)
wg.Done()
}(i)
}
waitJoin.Wait()
db.resumeRangeLookups()
wg.Wait()
db.assertLookupCountEq(t, expected, key)
}
// A request initially looks up the range descriptor ["a"-"b").
doLookup(t, db.cache, "aa")
db.assertLookupCountEq(t, 2, "aa")
// A split breaks up the range into ["a"-"an") and ["an"-"b").
db.splitRange(t, roachpb.RKey("an"))
// A request is sent to the stale descriptor on the right half
// such that a RangeKeyMismatchError is returned.
_, evictToken := doLookup(t, db.cache, "az")
// mismatchErrRange mocks out a RangeKeyMismatchError.Range response.
ranges, _, pErr := db.getDescriptors(roachpb.RKey("aa"), false, false)
if pErr != nil {
t.Fatal(pErr)
}
mismatchErrRange := ranges[0]
// The stale descriptor is evicted, the new descriptor from the error is
// replaced, and a new lookup is initialized.
if err := evictToken.EvictAndReplace(context.Background(), mismatchErrRange); err != nil {
t.Fatal(err)
}
pauseLookupResumeAndAssert("az", 2, evictToken)
// Both sides of the split are now correctly cached.
doLookup(t, db.cache, "aa")
db.assertLookupCountEq(t, 0, "aa")
doLookup(t, db.cache, "az")
db.assertLookupCountEq(t, 0, "az")
}
// flush writes all dirty nodes and the tree to the transaction.
func (tc *treeContext) flush(b *client.Batch) {
if tc.dirty {
b.Put(keys.RangeTreeRoot, tc.tree)
}
for key, cachedNode := range tc.nodes {
if cachedNode.dirty {
if cachedNode.node == nil {
b.Del(keys.RangeTreeNodeKey(roachpb.RKey(key)))
} else {
b.Put(keys.RangeTreeNodeKey(roachpb.RKey(key)), cachedNode.node)
}
}
}
}
// TestReplicateAfterSplit verifies that a new replica whose start key
// is not KeyMin replicating to a fresh store can apply snapshots correctly.
func TestReplicateAfterSplit(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 2)
defer mtc.Stop()
rangeID := roachpb.RangeID(1)
splitKey := roachpb.Key("m")
key := roachpb.Key("z")
store0 := mtc.stores[0]
// Make the split
splitArgs := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, err := client.SendWrapped(rg1(store0), nil, &splitArgs); err != nil {
t.Fatal(err)
}
rangeID2 := store0.LookupReplica(roachpb.RKey(key), nil).RangeID
if rangeID2 == rangeID {
t.Errorf("got same range id after split")
}
// Issue an increment for later check.
incArgs := incrementArgs(key, 11)
if _, err := client.SendWrappedWith(rg1(store0), nil, roachpb.Header{
RangeID: rangeID2,
}, &incArgs); err != nil {
t.Fatal(err)
}
// Now add the second replica.
mtc.replicateRange(rangeID2, 1)
if mtc.stores[1].LookupReplica(roachpb.RKey(key), nil).GetMaxBytes() == 0 {
t.Error("Range MaxBytes is not set after snapshot applied")
}
// Once it catches up, the effects of increment commands can be seen.
util.SucceedsWithin(t, replicaReadTimeout, func() error {
getArgs := getArgs(key)
// Reading on non-leader replica should use inconsistent read
if reply, err := client.SendWrappedWith(rg1(mtc.stores[1]), nil, roachpb.Header{
RangeID: rangeID2,
ReadConsistency: roachpb.INCONSISTENT,
}, &getArgs); err != nil {
return util.Errorf("failed to read data: %s", err)
} else if e, v := int64(11), mustGetInt(reply.(*roachpb.GetResponse).Value); v != e {
return util.Errorf("failed to read correct data: expected %d, got %d", e, v)
}
return nil
})
}
// A basic test of manual replication that used to fail because we weren't
// waiting for all of the stores to initialize.
func TestBasicManualReplication(t *testing.T) {
defer leaktest.AfterTest(t)()
tc := StartTestCluster(t, 3, ClusterArgs{ReplicationMode: ReplicationManual})
defer tc.Stopper().Stop()
desc, err := tc.AddReplicas(roachpb.RKey(keys.MinKey), tc.Target(1), tc.Target(2))
if err != nil {
t.Fatal(err)
}
if expected := 3; expected != len(desc.Replicas) {
t.Fatalf("expected %d replicas, got %+v", expected, desc.Replicas)
}
if err := tc.TransferRangeLease(desc, tc.Target(1)); err != nil {
t.Fatal(err)
}
// TODO(peter): Removing the range leader (tc.Target(1)) causes the test to
// take ~13s vs ~1.5s for removing a non-leader. Track down that slowness.
desc, err = tc.RemoveReplicas(desc.StartKey, tc.Target(0))
if err != nil {
t.Fatal(err)
}
if expected := 2; expected != len(desc.Replicas) {
t.Fatalf("expected %d replicas, got %+v", expected, desc.Replicas)
}
}
func TestRangeCacheAssumptions(t *testing.T) {
defer leaktest.AfterTest(t)()
expKeyMin := mustMeta(mustMeta(mustMeta(roachpb.RKey("test"))))
if !bytes.Equal(expKeyMin, roachpb.RKeyMin) {
t.Fatalf("RangeCache relies on RangeMetaKey returning KeyMin after two levels, but got %s", expKeyMin)
}
}
// TransactionKey returns a transaction key based on the provided
// transaction key and ID. The base key is encoded in order to
// guarantee that all transaction records for a range sort together.
func TransactionKey(key roachpb.Key, txnID *uuid.UUID) roachpb.Key {
rk, err := Addr(key)
if err != nil {
panic(err)
}
return MakeRangeKey(rk, localTransactionSuffix, roachpb.RKey(txnID.GetBytes()))
}
func TestObjectIDForKey(t *testing.T) {
defer leaktest.AfterTest(t)
testCases := []struct {
key roachpb.RKey
success bool
id uint32
}{
// Before the structured span.
{roachpb.RKeyMin, false, 0},
// Boundaries of structured span.
{roachpb.RKeyMax, false, 0},
// Valid, even if there are things after the ID.
{keys.MakeKey(keys.MakeTablePrefix(42), roachpb.RKey("\xff")), true, 42},
{keys.MakeTablePrefix(0), true, 0},
{keys.MakeTablePrefix(999), true, 999},
}
for tcNum, tc := range testCases {
id, success := config.ObjectIDForKey(tc.key)
if success != tc.success {
t.Errorf("#%d: expected success=%t", tcNum, tc.success)
continue
}
if id != tc.id {
t.Errorf("#%d: expected id=%d, got %d", tcNum, tc.id, id)
}
}
}
// TestStoreZoneUpdateAndRangeSplit verifies that modifying the zone
// configuration changes range max bytes and Range.maybeSplit() takes
// max bytes into account when deciding whether to enqueue a range for
// splitting. It further verifies that the range is in fact split on
// exceeding zone's RangeMaxBytes.
func TestStoreZoneUpdateAndRangeSplit(t *testing.T) {
defer leaktest.AfterTest(t)
t.Skip("#3762")
store, stopper := createTestStore(t)
config.TestingSetupZoneConfigHook(stopper)
defer stopper.Stop()
maxBytes := int64(1 << 16)
// Set max bytes.
descID := uint32(keys.MaxReservedDescID + 1)
config.TestingSetZoneConfig(descID, &config.ZoneConfig{RangeMaxBytes: maxBytes})
// Trigger gossip callback.
if err := store.Gossip().AddInfoProto(gossip.KeySystemConfig, &config.SystemConfig{}, 0); err != nil {
t.Fatal(err)
}
// Wait for the range to be split along table boundaries.
originalRange := store.LookupReplica(roachpb.RKey(keys.UserTableDataMin), nil)
var rng *storage.Replica
if err := util.IsTrueWithin(func() bool {
rng = store.LookupReplica(keys.MakeTablePrefix(descID), nil)
return rng.RangeID != originalRange.RangeID
}, splitTimeout); err != nil {
t.Fatalf("failed to notice range max bytes update: %s", err)
}
// Check range's max bytes settings.
if rng.GetMaxBytes() != maxBytes {
t.Fatalf("range max bytes mismatch, got: %d, expected: %d", rng.GetMaxBytes(), maxBytes)
}
// Make sure the second range goes to the end.
if !roachpb.RKeyMax.Equal(rng.Desc().EndKey) {
t.Fatalf("second range has split: %+v", rng.Desc())
}
// Look in the range after prefix we're writing to.
fillRange(store, rng.RangeID, keys.MakeTablePrefix(descID), maxBytes, t)
// Verify that the range is in fact split (give it a few seconds for very
// slow test machines).
var newRng *storage.Replica
util.SucceedsWithin(t, splitTimeout, func() error {
newRng = store.LookupReplica(keys.MakeTablePrefix(descID+1), nil)
if newRng.RangeID == rng.RangeID {
return util.Errorf("range has not yet split")
}
return nil
})
// Make sure the new range goes to the end.
if !roachpb.RKeyMax.Equal(newRng.Desc().EndKey) {
t.Fatalf("second range has split: %+v", rng.Desc())
}
}
func stripMeta(key roachpb.RKey) roachpb.RKey {
switch {
case bytes.HasPrefix(key, keys.Meta1Prefix):
return testutils.MakeKey(roachpb.RKey(keys.Meta2Prefix), key[len(keys.Meta1Prefix):])
case bytes.HasPrefix(key, keys.Meta2Prefix):
return key[len(keys.Meta2Prefix):]
}
// First range.
return nil
}
func TestKeyAddress(t *testing.T) {
testCases := []struct {
key roachpb.Key
expAddress roachpb.RKey
}{
{roachpb.Key{}, roachpb.RKeyMin},
{roachpb.Key("123"), roachpb.RKey("123")},
{RangeDescriptorKey(roachpb.RKey("foo")), roachpb.RKey("foo")},
{TransactionKey(roachpb.Key("baz"), uuid.NewV4()), roachpb.RKey("baz")},
{TransactionKey(roachpb.KeyMax, uuid.NewV4()), roachpb.RKeyMax},
{nil, nil},
}
for i, test := range testCases {
result := Addr(test.key)
if !result.Equal(test.expAddress) {
t.Errorf("%d: expected address for key %q doesn't match %q", i, test.key, test.expAddress)
}
}
}
// ComputeSplitKeys takes a start and end key and returns an array of keys
// at which to split the span [start, end).
// The only required splits are at each user table prefix.
func (s SystemConfig) ComputeSplitKeys(startKey, endKey roachpb.RKey) []roachpb.RKey {
testingLock.Lock()
tableSplitsDisabled := testingDisableTableSplits
testingLock.Unlock()
if tableSplitsDisabled {
return nil
}
tableStart := roachpb.RKey(keys.UserTableDataMin)
if !tableStart.Less(endKey) {
// This range is before the user tables span: no required splits.
return nil
}
startID, ok := ObjectIDForKey(startKey)
if !ok || startID <= keys.MaxReservedDescID {
// The start key is either:
// - not part of the structured data span
// - part of the system span
// In either case, start looking for splits at the first ID usable
// by the user data span.
startID = keys.MaxReservedDescID + 1
} else {
// The start key is either already a split key, or after the split
// key for its ID. We can skip straight to the next one.
startID++
}
// Find the largest object ID.
// We can't keep splitting until we reach endKey as it could be roachpb.KeyMax.
endID, err := s.GetLargestObjectID()
if err != nil {
log.Errorf("unable to determine largest object ID from system config: %s", err)
return nil
}
// Build key prefixes for sequential table IDs until we reach endKey.
var splitKeys []roachpb.RKey
var key roachpb.RKey
// endID could be smaller than startID if we don't have user tables.
for id := startID; id <= endID; id++ {
key = keys.MakeTablePrefix(id)
// Skip if the range starts on a split key.
if !startKey.Less(key) {
continue
}
// Handle the case where EndKey is already a table prefix.
if !key.Less(endKey) {
break
}
splitKeys = append(splitKeys, key)
}
return splitKeys
}
// Addr returns the address for the key, used to lookup the range containing
// the key. In the normal case, this is simply the key's value. However, for
// local keys, such as transaction records, range-spanning binary tree node
// pointers, the address is the trailing suffix of the key, with the local key
// prefix removed. In this way, local keys address to the same range as
// non-local keys, but are stored separately so that they don't collide with
// user-space or global system keys.
//
// However, not all local keys are addressable in the global map. Only range
// local keys incorporating a range key (start key or transaction key) are
// addressable (e.g. range metadata and txn records). Range local keys
// incorporating the Range ID are not (e.g. response cache entries, and range
// stats).
//
// TODO(pmattis): Should KeyAddress return an error when the key is malformed?
func Addr(k roachpb.Key) roachpb.RKey {
if k == nil {
return nil
}
if !bytes.HasPrefix(k, localPrefix) {
return roachpb.RKey(k)
}
if bytes.HasPrefix(k, LocalRangePrefix) {
k = k[len(LocalRangePrefix):]
_, k, err := encoding.DecodeBytes(k, nil)
if err != nil {
panic(err)
}
return roachpb.RKey(k)
}
log.Fatalf("local key %q malformed; should contain prefix %q",
k, LocalRangePrefix)
return nil
}
func TestApplySnapshotDenyPreemptive(t *testing.T) {
defer leaktest.AfterTest(t)()
var tc testContext
tc.Start(t)
defer tc.Stop()
key := roachpb.RKey("a")
realRng := tc.store.LookupReplica(key, nil)
// Use Raft to get a nontrivial term for our snapshot.
if pErr := realRng.redirectOnOrAcquireLease(context.Background()); pErr != nil {
t.Fatal(pErr)
}
snap, err := realRng.GetSnapshot()
if err != nil {
t.Fatal(err)
}
// Make sure that the Term is behind our first range term (raftInitialLogTerm)
snap.Metadata.Term--
// Create an uninitialized version of the first range. This is only ok
// because in the case we test, there's an error (and so we don't clobber
// our actual first range in the Store). If we want snapshots to apply
// successfully during tests, we need to adapt the snapshots to a new
// RangeID first and generally do a lot more work.
rng, err := NewReplica(&roachpb.RangeDescriptor{RangeID: 1}, tc.store, 0)
if err != nil {
t.Fatal(err)
}
if _, err := rng.applySnapshot(snap, raftpb.HardState{}); !testutils.IsError(
err, "cannot apply preemptive snapshot from past term",
) {
t.Fatal(err)
}
// Do something that extends the Raft log past what we have in the
// snapshot.
put := putArgs(roachpb.Key("a"), []byte("foo"))
if _, pErr := tc.SendWrapped(&put); pErr != nil {
t.Fatal(pErr)
}
snap.Metadata.Term++ // restore the "real" term of the snapshot
if _, err := rng.applySnapshot(snap, raftpb.HardState{}); !testutils.IsError(
err, "would erase acknowledged log entries",
) {
t.Fatal(err)
}
}
func TestRangeMetaKey(t *testing.T) {
defer leaktest.AfterTest(t)
testCases := []struct {
key, expKey roachpb.RKey
}{
{
key: roachpb.RKey{},
expKey: roachpb.RKeyMin,
},
{
key: roachpb.RKey("\x00\x00meta2\x00zonefoo"),
expKey: roachpb.RKey("\x00\x00meta1\x00zonefoo"),
},
{
key: roachpb.RKey("\x00\x00meta1\x00zonefoo"),
expKey: roachpb.RKeyMin,
},
{
key: roachpb.RKey("foo"),
expKey: roachpb.RKey("\x00\x00meta2foo"),
},
{
key: roachpb.RKey("foo"),
expKey: roachpb.RKey("\x00\x00meta2foo"),
},
{
key: roachpb.RKey("\x00\x00meta2foo"),
expKey: roachpb.RKey("\x00\x00meta1foo"),
},
{
key: roachpb.RKey("\x00\x00meta1foo"),
expKey: roachpb.RKeyMin,
},
}
for i, test := range testCases {
result := RangeMetaKey(test.key)
if !bytes.Equal(result, test.expKey) {
t.Errorf("%d: expected range meta for key %q doesn't match %q", i, test.key, test.expKey)
}
}
}
// TestRetryOnWrongReplicaError sets up a DistSender on a minimal gossip
// network and a mock of rpc.Send, and verifies that the DistSender correctly
// retries upon encountering a stale entry in its range descriptor cache.
func TestRetryOnWrongReplicaError(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
// Updated below, after it has first been returned.
badStartKey := roachpb.RKey("m")
newRangeDescriptor := testRangeDescriptor
goodStartKey := newRangeDescriptor.StartKey
newRangeDescriptor.StartKey = badStartKey
descStale := true
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
ba := getArgs(testAddress).(*roachpb.BatchRequest)
rs := keys.Range(*ba)
if _, ok := ba.GetArg(roachpb.RangeLookup); ok {
if !descStale && bytes.HasPrefix(rs.Key, keys.Meta2Prefix) {
t.Errorf("unexpected extra lookup for non-stale replica descriptor at %s",
rs.Key)
}
br := getReply().(*roachpb.BatchResponse)
r := &roachpb.RangeLookupResponse{}
r.Ranges = append(r.Ranges, newRangeDescriptor)
br.Add(r)
// If we just returned the stale descriptor, set up returning the
// good one next time.
if bytes.HasPrefix(rs.Key, keys.Meta2Prefix) {
if newRangeDescriptor.StartKey.Equal(badStartKey) {
newRangeDescriptor.StartKey = goodStartKey
} else {
descStale = false
}
}
return []proto.Message{br}, nil
}
// When the Scan first turns up, update the descriptor for future
// range descriptor lookups.
if !newRangeDescriptor.StartKey.Equal(goodStartKey) {
return nil, &roachpb.RangeKeyMismatchError{RequestStartKey: rs.Key.AsRawKey(),
RequestEndKey: rs.EndKey.AsRawKey()}
}
return []proto.Message{ba.CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
}
ds := NewDistSender(ctx, g)
scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 0)
if _, err := client.SendWrapped(ds, nil, scan); err != nil {
t.Errorf("scan encountered error: %s", err)
}
}