// prev gives the right boundary of the union of all requests which don't
// affect keys larger than the given key.
// TODO(tschottdorf): again, better on BatchRequest itself, but can't pull
// 'keys' into 'roachpb'.
func prev(ba roachpb.BatchRequest, k roachpb.RKey) roachpb.RKey {
candidate := roachpb.RKeyMin
for _, union := range ba.Requests {
h := union.GetInner().Header()
addr := keys.Addr(h.Key)
eAddr := keys.Addr(h.EndKey)
if len(eAddr) == 0 {
// Can probably avoid having to compute Next() here if
// we're in the mood for some more complexity.
eAddr = addr.Next()
}
if !eAddr.Less(k) {
if !k.Less(addr) {
// Range contains k, so won't be able to go lower.
return k
}
// Range is disjoint from [KeyMin,k).
continue
}
// We want the largest surviving candidate.
if candidate.Less(addr) {
candidate = addr
}
}
return candidate
}
// TestReplicateRange verifies basic replication functionality by creating two stores
// and a range, replicating the range to the second store, and reading its data there.
func TestReplicateRange(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 2)
defer mtc.Stop()
// Issue a command on the first node before replicating.
incArgs := incrementArgs([]byte("a"), 5)
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &incArgs); err != nil {
t.Fatal(err)
}
rng, err := mtc.stores[0].GetReplica(1)
if err != nil {
t.Fatal(err)
}
if err := rng.ChangeReplicas(roachpb.ADD_REPLICA,
roachpb.ReplicaDescriptor{
NodeID: mtc.stores[1].Ident.NodeID,
StoreID: mtc.stores[1].Ident.StoreID,
}, rng.Desc()); err != nil {
t.Fatal(err)
}
// Verify no intent remains on range descriptor key.
key := keys.RangeDescriptorKey(rng.Desc().StartKey)
desc := roachpb.RangeDescriptor{}
if ok, err := engine.MVCCGetProto(mtc.stores[0].Engine(), key, mtc.stores[0].Clock().Now(), true, nil, &desc); !ok || err != nil {
t.Fatalf("fetching range descriptor yielded %t, %s", ok, err)
}
// Verify that in time, no intents remain on meta addressing
// keys, and that range descriptor on the meta records is correct.
util.SucceedsWithin(t, 1*time.Second, func() error {
meta2 := keys.Addr(keys.RangeMetaKey(roachpb.RKeyMax))
meta1 := keys.Addr(keys.RangeMetaKey(meta2))
for _, key := range []roachpb.RKey{meta2, meta1} {
metaDesc := roachpb.RangeDescriptor{}
if ok, err := engine.MVCCGetProto(mtc.stores[0].Engine(), key.AsRawKey(), mtc.stores[0].Clock().Now(), true, nil, &metaDesc); !ok || err != nil {
return util.Errorf("failed to resolve %s", key.AsRawKey())
}
if !reflect.DeepEqual(metaDesc, desc) {
return util.Errorf("descs not equal: %+v != %+v", metaDesc, desc)
}
}
return nil
})
// Verify that the same data is available on the replica.
util.SucceedsWithin(t, replicaReadTimeout, func() error {
getArgs := getArgs([]byte("a"))
if reply, err := client.SendWrappedWith(rg1(mtc.stores[1]), nil, roachpb.Header{
ReadConsistency: roachpb.INCONSISTENT,
}, &getArgs); err != nil {
return util.Errorf("failed to read data: %s", err)
} else if e, v := int64(5), mustGetInt(reply.(*roachpb.GetResponse).Value); v != e {
return util.Errorf("failed to read correct data: expected %d, got %d", e, v)
}
return nil
})
}
func metaKey(key roachpb.RKey) []byte {
rk, err := keys.Addr(keys.RangeMetaKey(key))
if err != nil {
panic(err)
}
return rk
}
// prev gives the right boundary of the union of all requests which don't
// affect keys larger than the given key.
// TODO(tschottdorf): again, better on BatchRequest itself, but can't pull
// 'keys' into 'roachpb'.
func prev(ba roachpb.BatchRequest, k roachpb.RKey) (roachpb.RKey, error) {
candidate := roachpb.RKeyMin
for _, union := range ba.Requests {
h := union.GetInner().Header()
addr, err := keys.Addr(h.Key)
if err != nil {
return nil, err
}
eAddr, err := keys.AddrUpperBound(h.EndKey)
if err != nil {
return nil, err
}
if len(eAddr) == 0 {
eAddr = addr.Next()
}
if !eAddr.Less(k) {
if !k.Less(addr) {
// Range contains k, so won't be able to go lower.
return k, nil
}
// Range is disjoint from [KeyMin,k).
continue
}
// We want the largest surviving candidate.
if candidate.Less(addr) {
candidate = addr
}
}
return candidate, nil
}
// next gives the left boundary of the union of all requests which don't
// affect keys less than the given key.
// TODO(tschottdorf): again, better on BatchRequest itself, but can't pull
// 'keys' into 'proto'.
func next(ba roachpb.BatchRequest, k roachpb.RKey) (roachpb.RKey, error) {
candidate := roachpb.RKeyMax
for _, union := range ba.Requests {
h := union.GetInner().Header()
addr, err := keys.Addr(h.Key)
if err != nil {
return nil, err
}
if addr.Less(k) {
eAddr, err := keys.AddrUpperBound(h.EndKey)
if err != nil {
return nil, err
}
if k.Less(eAddr) {
// Starts below k, but continues beyond. Need to stay at k.
return k, nil
}
// Affects only [KeyMin,k).
continue
}
// We want the smallest of the surviving candidates.
if addr.Less(candidate) {
candidate = addr
}
}
return candidate, nil
}
// checkEndTransactionTrigger verifies that an EndTransactionRequest
// that includes intents for the SystemDB keys sets the proper trigger.
func checkEndTransactionTrigger(req roachpb.Request, _ roachpb.Header) error {
args, ok := req.(*roachpb.EndTransactionRequest)
if !ok {
return nil
}
if !args.Commit {
// This is a rollback: skip trigger verification.
return nil
}
modifiedSpanTrigger := args.InternalCommitTrigger.GetModifiedSpanTrigger()
modifiedSystemSpan := modifiedSpanTrigger != nil && modifiedSpanTrigger.SystemDBSpan
var hasSystemKey bool
for _, span := range args.IntentSpans {
addr := keys.Addr(span.Key)
if bytes.Compare(addr, keys.SystemDBSpan.Key) >= 0 && bytes.Compare(addr, keys.SystemDBSpan.EndKey) < 0 {
hasSystemKey = true
break
}
}
if hasSystemKey != modifiedSystemSpan {
return util.Errorf("EndTransaction hasSystemKey=%t, but hasSystemDBTrigger=%t",
hasSystemKey, modifiedSystemSpan)
}
return nil
}
func TestKeyAddress(t *testing.T) {
defer leaktest.AfterTest(t)()
testCases := []struct {
key roachpb.Key
}{
{MakeNameMetadataKey(0, "BAR")},
{MakeNameMetadataKey(1, "BAR")},
{MakeNameMetadataKey(1, "foo")},
{MakeNameMetadataKey(2, "foo")},
{MakeDescMetadataKey(123)},
{MakeDescMetadataKey(124)},
}
var lastKey roachpb.Key
for i, test := range testCases {
resultAddr, err := keys.Addr(test.key)
if err != nil {
t.Fatal(err)
}
result := resultAddr.AsRawKey()
if result.Compare(lastKey) <= 0 {
t.Errorf("%d: key address %q is <= %q", i, result, lastKey)
}
lastKey = result
}
}
func TestObjectIDForKey(t *testing.T) {
defer leaktest.AfterTest(t)
testCases := []struct {
key roachpb.RKey
success bool
id uint32
}{
// Before the structured span.
{roachpb.RKeyMin, false, 0},
{keys.Addr(keys.SystemMax), false, 0},
// Boundaries of structured span.
{keys.Addr(keys.TableDataPrefix), false, 0},
{roachpb.RKeyMax, false, 0},
// In system span, but no Uvarint ID.
{keys.MakeKey(keys.TableDataPrefix, roachpb.RKey("foo")), false, 0},
// Valid, even if there are things after the ID.
{keys.MakeKey(keys.MakeTablePrefix(42), roachpb.RKey("foo")), true, 42},
{keys.MakeTablePrefix(0), true, 0},
{keys.MakeTablePrefix(999), true, 999},
}
for tcNum, tc := range testCases {
id, success := config.ObjectIDForKey(tc.key)
if success != tc.success {
t.Errorf("#%d: expected success=%t", tcNum, tc.success)
continue
}
if id != tc.id {
t.Errorf("#%d: expected id=%d, got %d", tcNum, tc.id, id)
}
}
}
// SplitRange splits the range containing splitKey.
// The right range created by the split starts at the split key and extends to the
// original range's end key.
// Returns the new descriptors of the left and right ranges.
//
// splitKey must correspond to a SQL table key (it must end with a family ID /
// col ID).
func (tc *TestCluster) SplitRange(
splitKey roachpb.Key,
) (*roachpb.RangeDescriptor, *roachpb.RangeDescriptor, error) {
splitRKey, err := keys.Addr(splitKey)
if err != nil {
return nil, nil, err
}
origRangeDesc, err := tc.LookupRange(splitKey)
if err != nil {
return nil, nil, err
}
if origRangeDesc.StartKey.Equal(splitRKey) {
return nil, nil, errors.Errorf(
"cannot split range %+v at start key %q", origRangeDesc, splitKey)
}
splitReq := roachpb.AdminSplitRequest{
Span: roachpb.Span{
Key: splitKey,
},
SplitKey: splitKey,
}
_, pErr := client.SendWrapped(tc.Servers[0].GetDistSender(), nil, &splitReq)
if pErr != nil {
return nil, nil, errors.Errorf(
"%q: split unexpected error: %s", splitReq.SplitKey, pErr)
}
leftRangeDesc := new(roachpb.RangeDescriptor)
rightRangeDesc := new(roachpb.RangeDescriptor)
if err := tc.Servers[0].DB().GetProto(
keys.RangeDescriptorKey(origRangeDesc.StartKey), leftRangeDesc); err != nil {
return nil, nil, errors.Wrap(err, "could not look up left-hand side descriptor")
}
// The split point might not be exactly the one we requested (it can be
// adjusted slightly so we don't split in the middle of SQL rows). Update it
// to the real point.
splitRKey = leftRangeDesc.EndKey
if err := tc.Servers[0].DB().GetProto(
keys.RangeDescriptorKey(splitRKey), rightRangeDesc); err != nil {
return nil, nil, errors.Wrap(err, "could not look up right-hand side descriptor")
}
return leftRangeDesc, rightRangeDesc, nil
}
// checkEndTransactionTrigger verifies that an EndTransactionRequest
// that includes intents for the SystemDB keys sets the proper trigger.
func checkEndTransactionTrigger(args storagebase.FilterArgs) *roachpb.Error {
req, ok := args.Req.(*roachpb.EndTransactionRequest)
if !ok {
return nil
}
if !req.Commit {
// This is a rollback: skip trigger verification.
return nil
}
modifiedSpanTrigger := req.InternalCommitTrigger.GetModifiedSpanTrigger()
modifiedSystemConfigSpan := modifiedSpanTrigger != nil && modifiedSpanTrigger.SystemConfigSpan
var hasSystemKey bool
for _, span := range req.IntentSpans {
keyAddr, err := keys.Addr(span.Key)
if err != nil {
return roachpb.NewError(err)
}
if bytes.Compare(keyAddr, keys.SystemConfigSpan.Key) >= 0 &&
bytes.Compare(keyAddr, keys.SystemConfigSpan.EndKey) < 0 {
hasSystemKey = true
break
}
}
// If the transaction in question has intents in the system span, then
// modifiedSystemConfigSpan should always be true. However, it is possible
// for modifiedSystemConfigSpan to be set, even though no system keys are
// present. This can occur with certain conditional DDL statements (e.g.
// "CREATE TABLE IF NOT EXISTS"), which set the SystemConfigTrigger
// aggressively but may not actually end up changing the system DB depending
// on the current state.
// For more information, see the related comment at the beginning of
// planner.makePlan().
if hasSystemKey && !modifiedSystemConfigSpan {
return roachpb.NewError(util.Errorf("EndTransaction hasSystemKey=%t, but hasSystemConfigTrigger=%t",
hasSystemKey, modifiedSystemConfigSpan))
}
return nil
}
func runLsRanges(cmd *cobra.Command, args []string) {
if len(args) > 1 {
mustUsage(cmd)
return
}
var startKey roachpb.Key
{
k := roachpb.KeyMin.Next()
if len(args) > 0 {
k = roachpb.Key(args[0])
}
rk, err := keys.Addr(k)
if err != nil {
panic(err)
}
startKey = keys.RangeMetaKey(rk)
}
endKey := keys.Meta2Prefix.PrefixEnd()
kvDB, stopper := makeDBClient()
defer stopper.Stop()
rows, err := kvDB.Scan(startKey, endKey, maxResults)
if err != nil {
panicf("scan failed: %s\n", err)
}
for _, row := range rows {
desc := &roachpb.RangeDescriptor{}
if err := row.ValueProto(desc); err != nil {
panicf("%s: unable to unmarshal range descriptor\n", row.Key)
continue
}
fmt.Printf("%s-%s [%d]\n", desc.StartKey, desc.EndKey, desc.RangeID)
for i, replica := range desc.Replicas {
fmt.Printf("\t%d: node-id=%d store-id=%d\n",
i, replica.NodeID, replica.StoreID)
}
}
fmt.Printf("%d result(s)\n", len(rows))
}
func doLookupWithToken(
t *testing.T,
rc *rangeDescriptorCache,
key string,
evictToken *evictionToken,
considerIntents bool,
useReverseScan bool,
wg *sync.WaitGroup,
) (*roachpb.RangeDescriptor, *evictionToken) {
r, returnToken, pErr := rc.lookupRangeDescriptorInternal(
context.Background(), roachpb.RKey(key), evictToken, considerIntents, useReverseScan, wg)
if pErr != nil {
t.Fatalf("Unexpected error from LookupRangeDescriptor: %s", pErr)
}
keyAddr, err := keys.Addr(roachpb.Key(key))
if err != nil {
t.Fatal(err)
}
if (useReverseScan && !r.ContainsExclusiveEndKey(keyAddr)) || (!useReverseScan && !r.ContainsKey(keyAddr)) {
t.Fatalf("Returned range did not contain key: %s-%s, %s", r.StartKey, r.EndKey, key)
}
return r, returnToken
}
// TestStoreRangeSplit executes a split of a range and verifies that the
// resulting ranges respond to the right key ranges and that their stats
// and sequence cache have been properly accounted for.
func TestStoreRangeSplitIdempotency(t *testing.T) {
defer leaktest.AfterTest(t)
store, stopper := createTestStore(t)
defer stopper.Stop()
rangeID := roachpb.RangeID(1)
splitKey := roachpb.Key("m")
content := roachpb.Key("asdvb")
// First, write some values left and right of the proposed split key.
pArgs := putArgs([]byte("c"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
pArgs = putArgs([]byte("x"), content)
if _, err := client.SendWrapped(rg1(store), nil, &pArgs); err != nil {
t.Fatal(err)
}
// Increments are a good way of testing the sequence cache. Up here, we
// address them to the original range, then later to the one that contains
// the key.
txn := roachpb.NewTransaction("test", []byte("c"), 10, roachpb.SERIALIZABLE,
store.Clock().Now(), 0)
lIncArgs := incrementArgs([]byte("apoptosis"), 100)
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: txn,
}, &lIncArgs); err != nil {
t.Fatal(err)
}
rIncArgs := incrementArgs([]byte("wobble"), 10)
txn.Sequence++
if _, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: txn,
}, &rIncArgs); err != nil {
t.Fatal(err)
}
// Get the original stats for key and value bytes.
var ms engine.MVCCStats
if err := engine.MVCCGetRangeStats(store.Engine(), rangeID, &ms); err != nil {
t.Fatal(err)
}
keyBytes, valBytes := ms.KeyBytes, ms.ValBytes
// Split the range.
args := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
// Verify no intents remains on range descriptor keys.
for _, key := range []roachpb.Key{keys.RangeDescriptorKey(roachpb.RKeyMin), keys.RangeDescriptorKey(keys.Addr(splitKey))} {
if _, _, err := engine.MVCCGet(store.Engine(), key, store.Clock().Now(), true, nil); err != nil {
t.Fatal(err)
}
}
rng := store.LookupReplica(roachpb.RKeyMin, nil)
newRng := store.LookupReplica([]byte("m"), nil)
if !bytes.Equal(newRng.Desc().StartKey, splitKey) || !bytes.Equal(splitKey, rng.Desc().EndKey) {
t.Errorf("ranges mismatched, wanted %q=%q=%q", newRng.Desc().StartKey, splitKey, rng.Desc().EndKey)
}
if !bytes.Equal(newRng.Desc().EndKey, roachpb.RKeyMax) || !bytes.Equal(rng.Desc().StartKey, roachpb.RKeyMin) {
t.Errorf("new ranges do not cover KeyMin-KeyMax, but only %q-%q", rng.Desc().StartKey, newRng.Desc().EndKey)
}
// Try to get values from both left and right of where the split happened.
gArgs := getArgs([]byte("c"))
if reply, err := client.SendWrapped(rg1(store), nil, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
gArgs = getArgs([]byte("x"))
if reply, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: newRng.Desc().RangeID,
}, &gArgs); err != nil {
t.Fatal(err)
} else if replyBytes, err := reply.(*roachpb.GetResponse).Value.GetBytes(); err != nil {
t.Fatal(err)
} else if !bytes.Equal(replyBytes, content) {
t.Fatalf("actual value %q did not match expected value %q", replyBytes, content)
}
// Send out an increment request copied from above (same txn/sequence)
// which remains in the old range.
_, err := client.SendWrappedWith(rg1(store), nil, roachpb.Header{
Txn: txn,
}, &lIncArgs)
if _, ok := err.(*roachpb.TransactionRetryError); !ok {
t.Fatalf("unexpected sequence cache miss: %v", err)
}
// Send out the same increment copied from above (same txn/sequence), but
// now to the newly created range (which should hold that key).
_, err = client.SendWrappedWith(rg1(store), nil, roachpb.Header{
RangeID: newRng.Desc().RangeID,
Txn: txn,
}, &rIncArgs)
if _, ok := err.(*roachpb.TransactionRetryError); !ok {
t.Fatalf("unexpected sequence cache miss: %v", err)
}
// Compare stats of split ranges to ensure they are non zero and
// exceed the original range when summed.
var left, right engine.MVCCStats
if err := engine.MVCCGetRangeStats(store.Engine(), rangeID, &left); err != nil {
t.Fatal(err)
}
lKeyBytes, lValBytes := left.KeyBytes, left.ValBytes
if err := engine.MVCCGetRangeStats(store.Engine(), newRng.Desc().RangeID, &right); err != nil {
t.Fatal(err)
}
rKeyBytes, rValBytes := right.KeyBytes, right.ValBytes
if lKeyBytes == 0 || rKeyBytes == 0 {
t.Errorf("expected non-zero key bytes; got %d, %d", lKeyBytes, rKeyBytes)
}
if lValBytes == 0 || rValBytes == 0 {
t.Errorf("expected non-zero val bytes; got %d, %d", lValBytes, rValBytes)
}
if lKeyBytes+rKeyBytes <= keyBytes {
t.Errorf("left + right key bytes don't match; %d + %d <= %d", lKeyBytes, rKeyBytes, keyBytes)
}
if lValBytes+rValBytes <= valBytes {
t.Errorf("left + right val bytes don't match; %d + %d <= %d", lValBytes, rValBytes, valBytes)
}
}
func meta(k roachpb.RKey) roachpb.RKey {
return keys.Addr(keys.RangeMetaKey(k))
}
// TestAcceptsUnsplitRanges verifies that ranges that need to split are properly
// rejected when the queue has 'acceptsUnsplitRanges = false'.
func TestAcceptsUnsplitRanges(t *testing.T) {
defer leaktest.AfterTest(t)
g, stopper := gossipForTest(t)
defer stopper.Stop()
// This range can never be split due to zone configs boundaries.
neverSplits := &Replica{}
if err := neverSplits.setDesc(&roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKeyMin,
EndKey: keys.Addr(keys.UserTableDataMin),
}); err != nil {
t.Fatal(err)
}
// This range will need to be split after user db/table entries are created.
willSplit := &Replica{}
if err := willSplit.setDesc(&roachpb.RangeDescriptor{
RangeID: 2,
StartKey: keys.Addr(keys.UserTableDataMin),
EndKey: roachpb.RKeyMax,
}); err != nil {
t.Fatal(err)
}
var queued int32
testQueue := &testQueueImpl{
shouldQueueFn: func(now roachpb.Timestamp, r *Replica) (shouldQueue bool, priority float64) {
// Always queue ranges if they make it past the base queue's logic.
atomic.AddInt32(&queued, 1)
return true, float64(r.Desc().RangeID)
},
acceptUnsplit: false,
}
bq := makeBaseQueue("test", testQueue, g, 2)
mc := hlc.NewManualClock(0)
clock := hlc.NewClock(mc.UnixNano)
bq.Start(clock, stopper)
// Check our config.
sysCfg := g.GetSystemConfig()
if sysCfg == nil {
t.Fatal("nil config")
}
if sysCfg.NeedsSplit(neverSplits.Desc().StartKey, neverSplits.Desc().EndKey) {
t.Fatal("System config says range needs to be split")
}
if sysCfg.NeedsSplit(willSplit.Desc().StartKey, willSplit.Desc().EndKey) {
t.Fatal("System config says range needs to be split")
}
// There are no user db/table entries, everything should be added and
// processed as usual.
bq.MaybeAdd(neverSplits, roachpb.ZeroTimestamp)
bq.MaybeAdd(willSplit, roachpb.ZeroTimestamp)
if err := util.IsTrueWithin(func() bool {
return atomic.LoadInt32(&testQueue.processed) == 2
}, 250*time.Millisecond); err != nil {
t.Error(err)
}
if pc := atomic.LoadInt32(&queued); pc != 2 {
t.Errorf("expected queued count of 2; got %d", pc)
}
// Now add a user object, it will trigger a split.
// The range willSplit starts at the beginning of the user data range,
// which means keys.MaxReservedDescID+1.
config.TestingSetZoneConfig(keys.MaxReservedDescID+2, &config.ZoneConfig{RangeMaxBytes: 1 << 20})
// Check our config.
if sysCfg.NeedsSplit(neverSplits.Desc().StartKey, neverSplits.Desc().EndKey) {
t.Fatal("System config says range needs to be split")
}
if !sysCfg.NeedsSplit(willSplit.Desc().StartKey, willSplit.Desc().EndKey) {
t.Fatal("System config says range does not need to be split")
}
bq.MaybeAdd(neverSplits, roachpb.ZeroTimestamp)
bq.MaybeAdd(willSplit, roachpb.ZeroTimestamp)
if err := util.IsTrueWithin(func() bool {
return atomic.LoadInt32(&testQueue.processed) == 3
}, 250*time.Millisecond); err != nil {
t.Error(err)
}
if pc := atomic.LoadInt32(&queued); pc != 3 {
t.Errorf("expected queued count of 3; got %d", pc)
}
}
// TestGetZoneConfig exercises config.GetZoneConfig and the sql hook for it.
func TestGetZoneConfig(t *testing.T) {
defer leaktest.AfterTest(t)
// Disable splitting. We're using bad attributes in zone configs
// to be able to match.
defer config.TestingDisableTableSplits()()
s, sqlDB, _ := setup(t)
defer cleanup(s, sqlDB)
expectedCounter := uint32(keys.MaxReservedDescID + 1)
// Naming scheme for database and tables:
// db1 has tables tb11 and tb12
// db2 has tables tb21 and tb22
db1 := expectedCounter
if _, err := sqlDB.Exec(`CREATE DATABASE db1`); err != nil {
t.Fatal(err)
}
expectedCounter++
db2 := expectedCounter
if _, err := sqlDB.Exec(`CREATE DATABASE db2`); err != nil {
t.Fatal(err)
}
expectedCounter++
tb11 := expectedCounter
if _, err := sqlDB.Exec(`CREATE TABLE db1.tb1 (k INT PRIMARY KEY, v INT)`); err != nil {
t.Fatal(err)
}
expectedCounter++
tb12 := expectedCounter
if _, err := sqlDB.Exec(`CREATE TABLE db1.tb2 (k INT PRIMARY KEY, v INT)`); err != nil {
t.Fatal(err)
}
expectedCounter++
tb21 := expectedCounter
if _, err := sqlDB.Exec(`CREATE TABLE db2.tb1 (k INT PRIMARY KEY, v INT)`); err != nil {
t.Fatal(err)
}
expectedCounter++
tb22 := expectedCounter
if _, err := sqlDB.Exec(`CREATE TABLE db2.tb2 (k INT PRIMARY KEY, v INT)`); err != nil {
t.Fatal(err)
}
expectedCounter++
cfg, err := forceNewConfig(t, s)
if err != nil {
t.Fatalf("failed to get latest system config: %s", err)
}
// We have no custom zone configs.
testCases := []struct {
key roachpb.RKey
zoneCfg config.ZoneConfig
}{
{roachpb.RKeyMin, *config.DefaultZoneConfig},
{keys.Addr(keys.TableDataPrefix), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(1), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(keys.MaxReservedDescID), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(db1), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(db2), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(tb11), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(tb12), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(tb21), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(tb22), *config.DefaultZoneConfig},
}
for tcNum, tc := range testCases {
zoneCfg, err := cfg.GetZoneConfigForKey(tc.key)
if err != nil {
t.Fatalf("#%d: err=%s", tcNum, err)
}
if !reflect.DeepEqual(*zoneCfg, tc.zoneCfg) {
t.Errorf("#%d: bad zone config.\nexpected: %+v\ngot: %+v", tcNum, tc.zoneCfg, zoneCfg)
}
}
// Now set some zone configs. We don't have a nice way of using table
// names for this, so we do raw puts.
// Here is the list of dbs/tables and whether they have a custom zone config:
// db1: true
// tb1: true
// tb2: false
// db1: false
// tb1: true
// tb2: false
db1Cfg := config.ZoneConfig{ReplicaAttrs: []roachpb.Attributes{{[]string{"db1"}}}}
tb11Cfg := config.ZoneConfig{ReplicaAttrs: []roachpb.Attributes{{[]string{"db1.tb1"}}}}
tb21Cfg := config.ZoneConfig{ReplicaAttrs: []roachpb.Attributes{{[]string{"db2.tb1"}}}}
for objID, objZone := range map[uint32]config.ZoneConfig{
db1: db1Cfg,
tb11: tb11Cfg,
tb21: tb21Cfg,
} {
buf, err := proto.Marshal(&objZone)
if err != nil {
t.Fatal(err)
}
if _, err = sqlDB.Exec(`INSERT INTO system.zones VALUES ($1, $2)`, objID, buf); err != nil {
t.Fatalf("problem writing zone %+v: %s", objZone, err)
}
}
cfg, err = forceNewConfig(t, s)
if err != nil {
t.Fatalf("failed to get latest system config: %s", err)
}
testCases = []struct {
key roachpb.RKey
zoneCfg config.ZoneConfig
}{
{roachpb.RKeyMin, *config.DefaultZoneConfig},
{keys.Addr(keys.TableDataPrefix), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(1), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(keys.MaxReservedDescID), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(db1), db1Cfg},
{keys.MakeTablePrefix(db2), *config.DefaultZoneConfig},
{keys.MakeTablePrefix(tb11), tb11Cfg},
{keys.MakeTablePrefix(tb12), db1Cfg},
{keys.MakeTablePrefix(tb21), tb21Cfg},
{keys.MakeTablePrefix(tb22), *config.DefaultZoneConfig},
}
for tcNum, tc := range testCases {
zoneCfg, err := cfg.GetZoneConfigForKey(tc.key)
if err != nil {
t.Fatalf("#%d: err=%s", tcNum, err)
}
if !reflect.DeepEqual(*zoneCfg, tc.zoneCfg) {
t.Errorf("#%d: bad zone config.\nexpected: %+v\ngot: %+v", tcNum, tc.zoneCfg, zoneCfg)
}
}
}
// TestMultiRangeMergeStaleDescriptor simulates the situation in which the
// DistSender executes a multi-range scan which encounters the stale descriptor
// of a range which has since incorporated its right neighbor by means of a
// merge. It is verified that the DistSender scans the correct keyrange exactly
// once.
func TestMultiRangeMergeStaleDescriptor(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
// Assume we have two ranges, [a-b) and [b-KeyMax).
merged := false
// The stale first range descriptor which is unaware of the merge.
var FirstRange = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKey("b"),
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
// The merged descriptor, which will be looked up after having processed
// the stale range [a,b).
var mergedRange = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKeyMax,
Replicas: []roachpb.ReplicaDescriptor{
{
NodeID: 1,
StoreID: 1,
},
},
}
// Assume we have two key-value pairs, a=1 and c=2.
existingKVs := []roachpb.KeyValue{
{Key: roachpb.Key("a"), Value: roachpb.MakeValueFromString("1")},
{Key: roachpb.Key("c"), Value: roachpb.MakeValueFromString("2")},
}
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method != "Node.Batch" {
t.Fatalf("unexpected method:%s", method)
}
ba := getArgs(testAddress).(*roachpb.BatchRequest)
rs := keys.Range(*ba)
batchReply := getReply().(*roachpb.BatchResponse)
reply := &roachpb.ScanResponse{}
batchReply.Add(reply)
results := []roachpb.KeyValue{}
for _, curKV := range existingKVs {
if rs.Key.Less(keys.Addr(curKV.Key).Next()) && keys.Addr(curKV.Key).Less(rs.EndKey) {
results = append(results, curKV)
}
}
reply.Rows = results
return []proto.Message{batchReply}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(key roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
if !merged {
// Assume a range merge operation happened.
merged = true
return []roachpb.RangeDescriptor{FirstRange}, nil
}
return []roachpb.RangeDescriptor{mergedRange}, nil
}),
}
ds := NewDistSender(ctx, g)
scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 10).(*roachpb.ScanRequest)
// Set the Txn info to avoid an OpRequiresTxnError.
reply, err := client.SendWrappedWith(ds, nil, roachpb.Header{
Txn: &roachpb.Transaction{},
}, scan)
if err != nil {
t.Fatalf("scan encountered error: %s", err)
}
sr := reply.(*roachpb.ScanResponse)
if !reflect.DeepEqual(existingKVs, sr.Rows) {
t.Fatalf("expect get %v, actual get %v", existingKVs, sr.Rows)
}
}
// TestStoreRangeDownReplicate verifies that the replication queue will notice
// over-replicated ranges and remove replicas from them.
func TestStoreRangeDownReplicate(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 5)
defer mtc.Stop()
store0 := mtc.stores[0]
// Split off a range from the initial range for testing; there are
// complications if the metadata ranges are removed from store 1, this
// simplifies the test.
splitKey := roachpb.Key("m")
rightKey := roachpb.Key("z")
{
replica := store0.LookupReplica(roachpb.RKeyMin, nil)
mtc.replicateRange(replica.Desc().RangeID, 0, 1, 2)
desc := replica.Desc()
splitArgs := adminSplitArgs(splitKey, splitKey)
if _, err := replica.AdminSplit(splitArgs, desc); err != nil {
t.Fatal(err)
}
}
// Replicate the new range to all five stores.
replica := store0.LookupReplica(keys.Addr(rightKey), nil)
desc := replica.Desc()
mtc.replicateRange(desc.RangeID, 0, 3, 4)
// Initialize the gossip network.
var wg sync.WaitGroup
wg.Add(len(mtc.stores))
key := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
mtc.stores[0].Gossip().RegisterCallback(key, func(_ string, _ roachpb.Value) { wg.Done() })
for _, s := range mtc.stores {
s.GossipStore()
}
wg.Wait()
// storeIDset is used to compare the replica sets from different views (i.e.
// local range descriptors)
type storeIDset map[roachpb.StoreID]struct{}
makeStoreIDset := func(replicas []roachpb.ReplicaDescriptor) storeIDset {
idSet := make(storeIDset)
for _, r := range replicas {
idSet[r.StoreID] = struct{}{}
}
return idSet
}
// Function to see if the replication level of the new range has reached the
// expected equilibrium. If equilibrium has not been reached, this function
// returns the list of stores that *should* have a replica for the range.
checkReplication := func() (bool, storeIDset) {
// Query each store for a replica of the range, generating a real map of
// the replicas.
foundIDset := make(storeIDset)
foundLocalRangeDescs := make([]*roachpb.RangeDescriptor, 0, len(mtc.stores))
for _, s := range mtc.stores {
r := s.LookupReplica(keys.Addr(splitKey), nil)
if r != nil {
foundLocalRangeDescs = append(foundLocalRangeDescs, r.Desc())
foundIDset[s.StoreID()] = struct{}{}
}
}
// Fail immediately if there are less than three replicas.
replicaCount := len(foundIDset)
if replicaCount < 3 {
t.Fatalf("Removed too many replicas; expected at least three replicas, found %d", replicaCount)
}
// Look up the official range descriptor, make sure it agrees with the
// found replicas.
realRangeDesc := getRangeMetadata(keys.Addr(rightKey), mtc, t)
realIDset := makeStoreIDset(realRangeDesc.Replicas)
if !reflect.DeepEqual(realIDset, foundIDset) {
return false, realIDset
}
// Ensure the local range descriptors everywhere agree with reality.
for _, desc := range foundLocalRangeDescs {
localIDset := makeStoreIDset(desc.Replicas)
if !reflect.DeepEqual(localIDset, foundIDset) {
return false, realIDset
}
}
// If we have only three replicas, exit the loop.
if replicaCount == 3 {
return true, nil
}
return false, foundIDset
}
maxTimeout := time.After(10 * time.Second)
succeeded := false
for !succeeded {
select {
case <-maxTimeout:
t.Fatalf("Failed to achieve proper replication within 10 seconds")
case <-time.After(10 * time.Millisecond):
mtc.expireLeaderLeases()
// If our replication level matches the target, we have succeeded.
var idSet storeIDset
succeeded, idSet = checkReplication()
if succeeded {
break
}
// Kick off a manual ReplicaGC Scan on any store which is not part of the
// current replica set. Kick off a Replication scan on *one* store which
// is part of the replica set.
kickedOffReplicationQueue := false
for _, store := range mtc.stores {
if _, ok := idSet[store.StoreID()]; !ok {
store.ForceReplicaGCScan(t)
} else if !kickedOffReplicationQueue {
store.ForceReplicationScan(t)
kickedOffReplicationQueue = true
}
}
}
}
// Expire leader leases one more time, so that any remaining resolutions can
// get a leader lease.
mtc.expireLeaderLeases()
}
// truncate restricts all contained requests to the given key range
// and returns a new BatchRequest.
// All requests contained in that batch are "truncated" to the given
// span, inserting NoopRequest appropriately to replace requests which
// are left without a key range to operate on. The number of non-noop
// requests after truncation is returned.
func truncate(ba roachpb.BatchRequest, rs roachpb.RSpan) (roachpb.BatchRequest, int, error) {
truncateOne := func(args roachpb.Request) (bool, roachpb.Span, error) {
if _, ok := args.(*roachpb.NoopRequest); ok {
return true, emptySpan, nil
}
header := *args.Header()
if !roachpb.IsRange(args) {
// This is a point request.
if len(header.EndKey) > 0 {
return false, emptySpan, util.Errorf("%T is not a range command, but EndKey is set", args)
}
if !rs.ContainsKey(keys.Addr(header.Key)) {
return false, emptySpan, nil
}
return true, header, nil
}
// We're dealing with a range-spanning request.
keyAddr, endKeyAddr := keys.Addr(header.Key), keys.Addr(header.EndKey)
if l, r := !keyAddr.Equal(header.Key), !endKeyAddr.Equal(header.EndKey); l || r {
if !rs.ContainsKeyRange(keyAddr, endKeyAddr) {
return false, emptySpan, util.Errorf("local key range must not span ranges")
}
if !l || !r {
return false, emptySpan, util.Errorf("local key mixed with global key in range")
}
// Range-local local key range.
return true, header, nil
}
// Below, {end,}keyAddr equals header.{End,}Key, so nothing is local.
if keyAddr.Less(rs.Key) {
header.Key = rs.Key.AsRawKey() // "key" can't be local
keyAddr = rs.Key
}
if !endKeyAddr.Less(rs.EndKey) {
header.EndKey = rs.EndKey.AsRawKey() // "endKey" can't be local
endKeyAddr = rs.EndKey
}
// Check whether the truncation has left any keys in the range. If not,
// we need to cut it out of the request.
if !keyAddr.Less(endKeyAddr) {
return false, emptySpan, nil
}
return true, header, nil
}
var numNoop int
origRequests := ba.Requests
ba.Requests = make([]roachpb.RequestUnion, len(ba.Requests))
for pos, arg := range origRequests {
hasRequest, newHeader, err := truncateOne(arg.GetInner())
if !hasRequest {
// We omit this one, i.e. replace it with a Noop.
numNoop++
nReq := roachpb.RequestUnion{}
if !nReq.SetValue(&roachpb.NoopRequest{}) {
panic("RequestUnion excludes NoopRequest")
}
ba.Requests[pos] = nReq
} else {
// Keep the old one. If we must adjust the header, must copy.
// TODO(tschottdorf): this could wind up cloning big chunks of data.
// Can optimize by creating a new Request manually, but with the old
// data.
if newHeader.Equal(*origRequests[pos].GetInner().Header()) {
ba.Requests[pos] = origRequests[pos]
} else {
ba.Requests[pos] = *proto.Clone(&origRequests[pos]).(*roachpb.RequestUnion)
*ba.Requests[pos].GetInner().Header() = newHeader
}
}
if err != nil {
return roachpb.BatchRequest{}, 0, err
}
}
return ba, len(ba.Requests) - numNoop, nil
}
// truncate restricts all contained requests to the given key range
// and returns a new BatchRequest.
// All requests contained in that batch are "truncated" to the given
// span, inserting NoopRequest appropriately to replace requests which
// are left without a key range to operate on. The number of non-noop
// requests after truncation is returned.
func truncate(ba roachpb.BatchRequest, rs roachpb.RSpan) (roachpb.BatchRequest, int, error) {
truncateOne := func(args roachpb.Request) (bool, roachpb.Span, error) {
if _, ok := args.(*roachpb.NoopRequest); ok {
return true, emptySpan, nil
}
header := args.Header()
if !roachpb.IsRange(args) {
// This is a point request.
if len(header.EndKey) > 0 {
return false, emptySpan, errors.Errorf("%T is not a range command, but EndKey is set", args)
}
keyAddr, err := keys.Addr(header.Key)
if err != nil {
return false, emptySpan, err
}
if !rs.ContainsKey(keyAddr) {
return false, emptySpan, nil
}
return true, header, nil
}
// We're dealing with a range-spanning request.
local := false
keyAddr, err := keys.Addr(header.Key)
if err != nil {
return false, emptySpan, err
}
endKeyAddr, err := keys.Addr(header.EndKey)
if err != nil {
return false, emptySpan, err
}
if l, r := !keyAddr.Equal(header.Key), !endKeyAddr.Equal(header.EndKey); l || r {
if !l || !r {
return false, emptySpan, errors.Errorf("local key mixed with global key in range")
}
local = true
}
if keyAddr.Less(rs.Key) {
// rs.Key can't be local because it contains range split points, which
// are never local.
if !local {
header.Key = rs.Key.AsRawKey()
} else {
// The local start key should be truncated to the boundary of local keys which
// address to rs.Key.
header.Key = keys.MakeRangeKeyPrefix(rs.Key)
}
}
if !endKeyAddr.Less(rs.EndKey) {
// rs.EndKey can't be local because it contains range split points, which
// are never local.
if !local {
header.EndKey = rs.EndKey.AsRawKey()
} else {
// The local end key should be truncated to the boundary of local keys which
// address to rs.EndKey.
header.EndKey = keys.MakeRangeKeyPrefix(rs.EndKey)
}
}
// Check whether the truncation has left any keys in the range. If not,
// we need to cut it out of the request.
if header.Key.Compare(header.EndKey) >= 0 {
return false, emptySpan, nil
}
return true, header, nil
}
var numNoop int
origRequests := ba.Requests
ba.Requests = make([]roachpb.RequestUnion, len(ba.Requests))
for pos, arg := range origRequests {
hasRequest, newHeader, err := truncateOne(arg.GetInner())
if !hasRequest {
// We omit this one, i.e. replace it with a Noop.
numNoop++
union := roachpb.RequestUnion{}
union.MustSetInner(&noopRequest)
ba.Requests[pos] = union
} else {
// Keep the old one. If we must adjust the header, must copy.
if inner := origRequests[pos].GetInner(); newHeader.Equal(inner.Header()) {
ba.Requests[pos] = origRequests[pos]
} else {
shallowCopy := inner.ShallowCopy()
shallowCopy.SetHeader(newHeader)
union := &ba.Requests[pos] // avoid operating on copy
union.MustSetInner(shallowCopy)
}
}
if err != nil {
return roachpb.BatchRequest{}, 0, err
}
}
return ba, len(ba.Requests) - numNoop, nil
}
// TableStats is an endpoint that returns columns, indices, and other
// relevant details for the specified table.
func (s *adminServer) TableStats(ctx context.Context, req *serverpb.TableStatsRequest) (
*serverpb.TableStatsResponse, error,
) {
// Get table span.
var tableSpan roachpb.Span
var iexecutor sql.InternalExecutor
if err := s.server.db.Txn(func(txn *client.Txn) error {
var err error
tableSpan, err = iexecutor.GetTableSpan(s.getUser(req), txn, req.Database, req.Table)
return err
}); err != nil {
return nil, s.serverError(err)
}
startKey, err := keys.Addr(tableSpan.Key)
if err != nil {
return nil, s.serverError(err)
}
endKey, err := keys.Addr(tableSpan.EndKey)
if err != nil {
return nil, s.serverError(err)
}
// Get current range descriptors for table. This is done by scanning over
// meta2 keys for the range.
rangeDescKVs, err := s.server.db.Scan(keys.RangeMetaKey(startKey), keys.RangeMetaKey(endKey), 0)
if err != nil {
return nil, s.serverError(err)
}
// Extract a list of node IDs from the response.
nodeIDs := make(map[roachpb.NodeID]struct{})
for _, kv := range rangeDescKVs {
var rng roachpb.RangeDescriptor
if err := kv.Value.GetProto(&rng); err != nil {
return nil, s.serverError(err)
}
for _, repl := range rng.Replicas {
nodeIDs[repl.NodeID] = struct{}{}
}
}
// Construct TableStatsResponse by sending an RPC to every node involved.
tableStatResponse := serverpb.TableStatsResponse{
NodeCount: int64(len(nodeIDs)),
RangeCount: int64(len(rangeDescKVs)),
}
type nodeResponse struct {
nodeID roachpb.NodeID
resp *serverpb.SpanStatsResponse
err error
}
// Send a SpanStats query to each node. Set a timeout on the context for
// these queries.
responses := make(chan nodeResponse)
ctx, cancel := context.WithTimeout(ctx, base.NetworkTimeout)
defer cancel()
for nodeID := range nodeIDs {
nodeID := nodeID
if err := s.server.stopper.RunAsyncTask(func() {
var spanResponse *serverpb.SpanStatsResponse
client, err := s.server.status.dialNode(nodeID)
if err == nil {
req := serverpb.SpanStatsRequest{
StartKey: startKey,
EndKey: endKey,
NodeID: nodeID.String(),
}
spanResponse, err = client.SpanStats(ctx, &req)
}
response := nodeResponse{
nodeID: nodeID,
resp: spanResponse,
err: err,
}
select {
case responses <- response:
// Response processed.
case <-ctx.Done():
// Context completed, response no longer needed.
}
}); err != nil {
return nil, err
}
}
for remainingResponses := len(nodeIDs); remainingResponses > 0; remainingResponses-- {
select {
case resp := <-responses:
// For nodes which returned an error, note that the node's data
// is missing. For successful calls, aggregate statistics.
if resp.err != nil {
tableStatResponse.MissingNodes = append(
tableStatResponse.MissingNodes,
serverpb.TableStatsResponse_MissingNode{
NodeID: resp.nodeID.String(),
ErrorMessage: resp.err.Error(),
},
)
} else {
tableStatResponse.Stats.Add(resp.resp.TotalStats)
tableStatResponse.ReplicaCount += int64(resp.resp.RangeCount)
}
case <-ctx.Done():
return nil, ctx.Err()
}
}
return &tableStatResponse, nil
}
// TestStoreSplitReadRace prevents regression of #3148. It begins a couple of
// read requests and lets them complete while a split is happening; the reads
// hit the second half of the split. If the split happens non-atomically with
// respect to the reads (and in particular their update of the timestamp
// cache), then some of them may not be reflected in the timestamp cache of the
// new range, in which case this test would fail.
func TestStoreSplitReadRace(t *testing.T) {
defer leaktest.AfterTest(t)()
defer config.TestingDisableTableSplits()()
splitKey := roachpb.Key("a")
key := func(i int) roachpb.Key {
splitCopy := append([]byte(nil), splitKey.Next()...)
return append(splitCopy, []byte(fmt.Sprintf("%03d", i))...)
}
getContinues := make(chan struct{})
var getStarted sync.WaitGroup
sCtx := storage.TestStoreContext()
sCtx.TestingKnobs.TestingCommandFilter =
func(filterArgs storagebase.FilterArgs) *roachpb.Error {
if et, ok := filterArgs.Req.(*roachpb.EndTransactionRequest); ok {
st := et.InternalCommitTrigger.GetSplitTrigger()
if st == nil || !st.UpdatedDesc.EndKey.Equal(splitKey) {
return nil
}
close(getContinues)
} else if filterArgs.Req.Method() == roachpb.Get &&
bytes.HasPrefix(filterArgs.Req.Header().Key, splitKey.Next()) {
getStarted.Done()
<-getContinues
}
return nil
}
store, stopper, _ := createTestStoreWithContext(t, &sCtx)
defer stopper.Stop()
now := store.Clock().Now()
var wg sync.WaitGroup
ts := func(i int) hlc.Timestamp {
return now.Add(0, int32(1000+i))
}
const num = 10
for i := 0; i < num; i++ {
wg.Add(1)
getStarted.Add(1)
go func(i int) {
defer wg.Done()
args := getArgs(key(i))
var h roachpb.Header
h.Timestamp = ts(i)
if _, pErr := client.SendWrappedWith(rg1(store), nil, h, &args); pErr != nil {
t.Fatal(pErr)
}
}(i)
}
getStarted.Wait()
wg.Add(1)
func() {
defer wg.Done()
args := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, pErr := client.SendWrapped(rg1(store), nil, &args); pErr != nil {
t.Fatal(pErr)
}
}()
wg.Wait()
for i := 0; i < num; i++ {
var h roachpb.Header
h.Timestamp = now
args := putArgs(key(i), []byte("foo"))
keyAddr, err := keys.Addr(args.Key)
if err != nil {
t.Fatal(err)
}
h.RangeID = store.LookupReplica(keyAddr, nil).RangeID
_, respH, pErr := storage.SendWrapped(store, context.Background(), h, &args)
if pErr != nil {
t.Fatal(pErr)
}
if respH.Timestamp.Less(ts(i)) {
t.Fatalf("%d: expected Put to be forced higher than %s by timestamp caches, but wrote at %s", i, ts(i), respH.Timestamp)
}
}
}
func metaKey(key roachpb.RKey) []byte {
return keys.Addr(keys.RangeMetaKey(key))
}
// TestStoreSplitReadRace prevents regression of #3148. It begins a couple of
// read requests and lets them complete while a split is happening; the reads
// hit the second half of the split. If the split happens non-atomically with
// respect to the reads (and in particular their update of the timestamp
// cache), then some of them may not be reflected in the timestamp cache of the
// new range, in which case this test would fail.
func TestStoreSplitReadRace(t *testing.T) {
defer leaktest.AfterTest(t)
defer func() { storage.TestingCommandFilter = nil }()
splitKey := roachpb.Key("a")
key := func(i int) roachpb.Key {
return append(splitKey.Next(), []byte(fmt.Sprintf("%03d", i))...)
}
getContinues := make(chan struct{})
var getStarted sync.WaitGroup
storage.TestingCommandFilter = func(_ roachpb.StoreID, args roachpb.Request, h roachpb.Header) error {
if et, ok := args.(*roachpb.EndTransactionRequest); ok {
st := et.InternalCommitTrigger.GetSplitTrigger()
if st == nil || !st.UpdatedDesc.EndKey.Equal(splitKey) {
return nil
}
close(getContinues)
} else if args.Method() == roachpb.Get &&
bytes.HasPrefix(args.Header().Key, splitKey.Next()) {
getStarted.Done()
<-getContinues
}
return nil
}
store, stopper := createTestStore(t)
defer stopper.Stop()
now := store.Clock().Now()
var wg sync.WaitGroup
ts := func(i int) roachpb.Timestamp {
return now.Add(0, int32(1000+i))
}
const num = 10
for i := 0; i < num; i++ {
wg.Add(1)
getStarted.Add(1)
go func(i int) {
defer wg.Done()
args := getArgs(key(i))
var h roachpb.Header
h.Timestamp = ts(i)
if _, err := client.SendWrappedWith(rg1(store), nil, h, &args); err != nil {
t.Fatal(err)
}
}(i)
}
getStarted.Wait()
wg.Add(1)
func() {
defer wg.Done()
args := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, err := client.SendWrapped(rg1(store), nil, &args); err != nil {
t.Fatal(err)
}
}()
wg.Wait()
for i := 0; i < num; i++ {
var h roachpb.Header
h.Timestamp = now
args := putArgs(key(i), []byte("foo"))
h.RangeID = store.LookupReplica(keys.Addr(args.Key), nil).Desc().RangeID
reply, err := client.SendWrappedWith(store, nil, h, &args)
if err != nil {
t.Fatal(err)
}
if reply.Header().Timestamp.Less(ts(i)) {
t.Fatalf("%d: expected Put to be forced higher than %s by timestamp caches, but wrote at %s", i, ts(i), reply.Header().Timestamp)
}
}
}
// Test that a lease extension (a RequestLeaseRequest that doesn't change the
// lease holder) is not blocked by ongoing reads.
// Note that lease transfers are blocked by reads through their
// PostCommitTrigger.noConcurrentReads.
func TestLeaseExtensionNotBlockedByRead(t *testing.T) {
defer leaktest.AfterTest(t)()
readBlocked := make(chan struct{})
cmdFilter := func(fArgs storagebase.FilterArgs) *roachpb.Error {
if fArgs.Hdr.UserPriority == 42 {
// Signal that the read is blocked.
readBlocked <- struct{}{}
// Wait for read to be unblocked.
<-readBlocked
}
return nil
}
srv, _, _ := serverutils.StartServer(t,
base.TestServerArgs{
Knobs: base.TestingKnobs{
Store: &storage.StoreTestingKnobs{
TestingCommandFilter: cmdFilter,
},
},
})
s := srv.(*server.TestServer)
defer s.Stopper().Stop()
// Start a read and wait for it to block.
key := roachpb.Key("a")
errChan := make(chan error)
go func() {
getReq := roachpb.GetRequest{
Span: roachpb.Span{
Key: key,
},
}
if _, pErr := client.SendWrappedWith(s.GetDistSender(), nil,
roachpb.Header{UserPriority: 42},
&getReq); pErr != nil {
errChan <- pErr.GoError()
}
}()
select {
case err := <-errChan:
t.Fatal(err)
case <-readBlocked:
// Send the lease request.
// We change the key slightly, otherwise the lease request will be blocked
// by the read through the command queue.
// TODO(andrei): don't change the key anymore once lease requests don't go
// through the command queue any more.
leaseHdrKey := roachpb.Key(append(key, 0x00))
rKey, err := keys.Addr(leaseHdrKey)
if err != nil {
t.Fatal(err)
}
_, repDesc, err := s.Stores().LookupReplica(rKey, nil)
if err != nil {
t.Fatal(err)
}
leaseReq := roachpb.RequestLeaseRequest{
Span: roachpb.Span{
Key: leaseHdrKey,
},
Lease: roachpb.Lease{
Start: s.Clock().Now(),
StartStasis: s.Clock().Now().Add(time.Second.Nanoseconds(), 0),
Expiration: s.Clock().Now().Add(2*time.Second.Nanoseconds(), 0),
Replica: repDesc,
},
}
if _, pErr := client.SendWrapped(s.GetDistSender(), nil, &leaseReq); pErr != nil {
t.Fatal(pErr)
}
// Unblock the read.
readBlocked <- struct{}{}
}
}
// TableDetails is an endpoint that returns columns, indices, and other
// relevant details for the specified table.
func (s *adminServer) TableDetails(
ctx context.Context, req *serverpb.TableDetailsRequest,
) (*serverpb.TableDetailsResponse, error) {
args := sql.SessionArgs{User: s.getUser(req)}
session := sql.NewSession(ctx, args, s.server.sqlExecutor, nil)
// TODO(cdo): Use real placeholders for the table and database names when we've extended our SQL
// grammar to allow that.
escDBName := parser.Name(req.Database).String()
escTableName := parser.Name(req.Table).String()
escQualTable := fmt.Sprintf("%s.%s", escDBName, escTableName)
query := fmt.Sprintf("SHOW COLUMNS FROM %s; SHOW INDEX FROM %s; SHOW GRANTS ON TABLE %s; SHOW CREATE TABLE %s;",
escQualTable, escQualTable, escQualTable, escQualTable)
r := s.server.sqlExecutor.ExecuteStatements(session, query, nil)
if err := s.firstNotFoundError(r.ResultList); err != nil {
return nil, grpc.Errorf(codes.NotFound, "%s", err)
}
if err := s.checkQueryResults(r.ResultList, 4); err != nil {
return nil, err
}
var resp serverpb.TableDetailsResponse
// Marshal SHOW COLUMNS result.
//
// TODO(cdo): protobuf v3's default behavior for fields with zero values (e.g. empty strings)
// is to suppress them. So, if protobuf field "foo" is an empty string, "foo" won't show
// up in the marshalled JSON. I feel that this is counterintuitive, and this should be fixed
// for our API.
{
const (
fieldCol = "Field" // column name
typeCol = "Type"
nullCol = "Null"
defaultCol = "Default"
)
scanner := makeResultScanner(r.ResultList[0].Columns)
for _, row := range r.ResultList[0].Rows {
var col serverpb.TableDetailsResponse_Column
if err := scanner.Scan(row, fieldCol, &col.Name); err != nil {
return nil, err
}
if err := scanner.Scan(row, typeCol, &col.Type); err != nil {
return nil, err
}
if err := scanner.Scan(row, nullCol, &col.Nullable); err != nil {
return nil, err
}
isDefaultNull, err := scanner.IsNull(row, defaultCol)
if err != nil {
return nil, err
}
if !isDefaultNull {
if err := scanner.Scan(row, defaultCol, &col.DefaultValue); err != nil {
return nil, err
}
}
resp.Columns = append(resp.Columns, col)
}
}
// Marshal SHOW INDEX result.
{
const (
nameCol = "Name"
uniqueCol = "Unique"
seqCol = "Seq"
columnCol = "Column"
directionCol = "Direction"
storingCol = "Storing"
)
scanner := makeResultScanner(r.ResultList[1].Columns)
for _, row := range r.ResultList[1].Rows {
// Marshal grant, splitting comma-separated privileges into a proper slice.
var index serverpb.TableDetailsResponse_Index
if err := scanner.Scan(row, nameCol, &index.Name); err != nil {
return nil, err
}
if err := scanner.Scan(row, uniqueCol, &index.Unique); err != nil {
return nil, err
}
if err := scanner.Scan(row, seqCol, &index.Seq); err != nil {
return nil, err
}
if err := scanner.Scan(row, columnCol, &index.Column); err != nil {
return nil, err
}
if err := scanner.Scan(row, directionCol, &index.Direction); err != nil {
return nil, err
}
if err := scanner.Scan(row, storingCol, &index.Storing); err != nil {
return nil, err
}
resp.Indexes = append(resp.Indexes, index)
}
}
// Marshal SHOW GRANTS result.
{
const (
userCol = "User"
privilegesCol = "Privileges"
)
scanner := makeResultScanner(r.ResultList[2].Columns)
for _, row := range r.ResultList[2].Rows {
// Marshal grant, splitting comma-separated privileges into a proper slice.
var grant serverpb.TableDetailsResponse_Grant
var privileges string
if err := scanner.Scan(row, userCol, &grant.User); err != nil {
return nil, err
}
if err := scanner.Scan(row, privilegesCol, &privileges); err != nil {
return nil, err
}
grant.Privileges = strings.Split(privileges, ",")
resp.Grants = append(resp.Grants, grant)
}
}
// Marshal SHOW CREATE TABLE result.
{
const createTableCol = "CreateTable"
showResult := r.ResultList[3]
if len(showResult.Rows) != 1 {
return nil, s.serverErrorf("CreateTable response not available.")
}
scanner := makeResultScanner(showResult.Columns)
var createStmt string
if err := scanner.Scan(showResult.Rows[0], createTableCol, &createStmt); err != nil {
return nil, err
}
resp.CreateTableStatement = createStmt
}
// Get the number of ranges in the table. We get the key span for the table
// data. Then, we count the number of ranges that make up that key span.
{
var iexecutor sql.InternalExecutor
var tableSpan roachpb.Span
if err := s.server.db.Txn(func(txn *client.Txn) error {
var err error
tableSpan, err = iexecutor.GetTableSpan(s.getUser(req), txn, escDBName, escTableName)
return err
}); err != nil {
return nil, s.serverError(err)
}
tableRSpan := roachpb.RSpan{}
var err error
tableRSpan.Key, err = keys.Addr(tableSpan.Key)
if err != nil {
return nil, s.serverError(err)
}
tableRSpan.EndKey, err = keys.Addr(tableSpan.EndKey)
if err != nil {
return nil, s.serverError(err)
}
rangeCount, err := s.server.distSender.CountRanges(tableRSpan)
if err != nil {
return nil, s.serverError(err)
}
resp.RangeCount = rangeCount
}
// Query the zone configuration for this table.
{
path, err := s.queryDescriptorIDPath(session, []string{escDBName, escTableName})
if err != nil {
return nil, s.serverError(err)
}
id, zone, zoneExists, err := s.queryZonePath(session, path)
if err != nil {
return nil, s.serverError(err)
}
if !zoneExists {
zone = config.DefaultZoneConfig()
}
resp.ZoneConfig = zone
switch id {
case path[1]:
resp.ZoneConfigLevel = serverpb.ZoneConfigurationLevel_DATABASE
case path[2]:
resp.ZoneConfigLevel = serverpb.ZoneConfigurationLevel_TABLE
default:
resp.ZoneConfigLevel = serverpb.ZoneConfigurationLevel_CLUSTER
}
}
return &resp, nil
}
// TestStoreRangeDownReplicate verifies that the replication queue will notice
// over-replicated ranges and remove replicas from them.
func TestStoreRangeDownReplicate(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 5)
defer mtc.Stop()
store0 := mtc.stores[0]
// Split off a range from the initial range for testing; there are
// complications if the metadata ranges are removed from store 1, this
// simplifies the test.
splitKey := roachpb.Key("m")
rightKey := roachpb.Key("z")
{
replica := store0.LookupReplica(roachpb.RKeyMin, nil)
mtc.replicateRange(replica.RangeID, 1, 2)
desc := replica.Desc()
splitArgs := adminSplitArgs(splitKey, splitKey)
if _, err := replica.AdminSplit(splitArgs, desc); err != nil {
t.Fatal(err)
}
}
// Replicate the new range to all five stores.
replica := store0.LookupReplica(keys.Addr(rightKey), nil)
desc := replica.Desc()
mtc.replicateRange(desc.RangeID, 3, 4)
// Initialize the gossip network.
var wg sync.WaitGroup
wg.Add(len(mtc.stores))
key := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
mtc.stores[0].Gossip().RegisterCallback(key, func(_ string, _ roachpb.Value) { wg.Done() })
for _, s := range mtc.stores {
s.GossipStore()
}
wg.Wait()
maxTimeout := time.After(10 * time.Second)
succeeded := false
for !succeeded {
select {
case <-maxTimeout:
t.Fatalf("Failed to achieve proper replication within 10 seconds")
case <-time.After(10 * time.Millisecond):
mtc.expireLeaderLeases()
rangeDesc := getRangeMetadata(keys.Addr(rightKey), mtc, t)
if count := len(rangeDesc.Replicas); count < 3 {
t.Fatalf("Removed too many replicas; expected at least 3 replicas, found %d", count)
} else if count == 3 {
succeeded = true
break
}
// Run replication scans on every store; only the store with the
// leader lease will actually do anything. If we did not wait
// for the scan to complete here it could be interrupted by the
// next call to expireLeaderLeases.
for _, store := range mtc.stores {
store.ForceReplicationScanAndProcess()
}
}
}
// Expire leader leases one more time, so that any remaining resolutions can
// get a leader lease.
// TODO(bdarnell): understand why some tests need this.
mtc.expireLeaderLeases()
}
// truncate restricts all contained requests to the given key range.
// Even on error, the returned closure must be executed; it undoes any
// truncations performed.
// All requests contained in the batch are "truncated" to the given
// span, inserting NoopRequest appropriately to replace requests which
// are left without a key range to operate on. The number of non-noop
// requests after truncation is returned along with a closure which
// must be executed to undo the truncation, even in case of an error.
// TODO(tschottdorf): Consider returning a new BatchRequest, which has more
// overhead in the common case of a batch which never needs truncation but is
// less magical.
func truncate(br *roachpb.BatchRequest, rs roachpb.RSpan) (func(), int, error) {
truncateOne := func(args roachpb.Request) (bool, []func(), error) {
if _, ok := args.(*roachpb.NoopRequest); ok {
return true, nil, nil
}
header := args.Header()
if !roachpb.IsRange(args) {
// This is a point request.
if len(header.EndKey) > 0 {
return false, nil, util.Errorf("%T is not a range command, but EndKey is set", args)
}
if !rs.ContainsKey(keys.Addr(header.Key)) {
return true, nil, nil
}
return false, nil, nil
}
// We're dealing with a range-spanning request.
var undo []func()
keyAddr, endKeyAddr := keys.Addr(header.Key), keys.Addr(header.EndKey)
if l, r := !keyAddr.Equal(header.Key), !endKeyAddr.Equal(header.EndKey); l || r {
if !rs.ContainsKeyRange(keyAddr, endKeyAddr) {
return false, nil, util.Errorf("local key range must not span ranges")
}
if !l || !r {
return false, nil, util.Errorf("local key mixed with global key in range")
}
return false, nil, nil
}
// Below, {end,}keyAddr equals header.{End,}Key, so nothing is local.
if keyAddr.Less(rs.Key) {
{
origKey := header.Key
undo = append(undo, func() { header.Key = origKey })
}
header.Key = rs.Key.AsRawKey() // "key" can't be local
keyAddr = rs.Key
}
if !endKeyAddr.Less(rs.EndKey) {
{
origEndKey := header.EndKey
undo = append(undo, func() { header.EndKey = origEndKey })
}
header.EndKey = rs.EndKey.AsRawKey() // "endKey" can't be local
endKeyAddr = rs.EndKey
}
// Check whether the truncation has left any keys in the range. If not,
// we need to cut it out of the request.
return !keyAddr.Less(endKeyAddr), undo, nil
}
var fns []func()
gUndo := func() {
for _, f := range fns {
f()
}
}
var numNoop int
for pos, arg := range br.Requests {
omit, undo, err := truncateOne(arg.GetInner())
if omit {
numNoop++
nReq := &roachpb.RequestUnion{}
if !nReq.SetValue(&roachpb.NoopRequest{}) {
panic("RequestUnion excludes NoopRequest")
}
oReq := br.Requests[pos]
br.Requests[pos] = *nReq
posCpy := pos // for closure
undo = append(undo, func() {
br.Requests[posCpy] = oReq
})
}
fns = append(fns, undo...)
if err != nil {
return gUndo, 0, err
}
}
return gUndo, len(br.Requests) - numNoop, nil
}
// TableDetails is an endpoint that returns columns, indices, and other
// relevant details for the specified table.
func (s *adminServer) TableDetails(ctx context.Context, req *TableDetailsRequest) (
*TableDetailsResponse, error) {
session := sql.NewSession(sql.SessionArgs{User: s.getUser(req)}, s.sqlExecutor, nil)
// TODO(cdo): Use real placeholders for the table and database names when we've extended our SQL
// grammar to allow that.
escDbName := parser.Name(req.Database).String()
escTableName := parser.Name(req.Table).String()
escQualTable := fmt.Sprintf("%s.%s", escDbName, escTableName)
query := fmt.Sprintf("SHOW COLUMNS FROM %s; SHOW INDEX FROM %s; SHOW GRANTS ON TABLE %s",
escQualTable, escQualTable, escQualTable)
r := s.sqlExecutor.ExecuteStatements(ctx, session, query, nil)
if pErr := s.firstNotFoundError(r.ResultList); pErr != nil {
return nil, grpc.Errorf(codes.NotFound, "%s", pErr)
}
if err := s.checkQueryResults(r.ResultList, 3); err != nil {
return nil, err
}
var resp TableDetailsResponse
// Marshal SHOW COLUMNS result.
//
// TODO(cdo): protobuf v3's default behavior for fields with zero values (e.g. empty strings)
// is to suppress them. So, if protobuf field "foo" is an empty string, "foo" won't show
// up in the marshalled JSON. I feel that this is counterintuitive, and this should be fixed
// for our API.
{
const (
fieldCol = "Field" // column name
typeCol = "Type"
nullCol = "Null"
defaultCol = "Default"
)
scanner := makeResultScanner(r.ResultList[0].Columns)
for _, row := range r.ResultList[0].Rows {
var col TableDetailsResponse_Column
if err := scanner.Scan(row, fieldCol, &col.Name); err != nil {
return nil, err
}
if err := scanner.Scan(row, typeCol, &col.Type); err != nil {
return nil, err
}
if err := scanner.Scan(row, nullCol, &col.Nullable); err != nil {
return nil, err
}
isDefaultNull, err := scanner.IsNull(row, defaultCol)
if err != nil {
return nil, err
}
if !isDefaultNull {
if err := scanner.Scan(row, defaultCol, &col.Default); err != nil {
return nil, err
}
}
resp.Columns = append(resp.Columns, col)
}
}
// Marshal SHOW INDEX result.
{
const (
nameCol = "Name"
uniqueCol = "Unique"
seqCol = "Seq"
columnCol = "Column"
directionCol = "Direction"
storingCol = "Storing"
)
scanner := makeResultScanner(r.ResultList[1].Columns)
for _, row := range r.ResultList[1].Rows {
// Marshal grant, splitting comma-separated privileges into a proper slice.
var index TableDetailsResponse_Index
if err := scanner.Scan(row, nameCol, &index.Name); err != nil {
return nil, err
}
if err := scanner.Scan(row, uniqueCol, &index.Unique); err != nil {
return nil, err
}
if err := scanner.Scan(row, seqCol, &index.Seq); err != nil {
return nil, err
}
if err := scanner.Scan(row, columnCol, &index.Column); err != nil {
return nil, err
}
if err := scanner.Scan(row, directionCol, &index.Direction); err != nil {
return nil, err
}
if err := scanner.Scan(row, storingCol, &index.Storing); err != nil {
return nil, err
}
resp.Indexes = append(resp.Indexes, index)
}
}
// Marshal SHOW GRANTS result.
{
const (
userCol = "User"
privilegesCol = "Privileges"
)
scanner := makeResultScanner(r.ResultList[2].Columns)
for _, row := range r.ResultList[2].Rows {
// Marshal grant, splitting comma-separated privileges into a proper slice.
var grant TableDetailsResponse_Grant
var privileges string
if err := scanner.Scan(row, userCol, &grant.User); err != nil {
return nil, err
}
if err := scanner.Scan(row, privilegesCol, &privileges); err != nil {
return nil, err
}
grant.Privileges = strings.Split(privileges, ",")
resp.Grants = append(resp.Grants, grant)
}
}
// Get the number of ranges in the table. We get the key span for the table
// data. Then, we count the number of ranges that make up that key span.
{
var iexecutor sql.InternalExecutor
var tableSpan roachpb.Span
if pErr := s.db.Txn(func(txn *client.Txn) *roachpb.Error {
var pErr *roachpb.Error
tableSpan, pErr = iexecutor.GetTableSpan(s.getUser(req), txn, escDbName, escTableName)
return pErr
}); pErr != nil {
return nil, s.serverError(pErr.GoError())
}
tableRSpan := roachpb.RSpan{}
var err error
tableRSpan.Key, err = keys.Addr(tableSpan.Key)
if err != nil {
return nil, s.serverError(err)
}
tableRSpan.EndKey, err = keys.Addr(tableSpan.EndKey)
if err != nil {
return nil, s.serverError(err)
}
rangeCount, pErr := s.distSender.CountRanges(tableRSpan)
if pErr != nil {
return nil, s.serverError(pErr.GoError())
}
resp.RangeCount = rangeCount
}
return &resp, nil
}
func meta(k roachpb.RKey) (roachpb.RKey, error) {
return keys.Addr(keys.RangeMetaKey(k))
}