func (z *zeroSum) monkey(tableID uint32, d time.Duration) {
r := newRand()
zipf := z.accountDistribution(r)
for {
time.Sleep(time.Duration(rand.Float64() * float64(d)))
key := keys.MakeTablePrefix(tableID)
key = encoding.EncodeVarintAscending(key, int64(zipf.Uint64()))
key = keys.MakeRowSentinelKey(key)
switch r.Intn(2) {
case 0:
if err := z.Split(z.RandNode(r.Intn), key); err != nil {
if strings.Contains(err.Error(), "range is already split at key") ||
strings.Contains(err.Error(), storage.ErrMsgConflictUpdatingRangeDesc) {
continue
}
z.maybeLogError(err)
} else {
atomic.AddUint64(&z.stats.splits, 1)
}
case 1:
if transferred, err := z.TransferLease(z.RandNode(r.Intn), r, key); err != nil {
z.maybeLogError(err)
} else if transferred {
atomic.AddUint64(&z.stats.transfers, 1)
}
}
}
}
func TestSplitAtTableBoundary(t *testing.T) {
defer leaktest.AfterTest(t)()
testClusterArgs := base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
}
tc := testcluster.StartTestCluster(t, 3, testClusterArgs)
defer tc.Stopper().Stop()
runner := sqlutils.MakeSQLRunner(t, tc.Conns[0])
runner.Exec(`CREATE DATABASE test`)
runner.Exec(`CREATE TABLE test.t (k SERIAL PRIMARY KEY, v INT)`)
const tableIDQuery = `
SELECT tables.id FROM system.namespace tables
JOIN system.namespace dbs ON dbs.id = tables.parentid
WHERE dbs.name = $1 AND tables.name = $2
`
var tableID uint32
runner.QueryRow(tableIDQuery, "test", "t").Scan(&tableID)
tableStartKey := keys.MakeTablePrefix(tableID)
// Wait for new table to split.
testutils.SucceedsSoon(t, func() error {
desc, err := tc.LookupRange(keys.MakeRowSentinelKey(tableStartKey))
if err != nil {
t.Fatal(err)
}
if !desc.StartKey.Equal(tableStartKey) {
log.Infof(context.TODO(), "waiting on split results")
return errors.Errorf("expected range start key %s; got %s", tableStartKey, desc.StartKey)
}
return nil
})
}
// EncodeSecondaryIndex encodes key/values for a secondary index. colMap maps
// ColumnIDs to indices in `values`.
func EncodeSecondaryIndex(
tableDesc *TableDescriptor,
secondaryIndex *IndexDescriptor,
colMap map[ColumnID]int,
values []parser.Datum,
) (IndexEntry, error) {
secondaryIndexKeyPrefix := MakeIndexKeyPrefix(tableDesc, secondaryIndex.ID)
secondaryIndexKey, containsNull, err := EncodeIndexKey(
tableDesc, secondaryIndex, colMap, values, secondaryIndexKeyPrefix)
if err != nil {
return IndexEntry{}, err
}
// Add the implicit columns - they are encoded ascendingly which is done by
// passing nil for the encoding directions.
extraKey, _, err := EncodeColumns(secondaryIndex.ImplicitColumnIDs, nil,
colMap, values, nil)
if err != nil {
return IndexEntry{}, err
}
entry := IndexEntry{Key: secondaryIndexKey}
if !secondaryIndex.Unique || containsNull {
// If the index is not unique or it contains a NULL value, append
// extraKey to the key in order to make it unique.
entry.Key = append(entry.Key, extraKey...)
}
// Index keys are considered "sentinel" keys in that they do not have a
// column ID suffix.
entry.Key = keys.MakeRowSentinelKey(entry.Key)
if secondaryIndex.Unique {
// Note that a unique secondary index that contains a NULL column value
// will have extraKey appended to the key and stored in the value. We
// require extraKey to be appended to the key in order to make the key
// unique. We could potentially get rid of the duplication here but at
// the expense of complicating scanNode when dealing with unique
// secondary indexes.
entry.Value.SetBytes(extraKey)
} else {
// The zero value for an index-key is a 0-length bytes value.
entry.Value.SetBytes([]byte{})
}
return entry, nil
}
func fillTestRange(t testing.TB, rep *Replica, size int64) {
src := rand.New(rand.NewSource(0))
for i := int64(0); i < size/int64(keySize+valSize); i++ {
key := keys.MakeRowSentinelKey(randutil.RandBytes(src, keySize))
val := randutil.RandBytes(src, valSize)
pArgs := putArgs(key, val)
if _, pErr := client.SendWrappedWith(context.Background(), rep, roachpb.Header{
RangeID: rangeID,
}, &pArgs); pErr != nil {
t.Fatal(pErr)
}
}
rep.mu.Lock()
after := rep.mu.state.Stats.Total()
rep.mu.Unlock()
if after < size {
t.Fatalf("range not full after filling: wrote %d, but range at %d", size, after)
}
}
// splitRangeAtVal splits the range for a table with schema
// `CREATE TABLE test (k INT PRIMARY KEY)` at row with value pk (the row will be
// the first on the right of the split).
func splitRangeAtVal(
ts *server.TestServer, tableDesc *sqlbase.TableDescriptor, pk int,
) (roachpb.RangeDescriptor, roachpb.RangeDescriptor, error) {
if len(tableDesc.Indexes) != 0 {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{},
errors.Errorf("expected table with just a PK, got: %+v", tableDesc)
}
pik, err := sqlbase.MakePrimaryIndexKey(tableDesc, pk)
if err != nil {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{}, err
}
startKey := keys.MakeRowSentinelKey(pik)
leftRange, rightRange, err := ts.SplitRange(startKey)
if err != nil {
return roachpb.RangeDescriptor{}, roachpb.RangeDescriptor{},
errors.Wrapf(err, "failed to split at row: %d", pk)
}
return leftRange, rightRange, nil
}
func TestReplicateQueueRebalance(t *testing.T) {
defer leaktest.AfterTest(t)()
// Set the gossip stores interval lower to speed up rebalancing. With the
// default of 5s we have to wait ~5s for the rebalancing to start.
defer func(v time.Duration) {
gossip.GossipStoresInterval = v
}(gossip.GossipStoresInterval)
gossip.GossipStoresInterval = 100 * time.Millisecond
// TODO(peter): Remove when lease rebalancing is the default.
defer func(v bool) {
storage.EnableLeaseRebalancing = v
}(storage.EnableLeaseRebalancing)
storage.EnableLeaseRebalancing = true
const numNodes = 5
tc := testcluster.StartTestCluster(t, numNodes,
base.TestClusterArgs{ReplicationMode: base.ReplicationAuto},
)
defer tc.Stopper().Stop()
const newRanges = 5
for i := 0; i < newRanges; i++ {
tableID := keys.MaxReservedDescID + i + 1
splitKey := keys.MakeRowSentinelKey(keys.MakeTablePrefix(uint32(tableID)))
for {
if _, _, err := tc.SplitRange(splitKey); err != nil {
if testutils.IsError(err, "split at key .* failed: conflict updating range descriptors") ||
testutils.IsError(err, "range is already split at key") {
continue
}
t.Fatal(err)
}
break
}
}
countReplicas := func() []int {
counts := make([]int, len(tc.Servers))
for _, s := range tc.Servers {
err := s.Stores().VisitStores(func(s *storage.Store) error {
counts[s.StoreID()-1] += s.ReplicaCount()
return nil
})
if err != nil {
t.Fatal(err)
}
}
return counts
}
numRanges := newRanges + server.ExpectedInitialRangeCount()
numReplicas := numRanges * 3
const minThreshold = 0.9
minReplicas := int(math.Floor(minThreshold * (float64(numReplicas) / numNodes)))
util.SucceedsSoon(t, func() error {
counts := countReplicas()
for _, c := range counts {
if c < minReplicas {
err := errors.Errorf("not balanced: %d", counts)
log.Info(context.Background(), err)
return err
}
}
return nil
})
}
func TestSplitAt(t *testing.T) {
defer leaktest.AfterTest(t)()
params, _ := createTestServerParams()
s, db, _ := serverutils.StartServer(t, params)
defer s.Stopper().Stop()
r := sqlutils.MakeSQLRunner(t, db)
r.Exec("CREATE DATABASE d")
r.Exec(`CREATE TABLE d.t (
i INT,
s STRING,
PRIMARY KEY (i, s),
INDEX s_idx (s)
)`)
r.Exec(`CREATE TABLE d.i (k INT PRIMARY KEY)`)
tests := []struct {
in string
error string
args []interface{}
}{
{
in: "ALTER TABLE d.t SPLIT AT (2, 'b')",
},
{
in: "ALTER TABLE d.t SPLIT AT (2, 'b')",
error: "range is already split",
},
{
in: "ALTER TABLE d.t SPLIT AT ('c', 3)",
error: "argument of SPLIT AT must be type int, not type string",
},
{
in: "ALTER TABLE d.t SPLIT AT (4)",
error: "expected 2 expressions, got 1",
},
{
in: "ALTER TABLE d.t SPLIT AT (5, 'e')",
},
{
in: "ALTER TABLE d.t SPLIT AT (i, s)",
error: `name "i" is not defined`,
},
{
in: "ALTER INDEX d.t@s_idx SPLIT AT ('f')",
},
{
in: "ALTER INDEX d.t@not_present SPLIT AT ('g')",
error: `index "not_present" does not exist`,
},
{
in: "ALTER TABLE d.i SPLIT AT (avg(1))",
error: "unknown signature: avg(int) (desired <int>)",
},
{
in: "ALTER TABLE d.i SPLIT AT (avg(k))",
error: `avg(): name "k" is not defined`,
},
{
in: "ALTER TABLE d.i SPLIT AT ($1)",
args: []interface{}{8},
},
{
in: "ALTER TABLE d.i SPLIT AT ($1)",
error: "no value provided for placeholder: $1",
},
{
in: "ALTER TABLE d.i SPLIT AT ($1)",
args: []interface{}{"blah"},
error: "error in argument for $1: strconv.ParseInt",
},
{
in: "ALTER TABLE d.i SPLIT AT ($1::string)",
args: []interface{}{"1"},
error: "argument of SPLIT AT must be type int, not type string",
},
{
in: "ALTER TABLE d.i SPLIT AT ((SELECT 1))",
},
{
in: "ALTER TABLE d.i SPLIT AT ((SELECT 1, 2))",
error: "subquery must return only one column, found 2",
},
}
for _, tt := range tests {
var key roachpb.Key
var pretty string
err := db.QueryRow(tt.in, tt.args...).Scan(&key, &pretty)
if err != nil && tt.error == "" {
t.Fatalf("%s: unexpected error: %s", tt.in, err)
} else if tt.error != "" && err == nil {
t.Fatalf("%s: expected error: %s", tt.in, tt.error)
} else if err != nil && tt.error != "" {
if !strings.Contains(err.Error(), tt.error) {
t.Fatalf("%s: unexpected error: %s", tt.in, err)
}
} else {
// Successful split, verify it happened.
rng, err := s.(*server.TestServer).LookupRange(key)
if err != nil {
t.Fatal(err)
}
expect := roachpb.Key(keys.MakeRowSentinelKey(rng.StartKey))
if !expect.Equal(key) {
t.Fatalf("%s: expected range start %s, got %s", tt.in, pretty, expect)
}
}
}
}
// TestAmbiguousCommitDueToLeadershipChange verifies that an ambiguous
// commit error is returned from sql.Exec in situations where an
// EndTransaction is part of a batch and the disposition of the batch
// request is unknown after a network failure or timeout. The goal
// here is to prevent spurious transaction retries after the initial
// transaction actually succeeded. In cases where there's an
// auto-generated primary key, this can result in silent
// duplications. In cases where the primary key is specified in
// advance, it can result in violated uniqueness constraints, or
// duplicate key violations. See #6053, #7604, and #10023.
func TestAmbiguousCommitDueToLeadershipChange(t *testing.T) {
defer leaktest.AfterTest(t)()
t.Skip("#10341")
// Create a command filter which prevents EndTransaction from
// returning a response.
params := base.TestServerArgs{}
committed := make(chan struct{})
wait := make(chan struct{})
var tableStartKey atomic.Value
var responseCount int32
// Prevent the first conditional put on table 51 from returning to
// waiting client in order to simulate a lost update or slow network
// link.
params.Knobs.Store = &storage.StoreTestingKnobs{
TestingResponseFilter: func(ba roachpb.BatchRequest, br *roachpb.BatchResponse) *roachpb.Error {
req, ok := ba.GetArg(roachpb.ConditionalPut)
tsk := tableStartKey.Load()
if tsk == nil {
return nil
}
if !ok || !bytes.HasPrefix(req.Header().Key, tsk.([]byte)) {
return nil
}
// If this is the first write to the table, wait to respond to the
// client in order to simulate a retry.
if atomic.AddInt32(&responseCount, 1) == 1 {
close(committed)
<-wait
}
return nil
},
}
testClusterArgs := base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
ServerArgs: params,
}
const numReplicas = 3
tc := testcluster.StartTestCluster(t, numReplicas, testClusterArgs)
defer tc.Stopper().Stop()
sqlDB := sqlutils.MakeSQLRunner(t, tc.Conns[0])
sqlDB.Exec(`CREATE DATABASE test`)
sqlDB.Exec(`CREATE TABLE test.t (k SERIAL PRIMARY KEY, v INT)`)
tableID := sqlutils.QueryTableID(t, tc.Conns[0], "test", "t")
tableStartKey.Store(keys.MakeTablePrefix(tableID))
// Wait for new table to split.
util.SucceedsSoon(t, func() error {
startKey := tableStartKey.Load().([]byte)
desc, err := tc.LookupRange(keys.MakeRowSentinelKey(startKey))
if err != nil {
t.Fatal(err)
}
if !desc.StartKey.Equal(startKey) {
return errors.Errorf("expected range start key %s; got %s",
startKey, desc.StartKey)
}
return nil
})
// Lookup the lease.
tableRangeDesc, err := tc.LookupRange(keys.MakeRowSentinelKey(tableStartKey.Load().([]byte)))
if err != nil {
t.Fatal(err)
}
leaseHolder, err := tc.FindRangeLeaseHolder(
&tableRangeDesc,
&testcluster.ReplicationTarget{
NodeID: tc.Servers[0].GetNode().Descriptor.NodeID,
StoreID: tc.Servers[0].GetFirstStoreID(),
})
if err != nil {
t.Fatal(err)
}
// In a goroutine, send an insert which will commit but not return
// from the leader (due to the command filter we installed on node 0).
sqlErrCh := make(chan error, 1)
go func() {
// Use a connection other than through the node which is the current
// leaseholder to ensure that we use GRPC instead of the local server.
// If we use a local server, the hanging response we simulate takes
// up the dist sender thread of execution because local requests are
// executed synchronously.
sqlConn := tc.Conns[leaseHolder.NodeID%numReplicas]
_, err := sqlConn.Exec(`INSERT INTO test.t (v) VALUES (1)`)
sqlErrCh <- err
close(wait)
}()
// Wait until the insert has committed.
<-committed
// Find a node other than the current lease holder to transfer the lease to.
for i, s := range tc.Servers {
if leaseHolder.StoreID != s.GetFirstStoreID() {
if err := tc.TransferRangeLease(&tableRangeDesc, tc.Target(i)); err != nil {
t.Fatal(err)
}
break
}
}
// Wait for the error from the pending SQL insert.
if err := <-sqlErrCh; !testutils.IsError(err, "result is ambiguous") {
t.Errorf("expected ambiguous commit error; got %v", err)
}
// Verify a single row exists in the table.
var rowCount int
sqlDB.QueryRow(`SELECT COUNT(*) FROM test.t`).Scan(&rowCount)
if rowCount != 1 {
t.Errorf("expected 1 row but found %d", rowCount)
}
}
func TestManualReplication(t *testing.T) {
defer leaktest.AfterTest(t)()
tc := StartTestCluster(t, 3,
base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
UseDatabase: "t",
},
})
defer tc.Stopper().Stop()
s0 := sqlutils.MakeSQLRunner(t, tc.Conns[0])
s1 := sqlutils.MakeSQLRunner(t, tc.Conns[1])
s2 := sqlutils.MakeSQLRunner(t, tc.Conns[2])
s0.Exec(`CREATE DATABASE t`)
s0.Exec(`CREATE TABLE test (k INT PRIMARY KEY, v INT)`)
s0.Exec(`INSERT INTO test VALUES (5, 1), (4, 2), (1, 2)`)
if r := s1.Query(`SELECT * FROM test WHERE k = 5`); !r.Next() {
t.Fatal("no rows")
}
s2.ExecRowsAffected(3, `DELETE FROM test`)
// Split the table to a new range.
kvDB := tc.Servers[0].DB()
tableDesc := sqlbase.GetTableDescriptor(kvDB, "t", "test")
tableStartKey := keys.MakeRowSentinelKey(keys.MakeTablePrefix(uint32(tableDesc.ID)))
leftRangeDesc, tableRangeDesc, err := tc.SplitRange(tableStartKey)
if err != nil {
t.Fatal(err)
}
log.Infof(context.Background(), "After split got ranges: %+v and %+v.", leftRangeDesc, tableRangeDesc)
if len(tableRangeDesc.Replicas) == 0 {
t.Fatalf(
"expected replica on node 1, got no replicas: %+v", tableRangeDesc.Replicas)
}
if tableRangeDesc.Replicas[0].NodeID != 1 {
t.Fatalf(
"expected replica on node 1, got replicas: %+v", tableRangeDesc.Replicas)
}
// Replicate the table's range to all the nodes.
tableRangeDesc, err = tc.AddReplicas(
tableRangeDesc.StartKey.AsRawKey(), tc.Target(1), tc.Target(2),
)
if err != nil {
t.Fatal(err)
}
if len(tableRangeDesc.Replicas) != 3 {
t.Fatalf("expected 3 replicas, got %+v", tableRangeDesc.Replicas)
}
for i := 0; i < 3; i++ {
if _, ok := tableRangeDesc.GetReplicaDescriptor(
tc.Servers[i].GetFirstStoreID()); !ok {
t.Fatalf("expected replica on store %d, got %+v",
tc.Servers[i].GetFirstStoreID(), tableRangeDesc.Replicas)
}
}
// Transfer the lease to node 1.
leaseHolder, err := tc.FindRangeLeaseHolder(
tableRangeDesc,
&ReplicationTarget{
NodeID: tc.Servers[0].GetNode().Descriptor.NodeID,
StoreID: tc.Servers[0].GetFirstStoreID(),
})
if err != nil {
t.Fatal(err)
}
if leaseHolder.StoreID != tc.Servers[0].GetFirstStoreID() {
t.Fatalf("expected initial lease on server idx 0, but is on node: %+v",
leaseHolder)
}
err = tc.TransferRangeLease(tableRangeDesc, tc.Target(1))
if err != nil {
t.Fatal(err)
}
// Check that the lease holder has changed. We'll use the old lease holder as
// the hint, since it's guaranteed that the old lease holder has applied the
// new lease.
leaseHolder, err = tc.FindRangeLeaseHolder(
tableRangeDesc,
&ReplicationTarget{
NodeID: tc.Servers[0].GetNode().Descriptor.NodeID,
StoreID: tc.Servers[0].GetFirstStoreID(),
})
if err != nil {
t.Fatal(err)
}
if leaseHolder.StoreID != tc.Servers[1].GetFirstStoreID() {
t.Fatalf("expected lease on server idx 1 (node: %d store: %d), but is on node: %+v",
tc.Servers[1].GetNode().Descriptor.NodeID,
tc.Servers[1].GetFirstStoreID(),
leaseHolder)
}
}
func BenchmarkSstRekey(b *testing.B) {
// TODO(dan): DRY this with BenchmarkRocksDBSstFileReader.
dir, cleanupFn := testutils.TempDir(b, 1)
defer cleanupFn()
sstPath := filepath.Join(dir, "sst")
{
const maxEntries = 100000
const keyLen = 10
const valLen = 100
b.SetBytes(keyLen + valLen)
ts := hlc.Timestamp{WallTime: timeutil.Now().UnixNano()}
kv := engine.MVCCKeyValue{
Key: engine.MVCCKey{Key: roachpb.Key(make([]byte, keyLen)), Timestamp: ts},
Value: make([]byte, valLen),
}
sst := engine.MakeRocksDBSstFileWriter()
if err := sst.Open(sstPath); err != nil {
b.Fatal(sst)
}
var entries = b.N
if entries > maxEntries {
entries = maxEntries
}
for i := 0; i < entries; i++ {
payload := []byte(fmt.Sprintf("%09d", i))
kv.Key.Key = kv.Key.Key[:0]
kv.Key.Key = encoding.EncodeUvarintAscending(kv.Key.Key, uint64(i)) // tableID
kv.Key.Key = encoding.EncodeUvarintAscending(kv.Key.Key, 0) // indexID
kv.Key.Key = encoding.EncodeBytesAscending(kv.Key.Key, payload)
kv.Key.Key = keys.MakeRowSentinelKey(kv.Key.Key)
copy(kv.Value, payload)
if err := sst.Add(kv); err != nil {
b.Fatal(err)
}
}
if err := sst.Close(); err != nil {
b.Fatal(err)
}
}
const newTableID = 100
b.ResetTimer()
sst, err := engine.MakeRocksDBSstFileReader()
if err != nil {
b.Fatal(err)
}
if err := sst.AddFile(sstPath); err != nil {
b.Fatal(err)
}
defer sst.Close()
count := 0
iterateFn := sql.MakeRekeyMVCCKeyValFunc(newTableID, func(kv engine.MVCCKeyValue) (bool, error) {
count++
if count >= b.N {
return true, nil
}
return false, nil
})
for {
if err := sst.Iterate(engine.MVCCKey{Key: keys.MinKey}, engine.MVCCKey{Key: keys.MaxKey}, iterateFn); err != nil {
b.Fatal(err)
}
if count >= b.N {
break
}
}
}
// ComputeSplitKeys takes a start and end key and returns an array of keys
// at which to split the span [start, end).
// The only required splits are at each user table prefix.
func (s SystemConfig) ComputeSplitKeys(startKey, endKey roachpb.RKey) []roachpb.RKey {
tableStart := roachpb.RKey(keys.SystemConfigTableDataMax)
if !tableStart.Less(endKey) {
// This range is before the user tables span: no required splits.
return nil
}
startID, ok := ObjectIDForKey(startKey)
if !ok || startID <= keys.MaxSystemConfigDescID {
// The start key is either:
// - not part of the structured data span
// - part of the system span
// In either case, start looking for splits at the first ID usable
// by the user data span.
startID = keys.MaxSystemConfigDescID + 1
} else {
// The start key is either already a split key, or after the split
// key for its ID. We can skip straight to the next one.
startID++
}
// Build key prefixes for sequential table IDs until we reach endKey. Note
// that there are two disjoint sets of sequential keys: non-system reserved
// tables have sequential IDs, as do user tables, but the two ranges contain a
// gap.
var splitKeys []roachpb.RKey
var key roachpb.RKey
// appendSplitKeys generates all possible split keys between the given range
// of IDs and adds them to splitKeys.
appendSplitKeys := func(startID, endID uint32) {
// endID could be smaller than startID if we don't have user tables.
for id := startID; id <= endID; id++ {
key = keys.MakeRowSentinelKey(keys.MakeTablePrefix(id))
// Skip if this ID matches the startKey passed to ComputeSplitKeys.
if !startKey.Less(key) {
continue
}
// Handle the case where EndKey is already a table prefix.
if !key.Less(endKey) {
break
}
splitKeys = append(splitKeys, key)
}
}
// If the startKey falls within the non-system reserved range, compute those
// keys first.
if startID <= keys.MaxReservedDescID {
endID, err := s.GetLargestObjectID(keys.MaxReservedDescID)
if err != nil {
log.Errorf(context.TODO(), "unable to determine largest reserved object ID from system config: %s", err)
return nil
}
appendSplitKeys(startID, endID)
startID = keys.MaxReservedDescID + 1
}
// Append keys in the user space.
endID, err := s.GetLargestObjectID(0)
if err != nil {
log.Errorf(context.TODO(), "unable to determine largest object ID from system config: %s", err)
return nil
}
appendSplitKeys(startID, endID)
return splitKeys
}
func TestDistSQLJoinAndAgg(t *testing.T) {
defer leaktest.AfterTest(t)()
// This test sets up a distributed join between two tables:
// - a NumToSquare table of size N that maps integers from 1 to n to their
// squares
// - a NumToStr table of size N^2 that maps integers to their string
// representations. This table is split and distributed to all the nodes.
const n = 100
const numNodes = 5
tc := serverutils.StartTestCluster(t, numNodes,
base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
UseDatabase: "test",
},
})
defer tc.Stopper().Stop()
cdb := tc.Server(0).KVClient().(*client.DB)
sqlutils.CreateTable(
t, tc.ServerConn(0), "NumToSquare", "x INT PRIMARY KEY, xsquared INT",
n,
sqlutils.ToRowFn(sqlutils.RowIdxFn, func(row int) parser.Datum {
return parser.NewDInt(parser.DInt(row * row))
}),
)
sqlutils.CreateTable(
t, tc.ServerConn(0), "NumToStr", "y INT PRIMARY KEY, str STRING",
n*n,
sqlutils.ToRowFn(sqlutils.RowIdxFn, sqlutils.RowEnglishFn),
)
// Split the table into multiple ranges, with each range having a single
// replica on a certain node. This forces the query to be distributed.
//
// TODO(radu): this approach should be generalized into test infrastructure
// (perhaps by adding functionality to logic tests).
// TODO(radu): we should verify that the plan is indeed distributed as
// intended.
descNumToStr := sqlbase.GetTableDescriptor(cdb, "test", "NumToStr")
// split introduces a split and moves the right range to a given node.
split := func(val int, targetNode int) {
pik, err := sqlbase.MakePrimaryIndexKey(descNumToStr, val)
if err != nil {
t.Fatal(err)
}
splitKey := keys.MakeRowSentinelKey(pik)
_, rightRange, err := tc.Server(0).SplitRange(splitKey)
if err != nil {
t.Fatal(err)
}
splitKey = rightRange.StartKey.AsRawKey()
rightRange, err = tc.AddReplicas(splitKey, tc.Target(targetNode))
if err != nil {
t.Fatal(err)
}
// This transfer is necessary to avoid waiting for the lease to expire when
// removing the first replica.
if err := tc.TransferRangeLease(rightRange, tc.Target(targetNode)); err != nil {
t.Fatal(err)
}
if _, err := tc.RemoveReplicas(splitKey, tc.Target(0)); err != nil {
t.Fatal(err)
}
}
// split moves the right range, so we split things back to front.
for i := numNodes - 1; i > 0; i-- {
split(n*n/numNodes*i, i)
}
r := sqlutils.MakeSQLRunner(t, tc.ServerConn(0))
r.DB.SetMaxOpenConns(1)
r.Exec("SET DIST_SQL = ALWAYS")
res := r.QueryStr("SELECT x, str FROM NumToSquare JOIN NumToStr ON y = xsquared")
// Verify that res contains one entry for each integer, with the string
// representation of its square, e.g.:
// [1, one]
// [2, two]
// [3, nine]
// [4, one-six]
// (but not necessarily in order).
if len(res) != n {
t.Fatalf("expected %d rows, got %d", n, len(res))
}
resMap := make(map[int]string)
for _, row := range res {
if len(row) != 2 {
t.Fatalf("invalid row %v", row)
}
n, err := strconv.Atoi(row[0])
if err != nil {
t.Fatalf("error parsing row %v: %s", row, err)
}
resMap[n] = row[1]
}
for i := 1; i <= n; i++ {
if resMap[i] != sqlutils.IntToEnglish(i*i) {
t.Errorf("invalid string for %d: %s", i, resMap[i])
}
}
checkRes := func(exp int) bool {
return len(res) == 1 && len(res[0]) == 1 && res[0][0] == strconv.Itoa(exp)
}
// Sum the numbers in the NumToStr table.
res = r.QueryStr("SELECT SUM(y) FROM NumToStr")
if exp := n * n * (n*n + 1) / 2; !checkRes(exp) {
t.Errorf("expected [[%d]], got %s", exp, res)
}
// Count the rows in the NumToStr table.
res = r.QueryStr("SELECT COUNT(*) FROM NumToStr")
if !checkRes(n * n) {
t.Errorf("expected [[%d]], got %s", n*n, res)
}
// Count how many numbers contain the digit 5.
res = r.QueryStr("SELECT COUNT(*) FROM NumToStr WHERE str LIKE '%five%'")
exp := 0
for i := 1; i <= n*n; i++ {
for x := i; x > 0; x /= 10 {
if x%10 == 5 {
exp++
break
}
}
}
if !checkRes(exp) {
t.Errorf("expected [[%d]], got %s", exp, res)
}
}
func TestComputeSplits(t *testing.T) {
defer leaktest.AfterTest(t)()
const (
start = keys.MaxReservedDescID + 1
reservedStart = keys.MaxSystemConfigDescID + 1
)
schema := sqlbase.MakeMetadataSchema()
// Real system tables only.
baseSql := schema.GetInitialValues()
// Real system tables plus some user stuff.
allSql := append(schema.GetInitialValues(),
descriptor(start), descriptor(start+1), descriptor(start+5))
sort.Sort(roachpb.KeyValueByKey(allSql))
allUserSplits := []uint32{start, start + 1, start + 2, start + 3, start + 4, start + 5}
var allReservedSplits []uint32
for i := 0; i < schema.SystemDescriptorCount()-schema.SystemConfigDescriptorCount(); i++ {
allReservedSplits = append(allReservedSplits, reservedStart+uint32(i))
}
allSplits := append(allReservedSplits, allUserSplits...)
testCases := []struct {
values []roachpb.KeyValue
start, end roachpb.RKey
// Use ints in the testcase definitions, more readable.
splits []uint32
}{
// No data.
{nil, roachpb.RKeyMin, roachpb.RKeyMax, nil},
{nil, keys.MakeTablePrefix(start), roachpb.RKeyMax, nil},
{nil, keys.MakeTablePrefix(start), keys.MakeTablePrefix(start + 10), nil},
{nil, roachpb.RKeyMin, keys.MakeTablePrefix(start + 10), nil},
// Reserved descriptors.
{baseSql, roachpb.RKeyMin, roachpb.RKeyMax, allReservedSplits},
{baseSql, keys.MakeTablePrefix(start), roachpb.RKeyMax, nil},
{baseSql, keys.MakeTablePrefix(start), keys.MakeTablePrefix(start + 10), nil},
{baseSql, roachpb.RKeyMin, keys.MakeTablePrefix(start + 10), allReservedSplits},
{baseSql, keys.MakeTablePrefix(reservedStart), roachpb.RKeyMax, allReservedSplits[1:]},
{baseSql, keys.MakeTablePrefix(reservedStart), keys.MakeTablePrefix(start + 10), allReservedSplits[1:]},
{baseSql, roachpb.RKeyMin, keys.MakeTablePrefix(reservedStart + 2), allReservedSplits[:2]},
{baseSql, roachpb.RKeyMin, keys.MakeTablePrefix(reservedStart + 10), allReservedSplits},
{baseSql, keys.MakeTablePrefix(reservedStart), keys.MakeTablePrefix(reservedStart + 2), allReservedSplits[1:2]},
{baseSql, testutils.MakeKey(keys.MakeTablePrefix(reservedStart), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start+10), roachpb.RKey("foo")), allReservedSplits[1:]},
// Reserved + User descriptors.
{allSql, keys.MakeTablePrefix(start - 1), roachpb.RKeyMax, allUserSplits},
{allSql, keys.MakeTablePrefix(start), roachpb.RKeyMax, allUserSplits[1:]},
{allSql, keys.MakeTablePrefix(start), keys.MakeTablePrefix(start + 10), allUserSplits[1:]},
{allSql, keys.MakeTablePrefix(start - 1), keys.MakeTablePrefix(start + 10), allUserSplits},
{allSql, keys.MakeTablePrefix(start + 4), keys.MakeTablePrefix(start + 10), allUserSplits[5:]},
{allSql, keys.MakeTablePrefix(start + 5), keys.MakeTablePrefix(start + 10), nil},
{allSql, keys.MakeTablePrefix(start + 6), keys.MakeTablePrefix(start + 10), nil},
{allSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
keys.MakeTablePrefix(start + 10), allUserSplits[1:]},
{allSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
keys.MakeTablePrefix(start + 5), allUserSplits[1:5]},
{allSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start+5), roachpb.RKey("bar")), allUserSplits[1:5]},
{allSql, testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start), roachpb.RKey("morefoo")), nil},
{allSql, roachpb.RKeyMin, roachpb.RKeyMax, allSplits},
{allSql, keys.MakeTablePrefix(reservedStart + 1), roachpb.RKeyMax, allSplits[2:]},
{allSql, keys.MakeTablePrefix(reservedStart), keys.MakeTablePrefix(start + 10), allSplits[1:]},
{allSql, roachpb.RKeyMin, keys.MakeTablePrefix(start + 2), allSplits[:6]},
{allSql, testutils.MakeKey(keys.MakeTablePrefix(reservedStart), roachpb.RKey("foo")),
testutils.MakeKey(keys.MakeTablePrefix(start+5), roachpb.RKey("foo")), allSplits[1:9]},
}
cfg := config.SystemConfig{}
for tcNum, tc := range testCases {
cfg.Values = tc.values
splits := cfg.ComputeSplitKeys(tc.start, tc.end)
if len(splits) == 0 && len(tc.splits) == 0 {
continue
}
// Convert ints to actual keys.
expected := []roachpb.RKey{}
for _, s := range tc.splits {
expected = append(expected, keys.MakeRowSentinelKey(keys.MakeTablePrefix(s)))
}
if !reflect.DeepEqual(splits, expected) {
t.Errorf("#%d: bad splits:\ngot: %v\nexpected: %v", tcNum, splits, expected)
}
}
}