// TestGossipHandlesReplacedNode tests that we can shut down a node and
// replace it with a new node at the same address (simulating a node getting
// restarted after losing its data) without the cluster breaking.
func TestGossipHandlesReplacedNode(t *testing.T) {
defer leaktest.AfterTest(t)()
ctx := context.Background()
tc := testcluster.StartTestCluster(t, 3,
base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
})
defer tc.Stopper().Stop()
// Take down the first node of the cluster and replace it with a new one.
// We replace the first node rather than the second or third to be adversarial
// because it typically has the most leases on it.
oldNodeIdx := 0
newServerArgs := base.TestServerArgs{
Addr: tc.Servers[oldNodeIdx].ServingAddr(),
PartOfCluster: true,
JoinAddr: tc.Servers[1].ServingAddr(),
}
tc.StopServer(oldNodeIdx)
tc.AddServer(t, newServerArgs)
tc.WaitForStores(t, tc.Server(1).Gossip())
// Ensure that all servers still running are responsive. If the two remaining
// original nodes don't refresh their connection to the address of the first
// node, they can get stuck here.
for i := 1; i < 4; i++ {
kvClient := tc.Server(i).KVClient().(*client.DB)
if err := kvClient.Put(ctx, fmt.Sprintf("%d", i), i); err != nil {
t.Errorf("failed Put to node %d: %s", i, err)
}
}
}
func TestSplitAtTableBoundary(t *testing.T) {
defer leaktest.AfterTest(t)()
testClusterArgs := base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
}
tc := testcluster.StartTestCluster(t, 3, testClusterArgs)
defer tc.Stopper().Stop()
runner := sqlutils.MakeSQLRunner(t, tc.Conns[0])
runner.Exec(`CREATE DATABASE test`)
runner.Exec(`CREATE TABLE test.t (k SERIAL PRIMARY KEY, v INT)`)
const tableIDQuery = `
SELECT tables.id FROM system.namespace tables
JOIN system.namespace dbs ON dbs.id = tables.parentid
WHERE dbs.name = $1 AND tables.name = $2
`
var tableID uint32
runner.QueryRow(tableIDQuery, "test", "t").Scan(&tableID)
tableStartKey := keys.MakeTablePrefix(tableID)
// Wait for new table to split.
testutils.SucceedsSoon(t, func() error {
desc, err := tc.LookupRange(keys.MakeRowSentinelKey(tableStartKey))
if err != nil {
t.Fatal(err)
}
if !desc.StartKey.Equal(tableStartKey) {
log.Infof(context.TODO(), "waiting on split results")
return errors.Errorf("expected range start key %s; got %s", tableStartKey, desc.StartKey)
}
return nil
})
}
func backupRestoreTestSetup(
t testing.TB, numAccounts int,
) (
ctx context.Context,
tempDir string,
tc *testcluster.TestCluster,
kvDB *client.DB,
sqlDB *sqlutils.SQLRunner,
cleanup func(),
) {
ctx = context.Background()
dir, dirCleanupFn := testutils.TempDir(t, 1)
// Use ReplicationManual so we can force full replication, which is needed
// to later move the leases around.
tc = testcluster.StartTestCluster(t, backupRestoreClusterSize, base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
})
sqlDB = sqlutils.MakeSQLRunner(t, tc.Conns[0])
kvDB = tc.Server(0).KVClient().(*client.DB)
sqlDB.Exec(bankCreateDatabase)
sqlDB.Exec(bankCreateTable)
for _, insert := range bankDataInsertStmts(numAccounts) {
sqlDB.Exec(insert)
}
for _, split := range bankSplitStmts(numAccounts, backupRestoreDefaultRanges) {
sqlDB.Exec(split)
}
targets := make([]testcluster.ReplicationTarget, backupRestoreClusterSize-1)
for i := 1; i < backupRestoreClusterSize; i++ {
targets[i-1] = tc.Target(i)
}
txn := client.NewTxn(ctx, *kvDB)
rangeDescs, err := sql.AllRangeDescriptors(txn)
if err != nil {
t.Fatal(err)
}
for _, r := range rangeDescs {
if _, err := tc.AddReplicas(r.StartKey.AsRawKey(), targets...); err != nil {
t.Fatal(err)
}
}
cleanupFn := func() {
tc.Stopper().Stop()
dirCleanupFn()
}
return ctx, dir, tc, kvDB, sqlDB, cleanupFn
}
// TestParallelCreateTables tests that concurrent create table requests are
// correctly filled.
func TestParallelCreateTables(t *testing.T) {
defer leaktest.AfterTest(t)()
// This number has to be around 10 or else testrace will take too long to
// finish.
const numberOfTables = 10
const numberOfNodes = 3
tc := testcluster.StartTestCluster(t, numberOfNodes, base.TestClusterArgs{})
defer tc.Stopper().Stop()
if _, err := tc.ServerConn(0).Exec(`CREATE DATABASE "test"`); err != nil {
t.Fatal(err)
}
// Get the id descriptor generator count.
kvDB := tc.Servers[0].KVClient().(*client.DB)
var descIDStart int64
if descID, err := kvDB.Get(context.Background(), keys.DescIDGenerator); err != nil {
t.Fatal(err)
} else {
descIDStart = descID.ValueInt()
}
var wgStart sync.WaitGroup
var wgEnd sync.WaitGroup
wgStart.Add(numberOfTables)
wgEnd.Add(numberOfTables)
signal := make(chan struct{})
completed := make(chan int, numberOfTables)
for i := 0; i < numberOfTables; i++ {
db := tc.ServerConn(i % numberOfNodes)
go createTestTable(t, tc, i, db, &wgStart, &wgEnd, signal, completed)
}
// Wait until all goroutines are ready.
wgStart.Wait()
// Signal the create table goroutines to start.
close(signal)
// Wait until all create tables are finished.
wgEnd.Wait()
close(completed)
verifyTables(
t,
tc,
completed,
numberOfTables,
descIDStart,
)
}
// TestGossipHandlesReplacedNode tests that we can shut down a node and
// replace it with a new node at the same address (simulating a node getting
// restarted after losing its data) without the cluster breaking.
func TestGossipHandlesReplacedNode(t *testing.T) {
defer leaktest.AfterTest(t)()
ctx := context.Background()
// Shorten the raft tick interval and election timeout to make range leases
// much shorter than normal. This keeps us from having to wait so long for
// the replaced node's leases to time out, but has still shown itself to be
// long enough to avoid flakes.
serverArgs := base.TestServerArgs{
RaftTickInterval: 10 * time.Millisecond,
RaftElectionTimeoutTicks: 10,
RetryOptions: retry.Options{
InitialBackoff: 10 * time.Millisecond,
MaxBackoff: 50 * time.Millisecond,
},
}
tc := testcluster.StartTestCluster(t, 3,
base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
ServerArgs: serverArgs,
})
defer tc.Stopper().Stop()
// Take down a node other than the first node and replace it with a new one.
// Replacing the first node would be better from an adversarial testing
// perspective because it typically has the most leases on it, but that also
// causes the test to take significantly longer as a result.
oldNodeIdx := 0
newServerArgs := serverArgs
newServerArgs.Addr = tc.Servers[oldNodeIdx].ServingAddr()
newServerArgs.PartOfCluster = true
newServerArgs.JoinAddr = tc.Servers[1].ServingAddr()
tc.StopServer(oldNodeIdx)
tc.AddServer(t, newServerArgs)
tc.WaitForStores(t, tc.Server(1).Gossip())
// Ensure that all servers still running are responsive. If the two remaining
// original nodes don't refresh their connection to the address of the first
// node, they can get stuck here.
for i, server := range tc.Servers {
if i == oldNodeIdx {
continue
}
kvClient := server.KVClient().(*client.DB)
if err := kvClient.Put(ctx, fmt.Sprintf("%d", i), i); err != nil {
t.Errorf("failed Put to node %d: %s", i, err)
}
}
}
func benchmarkMultinodeCockroach(b *testing.B, f func(b *testing.B, db *gosql.DB)) {
defer tracing.Disable()()
tc := testcluster.StartTestCluster(b, 3,
base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
ServerArgs: base.TestServerArgs{
UseDatabase: "bench",
},
})
if _, err := tc.Conns[0].Exec(`CREATE DATABASE bench`); err != nil {
b.Fatal(err)
}
defer tc.Stopper().Stop()
f(b, tc.Conns[0])
}
func TestBackupRestoreOnce(t *testing.T) {
defer leaktest.AfterTest(t)()
// TODO(dan): Actually invalidate the descriptor cache and delete this line.
defer sql.TestDisableTableLeases()()
const numAccounts = 1000
ctx, dir, tc, kvDB, _, cleanupFn := backupRestoreTestSetup(t, numAccounts)
defer cleanupFn()
{
desc, err := sql.Backup(ctx, *kvDB, dir, tc.Server(0).Clock().Now())
if err != nil {
t.Fatal(err)
}
approxDataSize := int64(backupRestoreRowPayloadSize) * numAccounts
if max := approxDataSize * 2; desc.DataSize < approxDataSize || desc.DataSize > 2*max {
t.Errorf("expected data size in [%d,%d] but was %d", approxDataSize, max, desc.DataSize)
}
}
// Start a new cluster to restore into.
{
tcRestore := testcluster.StartTestCluster(t, backupRestoreClusterSize, base.TestClusterArgs{})
defer tcRestore.Stopper().Stop()
sqlDBRestore := sqlutils.MakeSQLRunner(t, tcRestore.Conns[0])
kvDBRestore := tcRestore.Server(0).KVClient().(*client.DB)
// Restore assumes the database exists.
sqlDBRestore.Exec(bankCreateDatabase)
table := parser.TableName{DatabaseName: "bench", TableName: "bank"}
if _, err := sql.Restore(ctx, *kvDBRestore, dir, table); err != nil {
t.Fatal(err)
}
var rowCount int
sqlDBRestore.QueryRow(`SELECT COUNT(*) FROM bench.bank`).Scan(&rowCount)
if rowCount != numAccounts {
t.Fatalf("expected %d rows but found %d", numAccounts, rowCount)
}
}
}
func TestReplicateQueueRebalance(t *testing.T) {
defer leaktest.AfterTest(t)()
// Set the gossip stores interval lower to speed up rebalancing. With the
// default of 5s we have to wait ~5s for the rebalancing to start.
defer func(v time.Duration) {
gossip.GossipStoresInterval = v
}(gossip.GossipStoresInterval)
gossip.GossipStoresInterval = 100 * time.Millisecond
// TODO(peter): Remove when lease rebalancing is the default.
defer func(v bool) {
storage.EnableLeaseRebalancing = v
}(storage.EnableLeaseRebalancing)
storage.EnableLeaseRebalancing = true
const numNodes = 5
tc := testcluster.StartTestCluster(t, numNodes,
base.TestClusterArgs{ReplicationMode: base.ReplicationAuto},
)
defer tc.Stopper().Stop()
const newRanges = 5
for i := 0; i < newRanges; i++ {
tableID := keys.MaxReservedDescID + i + 1
splitKey := keys.MakeRowSentinelKey(keys.MakeTablePrefix(uint32(tableID)))
for {
if _, _, err := tc.SplitRange(splitKey); err != nil {
if testutils.IsError(err, "split at key .* failed: conflict updating range descriptors") ||
testutils.IsError(err, "range is already split at key") {
continue
}
t.Fatal(err)
}
break
}
}
countReplicas := func() []int {
counts := make([]int, len(tc.Servers))
for _, s := range tc.Servers {
err := s.Stores().VisitStores(func(s *storage.Store) error {
counts[s.StoreID()-1] += s.ReplicaCount()
return nil
})
if err != nil {
t.Fatal(err)
}
}
return counts
}
numRanges := newRanges + server.ExpectedInitialRangeCount()
numReplicas := numRanges * 3
const minThreshold = 0.9
minReplicas := int(math.Floor(minThreshold * (float64(numReplicas) / numNodes)))
util.SucceedsSoon(t, func() error {
counts := countReplicas()
for _, c := range counts {
if c < minReplicas {
err := errors.Errorf("not balanced: %d", counts)
log.Info(context.Background(), err)
return err
}
}
return nil
})
}
// TestReplicateQueueDownReplicate verifies that the replication queue will notice
// over-replicated ranges and remove replicas from them.
func TestReplicateQueueDownReplicate(t *testing.T) {
defer leaktest.AfterTest(t)()
const replicaCount = 3
tc := testcluster.StartTestCluster(t, replicaCount+2,
base.TestClusterArgs{ReplicationMode: base.ReplicationAuto},
)
defer tc.Stopper().Stop()
// Split off a range from the initial range for testing; there are
// complications if the metadata ranges are moved.
testKey := roachpb.Key("m")
if _, _, err := tc.SplitRange(testKey); err != nil {
t.Fatal(err)
}
desc, err := tc.LookupRange(testKey)
if err != nil {
t.Fatal(err)
}
rangeID := desc.RangeID
countReplicas := func() int {
count := 0
for _, s := range tc.Servers {
if err := s.Stores().VisitStores(func(store *storage.Store) error {
if _, err := store.GetReplica(rangeID); err == nil {
count++
}
return nil
}); err != nil {
t.Fatal(err)
}
}
return count
}
// Up-replicate the new range to all servers to create redundant replicas.
// Add replicas to all of the nodes. Only 2 of these calls will succeed
// because the range is already replicated to the other 3 nodes.
util.SucceedsSoon(t, func() error {
for i := 0; i < tc.NumServers(); i++ {
_, err := tc.AddReplicas(testKey, tc.Target(i))
if err != nil {
if testutils.IsError(err, "unable to add replica .* which is already present") {
continue
}
return err
}
}
if c := countReplicas(); c != tc.NumServers() {
return errors.Errorf("replica count = %d", c)
}
return nil
})
// Ensure that the replicas for the new range down replicate.
util.SucceedsSoon(t, func() error {
if c := countReplicas(); c != replicaCount {
return errors.Errorf("replica count = %d", c)
}
return nil
})
}
// Test that resolving spans uses a node's range cache and lease holder cache.
// The idea is to test that resolving is not random, but predictable given the
// state of caches.
func TestSpanResolverUsesCaches(t *testing.T) {
defer leaktest.AfterTest(t)()
tc := testcluster.StartTestCluster(t, 4,
base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
ServerArgs: base.TestServerArgs{
UseDatabase: "t",
},
})
defer tc.Stopper().Stop()
rowRanges, _ := setupRanges(
tc.Conns[0], tc.Servers[0], tc.Servers[0].KVClient().(*client.DB), t)
// Replicate the row ranges on all of the first 3 nodes. Save the 4th node in
// a pristine state, with empty caches.
for i := 0; i < 3; i++ {
var err error
rowRanges[i], err = tc.AddReplicas(
rowRanges[i].StartKey.AsRawKey(), tc.Target(1), tc.Target(2))
if err != nil {
t.Fatal(err)
}
}
// Scatter the leases around; node i gets range i.
for i := 0; i < 3; i++ {
if err := tc.TransferRangeLease(rowRanges[i], tc.Target(i)); err != nil {
t.Fatal(err)
}
// Wait for everybody to apply the new lease, so that we can rely on the
// lease discovery done later by the SpanResolver to be up to date.
testutils.SucceedsSoon(t, func() error {
for j := 0; j < 3; j++ {
target := tc.Target(j)
rt, err := tc.FindRangeLeaseHolder(rowRanges[i], &target)
if err != nil {
return err
}
if rt != tc.Target(i) {
return errors.Errorf("node %d hasn't applied the lease yet", j)
}
}
return nil
})
}
// Create a SpanResolver using the 4th node, with empty caches.
s3 := tc.Servers[3]
lr := distsqlplan.NewSpanResolver(
s3.DistSender(), s3.Gossip(), s3.GetNode().Descriptor,
distsqlplan.BinPackingLeaseHolderChoice)
var spans []spanWithDir
for i := 0; i < 3; i++ {
spans = append(
spans,
spanWithDir{
Span: roachpb.Span{
Key: rowRanges[i].StartKey.AsRawKey(),
EndKey: rowRanges[i].EndKey.AsRawKey(),
},
dir: kv.Ascending,
})
}
// Resolve the spans. Since the LeaseHolderCache is empty, all the ranges
// should be grouped and "assigned" to replica 0.
replicas, err := resolveSpans(context.TODO(), lr.NewSpanResolverIterator(), spans...)
if err != nil {
t.Fatal(err)
}
if len(replicas) != 3 {
t.Fatalf("expected replies for 3 spans, got %d: %+v", len(replicas), replicas)
}
si := tc.Servers[0]
nodeID := si.GetNode().Descriptor.NodeID
storeID := si.GetFirstStoreID()
for i := 0; i < 3; i++ {
if len(replicas[i]) != 1 {
t.Fatalf("expected 1 range for span %s, got %d (%+v)",
spans[i].Span, len(replicas[i]), replicas[i])
}
rd := replicas[i][0].ReplicaDescriptor
if rd.NodeID != nodeID || rd.StoreID != storeID {
t.Fatalf("expected span %s to be on replica (%d, %d) but was on %s",
spans[i].Span, nodeID, storeID, rd)
}
}
// Now populate the cached on node 4 and query again. This time, we expect to see
// each span on its own range.
if err := populateCache(tc.Conns[3], 3 /* expectedNumRows */); err != nil {
t.Fatal(err)
}
replicas, err = resolveSpans(context.TODO(), lr.NewSpanResolverIterator(), spans...)
if err != nil {
t.Fatal(err)
}
var expected [][]rngInfo
for i := 0; i < 3; i++ {
expected = append(expected, []rngInfo{selectReplica(tc.Servers[i].NodeID(), rowRanges[i])})
}
if err = expectResolved(replicas, expected...); err != nil {
t.Fatal(err)
}
}
// TestAmbiguousCommitDueToLeadershipChange verifies that an ambiguous
// commit error is returned from sql.Exec in situations where an
// EndTransaction is part of a batch and the disposition of the batch
// request is unknown after a network failure or timeout. The goal
// here is to prevent spurious transaction retries after the initial
// transaction actually succeeded. In cases where there's an
// auto-generated primary key, this can result in silent
// duplications. In cases where the primary key is specified in
// advance, it can result in violated uniqueness constraints, or
// duplicate key violations. See #6053, #7604, and #10023.
func TestAmbiguousCommitDueToLeadershipChange(t *testing.T) {
defer leaktest.AfterTest(t)()
t.Skip("#10341")
// Create a command filter which prevents EndTransaction from
// returning a response.
params := base.TestServerArgs{}
committed := make(chan struct{})
wait := make(chan struct{})
var tableStartKey atomic.Value
var responseCount int32
// Prevent the first conditional put on table 51 from returning to
// waiting client in order to simulate a lost update or slow network
// link.
params.Knobs.Store = &storage.StoreTestingKnobs{
TestingResponseFilter: func(ba roachpb.BatchRequest, br *roachpb.BatchResponse) *roachpb.Error {
req, ok := ba.GetArg(roachpb.ConditionalPut)
tsk := tableStartKey.Load()
if tsk == nil {
return nil
}
if !ok || !bytes.HasPrefix(req.Header().Key, tsk.([]byte)) {
return nil
}
// If this is the first write to the table, wait to respond to the
// client in order to simulate a retry.
if atomic.AddInt32(&responseCount, 1) == 1 {
close(committed)
<-wait
}
return nil
},
}
testClusterArgs := base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
ServerArgs: params,
}
const numReplicas = 3
tc := testcluster.StartTestCluster(t, numReplicas, testClusterArgs)
defer tc.Stopper().Stop()
sqlDB := sqlutils.MakeSQLRunner(t, tc.Conns[0])
sqlDB.Exec(`CREATE DATABASE test`)
sqlDB.Exec(`CREATE TABLE test.t (k SERIAL PRIMARY KEY, v INT)`)
tableID := sqlutils.QueryTableID(t, tc.Conns[0], "test", "t")
tableStartKey.Store(keys.MakeTablePrefix(tableID))
// Wait for new table to split.
util.SucceedsSoon(t, func() error {
startKey := tableStartKey.Load().([]byte)
desc, err := tc.LookupRange(keys.MakeRowSentinelKey(startKey))
if err != nil {
t.Fatal(err)
}
if !desc.StartKey.Equal(startKey) {
return errors.Errorf("expected range start key %s; got %s",
startKey, desc.StartKey)
}
return nil
})
// Lookup the lease.
tableRangeDesc, err := tc.LookupRange(keys.MakeRowSentinelKey(tableStartKey.Load().([]byte)))
if err != nil {
t.Fatal(err)
}
leaseHolder, err := tc.FindRangeLeaseHolder(
&tableRangeDesc,
&testcluster.ReplicationTarget{
NodeID: tc.Servers[0].GetNode().Descriptor.NodeID,
StoreID: tc.Servers[0].GetFirstStoreID(),
})
if err != nil {
t.Fatal(err)
}
// In a goroutine, send an insert which will commit but not return
// from the leader (due to the command filter we installed on node 0).
sqlErrCh := make(chan error, 1)
go func() {
// Use a connection other than through the node which is the current
// leaseholder to ensure that we use GRPC instead of the local server.
// If we use a local server, the hanging response we simulate takes
// up the dist sender thread of execution because local requests are
// executed synchronously.
sqlConn := tc.Conns[leaseHolder.NodeID%numReplicas]
_, err := sqlConn.Exec(`INSERT INTO test.t (v) VALUES (1)`)
sqlErrCh <- err
close(wait)
}()
// Wait until the insert has committed.
<-committed
// Find a node other than the current lease holder to transfer the lease to.
for i, s := range tc.Servers {
if leaseHolder.StoreID != s.GetFirstStoreID() {
if err := tc.TransferRangeLease(&tableRangeDesc, tc.Target(i)); err != nil {
t.Fatal(err)
}
break
}
}
// Wait for the error from the pending SQL insert.
if err := <-sqlErrCh; !testutils.IsError(err, "result is ambiguous") {
t.Errorf("expected ambiguous commit error; got %v", err)
}
// Verify a single row exists in the table.
var rowCount int
sqlDB.QueryRow(`SELECT COUNT(*) FROM test.t`).Scan(&rowCount)
if rowCount != 1 {
t.Errorf("expected 1 row but found %d", rowCount)
}
}
func TestGossipFirstRange(t *testing.T) {
defer leaktest.AfterTest(t)()
tc := testcluster.StartTestCluster(t, 3,
base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
})
defer tc.Stopper().Stop()
errors := make(chan error)
descs := make(chan *roachpb.RangeDescriptor)
unregister := tc.Servers[0].Gossip().RegisterCallback(gossip.KeyFirstRangeDescriptor,
func(_ string, content roachpb.Value) {
var desc roachpb.RangeDescriptor
if err := content.GetProto(&desc); err != nil {
errors <- err
} else {
descs <- &desc
}
},
)
// Unregister the callback before attempting to stop the stopper to prevent
// deadlock. This is still flaky in theory since a callback can fire between
// the last read from the channels and this unregister, but testing has
// shown this solution to be sufficiently robust for now.
defer unregister()
// Wait for the specified descriptor to be gossiped for the first range. We
// loop because the timing of replica addition and lease transfer can cause
// extra gossiping of the first range.
waitForGossip := func(desc *roachpb.RangeDescriptor) {
for {
select {
case err := <-errors:
t.Fatal(err)
case gossiped := <-descs:
if reflect.DeepEqual(desc, gossiped) {
return
}
log.Infof(context.TODO(), "expected\n%+v\nbut found\n%+v", desc, gossiped)
}
}
}
// Expect an initial callback of the first range descriptor.
select {
case err := <-errors:
t.Fatal(err)
case <-descs:
}
// Add two replicas. The first range descriptor should be gossiped after each
// addition.
var desc *roachpb.RangeDescriptor
firstRangeKey := keys.MinKey
for i := 1; i <= 2; i++ {
var err error
if desc, err = tc.AddReplicas(firstRangeKey, tc.Target(i)); err != nil {
t.Fatal(err)
}
waitForGossip(desc)
}
// Transfer the lease to a new node. This should cause the first range to be
// gossiped again.
if err := tc.TransferRangeLease(desc, tc.Target(1)); err != nil {
t.Fatal(err)
}
waitForGossip(desc)
// Remove a non-lease holder replica.
desc, err := tc.RemoveReplicas(firstRangeKey, tc.Target(0))
if err != nil {
t.Fatal(err)
}
waitForGossip(desc)
// TODO(peter): Re-enable or remove when we've resolved the discussion
// about removing the lease-holder replica. See #7872.
// // Remove the lease holder replica.
// leaseHolder, err := tc.FindRangeLeaseHolder(desc, nil)
// desc, err = tc.RemoveReplicas(firstRangeKey, leaseHolder)
// if err != nil {
// t.Fatal(err)
// }
// select {
// case err := <-errors:
// t.Fatal(err)
// case gossiped := <-descs:
// if !reflect.DeepEqual(desc, gossiped) {
// t.Fatalf("expected\n%+v\nbut found\n%+v", desc, gossiped)
// }
// }
}
func TestLeaseInfoRequest(t *testing.T) {
defer leaktest.AfterTest(t)()
tc := testcluster.StartTestCluster(t, 3,
base.TestClusterArgs{
ReplicationMode: base.ReplicationManual,
})
defer tc.Stopper().Stop()
kvDB0 := tc.Servers[0].DB()
kvDB1 := tc.Servers[1].DB()
key := []byte("a")
rangeDesc := new(roachpb.RangeDescriptor)
var err error
*rangeDesc, err = tc.LookupRange(key)
if err != nil {
t.Fatal(err)
}
rangeDesc, err = tc.AddReplicas(
rangeDesc.StartKey.AsRawKey(), tc.Target(1), tc.Target(2),
)
if err != nil {
t.Fatal(err)
}
if len(rangeDesc.Replicas) != 3 {
t.Fatalf("expected 3 replicas, got %+v", rangeDesc.Replicas)
}
replicas := make([]roachpb.ReplicaDescriptor, 3)
for i := 0; i < 3; i++ {
var ok bool
replicas[i], ok = rangeDesc.GetReplicaDescriptor(tc.Servers[i].GetFirstStoreID())
if !ok {
t.Fatalf("expected to find replica in server %d", i)
}
}
// Lease should start on Server 0, since nobody told it to move.
leaseHolderReplica := LeaseInfo(t, kvDB0, *rangeDesc, roachpb.INCONSISTENT).Lease.Replica
if leaseHolderReplica != replicas[0] {
t.Fatalf("lease holder should be replica %+v, but is: %+v", replicas[0], leaseHolderReplica)
}
// Transfer the lease to Server 1 and check that LeaseInfoRequest gets the
// right answer.
err = tc.TransferRangeLease(rangeDesc, tc.Target(1))
if err != nil {
t.Fatal(err)
}
// An inconsistent LeaseInfoReqeust on the old lease holder should give us the
// right answer immediately, since the old holder has definitely applied the
// transfer before TransferRangeLease returned.
leaseHolderReplica = LeaseInfo(t, kvDB0, *rangeDesc, roachpb.INCONSISTENT).Lease.Replica
if leaseHolderReplica != replicas[1] {
t.Fatalf("lease holder should be replica %+v, but is: %+v",
replicas[1], leaseHolderReplica)
}
// A read on the new lease holder does not necessarily succeed immediately,
// since it might take a while for it to apply the transfer.
util.SucceedsSoon(t, func() error {
// We can't reliably do a CONSISTENT read here, even though we're reading
// from the supposed lease holder, because this node might initially be
// unaware of the new lease and so the request might bounce around for a
// while (see #8816).
leaseHolderReplica = LeaseInfo(t, kvDB1, *rangeDesc, roachpb.INCONSISTENT).Lease.Replica
if leaseHolderReplica != replicas[1] {
return errors.Errorf("lease holder should be replica %+v, but is: %+v",
replicas[1], leaseHolderReplica)
}
return nil
})
// Transfer the lease to Server 2 and check that LeaseInfoRequest gets the
// right answer.
err = tc.TransferRangeLease(rangeDesc, tc.Target(2))
if err != nil {
t.Fatal(err)
}
leaseHolderReplica = LeaseInfo(t, kvDB1, *rangeDesc, roachpb.INCONSISTENT).Lease.Replica
if leaseHolderReplica != replicas[2] {
t.Fatalf("lease holder should be replica %+v, but is: %+v", replicas[2], leaseHolderReplica)
}
// TODO(andrei): test the side-effect of LeaseInfoRequest when there's no
// active lease - the node getting the request is supposed to acquire the
// lease. This requires a way to expire leases; the TestCluster probably needs
// to use a mock clock.
}
func TestAdminAPITableStats(t *testing.T) {
defer leaktest.AfterTest(t)()
t.Skip("#8890")
const nodeCount = 3
tc := testcluster.StartTestCluster(t, nodeCount, base.TestClusterArgs{
ReplicationMode: base.ReplicationAuto,
ServerArgs: base.TestServerArgs{
ScanInterval: time.Millisecond,
ScanMaxIdleTime: time.Millisecond,
},
})
defer tc.Stopper().Stop()
server0 := tc.Server(0)
// Create clients (SQL, HTTP) connected to server 0.
db := tc.ServerConn(0)
client, err := server0.GetHTTPClient()
if err != nil {
t.Fatal(err)
}
client.Timeout = base.NetworkTimeout * 3
// Make a single table and insert some data. The database and test have
// names which require escaping, in order to verify that database and
// table names are being handled correctly.
if _, err := db.Exec(`CREATE DATABASE "test test"`); err != nil {
t.Fatal(err)
}
if _, err := db.Exec(`
CREATE TABLE "test test"."foo foo" (
id INT PRIMARY KEY,
val STRING
)`,
); err != nil {
t.Fatal(err)
}
for i := 0; i < 10; i++ {
if _, err := db.Exec(`
INSERT INTO "test test"."foo foo" VALUES(
$1, $2
)`, i, "test",
); err != nil {
t.Fatal(err)
}
}
url := server0.AdminURL() + "/_admin/v1/databases/test test/tables/foo foo/stats"
var tsResponse serverpb.TableStatsResponse
// The new SQL table may not yet have split into its own range. Wait for
// this to occur, and for full replication.
util.SucceedsSoon(t, func() error {
if err := httputil.GetJSON(client, url, &tsResponse); err != nil {
return err
}
if tsResponse.RangeCount != 1 {
return errors.Errorf("Table range not yet separated.")
}
if tsResponse.NodeCount != nodeCount {
return errors.Errorf("Table range not yet replicated to %d nodes.", 3)
}
if a, e := tsResponse.ReplicaCount, int64(nodeCount); a != e {
return errors.Errorf("expected %d replicas, found %d", e, a)
}
return nil
})
// These two conditions *must* be true, given that the above
// SucceedsSoon has succeeded.
if a, e := tsResponse.Stats.KeyCount, int64(20); a < e {
t.Fatalf("expected at least 20 total keys, found %d", a)
}
if len(tsResponse.MissingNodes) > 0 {
t.Fatalf("expected no missing nodes, found %v", tsResponse.MissingNodes)
}
// Kill a node, ensure it shows up in MissingNodes and that ReplicaCount is
// lower.
tc.StopServer(1)
if err := httputil.GetJSON(client, url, &tsResponse); err != nil {
t.Fatal(err)
}
if a, e := tsResponse.NodeCount, int64(nodeCount); a != e {
t.Errorf("expected %d nodes, found %d", e, a)
}
if a, e := tsResponse.RangeCount, int64(1); a != e {
t.Errorf("expected %d ranges, found %d", e, a)
}
if a, e := tsResponse.ReplicaCount, int64((nodeCount/2)+1); a != e {
t.Errorf("expected %d replicas, found %d", e, a)
}
if a, e := tsResponse.Stats.KeyCount, int64(10); a < e {
t.Errorf("expected at least 10 total keys, found %d", a)
}
if len(tsResponse.MissingNodes) != 1 {
t.Errorf("expected one missing node, found %v", tsResponse.MissingNodes)
}
// Call TableStats with a very low timeout. This tests that fan-out queries
// do not leak goroutines if the calling context is abandoned.
// Interestingly, the call can actually sometimes succeed, despite the small
// timeout; however, in aggregate (or in stress tests) this will suffice for
// detecting leaks.
client.Timeout = 1 * time.Nanosecond
_ = httputil.GetJSON(client, url, &tsResponse)
}