// Start starts the TestServer by bootstrapping an in-memory store
// (defaults to maximum of 100M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.ServingAddr() after Start() for client connections.
// Use TestServer.Stopper().Stop() to shutdown the server after the test
// completes.
func (ts *TestServer) Start(params base.TestServerArgs) error {
if ts.Ctx == nil {
panic("Ctx not set")
}
if params.Stopper == nil {
params.Stopper = stop.NewStopper()
}
if !params.PartOfCluster {
// Change the replication requirements so we don't get log spam about ranges
// not being replicated enough.
cfg := config.DefaultZoneConfig()
cfg.ReplicaAttrs = []roachpb.Attributes{{}}
fn := config.TestingSetDefaultZoneConfig(cfg)
params.Stopper.AddCloser(stop.CloserFn(fn))
}
// Needs to be called before NewServer to ensure resolvers are initialized.
if err := ts.Ctx.InitNode(); err != nil {
return err
}
// Ensure we have the correct number of engines. Add in-memory ones where
// needed. There must be at least one store/engine.
if params.StoresPerNode < 1 {
params.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < params.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, params.Stopper))
}
var err error
ts.Server, err = NewServer(*ts.Ctx, params.Stopper)
if err != nil {
return err
}
// Our context must be shared with our server.
ts.Ctx = &ts.Server.ctx
if err := ts.Server.Start(); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if stk, ok := ts.ctx.TestingKnobs.Store.(*storage.StoreTestingKnobs); ok &&
stk.DisableSplitQueue {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
// StartTestCluster starts up a TestCluster made up of `nodes` in-memory testing
// servers.
// The cluster should be stopped using cluster.Stopper().Stop().
func StartTestCluster(t testing.TB, nodes int, args ClusterArgs) *TestCluster {
if nodes < 1 {
t.Fatal("invalid cluster size: ", nodes)
}
if args.ServerArgs.JoinAddr != "" {
t.Fatal("can't specify a join addr when starting a cluster")
}
if args.ServerArgs.Stopper != nil {
t.Fatal("can't set individual server stoppers when starting a cluster")
}
storeKnobs := args.ServerArgs.Knobs.Store
if storeKnobs != nil &&
(storeKnobs.(*storage.StoreTestingKnobs).DisableSplitQueue ||
storeKnobs.(*storage.StoreTestingKnobs).DisableReplicateQueue) {
t.Fatal("can't disable an individual server's queues when starting a cluster; " +
"the cluster controls replication")
}
if args.Stopper == nil {
args.Stopper = stop.NewStopper()
args.ServerArgs.Stopper = args.Stopper
}
switch args.ReplicationMode {
case ReplicationFull:
// Force all ranges to be replicated everywhere.
cfg := config.DefaultZoneConfig()
cfg.ReplicaAttrs = make([]roachpb.Attributes, nodes)
fn := config.TestingSetDefaultZoneConfig(cfg)
args.Stopper.AddCloser(stop.CloserFn(fn))
case ReplicationManual:
if args.ServerArgs.Knobs.Store == nil {
args.ServerArgs.Knobs.Store = &storage.StoreTestingKnobs{}
}
storeKnobs := args.ServerArgs.Knobs.Store.(*storage.StoreTestingKnobs)
storeKnobs.DisableSplitQueue = true
storeKnobs.DisableReplicateQueue = true
default:
t.Fatal("unexpected replication mode")
}
tc := &TestCluster{}
args.ServerArgs.PartOfCluster = true
first, conn, _ := serverutils.StartServer(t, args.ServerArgs)
tc.Servers = append(tc.Servers, first.(*server.TestServer))
tc.Conns = append(tc.Conns, conn)
args.ServerArgs.JoinAddr = first.ServingAddr()
for i := 1; i < nodes; i++ {
s, conn, _ := serverutils.StartServer(t, args.ServerArgs)
tc.Servers = append(tc.Servers, s.(*server.TestServer))
tc.Conns = append(tc.Conns, conn)
}
tc.waitForStores(t)
return tc
}
// TestRangeSplitsWithWritePressure sets the zone config max bytes for
// a range to 256K and writes data until there are five ranges.
func TestRangeSplitsWithWritePressure(t *testing.T) {
defer leaktest.AfterTest(t)()
// Override default zone config.
cfg := config.DefaultZoneConfig()
cfg.RangeMaxBytes = 1 << 18
defer config.TestingSetDefaultZoneConfig(cfg)()
dbCtx := client.DefaultDBContext()
dbCtx.TxnRetryOptions = retry.Options{
InitialBackoff: 1 * time.Millisecond,
MaxBackoff: 10 * time.Millisecond,
Multiplier: 2,
}
s, _ := createTestDBWithContext(t, dbCtx)
// This is purely to silence log spam.
config.TestingSetupZoneConfigHook(s.Stopper)
defer s.Stop()
// Start test writer write about a 32K/key so there aren't too many writes necessary to split 64K range.
done := make(chan struct{})
var wg sync.WaitGroup
wg.Add(1)
go startTestWriter(s.DB, int64(0), 1<<15, &wg, nil, nil, done, t)
// Check that we split 5 times in allotted time.
util.SucceedsSoon(t, func() error {
// Scan the txn records.
rows, err := s.DB.Scan(keys.Meta2Prefix, keys.MetaMax, 0)
if err != nil {
return util.Errorf("failed to scan meta2 keys: %s", err)
}
if lr := len(rows); lr < 5 {
return util.Errorf("expected >= 5 scans; got %d", lr)
}
return nil
})
close(done)
wg.Wait()
// This write pressure test often causes splits while resolve
// intents are in flight, causing them to fail with range key
// mismatch errors. However, LocalSender should retry in these
// cases. Check here via MVCC scan that there are no dangling write
// intents. We do this using a SucceedsSoon construct to account
// for timing of finishing the test writer and a possibly-ongoing
// asynchronous split.
util.SucceedsSoon(t, func() error {
if _, _, err := engine.MVCCScan(context.Background(), s.Eng, keys.LocalMax, roachpb.KeyMax, 0, hlc.MaxTimestamp, true, nil); err != nil {
return util.Errorf("failed to verify no dangling intents: %s", err)
}
return nil
})
}
// StartWithStopper is the same as Start, but allows passing a stopper
// explicitly.
func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error {
if ts.Ctx == nil {
ts.Ctx = NewTestContext()
}
if stopper == nil {
stopper = stop.NewStopper()
}
// Change the replication requirements so we don't get log spam about ranges
// not being replicated enough.
cfg := config.DefaultZoneConfig()
cfg.ReplicaAttrs = []roachpb.Attributes{{}}
fn := config.TestingSetDefaultZoneConfig(cfg)
stopper.AddCloser(stop.CloserFn(fn))
// Needs to be called before NewServer to ensure resolvers are initialized.
if err := ts.Ctx.InitNode(); err != nil {
return err
}
var err error
ts.Server, err = NewServer(ts.Ctx, stopper)
if err != nil {
return err
}
// Ensure we have the correct number of engines. Add in-memory ones where
// needed. There must be at least one store/engine.
if ts.StoresPerNode < 1 {
ts.StoresPerNode = 1
}
for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ {
ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper))
}
if err := ts.Server.Start(); err != nil {
return err
}
// If enabled, wait for initial splits to complete before returning control.
// If initial splits do not complete, the server is stopped before
// returning.
if config.TestingTableSplitsDisabled() {
return nil
}
if err := ts.WaitForInitialSplits(); err != nil {
ts.Stop()
return err
}
return nil
}
func TestSkipLargeReplicaSnapshot(t *testing.T) {
defer leaktest.AfterTest(t)()
sCtx := TestStoreContext()
sCtx.TestingKnobs.DisableSplitQueue = true
store, _, stopper := createTestStoreWithContext(t, &sCtx)
defer stopper.Stop()
const snapSize = 1 << 20 // 1 MiB
cfg := config.DefaultZoneConfig()
cfg.RangeMaxBytes = snapSize
defer config.TestingSetDefaultZoneConfig(cfg)()
rep, err := store.GetReplica(rangeID)
if err != nil {
t.Fatal(err)
}
rep.SetMaxBytes(snapSize)
if pErr := rep.redirectOnOrAcquireLease(context.Background()); pErr != nil {
t.Fatal(pErr)
}
fillTestRange(t, rep, snapSize)
if _, err := rep.GetSnapshot(context.Background()); err != nil {
t.Fatal(err)
}
fillTestRange(t, rep, snapSize*2)
if _, err := rep.Snapshot(); err != raft.ErrSnapshotTemporarilyUnavailable {
rep.mu.Lock()
after := rep.mu.state.Stats.Total()
rep.mu.Unlock()
t.Fatalf(
"snapshot of a very large range (%d / %d, needsSplit: %v, exceeds snap limit: %v) should fail but got %v",
after, rep.GetMaxBytes(),
rep.needsSplitBySize(), rep.exceedsDoubleSplitSizeLocked(), err,
)
}
}
// Starts up a cluster made of up `nodes` in-memory testing servers,
// creates database `name and returns open gosql.DB connections to each
// node (to the named db), as well as a cleanup func that stops and
// cleans up all nodes and connections.
func SetupMultinodeTestCluster(
t testing.TB, nodes int, name string,
) (MultinodeTestCluster, []*gosql.DB, *stop.Stopper) {
if nodes < 1 {
t.Fatal("invalid cluster size: ", nodes)
}
stopper := stop.NewStopper()
// Force all ranges to be replicated everywhere. This is needed until #7297 is
// fixed, otherwise starting a cluster takes forever.
cfg := config.DefaultZoneConfig()
cfg.ReplicaAttrs = make([]roachpb.Attributes, nodes)
fn := config.TestingSetDefaultZoneConfig(cfg)
stopper.AddCloser(stop.CloserFn(fn))
var servers []serverutils.TestServerInterface
var conns []*gosql.DB
args := base.TestServerArgs{
Stopper: stopper,
PartOfCluster: true,
UseDatabase: name,
}
first, conn, _ := serverutils.StartServer(t, args)
servers = append(servers, first)
conns = append(conns, conn)
args.JoinAddr = first.ServingAddr()
for i := 1; i < nodes; i++ {
s, conn, _ := serverutils.StartServer(t, args)
servers = append(servers, s)
conns = append(conns, conn)
}
if _, err := conns[0].Exec(fmt.Sprintf(`CREATE DATABASE %s`, name)); err != nil {
t.Fatal(err)
}
testCluster := MultinodeTestCluster{Servers: servers}
return testCluster, conns, first.Stopper()
}
// TestRaftLogQueue verifies that the raft log queue correctly truncates the
// raft log.
func TestRaftLogQueue(t *testing.T) {
defer leaktest.AfterTest(t)()
var mtc multiTestContext
// Set maxBytes to something small so we can trigger the raft log truncation
// without adding 64MB of logs.
const maxBytes = 1 << 16
defer config.TestingSetDefaultZoneConfig(config.ZoneConfig{
RangeMaxBytes: maxBytes,
})()
// Turn off raft elections so the raft leader won't change out from under
// us in this test.
sc := storage.TestStoreContext()
sc.RaftTickInterval = time.Hour * 24
sc.RaftElectionTimeoutTicks = 1000000
mtc.storeContext = &sc
mtc.Start(t, 3)
defer mtc.Stop()
// Write a single value to ensure we have a leader.
pArgs := putArgs([]byte("key"), []byte("value"))
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil {
t.Fatal(err)
}
// Get the raft leader (and ensure one exists).
rangeID := mtc.stores[0].LookupReplica([]byte("a"), nil).RangeID
raftLeaderRepl := mtc.getRaftLeader(rangeID)
if raftLeaderRepl == nil {
t.Fatalf("could not find raft leader replica for range %d", rangeID)
}
originalIndex, err := raftLeaderRepl.GetFirstIndex()
if err != nil {
t.Fatal(err)
}
// Disable splits since we're increasing the raft log with puts.
for _, store := range mtc.stores {
store.DisableSplitQueue(true)
}
// Write a collection of values to increase the raft log.
value := bytes.Repeat([]byte("a"), 1000) // 1KB
for size := int64(0); size < 2*maxBytes; size += int64(len(value)) {
pArgs = putArgs([]byte(fmt.Sprintf("key-%d", size)), value)
if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil {
t.Fatal(err)
}
}
// Sadly, occasionally the queue has a race with the force processing so
// this succeeds within will captures those rare cases.
var afterTruncationIndex uint64
util.SucceedsSoon(t, func() error {
// Force a truncation check.
for _, store := range mtc.stores {
store.ForceRaftLogScanAndProcess()
}
// Ensure that firstIndex has increased indicating that the log
// truncation has occurred.
var err error
afterTruncationIndex, err = raftLeaderRepl.GetFirstIndex()
if err != nil {
t.Fatal(err)
}
if afterTruncationIndex <= originalIndex {
return errors.Errorf("raft log has not been truncated yet, afterTruncationIndex:%d originalIndex:%d",
afterTruncationIndex, originalIndex)
}
return nil
})
// Force a truncation check again to ensure that attempting to truncate an
// already truncated log has no effect.
for _, store := range mtc.stores {
store.ForceRaftLogScanAndProcess()
}
after2ndTruncationIndex, err := raftLeaderRepl.GetFirstIndex()
if err != nil {
t.Fatal(err)
}
if afterTruncationIndex > after2ndTruncationIndex {
t.Fatalf("second truncation destroyed state: afterTruncationIndex:%d after2ndTruncationIndex:%d",
afterTruncationIndex, after2ndTruncationIndex)
}
}
