// Start starts the TestServer by bootstrapping an in-memory store // (defaults to maximum of 100M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.ServingAddr() after Start() for client connections. // Use TestServer.Stopper().Stop() to shutdown the server after the test // completes. func (ts *TestServer) Start(params base.TestServerArgs) error { if ts.Ctx == nil { panic("Ctx not set") } if params.Stopper == nil { params.Stopper = stop.NewStopper() } if !params.PartOfCluster { // Change the replication requirements so we don't get log spam about ranges // not being replicated enough. cfg := config.DefaultZoneConfig() cfg.ReplicaAttrs = []roachpb.Attributes{{}} fn := config.TestingSetDefaultZoneConfig(cfg) params.Stopper.AddCloser(stop.CloserFn(fn)) } // Needs to be called before NewServer to ensure resolvers are initialized. if err := ts.Ctx.InitNode(); err != nil { return err } // Ensure we have the correct number of engines. Add in-memory ones where // needed. There must be at least one store/engine. if params.StoresPerNode < 1 { params.StoresPerNode = 1 } for i := len(ts.Ctx.Engines); i < params.StoresPerNode; i++ { ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, params.Stopper)) } var err error ts.Server, err = NewServer(*ts.Ctx, params.Stopper) if err != nil { return err } // Our context must be shared with our server. ts.Ctx = &ts.Server.ctx if err := ts.Server.Start(); err != nil { return err } // If enabled, wait for initial splits to complete before returning control. // If initial splits do not complete, the server is stopped before // returning. if stk, ok := ts.ctx.TestingKnobs.Store.(*storage.StoreTestingKnobs); ok && stk.DisableSplitQueue { return nil } if err := ts.WaitForInitialSplits(); err != nil { ts.Stop() return err } return nil }
// StartTestCluster starts up a TestCluster made up of `nodes` in-memory testing // servers. // The cluster should be stopped using cluster.Stopper().Stop(). func StartTestCluster(t testing.TB, nodes int, args ClusterArgs) *TestCluster { if nodes < 1 { t.Fatal("invalid cluster size: ", nodes) } if args.ServerArgs.JoinAddr != "" { t.Fatal("can't specify a join addr when starting a cluster") } if args.ServerArgs.Stopper != nil { t.Fatal("can't set individual server stoppers when starting a cluster") } storeKnobs := args.ServerArgs.Knobs.Store if storeKnobs != nil && (storeKnobs.(*storage.StoreTestingKnobs).DisableSplitQueue || storeKnobs.(*storage.StoreTestingKnobs).DisableReplicateQueue) { t.Fatal("can't disable an individual server's queues when starting a cluster; " + "the cluster controls replication") } if args.Stopper == nil { args.Stopper = stop.NewStopper() args.ServerArgs.Stopper = args.Stopper } switch args.ReplicationMode { case ReplicationFull: // Force all ranges to be replicated everywhere. cfg := config.DefaultZoneConfig() cfg.ReplicaAttrs = make([]roachpb.Attributes, nodes) fn := config.TestingSetDefaultZoneConfig(cfg) args.Stopper.AddCloser(stop.CloserFn(fn)) case ReplicationManual: if args.ServerArgs.Knobs.Store == nil { args.ServerArgs.Knobs.Store = &storage.StoreTestingKnobs{} } storeKnobs := args.ServerArgs.Knobs.Store.(*storage.StoreTestingKnobs) storeKnobs.DisableSplitQueue = true storeKnobs.DisableReplicateQueue = true default: t.Fatal("unexpected replication mode") } tc := &TestCluster{} args.ServerArgs.PartOfCluster = true first, conn, _ := serverutils.StartServer(t, args.ServerArgs) tc.Servers = append(tc.Servers, first.(*server.TestServer)) tc.Conns = append(tc.Conns, conn) args.ServerArgs.JoinAddr = first.ServingAddr() for i := 1; i < nodes; i++ { s, conn, _ := serverutils.StartServer(t, args.ServerArgs) tc.Servers = append(tc.Servers, s.(*server.TestServer)) tc.Conns = append(tc.Conns, conn) } tc.waitForStores(t) return tc }
// TestRangeSplitsWithWritePressure sets the zone config max bytes for // a range to 256K and writes data until there are five ranges. func TestRangeSplitsWithWritePressure(t *testing.T) { defer leaktest.AfterTest(t)() // Override default zone config. cfg := config.DefaultZoneConfig() cfg.RangeMaxBytes = 1 << 18 defer config.TestingSetDefaultZoneConfig(cfg)() dbCtx := client.DefaultDBContext() dbCtx.TxnRetryOptions = retry.Options{ InitialBackoff: 1 * time.Millisecond, MaxBackoff: 10 * time.Millisecond, Multiplier: 2, } s, _ := createTestDBWithContext(t, dbCtx) // This is purely to silence log spam. config.TestingSetupZoneConfigHook(s.Stopper) defer s.Stop() // Start test writer write about a 32K/key so there aren't too many writes necessary to split 64K range. done := make(chan struct{}) var wg sync.WaitGroup wg.Add(1) go startTestWriter(s.DB, int64(0), 1<<15, &wg, nil, nil, done, t) // Check that we split 5 times in allotted time. util.SucceedsSoon(t, func() error { // Scan the txn records. rows, err := s.DB.Scan(keys.Meta2Prefix, keys.MetaMax, 0) if err != nil { return util.Errorf("failed to scan meta2 keys: %s", err) } if lr := len(rows); lr < 5 { return util.Errorf("expected >= 5 scans; got %d", lr) } return nil }) close(done) wg.Wait() // This write pressure test often causes splits while resolve // intents are in flight, causing them to fail with range key // mismatch errors. However, LocalSender should retry in these // cases. Check here via MVCC scan that there are no dangling write // intents. We do this using a SucceedsSoon construct to account // for timing of finishing the test writer and a possibly-ongoing // asynchronous split. util.SucceedsSoon(t, func() error { if _, _, err := engine.MVCCScan(context.Background(), s.Eng, keys.LocalMax, roachpb.KeyMax, 0, hlc.MaxTimestamp, true, nil); err != nil { return util.Errorf("failed to verify no dangling intents: %s", err) } return nil }) }
// StartWithStopper is the same as Start, but allows passing a stopper // explicitly. func (ts *TestServer) StartWithStopper(stopper *stop.Stopper) error { if ts.Ctx == nil { ts.Ctx = NewTestContext() } if stopper == nil { stopper = stop.NewStopper() } // Change the replication requirements so we don't get log spam about ranges // not being replicated enough. cfg := config.DefaultZoneConfig() cfg.ReplicaAttrs = []roachpb.Attributes{{}} fn := config.TestingSetDefaultZoneConfig(cfg) stopper.AddCloser(stop.CloserFn(fn)) // Needs to be called before NewServer to ensure resolvers are initialized. if err := ts.Ctx.InitNode(); err != nil { return err } var err error ts.Server, err = NewServer(ts.Ctx, stopper) if err != nil { return err } // Ensure we have the correct number of engines. Add in-memory ones where // needed. There must be at least one store/engine. if ts.StoresPerNode < 1 { ts.StoresPerNode = 1 } for i := len(ts.Ctx.Engines); i < ts.StoresPerNode; i++ { ts.Ctx.Engines = append(ts.Ctx.Engines, engine.NewInMem(roachpb.Attributes{}, 100<<20, ts.Server.stopper)) } if err := ts.Server.Start(); err != nil { return err } // If enabled, wait for initial splits to complete before returning control. // If initial splits do not complete, the server is stopped before // returning. if config.TestingTableSplitsDisabled() { return nil } if err := ts.WaitForInitialSplits(); err != nil { ts.Stop() return err } return nil }
func TestSkipLargeReplicaSnapshot(t *testing.T) { defer leaktest.AfterTest(t)() sCtx := TestStoreContext() sCtx.TestingKnobs.DisableSplitQueue = true store, _, stopper := createTestStoreWithContext(t, &sCtx) defer stopper.Stop() const snapSize = 1 << 20 // 1 MiB cfg := config.DefaultZoneConfig() cfg.RangeMaxBytes = snapSize defer config.TestingSetDefaultZoneConfig(cfg)() rep, err := store.GetReplica(rangeID) if err != nil { t.Fatal(err) } rep.SetMaxBytes(snapSize) if pErr := rep.redirectOnOrAcquireLease(context.Background()); pErr != nil { t.Fatal(pErr) } fillTestRange(t, rep, snapSize) if _, err := rep.GetSnapshot(context.Background()); err != nil { t.Fatal(err) } fillTestRange(t, rep, snapSize*2) if _, err := rep.Snapshot(); err != raft.ErrSnapshotTemporarilyUnavailable { rep.mu.Lock() after := rep.mu.state.Stats.Total() rep.mu.Unlock() t.Fatalf( "snapshot of a very large range (%d / %d, needsSplit: %v, exceeds snap limit: %v) should fail but got %v", after, rep.GetMaxBytes(), rep.needsSplitBySize(), rep.exceedsDoubleSplitSizeLocked(), err, ) } }
// Starts up a cluster made of up `nodes` in-memory testing servers, // creates database `name and returns open gosql.DB connections to each // node (to the named db), as well as a cleanup func that stops and // cleans up all nodes and connections. func SetupMultinodeTestCluster( t testing.TB, nodes int, name string, ) (MultinodeTestCluster, []*gosql.DB, *stop.Stopper) { if nodes < 1 { t.Fatal("invalid cluster size: ", nodes) } stopper := stop.NewStopper() // Force all ranges to be replicated everywhere. This is needed until #7297 is // fixed, otherwise starting a cluster takes forever. cfg := config.DefaultZoneConfig() cfg.ReplicaAttrs = make([]roachpb.Attributes, nodes) fn := config.TestingSetDefaultZoneConfig(cfg) stopper.AddCloser(stop.CloserFn(fn)) var servers []serverutils.TestServerInterface var conns []*gosql.DB args := base.TestServerArgs{ Stopper: stopper, PartOfCluster: true, UseDatabase: name, } first, conn, _ := serverutils.StartServer(t, args) servers = append(servers, first) conns = append(conns, conn) args.JoinAddr = first.ServingAddr() for i := 1; i < nodes; i++ { s, conn, _ := serverutils.StartServer(t, args) servers = append(servers, s) conns = append(conns, conn) } if _, err := conns[0].Exec(fmt.Sprintf(`CREATE DATABASE %s`, name)); err != nil { t.Fatal(err) } testCluster := MultinodeTestCluster{Servers: servers} return testCluster, conns, first.Stopper() }
// TestRaftLogQueue verifies that the raft log queue correctly truncates the // raft log. func TestRaftLogQueue(t *testing.T) { defer leaktest.AfterTest(t)() var mtc multiTestContext // Set maxBytes to something small so we can trigger the raft log truncation // without adding 64MB of logs. const maxBytes = 1 << 16 defer config.TestingSetDefaultZoneConfig(config.ZoneConfig{ RangeMaxBytes: maxBytes, })() // Turn off raft elections so the raft leader won't change out from under // us in this test. sc := storage.TestStoreContext() sc.RaftTickInterval = time.Hour * 24 sc.RaftElectionTimeoutTicks = 1000000 mtc.storeContext = &sc mtc.Start(t, 3) defer mtc.Stop() // Write a single value to ensure we have a leader. pArgs := putArgs([]byte("key"), []byte("value")) if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil { t.Fatal(err) } // Get the raft leader (and ensure one exists). rangeID := mtc.stores[0].LookupReplica([]byte("a"), nil).RangeID raftLeaderRepl := mtc.getRaftLeader(rangeID) if raftLeaderRepl == nil { t.Fatalf("could not find raft leader replica for range %d", rangeID) } originalIndex, err := raftLeaderRepl.GetFirstIndex() if err != nil { t.Fatal(err) } // Disable splits since we're increasing the raft log with puts. for _, store := range mtc.stores { store.DisableSplitQueue(true) } // Write a collection of values to increase the raft log. value := bytes.Repeat([]byte("a"), 1000) // 1KB for size := int64(0); size < 2*maxBytes; size += int64(len(value)) { pArgs = putArgs([]byte(fmt.Sprintf("key-%d", size)), value) if _, err := client.SendWrapped(rg1(mtc.stores[0]), nil, &pArgs); err != nil { t.Fatal(err) } } // Sadly, occasionally the queue has a race with the force processing so // this succeeds within will captures those rare cases. var afterTruncationIndex uint64 util.SucceedsSoon(t, func() error { // Force a truncation check. for _, store := range mtc.stores { store.ForceRaftLogScanAndProcess() } // Ensure that firstIndex has increased indicating that the log // truncation has occurred. var err error afterTruncationIndex, err = raftLeaderRepl.GetFirstIndex() if err != nil { t.Fatal(err) } if afterTruncationIndex <= originalIndex { return errors.Errorf("raft log has not been truncated yet, afterTruncationIndex:%d originalIndex:%d", afterTruncationIndex, originalIndex) } return nil }) // Force a truncation check again to ensure that attempting to truncate an // already truncated log has no effect. for _, store := range mtc.stores { store.ForceRaftLogScanAndProcess() } after2ndTruncationIndex, err := raftLeaderRepl.GetFirstIndex() if err != nil { t.Fatal(err) } if afterTruncationIndex > after2ndTruncationIndex { t.Fatalf("second truncation destroyed state: afterTruncationIndex:%d after2ndTruncationIndex:%d", afterTruncationIndex, after2ndTruncationIndex) } }