func (m *multiTestContext) Start(t *testing.T, numStores int) { if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(m.clock, rpc.LoadInsecureTLSConfig()) m.gossip = gossip.New(rpcContext, gossip.TestInterval, "") } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport() } if m.sender == nil { m.sender = kv.NewLocalSender() } if m.db == nil { txnSender := kv.NewTxnCoordSender(m.sender, m.clock, false) m.db = client.NewKV(txnSender, nil) m.db.User = storage.UserRoot } for i := 0; i < numStores; i++ { m.addStore(t) } }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool) *storage.Store { rpcContext := rpc.NewContext(hlc.NewClock(hlc.UnixNano), rpc.LoadInsecureTLSConfig()) g := gossip.New(rpcContext, gossip.TestInterval, "") lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, clock, false) db := client.NewKV(sender, nil) db.User = storage.UserRoot // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(clock, eng, db, g, multiraft.NewLocalRPCTransport()) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}); err != nil { t.Fatal(err) } } lSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(); err != nil { t.Fatal(err) } } if err := store.Start(); err != nil { t.Fatal(err) } return store }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool, context *storage.StoreContext) (*storage.Store, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) if context == nil { // make a copy ctx := storage.TestStoreContext context = &ctx } context.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, clock, false, nil, stopper) context.Clock = clock context.DB = client.NewDB(sender) context.Transport = multiraft.NewLocalRPCTransport(stopper) // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(*context, eng, &proto.NodeDescriptor{NodeID: 1}) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } } lSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(nil); err != nil { t.Fatal(err) } } if err := store.Start(stopper); err != nil { t.Fatal(err) } return store, stopper }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) ( *rpc.Server, *hlc.Clock, *Node, *stop.Stopper) { ctx := storage.StoreContext{} stopper := stop.NewStopper() ctx.Clock = hlc.NewClock(hlc.UnixNano) nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper) ctx.ScanInterval = 10 * time.Hour rpcServer := rpc.NewServer(addr, nodeRPCContext) if err := rpcServer.Start(); err != nil { t.Fatal(err) } g := gossip.New(nodeRPCContext, testContext.GossipInterval, testContext.GossipBootstrapResolvers) if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS == addr { gossipBS = rpcServer.Addr() } g.SetResolvers([]resolver.Resolver{resolver.NewResolverFromAddress(gossipBS)}) g.Start(rpcServer, stopper) } ctx.Gossip = g sender := kv.NewDistSender(&kv.DistSenderContext{Clock: ctx.Clock}, g) ctx.DB = client.NewDB(sender) // TODO(bdarnell): arrange to have the transport closed. // (or attach LocalRPCTransport.Close to the stopper) ctx.Transport = multiraft.NewLocalRPCTransport(stopper) ctx.EventFeed = util.NewFeed(stopper) node := NewNode(ctx) return rpcServer, ctx.Clock, node, stopper }
// createCluster generates a new cluster using the provided stopper and the // number of nodes supplied. Each node will have one store to start. func createCluster(stopper *stop.Stopper, nodeCount int) *Cluster { rand, seed := randutil.NewPseudoRand() clock := hlc.NewClock(hlc.UnixNano) rpcContext := rpc.NewContext(&base.Context{}, clock, stopper) g := gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) storePool := storage.NewStorePool(g, storage.TestTimeUntilStoreDeadOff, stopper) c := &Cluster{ stopper: stopper, clock: clock, rpc: rpcContext, gossip: g, storePool: storePool, allocator: storage.MakeAllocator(storePool, storage.RebalancingOptions{}), storeGossiper: gossiputil.NewStoreGossiper(g), nodes: make(map[proto.NodeID]*Node), stores: make(map[proto.StoreID]*Store), ranges: make(map[proto.RangeID]*Range), rand: rand, seed: seed, } // Add the nodes. for i := 0; i < nodeCount; i++ { c.addNewNodeWithStore() } // Add a single range and add to this first node's first store. firstRange := c.addRange() firstRange.attachRangeToStore(c.stores[proto.StoreID(0)]) return c }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool, sCtx *storage.StoreContext) (*storage.Store, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, clock, stopper) if sCtx == nil { // make a copy ctx := storage.TestStoreContext sCtx = &ctx } nodeDesc := &proto.NodeDescriptor{NodeID: 1} sCtx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) localSender := kv.NewLocalSender() rpcSend := func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { call := proto.Call{ Args: getArgs(nil /* net.Addr */).(proto.Request), Reply: getReply().(proto.Response), } localSender.Send(context.Background(), call) return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError() } // Mostly makes sure that we don't see a warning per request. { if err := sCtx.Gossip.AddInfoProto(gossip.MakeNodeIDKey(nodeDesc.NodeID), nodeDesc, time.Hour); err != nil { t.Fatal(err) } if err := sCtx.Gossip.SetNodeDescriptor(nodeDesc); err != nil { t.Fatal(err) } } distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: clock, RPCSend: rpcSend, // defined above RangeDescriptorDB: localSender, // for descriptor lookup }, sCtx.Gossip) sender := kv.NewTxnCoordSender(distSender, clock, false, nil, stopper) sCtx.Clock = clock sCtx.DB = client.NewDB(sender) sCtx.Transport = multiraft.NewLocalRPCTransport(stopper) // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(*sCtx, eng, nodeDesc) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } } localSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(sql.GetInitialSystemValues()); err != nil { t.Fatal(err) } } if err := store.Start(stopper); err != nil { t.Fatal(err) } return store, stopper }
// createTestStoreWithoutStart creates a test store using an in-memory // engine without starting the store. It returns the store, the store // clock's manual unix nanos time and a stopper. The caller is // responsible for stopping the stopper upon completion. func createTestStoreWithoutStart(t *testing.T) (*Store, *hlc.ManualClock, *stop.Stopper) { stopper := stop.NewStopper() // Setup fake zone config handler. config.TestingSetupZoneConfigHook(stopper) rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) ctx := TestStoreContext ctx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) ctx.StorePool = NewStorePool(ctx.Gossip, TestTimeUntilStoreDeadOff, stopper) manual := hlc.NewManualClock(0) ctx.Clock = hlc.NewClock(manual.UnixNano) eng := engine.NewInMem(roachpb.Attributes{}, 10<<20, stopper) ctx.Transport = multiraft.NewLocalRPCTransport(stopper) stopper.AddCloser(ctx.Transport) sender := &testSender{} ctx.DB = client.NewDB(sender) store := NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1}) sender.store = store if err := store.Bootstrap(roachpb.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } if err := store.BootstrapRange(nil); err != nil { t.Fatal(err) } return store, manual, stopper }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server and node are returned. If gossipBS is not // nil, the gossip bootstrap address is set to gossipBS. func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) ( *rpc.Server, *Node) { tlsConfig, err := rpc.LoadTestTLSConfig("..") if err != nil { t.Fatal(err) } rpcServer := rpc.NewServer(addr, tlsConfig) if err := rpcServer.Start(); err != nil { t.Fatal(err) } g := gossip.New(tlsConfig) if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS == addr { gossipBS = rpcServer.Addr() } g.SetBootstrap([]net.Addr{gossipBS}) g.Start(rpcServer) } clock := hlc.NewClock(hlc.UnixNano) db := kv.NewDB(kv.NewDistKV(g), clock) node := NewNode(db, g) if err := node.start(rpcServer, clock, engines, proto.Attributes{}); err != nil { t.Fatal(err) } return rpcServer, node }
func newServer() (*server, error) { // Determine hostname in case it hasn't been specified in -rpc or -http. host, err := os.Hostname() if err != nil { host = "127.0.0.1" } // Resolve if strings.HasPrefix(*rpcAddr, ":") { *rpcAddr = host + *rpcAddr } addr, err := net.ResolveTCPAddr("tcp", *rpcAddr) if err != nil { return nil, util.Errorf("unable to resolve RPC address %q: %v", *rpcAddr, err) } s := &server{ host: host, mux: http.NewServeMux(), rpc: rpc.NewServer(addr), } s.gossip = gossip.New() s.kvDB = kv.NewDB(s.gossip) s.kvREST = rest.NewRESTServer(s.kvDB) s.node = NewNode(s.kvDB, s.gossip) s.admin = newAdminServer(s.kvDB) s.status = newStatusServer(s.kvDB) s.structuredDB = structured.NewDB(s.kvDB) s.structuredREST = structured.NewRESTServer(s.structuredDB) return s, nil }
// createTestStorePool creates a stopper, gossip and storePool for use in // tests. Stopper must be stopped by the caller. func createTestStorePool(timeUntilStoreDead time.Duration) (*stop.Stopper, *gossip.Gossip, *StorePool) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) g := gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) storePool := NewStorePool(g, timeUntilStoreDead, stopper) return stopper, g, storePool }
func gossipForTest(t *testing.T) (*gossip.Gossip, *stop.Stopper) { stopper := stop.NewStopper() // Setup fake zone config handler. config.TestingSetupZoneConfigHook(stopper) rpcContext := rpc.NewContext(nil, nil, stopper) g := gossip.New(rpcContext, nil, stopper) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) // Put an empty system config into gossip. if err := g.AddInfoProto(gossip.KeySystemConfig, &config.SystemConfig{}, 0); err != nil { t.Fatal(err) } // Wait for SystemConfig. util.SucceedsSoon(t, func() error { if _, ok := g.GetSystemConfig(); !ok { return util.Errorf("expected system config to be set") } return nil }) return g, stopper }
func gossipForTest(t *testing.T) (*gossip.Gossip, *stop.Stopper) { stopper := stop.NewStopper() // Setup fake zone config handler. config.TestingSetupZoneConfigHook(stopper) rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) g := gossip.New(rpcContext, gossip.TestBootstrap, stopper) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) // Put an empty system config into gossip. if err := g.AddInfoProto(gossip.KeySystemConfig, &config.SystemConfig{}, 0); err != nil { t.Fatal(err) } // Wait for SystemConfig. util.SucceedsSoon(t, func() error { if g.GetSystemConfig() == nil { return util.Errorf("expected non-nil system config") } return nil }) return g, stopper }
// createTestStoreWithoutStart creates a test store using an in-memory // engine without starting the store. It returns the store, the store // clock's manual unix nanos time and a stopper. The caller is // responsible for stopping the stopper upon completion. func createTestStoreWithoutStart(t *testing.T) (*Store, *hlc.ManualClock, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(rootTestBaseContext, hlc.NewClock(hlc.UnixNano), stopper) ctx := TestStoreContext ctx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) manual := hlc.NewManualClock(0) ctx.Clock = hlc.NewClock(manual.UnixNano) eng := engine.NewInMem(proto.Attributes{}, 10<<20) ctx.Transport = multiraft.NewLocalRPCTransport() stopper.AddCloser(ctx.Transport) sender := &testSender{} var err error if ctx.DB, err = client.Open("//root@", client.SenderOpt(sender)); err != nil { t.Fatal(err) } store := NewStore(ctx, eng, &proto.NodeDescriptor{NodeID: 1}) sender.store = store if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } if err := store.BootstrapRange(); err != nil { t.Fatal(err) } return store, manual, stopper }
func gossipForTest(t *testing.T) (*gossip.Gossip, *stop.Stopper) { stopper := stop.NewStopper() // Setup fake zone config handler. config.TestingSetupZoneConfigHook(stopper) rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) g := gossip.New(rpcContext, gossip.TestBootstrap) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) // Put an empty system config into gossip. if err := g.AddInfoProto(gossip.KeySystemConfig, &config.SystemConfig{}, 0); err != nil { t.Fatal(err) } // Wait for SystemConfig. if err := util.IsTrueWithin(func() bool { return g.GetSystemConfig() != nil }, 100*time.Millisecond); err != nil { t.Fatal(err) } return g, stopper }
// Start starts the test cluster by bootstrapping an in-memory store // (defaults to maximum of 50M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.Addr after Start() for client connections. Use Stop() // to shutdown the server after the test completes. func (ltc *LocalTestCluster) Start(t util.Tester) { ltc.Manual = hlc.NewManualClock(0) ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano) ltc.Stopper = stop.NewStopper() rpcContext := rpc.NewContext(testutils.NewRootTestBaseContext(), ltc.Clock, ltc.Stopper) ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20) ltc.lSender = newRetryableLocalSender(NewLocalSender()) ltc.Sender = NewTxnCoordSender(ltc.lSender, ltc.Clock, false, nil, ltc.Stopper) var err error if ltc.DB, err = client.Open("//root@", client.SenderOpt(ltc.Sender)); err != nil { t.Fatal(err) } transport := multiraft.NewLocalRPCTransport(ltc.Stopper) ltc.Stopper.AddCloser(transport) ctx := storage.TestStoreContext ctx.Clock = ltc.Clock ctx.DB = ltc.DB ctx.Gossip = ltc.Gossip ctx.Transport = transport ltc.Store = storage.NewStore(ctx, ltc.Eng, &proto.NodeDescriptor{NodeID: 1}) if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.lSender.AddStore(ltc.Store) if err := ltc.Store.BootstrapRange(nil); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } if err := ltc.Store.Start(ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } }
// Start starts the test cluster by bootstrapping an in-memory store // (defaults to maximum of 50M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.Addr after Start() for client connections. Use Stop() // to shutdown the server after the test completes. func (ltc *LocalTestCluster) Start(t util.Tester) { nodeDesc := &proto.NodeDescriptor{NodeID: 1} ltc.tester = t ltc.Manual = hlc.NewManualClock(0) ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano) ltc.Stopper = stop.NewStopper() rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper) ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20, ltc.Stopper) ltc.localSender = NewLocalSender() var rpcSend rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { // TODO(tschottdorf): remove getReply(). br, pErr := ltc.localSender.Send(context.Background(), *getArgs(nil).(*proto.BatchRequest)) if br == nil { br = &proto.BatchResponse{} } if br.Error != nil { panic(proto.ErrorUnexpectedlySet(ltc.localSender, br)) } br.Error = pErr return []gogoproto.Message{br}, nil } ltc.distSender = NewDistSender(&DistSenderContext{ Clock: ltc.Clock, RangeDescriptorCacheSize: defaultRangeDescriptorCacheSize, RangeLookupMaxRanges: defaultRangeLookupMaxRanges, LeaderCacheSize: defaultLeaderCacheSize, RPCRetryOptions: &defaultRPCRetryOptions, nodeDescriptor: nodeDesc, RPCSend: rpcSend, // defined above RangeDescriptorDB: ltc.localSender, // for descriptor lookup }, ltc.Gossip) ltc.Sender = NewTxnCoordSender(ltc.distSender, ltc.Clock, false /* !linearizable */, nil /* tracer */, ltc.Stopper) ltc.DB = client.NewDB(ltc.Sender) transport := multiraft.NewLocalRPCTransport(ltc.Stopper) ltc.Stopper.AddCloser(transport) ctx := storage.TestStoreContext ctx.Clock = ltc.Clock ctx.DB = ltc.DB ctx.Gossip = ltc.Gossip ctx.Transport = transport ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc) if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.localSender.AddStore(ltc.Store) if err := ltc.Store.BootstrapRange(nil); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } if err := ltc.Store.Start(ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } }
// createTestAllocator creates a stopper, gossip, store pool and allocator for // use in tests. Stopper must be stopped by the caller. func createTestAllocator() (*stop.Stopper, *gossip.Gossip, *StorePool, Allocator) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) g := gossip.New(rpcContext, gossip.TestBootstrap) storePool := NewStorePool(g, TestTimeUntilStoreDeadOff, stopper) a := MakeAllocator(storePool, RebalancingOptions{AllowRebalance: true}) return stopper, g, storePool, a }
// createTestAllocator creates a stopper, gossip, store pool and allocator for // use in tests. Stopper must be stopped by the caller. func createTestAllocator() (*stop.Stopper, *gossip.Gossip, *StorePool, allocator) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) g := gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) storePool := NewStorePool(g, TestTimeUntilStoreDeadOff, stopper) a := makeAllocator(storePool) return stopper, g, storePool, a }
func (m *multiTestContext) Start(t *testing.T, numStores int) { m.t = t if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(&base.Context{}, m.clock, nil) m.gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) } if m.clientStopper == nil { m.clientStopper = stop.NewStopper() } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport(m.clientStopper) } if m.storePool == nil { if m.timeUntilStoreDead == 0 { m.timeUntilStoreDead = storage.TestTimeUntilStoreDeadOff } m.storePool = storage.NewStorePool(m.gossip, m.timeUntilStoreDead, m.clientStopper) } // Always create the first sender. m.senders = append(m.senders, kv.NewLocalSender()) rpcSend := func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { call := proto.Call{ Args: getArgs(nil /* net.Addr */).(proto.Request), Reply: getReply().(proto.Response), } m.senders[0].Send(context.Background(), call) return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError() } if m.db == nil { distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: m.clock, RangeDescriptorDB: m.senders[0], RPCSend: rpcSend, }, m.gossip) sender := kv.NewTxnCoordSender(distSender, m.clock, false, nil, m.clientStopper) m.db = client.NewDB(sender) } for i := 0; i < numStores; i++ { m.addStore() } if m.transportStopper == nil { m.transportStopper = stop.NewStopper() } m.transportStopper.AddCloser(m.transport) }
// createTestStorePool creates a stopper, gossip and storePool for use in // tests. Stopper must be stopped by the caller. func createTestStorePool(timeUntilStoreDead time.Duration) (*stop.Stopper, *gossip.Gossip, *StorePool) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) g := gossip.New(rpcContext, gossip.TestBootstrap) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) storePool := NewStorePool(g, timeUntilStoreDead, stopper) return stopper, g, storePool }
// createTestAllocator creates a stopper, gossip, store pool and allocator for // use in tests. Stopper must be stopped by the caller. func createTestAllocator() (*stop.Stopper, *gossip.Gossip, *StorePool, Allocator) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) g := gossip.New(rpcContext, gossip.TestBootstrap) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) storePool := NewStorePool(g, TestTimeUntilStoreDeadOff, stopper) a := MakeAllocator(storePool, AllocatorOptions{AllowRebalance: true}) return stopper, g, storePool, a }
// createTestRange creates a new range initialized to the full extent // of the keyspace. The gossip instance is also returned for testing. func createTestRange(engine Engine, t *testing.T) (*Range, *gossip.Gossip) { rm := RangeMetadata{ RangeID: 0, StartKey: KeyMin, EndKey: KeyMax, Replicas: testRangeLocations, } g := gossip.New(rpc.NewServer(&net.UnixAddr{"fake", "unix"})) return NewRange(rm, engine, nil, g), g }
// NewNetwork creates nodeCount gossip nodes. The networkType should // be set to either "tcp" or "unix". The gossipInterval should be set // to a compressed simulation timescale, though large enough to give // the concurrent goroutines enough time to pass data back and forth // in order to yield accurate estimates of how old data actually ends // up being at the various nodes (e.g. DefaultTestGossipInterval). // TODO: This method should take `stopper` as an argument. func NewNetwork(nodeCount int, networkType string, gossipInterval time.Duration) *Network { clock := hlc.NewClock(hlc.UnixNano) log.Infof("simulating gossip network with %d nodes", nodeCount) stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper) nodes := make([]*Node, nodeCount) for i := range nodes { server := rpc.NewServer(util.CreateTestAddr(networkType), rpcContext) if err := server.Start(); err != nil { log.Fatal(err) } nodes[i] = &Node{Server: server} } var numResolvers int if len(nodes) > 3 { numResolvers = 3 } else { numResolvers = len(nodes) } for i, leftNode := range nodes { // Build new resolvers for each instance or we'll get data races. var resolvers []resolver.Resolver for _, rightNode := range nodes[:numResolvers] { resolvers = append(resolvers, resolver.NewResolverFromAddress(rightNode.Server.Addr())) } gossipNode := gossip.New(rpcContext, gossipInterval, resolvers) addr := leftNode.Server.Addr() if err := gossipNode.SetNodeDescriptor(&roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(i + 1), Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), }); err != nil { log.Fatal(err) } gossipNode.Start(leftNode.Server, stopper) stopper.AddCloser(leftNode.Server) leftNode.Gossip = gossipNode } return &Network{ Nodes: nodes, NetworkType: networkType, GossipInterval: gossipInterval, Stopper: stopper, } }
// createCluster generates a new cluster using the provided stopper and the // number of nodes supplied. Each node will have one store to start. func createCluster( stopper *stop.Stopper, nodeCount int, epochWriter, actionWriter io.Writer, script Script, rand *rand.Rand, ) *Cluster { clock := hlc.NewClock(hlc.UnixNano) rpcContext := rpc.NewContext(nil, clock, stopper) g := gossip.New(rpcContext, nil, stopper) // NodeID is required for Gossip, so set it to -1 for the cluster Gossip // instance to prevent conflicts with real NodeIDs. g.SetNodeID(-1) storePool := storage.NewStorePool(g, clock, storage.TestTimeUntilStoreDeadOff, stopper) c := &Cluster{ stopper: stopper, clock: clock, rpc: rpcContext, gossip: g, storePool: storePool, allocator: storage.MakeAllocator(storePool, storage.AllocatorOptions{ AllowRebalance: true, Deterministic: true, }), storeGossiper: gossiputil.NewStoreGossiper(g), nodes: make(map[roachpb.NodeID]*Node), stores: make(map[roachpb.StoreID]*Store), ranges: make(map[roachpb.RangeID]*Range), rangeIDsByStore: make(map[roachpb.StoreID]roachpb.RangeIDSlice), rand: rand, epochWriter: tabwriter.NewWriter(epochWriter, 8, 1, 2, ' ', 0), actionWriter: tabwriter.NewWriter(actionWriter, 8, 1, 2, ' ', 0), script: script, epoch: -1, } // Add the nodes. for i := 0; i < nodeCount; i++ { c.addNewNodeWithStore() } // Add a single range and add to this first node's first store. firstRange := c.addRange() firstRange.addReplica(c.stores[0]) c.calculateRangeIDsByStore() // Output the first epoch header. c.epoch = 0 c.OutputEpochHeader() c.OutputEpoch() c.flush() return c }
// createTestRange creates a new range initialized to the full extent // of the keyspace. The gossip instance is also returned for testing. func createTestRange(engine engine.Engine, t *testing.T) (*Range, *gossip.Gossip) { rm := RangeMetadata{ RangeID: 0, RangeDescriptor: testRangeDescriptor, } g := gossip.New() clock := hlc.NewHLClock(hlc.UnixNano) r := NewRange(rm, clock, engine, nil, g) r.Start() return r, g }
// NewNetwork creates nodeCount gossip nodes. The networkType should // be set to either "tcp" or "unix". func NewNetwork(nodeCount int, networkType string) *Network { clock := hlc.NewClock(hlc.UnixNano) log.Infof("simulating gossip network with %d nodes", nodeCount) stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{Insecure: true}, clock, stopper) tlsConfig, err := rpcContext.GetServerTLSConfig() if err != nil { log.Fatal(err) } nodes := make([]*Node, nodeCount) for i := range nodes { server := rpc.NewServer(rpcContext) testAddr := util.CreateTestAddr(networkType) ln, err := util.ListenAndServe(stopper, server, testAddr, tlsConfig) if err != nil { log.Fatal(err) } nodes[i] = &Node{Server: server, Addr: ln.Addr()} } for i, leftNode := range nodes { // Build new resolvers for each instance or we'll get data races. resolvers := []resolver.Resolver{resolver.NewResolverFromAddress(nodes[0].Addr)} gossipNode := gossip.New(rpcContext, resolvers) addr := leftNode.Addr gossipNode.SetNodeID(roachpb.NodeID(i + 1)) if err := gossipNode.SetNodeDescriptor(&roachpb.NodeDescriptor{ NodeID: roachpb.NodeID(i + 1), Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()), }); err != nil { log.Fatal(err) } if err := gossipNode.AddInfo(addr.String(), encoding.EncodeUint64(nil, 0), time.Hour); err != nil { log.Fatal(err) } gossipNode.Start(leftNode.Server, addr, stopper) gossipNode.EnableSimulationCycler(true) leftNode.Gossip = gossipNode } return &Network{ Nodes: nodes, NetworkType: networkType, Stopper: stopper, } }
// createTestStorePool creates a stopper, gossip and storePool for use in // tests. Stopper must be stopped by the caller. func createTestStorePool(timeUntilStoreDead time.Duration) (*stop.Stopper, *gossip.Gossip, *hlc.ManualClock, *StorePool) { stopper := stop.NewStopper() mc := hlc.NewManualClock(0) clock := hlc.NewClock(mc.UnixNano) rpcContext := rpc.NewContext(nil, clock, stopper) g := gossip.New(rpcContext, nil, stopper) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) storePool := NewStorePool(g, clock, timeUntilStoreDead, stopper) return stopper, g, mc, storePool }
// createTestRange creates a new range initialized to the full extent // of the keyspace. The gossip instance is also returned for testing. func createTestRange(engine engine.Engine, t *testing.T) (*Range, *gossip.Gossip) { rm := &proto.RangeMetadata{ RangeID: 0, RangeDescriptor: testRangeDescriptor, } g := gossip.New(rpc.LoadInsecureTLSConfig()) clock := hlc.NewClock(hlc.UnixNano) r := NewRange(rm, clock, engine, nil, g) r.Start() return r, g }
// CreateNode creates a simulation node and starts an RPC server for it. func (n *Network) CreateNode() (*Node, error) { server := rpc.NewServer(n.rpcContext) ln, err := util.ListenAndServeGRPC(n.Stopper, server, util.TestAddr) if err != nil { return nil, err } node := &Node{Server: server, Addr: ln.Addr()} node.Gossip = gossip.New(n.rpcContext, nil, n.Stopper) n.Nodes = append(n.Nodes, node) return node, nil }
// createTestAllocator creates a stopper, gossip, store pool and allocator for // use in tests. Stopper must be stopped by the caller. func createTestAllocator() (*stop.Stopper, *gossip.Gossip, *StorePool, Allocator, *hlc.ManualClock) { stopper := stop.NewStopper() manualClock := hlc.NewManualClock(hlc.UnixNano()) clock := hlc.NewClock(manualClock.UnixNano) rpcContext := rpc.NewContext(nil, clock, stopper) g := gossip.New(rpcContext, nil, stopper) // Have to call g.SetNodeID before call g.AddInfo g.SetNodeID(roachpb.NodeID(1)) storePool := NewStorePool(g, clock, TestTimeUntilStoreDeadOff, stopper) a := MakeAllocator(storePool, AllocatorOptions{AllowRebalance: true}) return stopper, g, storePool, a, manualClock }