// NewNode returns a new instance of Node. func NewNode(ctx storage.StoreContext) *Node { return &Node{ ctx: ctx, status: status.NewNodeStatusMonitor(), lSender: kv.NewLocalSender(), } }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool, context *storage.StoreContext) (*storage.Store, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) if context == nil { // make a copy ctx := storage.TestStoreContext context = &ctx } context.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, clock, false, nil, stopper) context.Clock = clock context.DB = client.NewDB(sender) context.Transport = multiraft.NewLocalRPCTransport(stopper) // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(*context, eng, &proto.NodeDescriptor{NodeID: 1}) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } } lSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(nil); err != nil { t.Fatal(err) } } if err := store.Start(stopper); err != nil { t.Fatal(err) } return store, stopper }
// BootstrapCluster bootstraps a multiple stores using the provided engines and // cluster ID. The first bootstrapped store contains a single range spanning // all keys. Initial range lookup metadata is populated for the range. // // Returns a KV client for unittest purposes. Caller should close the returned // client. func BootstrapCluster(clusterID string, engines []engine.Engine, stopper *stop.Stopper) (*client.DB, error) { ctx := storage.StoreContext{} ctx.ScanInterval = 10 * time.Minute ctx.Clock = hlc.NewClock(hlc.UnixNano) // Create a KV DB with a local sender. lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, ctx.Clock, false, nil, stopper) ctx.DB = client.NewDB(sender) ctx.Transport = multiraft.NewLocalRPCTransport(stopper) for i, eng := range engines { sIdent := roachpb.StoreIdent{ ClusterID: clusterID, NodeID: 1, StoreID: roachpb.StoreID(i + 1), } // The bootstrapping store will not connect to other nodes so its // StoreConfig doesn't really matter. s := storage.NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1}) // Verify the store isn't already part of a cluster. if len(s.Ident.ClusterID) > 0 { return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID) } // Bootstrap store to persist the store ident. if err := s.Bootstrap(sIdent, stopper); err != nil { return nil, err } // Create first range, writing directly to engine. Note this does // not create the range, just its data. Only do this if this is the // first store. if i == 0 { // TODO(marc): this is better than having storage/ import sql, but still // not great. Find a better place to keep those. initialValues := sql.GetInitialSystemValues() if err := s.BootstrapRange(initialValues); err != nil { return nil, err } } if err := s.Start(stopper); err != nil { return nil, err } lSender.AddStore(s) // Initialize node and store ids. Only initialize the node once. if i == 0 { if nodeID, err := allocateNodeID(ctx.DB); nodeID != sIdent.NodeID || err != nil { return nil, util.Errorf("expected to initialize node id allocator to %d, got %d: %s", sIdent.NodeID, nodeID, err) } } if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, ctx.DB); storeID != sIdent.StoreID || err != nil { return nil, util.Errorf("expected to initialize store id allocator to %d, got %d: %s", sIdent.StoreID, storeID, err) } } return ctx.DB, nil }
func (m *multiTestContext) Start(t *testing.T, numStores int) { if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(m.clock, rpc.LoadInsecureTLSConfig()) m.gossip = gossip.New(rpcContext, gossip.TestInterval, "") } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport() } if m.sender == nil { m.sender = kv.NewLocalSender() } if m.db == nil { txnSender := kv.NewTxnCoordSender(m.sender, m.clock, false) m.db = client.NewKV(txnSender, nil) m.db.User = storage.UserRoot } for i := 0; i < numStores; i++ { m.addStore(t) } }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool, sCtx *storage.StoreContext) (*storage.Store, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, clock, stopper) if sCtx == nil { // make a copy ctx := storage.TestStoreContext sCtx = &ctx } nodeDesc := &proto.NodeDescriptor{NodeID: 1} sCtx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) localSender := kv.NewLocalSender() rpcSend := func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { call := proto.Call{ Args: getArgs(nil /* net.Addr */).(proto.Request), Reply: getReply().(proto.Response), } localSender.Send(context.Background(), call) return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError() } // Mostly makes sure that we don't see a warning per request. { if err := sCtx.Gossip.AddInfoProto(gossip.MakeNodeIDKey(nodeDesc.NodeID), nodeDesc, time.Hour); err != nil { t.Fatal(err) } if err := sCtx.Gossip.SetNodeDescriptor(nodeDesc); err != nil { t.Fatal(err) } } distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: clock, RPCSend: rpcSend, // defined above RangeDescriptorDB: localSender, // for descriptor lookup }, sCtx.Gossip) sender := kv.NewTxnCoordSender(distSender, clock, false, nil, stopper) sCtx.Clock = clock sCtx.DB = client.NewDB(sender) sCtx.Transport = multiraft.NewLocalRPCTransport(stopper) // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(*sCtx, eng, nodeDesc) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } } localSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(sql.GetInitialSystemValues()); err != nil { t.Fatal(err) } } if err := store.Start(stopper); err != nil { t.Fatal(err) } return store, stopper }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool) *storage.Store { rpcContext := rpc.NewContext(hlc.NewClock(hlc.UnixNano), rpc.LoadInsecureTLSConfig()) g := gossip.New(rpcContext, gossip.TestInterval, "") lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, clock, false) db := client.NewKV(sender, nil) db.User = storage.UserRoot // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(clock, eng, db, g, multiraft.NewLocalRPCTransport()) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}); err != nil { t.Fatal(err) } } lSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(); err != nil { t.Fatal(err) } } if err := store.Start(); err != nil { t.Fatal(err) } return store }
// NewNode returns a new instance of Node. func NewNode(ctx storage.StoreContext) *Node { return &Node{ ctx: ctx, status: status.NewNodeStatusMonitor(), lSender: kv.NewLocalSender(), completedScan: sync.NewCond(&sync.Mutex{}), } }
// AddStore creates a new store on the same Transport but doesn't create any ranges. func (m *multiTestContext) addStore() { idx := len(m.stores) var clock *hlc.Clock if len(m.clocks) > idx { clock = m.clocks[idx] } else { clock = m.clock m.clocks = append(m.clocks, clock) } var eng engine.Engine var needBootstrap bool if len(m.engines) > idx { eng = m.engines[idx] } else { eng = engine.NewInMem(proto.Attributes{}, 1<<20) m.engines = append(m.engines, eng) needBootstrap = true // Add an extra refcount to the engine so the underlying rocksdb instances // aren't closed when stopping and restarting the stores. // These refcounts are removed in Stop(). if err := eng.Open(); err != nil { m.t.Fatal(err) } } stopper := stop.NewStopper() ctx := m.makeContext(idx) store := storage.NewStore(ctx, eng, &proto.NodeDescriptor{NodeID: proto.NodeID(idx + 1)}) if needBootstrap { err := store.Bootstrap(proto.StoreIdent{ NodeID: proto.NodeID(idx + 1), StoreID: proto.StoreID(idx + 1), }, stopper) if err != nil { m.t.Fatal(err) } // Bootstrap the initial range on the first store if idx == 0 { if err := store.BootstrapRange(nil); err != nil { m.t.Fatal(err) } } } if err := store.Start(stopper); err != nil { m.t.Fatal(err) } store.WaitForInit() m.stores = append(m.stores, store) if len(m.senders) == idx { m.senders = append(m.senders, kv.NewLocalSender()) } m.senders[idx].AddStore(store) // Save the store identities for later so we can use them in // replication operations even while the store is stopped. m.idents = append(m.idents, store.Ident) m.stoppers = append(m.stoppers, stopper) }
func (m *multiTestContext) Start(t *testing.T, numStores int) { m.t = t if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(&base.Context{}, m.clock, nil) m.gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) } if m.clientStopper == nil { m.clientStopper = stop.NewStopper() } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport(m.clientStopper) } if m.storePool == nil { if m.timeUntilStoreDead == 0 { m.timeUntilStoreDead = storage.TestTimeUntilStoreDeadOff } m.storePool = storage.NewStorePool(m.gossip, m.timeUntilStoreDead, m.clientStopper) } // Always create the first sender. m.senders = append(m.senders, kv.NewLocalSender()) rpcSend := func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { call := proto.Call{ Args: getArgs(nil /* net.Addr */).(proto.Request), Reply: getReply().(proto.Response), } m.senders[0].Send(context.Background(), call) return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError() } if m.db == nil { distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: m.clock, RangeDescriptorDB: m.senders[0], RPCSend: rpcSend, }, m.gossip) sender := kv.NewTxnCoordSender(distSender, m.clock, false, nil, m.clientStopper) m.db = client.NewDB(sender) } for i := 0; i < numStores; i++ { m.addStore() } if m.transportStopper == nil { m.transportStopper = stop.NewStopper() } m.transportStopper.AddCloser(m.transport) }
// NewNode returns a new instance of Node, interpreting command line // flags to initialize the appropriate Store or set of // Stores. Registers the storage instance for the RPC service "Node". func NewNode(db *client.KV, gossip *gossip.Gossip) *Node { n := &Node{ gossip: gossip, db: db, lSender: kv.NewLocalSender(), closer: make(chan struct{}), } return n }
// BootstrapCluster bootstraps a store using the provided engine and // cluster ID. The bootstrapped store contains a single range spanning // all keys. Initial range lookup metadata is populated for the range. // // Returns a KV client for unittest purposes. Caller should close // the returned client. func BootstrapCluster(clusterID string, eng engine.Engine) (*client.KV, error) { sIdent := proto.StoreIdent{ ClusterID: clusterID, NodeID: 1, StoreID: 1, } clock := hlc.NewClock(hlc.UnixNano) // Create a KV DB with a local sender. lSender := kv.NewLocalSender() localDB := client.NewKV(kv.NewTxnCoordSender(lSender, clock, false), nil) // TODO(bdarnell): arrange to have the transport closed. s := storage.NewStore(clock, eng, localDB, nil, multiraft.NewLocalRPCTransport()) // Verify the store isn't already part of a cluster. if len(s.Ident.ClusterID) > 0 { return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID) } // Bootstrap store to persist the store ident. if err := s.Bootstrap(sIdent); err != nil { return nil, err } // Create first range. if err := s.BootstrapRange(); err != nil { return nil, err } if err := s.Start(); err != nil { return nil, err } lSender.AddStore(s) // Initialize node and store ids after the fact to account // for use of node ID = 1 and store ID = 1. if nodeID, err := allocateNodeID(localDB); nodeID != sIdent.NodeID || err != nil { return nil, util.Errorf("expected to intialize node id allocator to %d, got %d: %v", sIdent.NodeID, nodeID, err) } if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, localDB); storeID != sIdent.StoreID || err != nil { return nil, util.Errorf("expected to intialize store id allocator to %d, got %d: %v", sIdent.StoreID, storeID, err) } return localDB, nil }
func (m *multiTestContext) Start(t *testing.T, numStores int) { m.t = t if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(&base.Context{}, m.clock, nil) m.gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) } if m.clientStopper == nil { m.clientStopper = stop.NewStopper() } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport(m.clientStopper) } if m.storePool == nil { m.storePool = storage.NewStorePool(m.gossip, storage.TestTimeUntilStoreDeadOff, m.clientStopper) } // Always create the first sender. m.senders = append(m.senders, kv.NewLocalSender()) if m.db == nil { sender := kv.NewTxnCoordSender(m.senders[0], m.clock, false, nil, m.clientStopper) var err error if m.db, err = client.Open("//", client.SenderOpt(sender)); err != nil { t.Fatal(err) } } for i := 0; i < numStores; i++ { m.addStore() } if m.transportStopper == nil { m.transportStopper = stop.NewStopper() } m.transportStopper.AddCloser(m.transport) }