func TestLocalKVLookupReplica(t *testing.T) {
manual := hlc.ManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
eng := engine.NewInMem(proto.Attributes{}, 1<<20)
kv := NewLocalKV()
db := NewDB(kv, clock)
store := storage.NewStore(clock, eng, db, nil)
if err := store.Bootstrap(proto.StoreIdent{StoreID: 1}); err != nil {
t.Fatal(err)
}
kv.AddStore(store)
meta := store.BootstrapRangeMetadata()
meta.StartKey = engine.KeySystemPrefix
meta.EndKey = engine.PrefixEndKey(engine.KeySystemPrefix)
if _, err := store.CreateRange(meta); err != nil {
t.Fatal(err)
}
if err := store.Init(); err != nil {
t.Fatal(err)
}
// Create two new stores with ranges we care about.
var s [2]*storage.Store
ranges := []struct {
storeID int32
start, end engine.Key
}{
{2, engine.Key("a"), engine.Key("c")},
{3, engine.Key("x"), engine.Key("z")},
}
for i, rng := range ranges {
s[i] = storage.NewStore(clock, eng, db, nil)
s[i].Ident.StoreID = rng.storeID
replica := proto.Replica{StoreID: rng.storeID}
_, err := s[i].CreateRange(store.NewRangeMetadata(rng.start, rng.end, []proto.Replica{replica}))
if err != nil {
t.Fatal(err)
}
kv.AddStore(s[i])
}
if r, err := kv.lookupReplica(engine.Key("a"), engine.Key("c")); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if r, err := kv.lookupReplica(engine.Key("b"), nil); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if r, err := kv.lookupReplica(engine.Key("b"), engine.Key("d")); r != nil || err == nil {
t.Errorf("expected store 0 and error got %d", r.StoreID)
}
if r, err := kv.lookupReplica(engine.Key("x"), engine.Key("z")); r.StoreID != s[1].Ident.StoreID {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
if r, err := kv.lookupReplica(engine.Key("y"), nil); r.StoreID != s[1].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
}
// createTestStoreWithEngine creates a test store using the given engine and clock.
// The caller is responsible for closing the store on exit.
func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock,
bootstrap bool, context *storage.StoreContext) (*storage.Store, *stop.Stopper) {
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper)
if context == nil {
// make a copy
ctx := storage.TestStoreContext
context = &ctx
}
context.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
lSender := kv.NewLocalSender()
sender := kv.NewTxnCoordSender(lSender, clock, false, nil, stopper)
context.Clock = clock
context.DB = client.NewDB(sender)
context.Transport = multiraft.NewLocalRPCTransport(stopper)
// TODO(bdarnell): arrange to have the transport closed.
store := storage.NewStore(*context, eng, &proto.NodeDescriptor{NodeID: 1})
if bootstrap {
if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil {
t.Fatal(err)
}
}
lSender.AddStore(store)
if bootstrap {
if err := store.BootstrapRange(nil); err != nil {
t.Fatal(err)
}
}
if err := store.Start(stopper); err != nil {
t.Fatal(err)
}
return store, stopper
}
// initStores initializes the Stores map from ID to Store. Stores are
// added to the local sender if already bootstrapped. A bootstrapped
// Store has a valid ident with cluster, node and Store IDs set. If
// the Store doesn't yet have a valid ident, it's added to the
// bootstraps list for initialization once the cluster and node IDs
// have been determined.
func (n *Node) initStores(engines []engine.Engine, stopper *stop.Stopper) error {
bootstraps := list.New()
if len(engines) == 0 {
return util.Errorf("no engines")
}
for _, e := range engines {
s := storage.NewStore(n.ctx, e, &n.Descriptor)
// Initialize each store in turn, handling un-bootstrapped errors by
// adding the store to the bootstraps list.
if err := s.Start(stopper); err != nil {
if _, ok := err.(*storage.NotBootstrappedError); ok {
log.Infof("store %s not bootstrapped", s)
bootstraps.PushBack(s)
continue
}
return util.Errorf("failed to start store: %s", err)
}
if s.Ident.ClusterID == "" || s.Ident.NodeID == 0 {
return util.Errorf("unidentified store: %s", s)
}
capacity, err := s.Capacity()
if err != nil {
return util.Errorf("could not query store capacity: %s", err)
}
log.Infof("initialized store %s: %+v", s, capacity)
n.stores.AddStore(s)
}
// Verify all initialized stores agree on cluster and node IDs.
if err := n.validateStores(); err != nil {
return err
}
// Set the stores map as the gossip persistent storage, so that
// gossip can bootstrap using the most recently persisted set of
// node addresses.
if err := n.ctx.Gossip.SetStorage(n.stores); err != nil {
return fmt.Errorf("failed to initialize the gossip interface: %s", err)
}
// Connect gossip before starting bootstrap. For new nodes, connecting
// to the gossip network is necessary to get the cluster ID.
n.connectGossip()
// If no NodeID has been assigned yet, allocate a new node ID by
// supplying 0 to initNodeID.
if n.Descriptor.NodeID == 0 {
n.initNodeID(0)
}
// Bootstrap any uninitialized stores asynchronously.
if bootstraps.Len() > 0 {
stopper.RunAsyncTask(func() {
n.bootstrapStores(bootstraps, stopper)
})
}
return nil
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(testutils.NewRootTestBaseContext(), ltc.Clock, ltc.Stopper)
ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20)
ltc.lSender = newRetryableLocalSender(NewLocalSender())
ltc.Sender = NewTxnCoordSender(ltc.lSender, ltc.Clock, false, nil, ltc.Stopper)
var err error
if ltc.DB, err = client.Open("//root@", client.SenderOpt(ltc.Sender)); err != nil {
t.Fatal(err)
}
transport := multiraft.NewLocalRPCTransport(ltc.Stopper)
ltc.Stopper.AddCloser(transport)
ctx := storage.TestStoreContext
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ltc.Store = storage.NewStore(ctx, ltc.Eng, &proto.NodeDescriptor{NodeID: 1})
if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.lSender.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
}
// createTestStoreWithEngine creates a test store using the given engine and clock.
// The caller is responsible for closing the store on exit.
func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock,
bootstrap bool) *storage.Store {
rpcContext := rpc.NewContext(hlc.NewClock(hlc.UnixNano), rpc.LoadInsecureTLSConfig())
g := gossip.New(rpcContext, gossip.TestInterval, "")
lSender := kv.NewLocalSender()
sender := kv.NewTxnCoordSender(lSender, clock, false)
db := client.NewKV(sender, nil)
db.User = storage.UserRoot
// TODO(bdarnell): arrange to have the transport closed.
store := storage.NewStore(clock, eng, db, g, multiraft.NewLocalRPCTransport())
if bootstrap {
if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}); err != nil {
t.Fatal(err)
}
}
lSender.AddStore(store)
if bootstrap {
if err := store.BootstrapRange(); err != nil {
t.Fatal(err)
}
}
if err := store.Start(); err != nil {
t.Fatal(err)
}
return store
}
// createTestStoreWithEngine creates a test store using the given engine and clock.
// The caller is responsible for closing the store on exit.
func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock,
bootstrap bool, sCtx *storage.StoreContext) (*storage.Store, *stop.Stopper) {
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(&base.Context{}, clock, stopper)
if sCtx == nil {
// make a copy
ctx := storage.TestStoreContext
sCtx = &ctx
}
nodeDesc := &proto.NodeDescriptor{NodeID: 1}
sCtx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
localSender := kv.NewLocalSender()
rpcSend := func(_ rpc.Options, _ string, _ []net.Addr,
getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message,
_ *rpc.Context) ([]gogoproto.Message, error) {
call := proto.Call{
Args: getArgs(nil /* net.Addr */).(proto.Request),
Reply: getReply().(proto.Response),
}
localSender.Send(context.Background(), call)
return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError()
}
// Mostly makes sure that we don't see a warning per request.
{
if err := sCtx.Gossip.AddInfoProto(gossip.MakeNodeIDKey(nodeDesc.NodeID), nodeDesc, time.Hour); err != nil {
t.Fatal(err)
}
if err := sCtx.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatal(err)
}
}
distSender := kv.NewDistSender(&kv.DistSenderContext{
Clock: clock,
RPCSend: rpcSend, // defined above
RangeDescriptorDB: localSender, // for descriptor lookup
}, sCtx.Gossip)
sender := kv.NewTxnCoordSender(distSender, clock, false, nil, stopper)
sCtx.Clock = clock
sCtx.DB = client.NewDB(sender)
sCtx.Transport = multiraft.NewLocalRPCTransport(stopper)
// TODO(bdarnell): arrange to have the transport closed.
store := storage.NewStore(*sCtx, eng, nodeDesc)
if bootstrap {
if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil {
t.Fatal(err)
}
}
localSender.AddStore(store)
if bootstrap {
if err := store.BootstrapRange(sql.GetInitialSystemValues()); err != nil {
t.Fatal(err)
}
}
if err := store.Start(stopper); err != nil {
t.Fatal(err)
}
return store, stopper
}
// BootstrapCluster bootstraps a multiple stores using the provided engines and
// cluster ID. The first bootstrapped store contains a single range spanning
// all keys. Initial range lookup metadata is populated for the range.
//
// Returns a KV client for unittest purposes. Caller should close the returned
// client.
func BootstrapCluster(clusterID string, engines []engine.Engine, stopper *stop.Stopper) (*client.DB, error) {
ctx := storage.StoreContext{}
ctx.ScanInterval = 10 * time.Minute
ctx.Clock = hlc.NewClock(hlc.UnixNano)
// Create a KV DB with a local sender.
lSender := kv.NewLocalSender()
sender := kv.NewTxnCoordSender(lSender, ctx.Clock, false, nil, stopper)
ctx.DB = client.NewDB(sender)
ctx.Transport = multiraft.NewLocalRPCTransport(stopper)
for i, eng := range engines {
sIdent := roachpb.StoreIdent{
ClusterID: clusterID,
NodeID: 1,
StoreID: roachpb.StoreID(i + 1),
}
// The bootstrapping store will not connect to other nodes so its
// StoreConfig doesn't really matter.
s := storage.NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1})
// Verify the store isn't already part of a cluster.
if len(s.Ident.ClusterID) > 0 {
return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent, stopper); err != nil {
return nil, err
}
// Create first range, writing directly to engine. Note this does
// not create the range, just its data. Only do this if this is the
// first store.
if i == 0 {
// TODO(marc): this is better than having storage/ import sql, but still
// not great. Find a better place to keep those.
initialValues := sql.GetInitialSystemValues()
if err := s.BootstrapRange(initialValues); err != nil {
return nil, err
}
}
if err := s.Start(stopper); err != nil {
return nil, err
}
lSender.AddStore(s)
// Initialize node and store ids. Only initialize the node once.
if i == 0 {
if nodeID, err := allocateNodeID(ctx.DB); nodeID != sIdent.NodeID || err != nil {
return nil, util.Errorf("expected to initialize node id allocator to %d, got %d: %s",
sIdent.NodeID, nodeID, err)
}
}
if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, ctx.DB); storeID != sIdent.StoreID || err != nil {
return nil, util.Errorf("expected to initialize store id allocator to %d, got %d: %s",
sIdent.StoreID, storeID, err)
}
}
return ctx.DB, nil
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {
nodeDesc := &proto.NodeDescriptor{NodeID: 1}
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper)
ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap)
ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20, ltc.Stopper)
ltc.localSender = NewLocalSender()
var rpcSend rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr,
getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message,
_ *rpc.Context) ([]gogoproto.Message, error) {
// TODO(tschottdorf): remove getReply().
br, pErr := ltc.localSender.Send(context.Background(), *getArgs(nil).(*proto.BatchRequest))
if br == nil {
br = &proto.BatchResponse{}
}
if br.Error != nil {
panic(proto.ErrorUnexpectedlySet(ltc.localSender, br))
}
br.Error = pErr
return []gogoproto.Message{br}, nil
}
ltc.distSender = NewDistSender(&DistSenderContext{
Clock: ltc.Clock,
RangeDescriptorCacheSize: defaultRangeDescriptorCacheSize,
RangeLookupMaxRanges: defaultRangeLookupMaxRanges,
LeaderCacheSize: defaultLeaderCacheSize,
RPCRetryOptions: &defaultRPCRetryOptions,
nodeDescriptor: nodeDesc,
RPCSend: rpcSend, // defined above
RangeDescriptorDB: ltc.localSender, // for descriptor lookup
}, ltc.Gossip)
ltc.Sender = NewTxnCoordSender(ltc.distSender, ltc.Clock, false /* !linearizable */, nil /* tracer */, ltc.Stopper)
ltc.DB = client.NewDB(ltc.Sender)
transport := multiraft.NewLocalRPCTransport(ltc.Stopper)
ltc.Stopper.AddCloser(transport)
ctx := storage.TestStoreContext
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.localSender.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
}
// AddStore creates a new store on the same Transport but doesn't create any ranges.
func (m *multiTestContext) addStore() {
idx := len(m.stores)
var clock *hlc.Clock
if len(m.clocks) > idx {
clock = m.clocks[idx]
} else {
clock = m.clock
m.clocks = append(m.clocks, clock)
}
var eng engine.Engine
var needBootstrap bool
if len(m.engines) > idx {
eng = m.engines[idx]
} else {
eng = engine.NewInMem(proto.Attributes{}, 1<<20)
m.engines = append(m.engines, eng)
needBootstrap = true
// Add an extra refcount to the engine so the underlying rocksdb instances
// aren't closed when stopping and restarting the stores.
// These refcounts are removed in Stop().
if err := eng.Open(); err != nil {
m.t.Fatal(err)
}
}
stopper := stop.NewStopper()
ctx := m.makeContext(idx)
store := storage.NewStore(ctx, eng, &proto.NodeDescriptor{NodeID: proto.NodeID(idx + 1)})
if needBootstrap {
err := store.Bootstrap(proto.StoreIdent{
NodeID: proto.NodeID(idx + 1),
StoreID: proto.StoreID(idx + 1),
}, stopper)
if err != nil {
m.t.Fatal(err)
}
// Bootstrap the initial range on the first store
if idx == 0 {
if err := store.BootstrapRange(nil); err != nil {
m.t.Fatal(err)
}
}
}
if err := store.Start(stopper); err != nil {
m.t.Fatal(err)
}
store.WaitForInit()
m.stores = append(m.stores, store)
if len(m.senders) == idx {
m.senders = append(m.senders, kv.NewLocalSender())
}
m.senders[idx].AddStore(store)
// Save the store identities for later so we can use them in
// replication operations even while the store is stopped.
m.idents = append(m.idents, store.Ident)
m.stoppers = append(m.stoppers, stopper)
}
func TestLocalSenderLookupReplica(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
ctx := storage.TestStoreContext
manualClock := hlc.NewManualClock(0)
ctx.Clock = hlc.NewClock(manualClock.UnixNano)
ls := NewLocalSender()
// Create two new stores with ranges we care about.
var e [2]engine.Engine
var s [2]*storage.Store
ranges := []struct {
storeID proto.StoreID
start, end proto.Key
}{
{2, proto.Key("a"), proto.Key("c")},
{3, proto.Key("x"), proto.Key("z")},
}
for i, rng := range ranges {
e[i] = engine.NewInMem(proto.Attributes{}, 1<<20)
ctx.Transport = multiraft.NewLocalRPCTransport(stopper)
defer ctx.Transport.Close()
s[i] = storage.NewStore(ctx, e[i], &proto.NodeDescriptor{NodeID: 1})
s[i].Ident.StoreID = rng.storeID
desc := &proto.RangeDescriptor{
RangeID: proto.RangeID(i),
StartKey: rng.start,
EndKey: rng.end,
Replicas: []proto.Replica{{StoreID: rng.storeID}},
}
newRng, err := storage.NewReplica(desc, s[i])
if err != nil {
t.Fatal(err)
}
if err := s[i].AddRangeTest(newRng); err != nil {
t.Error(err)
}
ls.AddStore(s[i])
}
if _, r, err := ls.lookupReplica(proto.Key("a"), proto.Key("c")); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(proto.Key("b"), nil); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(proto.Key("b"), proto.Key("d")); r != nil || err == nil {
t.Errorf("expected store 0 and error got %d", r.StoreID)
}
if _, r, err := ls.lookupReplica(proto.Key("x"), proto.Key("z")); r.StoreID != s[1].Ident.StoreID {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(proto.Key("y"), nil); r.StoreID != s[1].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
}
// BootstrapCluster bootstraps a store using the provided engine and
// cluster ID. The bootstrapped store contains a single range spanning
// all keys. Initial range lookup metadata is populated for the range.
// Returns a direct-access kv.LocalDB for unittest purposes only.
func BootstrapCluster(clusterID string, engine storage.Engine) (*kv.LocalDB, error) {
sIdent := storage.StoreIdent{
ClusterID: clusterID,
NodeID: 1,
StoreID: 1,
}
s := storage.NewStore(engine, nil)
// Verify the store isn't already part of a cluster.
if s.Ident.ClusterID != "" {
return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent); err != nil {
return nil, err
}
if err := s.Init(); err != nil {
return nil, err
}
// Create first range.
rng, err := s.CreateRange(storage.KeyMin, storage.KeyMax)
if err != nil {
return nil, err
}
if rng.Meta.RangeID != 1 {
return nil, util.Errorf("expected range id of 1, got %d", rng.Meta.RangeID)
}
// Create a local DB to directly modify the new range.
localDB := kv.NewLocalDB(rng)
// Initialize meta1 and meta2 range addressing records.
replica := storage.Replica{
NodeID: 1,
StoreID: 1,
RangeID: 1,
Datacenter: getDatacenter(),
DiskType: engine.Type(),
}
kv.BootstrapRangeLocations(localDB, replica)
// Initialize node and store ids after the fact to account
// for use of node ID = 1 and store ID = 1.
if nodeID, err := allocateNodeID(localDB); nodeID != sIdent.NodeID || err != nil {
return nil, util.Errorf("expected to intialize node id allocator to %d, got %d: %v",
sIdent.NodeID, nodeID, err)
}
if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, localDB); storeID != sIdent.StoreID || err != nil {
return nil, util.Errorf("expected to intialize store id allocator to %d, got %d: %v",
sIdent.StoreID, storeID, err)
}
return localDB, nil
}
// BootstrapCluster bootstraps a store using the provided engine and
// cluster ID. The bootstrapped store contains a single range spanning
// all keys. Initial range lookup metadata is populated for the range.
//
// Returns a kv.DB for unittest purposes only.
func BootstrapCluster(clusterID string, eng engine.Engine) (*kv.DB, error) {
sIdent := proto.StoreIdent{
ClusterID: clusterID,
NodeID: 1,
StoreID: 1,
}
clock := hlc.NewClock(hlc.UnixNano)
now := clock.Now()
s := storage.NewStore(clock, eng, nil, nil)
// Verify the store isn't already part of a cluster.
if len(s.Ident.ClusterID) > 0 {
return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent); err != nil {
return nil, err
}
// Create first range.
rng, err := s.CreateRange(s.BootstrapRangeMetadata())
if err != nil {
return nil, err
}
// Create a KV DB with a local KV to directly modify the new range.
localKV := kv.NewLocalKV()
localKV.AddStore(s)
localDB := kv.NewDB(localKV, clock)
// Initialize range addressing records and default administrative configs.
desc := &rng.Meta.RangeDescriptor
if err := storage.BootstrapRangeDescriptor(localDB, desc, now); err != nil {
return nil, err
}
// Write default configs to local DB.
if err := storage.BootstrapConfigs(localDB, now); err != nil {
return nil, err
}
// Initialize node and store ids after the fact to account
// for use of node ID = 1 and store ID = 1.
if nodeID, err := allocateNodeID(localDB); nodeID != sIdent.NodeID || err != nil {
return nil, util.Errorf("expected to intialize node id allocator to %d, got %d: %v",
sIdent.NodeID, nodeID, err)
}
if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, localDB); storeID != sIdent.StoreID || err != nil {
return nil, util.Errorf("expected to intialize store id allocator to %d, got %d: %v",
sIdent.StoreID, storeID, err)
}
return localDB, nil
}
// restartStore restarts a store previously stopped with StopStore.
func (m *multiTestContext) restartStore(i int) {
m.stoppers[i] = stop.NewStopper()
ctx := m.makeContext(i)
m.stores[i] = storage.NewStore(ctx, m.engines[i], &proto.NodeDescriptor{NodeID: proto.NodeID(i + 1)})
if err := m.stores[i].Start(m.stoppers[i]); err != nil {
m.t.Fatal(err)
}
// The sender is assumed to still exist.
m.senders[i].AddStore(m.stores[i])
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Context, initSender InitSenderFn) {
nodeID := roachpb.NodeID(1)
nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
tracer := tracing.NewTracer()
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(baseCtx, ltc.Clock, ltc.Stopper)
server := rpc.NewServer(rpcContext) // never started
ltc.Gossip = gossip.New(
context.Background(), rpcContext, server, nil, ltc.Stopper, metric.NewRegistry())
ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20, ltc.Stopper)
ltc.Stores = storage.NewStores(ltc.Clock)
ltc.Sender = initSender(nodeDesc, tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper,
ltc.Gossip)
if ltc.DBContext == nil {
dbCtx := client.DefaultDBContext()
ltc.DBContext = &dbCtx
}
ltc.DB = client.NewDBWithContext(ltc.Sender, *ltc.DBContext)
transport := storage.NewDummyRaftTransport()
ctx := storage.TestStoreContext()
if ltc.RangeRetryOptions != nil {
ctx.RangeRetryOptions = *ltc.RangeRetryOptions
}
ctx.Ctx = tracing.WithTracer(context.Background(), tracer)
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Stores.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(context.Background(), ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Gossip.SetNodeID(nodeDesc.NodeID)
if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatalf("unable to set node descriptor: %s", err)
}
}
// initStores initializes the Stores map from id to Store. Stores are
// added to the local sender if already bootstrapped. A bootstrapped
// Store has a valid ident with cluster, node and Store IDs set. If
// the Store doesn't yet have a valid ident, it's added to the
// bootstraps list for initialization once the cluster and node IDs
// have been determined.
func (n *Node) initStores(clock *hlc.Clock, engines []engine.Engine) error {
bootstraps := list.New()
if len(engines) == 0 {
return util.Error("no engines")
}
for _, e := range engines {
// TODO(bdarnell): use a real transport here instead of NewLocalRPCTransport.
// TODO(bdarnell): arrange to have the transport closed.
s := storage.NewStore(clock, e, n.db, n.gossip, multiraft.NewLocalRPCTransport())
// Initialize each store in turn, handling un-bootstrapped errors by
// adding the store to the bootstraps list.
if err := s.Start(); err != nil {
if _, ok := err.(*storage.NotBootstrappedError); ok {
bootstraps.PushBack(s)
continue
}
return err
}
if s.Ident.ClusterID != "" {
if s.Ident.StoreID == 0 {
return util.Error("cluster id set for node ident but missing store id")
}
capacity, err := s.Capacity()
if err != nil {
return err
}
log.Infof("initialized store %s: %+v", s, capacity)
n.lSender.AddStore(s)
}
}
// Verify all initialized stores agree on cluster and node IDs.
if err := n.validateStores(); err != nil {
return err
}
// Connect gossip before starting bootstrap. For new nodes, connecting
// to the gossip network is necessary to get the cluster ID.
n.connectGossip()
// Bootstrap any uninitialized stores asynchronously.
if bootstraps.Len() > 0 {
go n.bootstrapStores(bootstraps)
}
return nil
}
// createTestDB creates a test kv.DB using a LocalKV object built with
// a store using an in-memory engine. Returns the created kv.DB and
// associated clock's manual time.
func createTestDB(t *testing.T) (*DB, *hlc.Clock, *hlc.ManualClock) {
manual := hlc.ManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
eng := engine.NewInMem(proto.Attributes{}, 1<<20)
store := storage.NewStore(clock, eng, nil)
store.Ident.StoreID = 1
replica := proto.Replica{StoreID: 1, RangeID: 1}
_, err := store.CreateRange(engine.KeyMin, engine.KeyMax, []proto.Replica{replica})
if err != nil {
t.Fatal(err)
}
kv := NewLocalKV()
kv.AddStore(store)
db := NewDB(kv, clock)
return db, clock, &manual
}
// initStoreMap initializes the Stores map from id to Store. Stores are
// added to the storeMap if the Store is already bootstrapped. A
// bootstrapped Store has a valid ident with cluster, node and Store
// IDs set. If the Store doesn't yet have a valid ident, it's added to
// the bootstraps list for initialization once the cluster and node
// IDs have been determined.
func (n *Node) initStoreMap(engines []storage.Engine) error {
bootstraps := list.New()
n.mu.Lock()
defer n.mu.Unlock()
for _, engine := range engines {
s := storage.NewStore(engine, n.gossip)
// If not bootstrapped, add to list.
if !s.IsBootstrapped() {
bootstraps.PushBack(s)
continue
}
// Otherwise, initialize each store in turn.
if err := s.Init(); err != nil {
return err
}
if s.Ident.ClusterID != "" {
if s.Ident.StoreID == 0 {
return util.Error("cluster id set for node ident but missing store id")
}
capacity, err := s.Capacity()
if err != nil {
return err
}
glog.Infof("initialized store %s: %+v", s, capacity)
n.storeMap[s.Ident.StoreID] = s
}
}
// Verify all initialized stores agree on cluster and node IDs.
if err := n.validateStores(); err != nil {
return err
}
// Connect gossip before starting bootstrap. For new nodes, connecting
// to the gossip network is necessary to get the cluster ID.
n.connectGossip()
// Bootstrap any uninitialized stores asynchronously.
if bootstraps.Len() > 0 {
go n.bootstrapStores(bootstraps)
}
return nil
}
// BootstrapCluster bootstraps a store using the provided engine and
// cluster ID. The bootstrapped store contains a single range spanning
// all keys. Initial range lookup metadata is populated for the range.
//
// Returns a KV client for unittest purposes. Caller should close
// the returned client.
func BootstrapCluster(clusterID string, eng engine.Engine) (*client.KV, error) {
sIdent := proto.StoreIdent{
ClusterID: clusterID,
NodeID: 1,
StoreID: 1,
}
clock := hlc.NewClock(hlc.UnixNano)
// Create a KV DB with a local sender.
lSender := kv.NewLocalSender()
localDB := client.NewKV(kv.NewTxnCoordSender(lSender, clock, false), nil)
// TODO(bdarnell): arrange to have the transport closed.
s := storage.NewStore(clock, eng, localDB, nil, multiraft.NewLocalRPCTransport())
// Verify the store isn't already part of a cluster.
if len(s.Ident.ClusterID) > 0 {
return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent); err != nil {
return nil, err
}
// Create first range.
if err := s.BootstrapRange(); err != nil {
return nil, err
}
if err := s.Start(); err != nil {
return nil, err
}
lSender.AddStore(s)
// Initialize node and store ids after the fact to account
// for use of node ID = 1 and store ID = 1.
if nodeID, err := allocateNodeID(localDB); nodeID != sIdent.NodeID || err != nil {
return nil, util.Errorf("expected to intialize node id allocator to %d, got %d: %v",
sIdent.NodeID, nodeID, err)
}
if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, localDB); storeID != sIdent.StoreID || err != nil {
return nil, util.Errorf("expected to intialize store id allocator to %d, got %d: %v",
sIdent.StoreID, storeID, err)
}
return localDB, nil
}
// createTestDB creates a test kv.DB using a LocalKV object built with
// a store using an in-memory engine. Returns the created kv.DB and
// associated clock's manual time.
func createTestDB(t *testing.T) (*DB, *hlc.Clock, *hlc.ManualClock) {
manual := hlc.ManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
eng := engine.NewInMem(proto.Attributes{}, 1<<20)
kv := NewLocalKV()
db := NewDB(kv, clock)
store := storage.NewStore(clock, eng, db, nil)
if err := store.Bootstrap(proto.StoreIdent{StoreID: 1}); err != nil {
t.Fatal(err)
}
kv.AddStore(store)
_, err := store.CreateRange(store.BootstrapRangeMetadata())
if err != nil {
t.Fatal(err)
}
if err := store.Init(); err != nil {
t.Fatal(err)
}
return db, clock, &manual
}
func startServer() *kvTestServer {
once.Do(func() {
server = &kvTestServer{}
g := gossip.New()
localDB := NewLocalDB()
engine := storage.NewInMem(storage.Attributes{}, 1<<20)
store := storage.NewStore(engine, g)
_, err := store.CreateRange(storage.KeyMin, storage.KeyMax, []storage.Replica{storage.Replica{RangeID: 1}})
if err != nil {
panic(err)
}
localDB.AddStore(store)
BootstrapConfigs(localDB)
server.db = localDB
server.rest = NewRESTServer(server.db)
server.httpServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
server.rest.HandleAction(w, r)
}))
})
return server
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Context, initSender InitSenderFn) {
nodeID := roachpb.NodeID(1)
nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
tracer := tracing.NewTracer()
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(baseCtx, ltc.Clock, ltc.Stopper)
ltc.Gossip = gossip.New(rpcContext, nil, ltc.Stopper)
ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20, ltc.Stopper)
ltc.Stores = storage.NewStores(ltc.Clock)
ltc.Sender = initSender(nodeDesc, tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper,
ltc.Gossip)
ltc.DB = client.NewDB(ltc.Sender)
transport := storage.NewDummyRaftTransport()
ctx := storage.TestStoreContext()
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ctx.Tracer = tracer
ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Stores.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Gossip.SetNodeID(nodeDesc.NodeID)
if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatalf("unable to set node descriptor: %s", err)
}
}
// initStoreMap initializes the Stores map from id to Store. Stores are
// added to the storeMap if the Store is already bootstrapped. A
// bootstrapped Store has a valid ident with cluster, node and Store
// IDs set. If the Store doesn't yet have a valid ident, it's added to
// the bootstraps list for initialization once the cluster and node
// IDs have been determined.
func (n *Node) initStoreMap(engines []storage.Engine) error {
bootstraps := list.New()
for _, engine := range engines {
s := storage.NewStore(engine, n.gossip)
if err := s.Init(); err != nil {
return err
}
// If Stores have been bootstrapped, their ident will be
// non-empty. Add these to Store map; otherwise, add to
// bootstraps list.
if s.Ident.ClusterID != "" {
if s.Ident.StoreID == 0 {
return util.Error("cluster id set for node ident but missing store id")
}
capacity, err := s.Capacity()
if err != nil {
return err
}
glog.Infof("Initialized store %s: %s", s.Ident, capacity)
n.storeMap[s.Ident.StoreID] = s
} else {
bootstraps.PushBack(s)
}
}
if err := n.validateStores(); err != nil {
return err
}
// Bootstrap any uninitialized stores asynchronously. We may have to
// wait until we've successfully joined the gossip network in order
// to initialize if this node is not yet aware of the cluster it's
// joining.
if bootstraps.Len() > 0 {
go n.bootstrapStores(bootstraps)
}
return nil
}
// AddStore creates a new store on the same Transport but doesn't create any ranges.
func (m *multiTestContext) addStore(t *testing.T) {
eng := engine.NewInMem(proto.Attributes{}, 1<<20)
store := storage.NewStore(m.clock, eng, m.db, m.gossip, m.transport)
err := store.Bootstrap(proto.StoreIdent{
NodeID: proto.NodeID(len(m.stores) + 1),
StoreID: proto.StoreID(len(m.stores) + 1),
})
if err != nil {
t.Fatal(err)
}
if len(m.stores) == 0 {
// Bootstrap the initial range on the first store
if err := store.BootstrapRange(); err != nil {
t.Fatal(err)
}
}
if err := store.Start(); err != nil {
t.Fatal(err)
}
m.engines = append(m.engines, eng)
m.stores = append(m.stores, store)
m.sender.AddStore(store)
}
func TestLocalSenderLookupReplica(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
ctx := storage.TestStoreContext
manualClock := hlc.NewManualClock(0)
ctx.Clock = hlc.NewClock(manualClock.UnixNano)
ls := NewLocalSender()
// Create two new stores with ranges we care about.
var e [2]engine.Engine
var s [2]*storage.Store
var d [2]*roachpb.RangeDescriptor
ranges := []struct {
storeID roachpb.StoreID
start, end roachpb.RKey
}{
{2, roachpb.RKeyMin, roachpb.RKey("c")},
{3, roachpb.RKey("x"), roachpb.RKey("z")},
}
for i, rng := range ranges {
e[i] = engine.NewInMem(roachpb.Attributes{}, 1<<20, stopper)
ctx.Transport = multiraft.NewLocalRPCTransport(stopper)
defer ctx.Transport.Close()
s[i] = storage.NewStore(ctx, e[i], &roachpb.NodeDescriptor{NodeID: 1})
s[i].Ident.StoreID = rng.storeID
d[i] = &roachpb.RangeDescriptor{
RangeID: roachpb.RangeID(i),
StartKey: rng.start,
EndKey: rng.end,
Replicas: []roachpb.ReplicaDescriptor{{StoreID: rng.storeID}},
}
newRng, err := storage.NewReplica(d[i], s[i])
if err != nil {
t.Fatal(err)
}
if err := s[i].AddReplicaTest(newRng); err != nil {
t.Error(err)
}
ls.AddStore(s[i])
}
if _, r, err := ls.lookupReplica(roachpb.RKey("a"), roachpb.RKey("c")); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(roachpb.RKey("b"), nil); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(roachpb.RKey("b"), roachpb.RKey("d")); r != nil || err == nil {
t.Errorf("expected store 0 and error got %d", r.StoreID)
}
if _, r, err := ls.lookupReplica(roachpb.RKey("x"), roachpb.RKey("z")); r.StoreID != s[1].Ident.StoreID {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(roachpb.RKey("y"), nil); r.StoreID != s[1].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
if desc, err := ls.firstRange(); err != nil || !reflect.DeepEqual(desc, d[0]) {
t.Fatalf("first range not as expected: error=%v, desc=%+v", err, desc)
}
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {
nodeID := roachpb.NodeID(1)
nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper)
ltc.Gossip = gossip.New(rpcContext, gossip.TestBootstrap, ltc.Stopper)
ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20, ltc.Stopper)
ltc.stores = storage.NewStores(ltc.Clock)
var rpcSend rpcSendFn = func(_ SendOptions, _ string, _ []net.Addr,
getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message,
_ *rpc.Context) (proto.Message, error) {
// TODO(tschottdorf): remove getReply().
if ltc.Latency > 0 {
time.Sleep(ltc.Latency)
}
br, pErr := ltc.stores.Send(context.Background(), *getArgs(nil).(*roachpb.BatchRequest))
if br == nil {
br = &roachpb.BatchResponse{}
}
if br.Error != nil {
panic(roachpb.ErrorUnexpectedlySet(ltc.stores, br))
}
br.Error = pErr
return br, nil
}
retryOpts := GetDefaultDistSenderRetryOptions()
retryOpts.Closer = ltc.Stopper.ShouldDrain()
ltc.distSender = NewDistSender(&DistSenderContext{
Clock: ltc.Clock,
RangeDescriptorCacheSize: defaultRangeDescriptorCacheSize,
RangeLookupMaxRanges: defaultRangeLookupMaxRanges,
LeaderCacheSize: defaultLeaderCacheSize,
RPCRetryOptions: &retryOpts,
nodeDescriptor: nodeDesc,
RPCSend: rpcSend, // defined above
RangeDescriptorDB: ltc.stores, // for descriptor lookup
}, ltc.Gossip)
ltc.Sender = NewTxnCoordSender(ltc.distSender, ltc.Clock, false /* !linearizable */, nil /* tracer */, ltc.Stopper)
ltc.DB = client.NewDB(ltc.Sender)
transport := storage.NewLocalRPCTransport(ltc.Stopper)
ltc.Stopper.AddCloser(transport)
ctx := storage.TestStoreContext
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.stores.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Gossip.SetNodeID(nodeDesc.NodeID)
if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatalf("unable to set node descriptor: %s", err)
}
}
// initStores initializes the Stores map from ID to Store. Stores are
// added to the local sender if already bootstrapped. A bootstrapped
// Store has a valid ident with cluster, node and Store IDs set. If
// the Store doesn't yet have a valid ident, it's added to the
// bootstraps list for initialization once the cluster and node IDs
// have been determined.
func (n *Node) initStores(engines []engine.Engine, stopper *stop.Stopper) error {
var bootstraps []*storage.Store
if len(engines) == 0 {
return util.Errorf("no engines")
}
for _, e := range engines {
s := storage.NewStore(n.ctx, e, &n.Descriptor)
// Initialize each store in turn, handling un-bootstrapped errors by
// adding the store to the bootstraps list.
if err := s.Start(stopper); err != nil {
if _, ok := err.(*storage.NotBootstrappedError); ok {
log.Infof("store %s not bootstrapped", s)
bootstraps = append(bootstraps, s)
continue
}
return util.Errorf("failed to start store: %s", err)
}
if s.Ident.ClusterID == *uuid.EmptyUUID || s.Ident.NodeID == 0 {
return util.Errorf("unidentified store: %s", s)
}
capacity, err := s.Capacity()
if err != nil {
return util.Errorf("could not query store capacity: %s", err)
}
log.Infof("initialized store %s: %+v", s, capacity)
n.stores.AddStore(s)
}
// If there are no initialized stores and no gossip resolvers,
// bootstrap this node as the seed of a new cluster.
if n.stores.GetStoreCount() == 0 {
resolvers := n.ctx.Gossip.GetResolvers()
// Check for the case of uninitialized node having only itself specified as join host.
switch len(resolvers) {
case 0:
return errNeedsBootstrap
case 1:
if resolvers[0].Addr() == n.Descriptor.Address.String() {
return errCannotJoinSelf
}
}
}
// Verify all initialized stores agree on cluster and node IDs.
if err := n.validateStores(); err != nil {
return err
}
// Set the stores map as the gossip persistent storage, so that
// gossip can bootstrap using the most recently persisted set of
// node addresses.
if err := n.ctx.Gossip.SetStorage(n.stores); err != nil {
return fmt.Errorf("failed to initialize the gossip interface: %s", err)
}
// Connect gossip before starting bootstrap. For new nodes, connecting
// to the gossip network is necessary to get the cluster ID.
n.connectGossip()
// If no NodeID has been assigned yet, allocate a new node ID by
// supplying 0 to initNodeID.
if n.Descriptor.NodeID == 0 {
n.initNodeID(0)
}
// Bootstrap any uninitialized stores asynchronously.
if len(bootstraps) > 0 {
stopper.RunAsyncTask(func() {
n.bootstrapStores(bootstraps, stopper)
})
}
return nil
}
// bootstrapCluster bootstraps a multiple stores using the provided
// engines and cluster ID. The first bootstrapped store contains a
// single range spanning all keys. Initial range lookup metadata is
// populated for the range. Returns the cluster ID.
func bootstrapCluster(engines []engine.Engine) (uuid.UUID, error) {
clusterID := uuid.MakeV4()
stopper := stop.NewStopper()
defer stopper.Stop()
ctx := storage.StoreContext{}
ctx.ScanInterval = 10 * time.Minute
ctx.Clock = hlc.NewClock(hlc.UnixNano)
ctx.Tracer = tracing.NewTracer()
// Create a KV DB with a local sender.
stores := storage.NewStores(ctx.Clock)
sender := kv.NewTxnCoordSender(stores, ctx.Clock, false, ctx.Tracer, stopper)
ctx.DB = client.NewDB(sender)
ctx.Transport = storage.NewLocalRPCTransport(stopper)
for i, eng := range engines {
sIdent := roachpb.StoreIdent{
ClusterID: clusterID,
NodeID: 1,
StoreID: roachpb.StoreID(i + 1),
}
// The bootstrapping store will not connect to other nodes so its
// StoreConfig doesn't really matter.
s := storage.NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1})
// Verify the store isn't already part of a cluster.
if s.Ident.ClusterID != *uuid.EmptyUUID {
return uuid.UUID{}, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent, stopper); err != nil {
return uuid.UUID{}, err
}
// Create first range, writing directly to engine. Note this does
// not create the range, just its data. Only do this if this is the
// first store.
if i == 0 {
initialValues := GetBootstrapSchema().GetInitialValues()
if err := s.BootstrapRange(initialValues); err != nil {
return uuid.UUID{}, err
}
}
if err := s.Start(stopper); err != nil {
return uuid.UUID{}, err
}
stores.AddStore(s)
// Initialize node and store ids. Only initialize the node once.
if i == 0 {
if nodeID, err := allocateNodeID(ctx.DB); nodeID != sIdent.NodeID || err != nil {
return uuid.UUID{}, util.Errorf("expected to initialize node id allocator to %d, got %d: %s",
sIdent.NodeID, nodeID, err)
}
}
if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, ctx.DB); storeID != sIdent.StoreID || err != nil {
return uuid.UUID{}, util.Errorf("expected to initialize store id allocator to %d, got %d: %s",
sIdent.StoreID, storeID, err)
}
}
return clusterID, nil
}
// BootstrapCluster bootstraps a store using the provided engine and
// cluster ID. The bootstrapped store contains a single range spanning
// all keys. Initial range lookup metadata is populated for the range.
//
// Returns a direct-access kv.LocalDB for unittest purposes only.
func BootstrapCluster(clusterID string, engine storage.Engine) (*kv.LocalDB, error) {
sIdent := storage.StoreIdent{
ClusterID: clusterID,
NodeID: 1,
StoreID: 1,
}
s := storage.NewStore(engine, nil)
defer s.Close()
// Verify the store isn't already part of a cluster.
if s.Ident.ClusterID != "" {
return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent); err != nil {
return nil, err
}
if err := s.Init(); err != nil {
return nil, err
}
// Create first range.
replica := storage.Replica{
NodeID: 1,
StoreID: 1,
RangeID: 1,
Attrs: storage.Attributes{},
}
rng, err := s.CreateRange(storage.KeyMin, storage.KeyMax, []storage.Replica{replica})
if err != nil {
return nil, err
}
if rng.Meta.RangeID != 1 {
return nil, util.Errorf("expected range id of 1, got %d", rng.Meta.RangeID)
}
// Create a local DB to directly modify the new range.
localDB := kv.NewLocalDB(rng)
// Initialize range addressing records and default administrative configs.
if err := kv.BootstrapRangeDescriptor(localDB, replica); err != nil {
return nil, err
}
if err := kv.BootstrapConfigs(localDB); err != nil {
return nil, err
}
// Initialize node and store ids after the fact to account
// for use of node ID = 1 and store ID = 1.
if nodeID, err := allocateNodeID(localDB); nodeID != sIdent.NodeID || err != nil {
return nil, util.Errorf("expected to intialize node id allocator to %d, got %d: %v",
sIdent.NodeID, nodeID, err)
}
if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, localDB); storeID != sIdent.StoreID || err != nil {
return nil, util.Errorf("expected to intialize store id allocator to %d, got %d: %v",
sIdent.StoreID, storeID, err)
}
return localDB, nil
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester) {
nodeID := roachpb.NodeID(1)
nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper)
ltc.Gossip = gossip.New(rpcContext, gossip.TestBootstrap, ltc.Stopper)
ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20, ltc.Stopper)
ltc.stores = storage.NewStores(ltc.Clock)
tracer := tracing.NewTracer()
var rpcSend rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (proto.Message, error) {
if ltc.Latency > 0 {
time.Sleep(ltc.Latency)
}
sp := tracer.StartSpan("node")
defer sp.Finish()
ctx := opentracing.ContextWithSpan(context.Background(), sp)
sp.LogEvent(args.String())
br, pErr := ltc.stores.Send(ctx, args)
if br == nil {
br = &roachpb.BatchResponse{}
}
if br.Error != nil {
panic(roachpb.ErrorUnexpectedlySet(ltc.stores, br))
}
br.Error = pErr
if pErr != nil {
sp.LogEvent("error: " + pErr.String())
}
return br, nil
}
retryOpts := GetDefaultDistSenderRetryOptions()
retryOpts.Closer = ltc.Stopper.ShouldDrain()
ltc.distSender = NewDistSender(&DistSenderContext{
Clock: ltc.Clock,
RangeDescriptorCacheSize: defaultRangeDescriptorCacheSize,
RangeLookupMaxRanges: defaultRangeLookupMaxRanges,
LeaderCacheSize: defaultLeaderCacheSize,
RPCRetryOptions: &retryOpts,
nodeDescriptor: nodeDesc,
RPCSend: rpcSend, // defined above
RangeDescriptorDB: ltc.stores, // for descriptor lookup
}, ltc.Gossip)
ltc.Sender = NewTxnCoordSender(ltc.distSender, ltc.Clock, false /* !linearizable */, tracer,
ltc.Stopper, NewTxnMetrics(metric.NewRegistry()))
ltc.DB = client.NewDB(ltc.Sender)
transport := storage.NewDummyRaftTransport()
ctx := storage.TestStoreContext()
ctx.Clock = ltc.Clock
ctx.DB = ltc.DB
ctx.Gossip = ltc.Gossip
ctx.Transport = transport
ctx.Tracer = tracer
ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}, ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.stores.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Gossip.SetNodeID(nodeDesc.NodeID)
if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatalf("unable to set node descriptor: %s", err)
}
}
func TestLocalSenderLookupReplica(t *testing.T) {
manualClock := hlc.NewManualClock(0)
clock := hlc.NewClock(manualClock.UnixNano)
eng := engine.NewInMem(proto.Attributes{}, 1<<20)
ls := NewLocalSender()
db := client.NewKV(NewTxnCoordSender(ls, clock, false), nil)
transport := multiraft.NewLocalRPCTransport()
defer transport.Close()
store := storage.NewStore(clock, eng, db, nil, transport)
if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}); err != nil {
t.Fatal(err)
}
ls.AddStore(store)
if err := store.BootstrapRange(); err != nil {
t.Fatal(err)
}
if err := store.Start(); err != nil {
t.Fatal(err)
}
defer store.Stop()
rng := splitTestRange(store, engine.KeyMin, proto.Key("a"), t)
if err := store.RemoveRange(rng); err != nil {
t.Fatal(err)
}
// Create two new stores with ranges we care about.
var e [2]engine.Engine
var s [2]*storage.Store
ranges := []struct {
storeID proto.StoreID
start, end proto.Key
}{
{2, proto.Key("a"), proto.Key("c")},
{3, proto.Key("x"), proto.Key("z")},
}
for i, rng := range ranges {
e[i] = engine.NewInMem(proto.Attributes{}, 1<<20)
transport := multiraft.NewLocalRPCTransport()
defer transport.Close()
s[i] = storage.NewStore(clock, e[i], db, nil, transport)
s[i].Ident.StoreID = rng.storeID
if err := s[i].Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: rng.storeID}); err != nil {
t.Fatal(err)
}
if err := s[i].Start(); err != nil {
t.Fatal(err)
}
defer s[i].Stop()
desc, err := store.NewRangeDescriptor(rng.start, rng.end, []proto.Replica{{StoreID: rng.storeID}})
if err != nil {
t.Fatal(err)
}
newRng, err := storage.NewRange(desc, s[i])
if err != nil {
t.Fatal(err)
}
if err := s[i].AddRange(newRng); err != nil {
t.Error(err)
}
ls.AddStore(s[i])
}
if _, r, err := ls.lookupReplica(proto.Key("a"), proto.Key("c")); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(proto.Key("b"), nil); r.StoreID != s[0].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[0].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(proto.Key("b"), proto.Key("d")); r != nil || err == nil {
t.Errorf("expected store 0 and error got %d", r.StoreID)
}
if _, r, err := ls.lookupReplica(proto.Key("x"), proto.Key("z")); r.StoreID != s[1].Ident.StoreID {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
if _, r, err := ls.lookupReplica(proto.Key("y"), nil); r.StoreID != s[1].Ident.StoreID || err != nil {
t.Errorf("expected store %d; got %d: %v", s[1].Ident.StoreID, r.StoreID, err)
}
}
