// TestBootstrapNewStore starts a cluster with two unbootstrapped
// stores and verifies both stores are added.
func TestBootstrapNewStore(t *testing.T) {
engine := storage.NewInMem(storage.Attributes{}, 1<<20)
localDB, err := BootstrapCluster("cluster-1", engine)
if err != nil {
t.Fatal(err)
}
localDB.Close()
// Start a new node with two new stores which will require bootstrapping.
engines := []storage.Engine{
engine,
storage.NewInMem(storage.Attributes{}, 1<<20),
storage.NewInMem(storage.Attributes{}, 1<<20),
}
server, node := createTestNode(util.CreateTestAddr("tcp"), engines, nil, t)
defer server.Close()
// Non-initialized stores (in this case the new in-memory-based
// store) will be bootstrapped by the node upon start. This happens
// in a goroutine, so we'll have to wait a bit (maximum 10ms) until
// we can find the new node.
if err := util.IsTrueWithin(func() bool { return node.localDB.GetStoreCount() == 3 }, 50*time.Millisecond); err != nil {
t.Error(err)
}
}
// initEngine parses the engine specification according to the
// dataDirRE regexp and instantiates an engine of correct type.
func initEngine(spec string) (storage.Engine, error) {
// Error if regexp doesn't match.
matches := dataDirRE.FindStringSubmatch(spec)
if matches == nil || len(matches) != 3 {
return nil, util.Errorf("invalid engine specification %q", spec)
}
var engine storage.Engine
var err error
if matches[1] == "mem" {
size, err := strconv.ParseInt(matches[2], 10, 64)
if err != nil {
return nil, util.Errorf("unable to init in-memory storage %q", spec)
}
engine = storage.NewInMem(size)
} else {
var typ storage.DiskType
switch matches[2] {
case "hdd":
typ = storage.HDD
case "ssd":
typ = storage.SSD
default:
return nil, util.Errorf("unhandled disk type %q", matches[1])
}
engine, err = storage.NewRocksDB(typ, matches[2])
if err != nil {
return nil, util.Errorf("unable to init rocksdb with data dir %q", matches[2])
}
}
return engine, nil
}
func startNewServer() *kvTestServer {
s := &kvTestServer{}
// Initialize engine, store, and localDB.
engine := storage.NewInMem(storage.Attributes{}, 1<<20)
localDB, err := server.BootstrapCluster("test-cluster", engine)
if err != nil {
panic(err)
}
s.db = localDB
// Rip through the stores (should be just one) and grab the first range (there should also just be one).
localDB.VisitStores(func(store *storage.Store) error {
rs := store.GetRanges()
if len(rs) > 0 {
s.firstRange = rs[0]
}
return nil
})
if s.firstRange == nil {
panic("Internal Error: Expected to find a range while initializing test server!")
}
// Initialize the REST server.
s.rest = rest.NewRESTServer(s.db)
s.httpServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
s.rest.HandleAction(w, r)
}))
return s
}
// TestBootstrap verifies the results of bootstrapping a cluster. Uses
// an in memory engine.
func TestBootstrapCluster(t *testing.T) {
engine := storage.NewInMem(1 << 20)
localDB, err := BootstrapCluster("cluster-1", engine)
if err != nil {
t.Fatal(err)
}
// Scan the complete contents of the local database.
sr := <-localDB.Scan(&storage.ScanRequest{
StartKey: storage.KeyMin,
EndKey: storage.KeyMax,
MaxResults: math.MaxInt64,
})
if sr.Error != nil {
t.Fatal(sr.Error)
}
var keys []storage.Key
for _, kv := range sr.Rows {
keys = append(keys, kv.Key)
}
var expectedKeys = []storage.Key{
storage.Key("\x00\x00\x00range-1"),
storage.Key("\x00\x00\x00range-id-generator"),
storage.Key("\x00\x00\x00store-ident"),
storage.Key("\x00\x00meta1\xff"),
storage.Key("\x00\x00meta2\xff"),
storage.Key("\x00node-id-generator"),
storage.Key("\x00store-id-generator-1"),
}
if !reflect.DeepEqual(keys, expectedKeys) {
t.Errorf("expected keys mismatch:\n%s\n -- vs. -- \n\n%s",
formatKeys(keys), formatKeys(expectedKeys))
}
// TODO(spencer): check values.
}
// TestNodeJoin verifies a new node is able to join a bootstrapped
// cluster consisting of one node.
func TestNodeJoin(t *testing.T) {
engine := storage.NewInMem(storage.Attributes{}, 1<<20)
localDB, err := BootstrapCluster("cluster-1", engine)
if err != nil {
t.Fatal(err)
}
localDB.Close()
// Set an aggressive gossip interval to make sure information is exchanged tout de suite.
*gossip.GossipInterval = 10 * time.Millisecond
// Start the bootstrap node.
engines1 := []storage.Engine{engine}
addr1 := util.CreateTestAddr("tcp")
server1, node1 := createTestNode(addr1, engines1, addr1, t)
defer server1.Close()
// Create a new node.
engines2 := []storage.Engine{storage.NewInMem(storage.Attributes{}, 1<<20)}
server2, node2 := createTestNode(util.CreateTestAddr("tcp"), engines2, server1.Addr(), t)
defer server2.Close()
// Verify new node is able to bootstrap its store.
if err := util.IsTrueWithin(func() bool { return node2.localDB.GetStoreCount() == 1 }, 50*time.Millisecond); err != nil {
t.Fatal(err)
}
// Verify node1 sees node2 via gossip and vice versa.
node1Key := gossip.MakeNodeIDGossipKey(node1.Descriptor.NodeID)
node2Key := gossip.MakeNodeIDGossipKey(node2.Descriptor.NodeID)
if err := util.IsTrueWithin(func() bool {
if val, err := node1.gossip.GetInfo(node2Key); err != nil {
return false
} else if val.(net.Addr).String() != server2.Addr().String() {
t.Error("addr2 gossip %s doesn't match addr2 address %s", val.(net.Addr).String(), server2.Addr().String())
}
if val, err := node2.gossip.GetInfo(node1Key); err != nil {
return false
} else if val.(net.Addr).String() != server1.Addr().String() {
t.Error("addr1 gossip %s doesn't match addr1 address %s", val.(net.Addr).String(), server1.Addr().String())
}
return true
}, 50*time.Millisecond); err != nil {
t.Error(err)
}
}
// startAdminServer launches a new admin server using minimal engine
// and local database setup. Returns the new http test server, which
// should be cleaned up by caller via httptest.Server.Close(). The
// Cockroach KV client address is set to the address of the test server.
func startAdminServer() *httptest.Server {
db, err := BootstrapCluster("cluster-1", storage.NewInMem(storage.Attributes{}, 1<<20))
if err != nil {
glog.Fatal(err)
}
admin := newAdminServer(db)
httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
admin.handleZoneAction(w, r)
}))
if strings.HasPrefix(httpServer.URL, "http://") {
*kv.Addr = strings.TrimPrefix(httpServer.URL, "http://")
} else if strings.HasPrefix(httpServer.URL, "https://") {
*kv.Addr = strings.TrimPrefix(httpServer.URL, "https://")
}
return httpServer
}
func startServer() *kvTestServer {
once.Do(func() {
meta := storage.RangeMetadata{
RangeID: 1,
StartKey: storage.KeyMin,
EndKey: storage.KeyMax,
}
server = &kvTestServer{}
server.db = NewLocalDB(storage.NewRange(meta, storage.NewInMem(1<<30), nil, nil))
server.rest = NewRESTServer(server.db)
server.httpServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
server.rest.HandleAction(w, r)
}))
})
return server
}
// initEngine parses the store attributes as a colon-separated list
// and instantiates an engine based on the dir parameter. If dir parses
// to an integer, it's taken to mean an in-memory engine; otherwise,
// dir is treated as a path and a RocksDB engine is created.
func initEngine(attrsStr, path string) (storage.Engine, error) {
attrs := parseAttributes(attrsStr)
var engine storage.Engine
if size, err := strconv.ParseUint(path, 10, 64); err == nil {
if size == 0 {
return nil, util.Errorf("unable to initialize an in-memory store with capacity 0")
}
engine = storage.NewInMem(attrs, int64(size))
} else {
engine, err = storage.NewRocksDB(attrs, path)
if err != nil {
return nil, util.Errorf("unable to init rocksdb with data dir %q: %v", path, err)
}
}
return engine, nil
}
func startServer() *server {
serverTestOnce.Do(func() {
resetTestData()
s, err := newServer()
if err != nil {
glog.Fatal(err)
}
engines := []storage.Engine{storage.NewInMem(1 << 20)}
if _, err := BootstrapCluster("cluster-1", engines[0]); err != nil {
glog.Fatal(err)
}
s.gossip.SetBootstrap([]net.Addr{s.rpc.Addr()})
go func() {
glog.Fatal(s.start(engines)) // TODO(spencer): should shutdown server.
}()
glog.Infof("Test server listening on http: %s, rpc: %s", *httpAddr, *rpcAddr)
})
return s
}
func startServer() *kvTestServer {
once.Do(func() {
server = &kvTestServer{}
g := gossip.New()
localDB := NewLocalDB()
engine := storage.NewInMem(storage.Attributes{}, 1<<20)
store := storage.NewStore(engine, g)
_, err := store.CreateRange(storage.KeyMin, storage.KeyMax, []storage.Replica{storage.Replica{RangeID: 1}})
if err != nil {
panic(err)
}
localDB.AddStore(store)
BootstrapConfigs(localDB)
server.db = localDB
server.rest = NewRESTServer(server.db)
server.httpServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
server.rest.HandleAction(w, r)
}))
})
return server
}
func startServer() *server {
serverTestOnce.Do(func() {
s, err := newServer()
if err != nil {
glog.Fatal(err)
}
engines := []storage.Engine{storage.NewInMem(storage.Attributes{}, 1<<20)}
if _, err := BootstrapCluster("cluster-1", engines[0]); err != nil {
glog.Fatal(err)
}
err = s.start(engines, true) // TODO(spencer): should shutdown server.
if err != nil {
glog.Fatalf("Could not start server: %s", err)
}
// Update the configuration variables to reflect the actual
// sockets bound during this test.
*httpAddr = (*s.httpListener).Addr().String()
*rpcAddr = s.rpc.Addr().String()
glog.Infof("Test server listening on http: %s, rpc: %s", *httpAddr, *rpcAddr)
})
return s
}