func (m *multiTestContext) Start(t *testing.T, numStores int) { if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(m.clock, rpc.LoadInsecureTLSConfig()) m.gossip = gossip.New(rpcContext, gossip.TestInterval, "") } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport() } if m.sender == nil { m.sender = kv.NewLocalSender() } if m.db == nil { txnSender := kv.NewTxnCoordSender(m.sender, m.clock, false) m.db = client.NewKV(txnSender, nil) m.db.User = storage.UserRoot } for i := 0; i < numStores; i++ { m.addStore(t) } }
// SimulateNetwork creates nodeCount gossip nodes. The network should // be set to either "tcp" or "unix". The gossipInterval should be set // to a compressed simulation timescale, though large enough to give // the concurrent goroutines enough time to pass data back and forth // in order to yield accurate estimates of how old data actually ends // up being at the various nodes. After each gossipInterval period, // simCallback is invoked; when it returns false, the simulation // ends. If it returns true, the simulation continues another cycle. // // Node0 gossips the node count as well as the gossip sentinel. The // gossip bootstrap hosts are set to the first three nodes (or fewer if // less than three are available). // // At each cycle of the simulation, node 0 gossips the sentinel. If // the simulation requires other nodes to gossip, this should be done // via simCallback. // // The simulation callback receives a map of nodes, keyed by node address. func SimulateNetwork(nodeCount int, network string, gossipInterval time.Duration, simCallback func(cycle int, nodes map[string]*Gossip) bool) { // seed the random number generator for non-determinism across // multiple runs. rand.Seed(time.Now().UTC().UnixNano()) tlsConfig := rpc.LoadInsecureTLSConfig() log.Infof("simulating network with %d nodes", nodeCount) servers := make([]*rpc.Server, nodeCount) addrs := make([]net.Addr, nodeCount) for i := 0; i < nodeCount; i++ { addr := util.CreateTestAddr(network) servers[i] = rpc.NewServer(addr, tlsConfig) if err := servers[i].Start(); err != nil { log.Fatal(err) } addrs[i] = servers[i].Addr() } var bootstrap []net.Addr if nodeCount < 3 { bootstrap = addrs } else { bootstrap = addrs[:3] } nodes := make(map[string]*Gossip, nodeCount) for i := 0; i < nodeCount; i++ { node := New(tlsConfig) node.Name = fmt.Sprintf("Node%d", i) node.SetBootstrap(bootstrap) node.SetInterval(gossipInterval) node.Start(servers[i]) // Node 0 gossips node count. if i == 0 { node.AddInfo(KeyNodeCount, int64(nodeCount), time.Hour) } nodes[addrs[i].String()] = node } gossipTimeout := time.Tick(gossipInterval) var complete bool for cycle := 0; !complete; cycle++ { select { case <-gossipTimeout: // Node 0 gossips sentinel every cycle. nodes[addrs[0].String()].AddInfo(KeySentinel, int64(cycle), time.Hour) if !simCallback(cycle, nodes) { complete = true } } } // Stop all servers & nodes. for i := 0; i < nodeCount; i++ { servers[i].Close() nodes[addrs[i].String()].Stop() } }
// TestGossipGroupsInfoStore verifies gossiping of groups via the // gossip instance infostore. func TestGossipGroupsInfoStore(t *testing.T) { g := New(rpc.LoadInsecureTLSConfig()) // For int64. g.RegisterGroup("i", 3, MinGroup) for i := 0; i < 3; i++ { g.AddInfo(fmt.Sprintf("i.%d", i), int64(i), time.Hour) } values, err := g.GetGroupInfos("i") if err != nil { t.Errorf("error fetching int64 group: %v", err) } if len(values) != 3 { t.Errorf("incorrect number of values in group: %v", values) } for i := 0; i < 3; i++ { if values[i].(int64) != int64(i) { t.Errorf("index %d has incorrect value: %d, expected %d", i, values[i].(int64), i) } } if _, err := g.GetGroupInfos("i2"); err == nil { t.Errorf("expected error fetching nonexistent key \"i2\"") } // For float64. g.RegisterGroup("f", 3, MinGroup) for i := 0; i < 3; i++ { g.AddInfo(fmt.Sprintf("f.%d", i), float64(i), time.Hour) } values, err = g.GetGroupInfos("f") if err != nil { t.Errorf("error fetching float64 group: %v", err) } if len(values) != 3 { t.Errorf("incorrect number of values in group: %v", values) } for i := 0; i < 3; i++ { if values[i].(float64) != float64(i) { t.Errorf("index %d has incorrect value: %f, expected %d", i, values[i].(float64), i) } } // For string. g.RegisterGroup("s", 3, MinGroup) for i := 0; i < 3; i++ { g.AddInfo(fmt.Sprintf("s.%d", i), fmt.Sprintf("%d", i), time.Hour) } values, err = g.GetGroupInfos("s") if err != nil { t.Errorf("error fetching string group: %v", err) } if len(values) != 3 { t.Errorf("incorrect number of values in group: %v", values) } for i := 0; i < 3; i++ { if values[i].(string) != fmt.Sprintf("%d", i) { t.Errorf("index %d has incorrect value: %d, expected %s", i, values[i], fmt.Sprintf("%d", i)) } } }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool) *storage.Store { rpcContext := rpc.NewContext(hlc.NewClock(hlc.UnixNano), rpc.LoadInsecureTLSConfig()) g := gossip.New(rpcContext, gossip.TestInterval, "") lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, clock, false) db := client.NewKV(sender, nil) db.User = storage.UserRoot // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(clock, eng, db, g, multiraft.NewLocalRPCTransport()) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}); err != nil { t.Fatal(err) } } lSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(); err != nil { t.Fatal(err) } } if err := store.Start(); err != nil { t.Fatal(err) } return store }
// createTestRange creates a new range initialized to the full extent // of the keyspace. The gossip instance is also returned for testing. func createTestRange(engine engine.Engine, t *testing.T) (*Range, *gossip.Gossip) { rm := &proto.RangeMetadata{ RangeID: 0, RangeDescriptor: testRangeDescriptor, } g := gossip.New(rpc.LoadInsecureTLSConfig()) clock := hlc.NewClock(hlc.UnixNano) r := NewRange(rm, clock, engine, nil, g) r.Start() return r, g }
// NewNetwork creates nodeCount gossip nodes. The networkType should // be set to either "tcp" or "unix". The gossipInterval should be set // to a compressed simulation timescale, though large enough to give // the concurrent goroutines enough time to pass data back and forth // in order to yield accurate estimates of how old data actually ends // up being at the various nodes (e.g. DefaultTestGossipInterval). func NewNetwork(nodeCount int, networkType string, gossipInterval time.Duration, gossipBootstrap string) *Network { tlsConfig := rpc.LoadInsecureTLSConfig() clock := hlc.NewClock(hlc.UnixNano) rpcContext := rpc.NewContext(clock, tlsConfig) log.Infof("simulating gossip network with %d nodes", nodeCount) servers := make([]*rpc.Server, nodeCount) addrs := make([]net.Addr, nodeCount) for i := 0; i < nodeCount; i++ { addr := util.CreateTestAddr(networkType) servers[i] = rpc.NewServer(addr, rpcContext) if err := servers[i].Start(); err != nil { log.Fatal(err) } addrs[i] = servers[i].Addr() } var bootstrap []net.Addr if nodeCount < 3 { bootstrap = addrs } else { bootstrap = addrs[:3] } nodes := make([]*Node, nodeCount) for i := 0; i < nodeCount; i++ { node := gossip.New(rpcContext, gossipInterval, gossipBootstrap) node.Name = fmt.Sprintf("Node%d", i) node.SetBootstrap(bootstrap) node.SetInterval(gossipInterval) node.Start(servers[i]) // Node 0 gossips node count. if i == 0 { node.AddInfo(gossip.KeyNodeCount, int64(nodeCount), time.Hour) } nodes[i] = &Node{Gossip: node, Addr: addrs[i], Server: servers[i]} } return &Network{ Nodes: nodes, Addrs: addrs, NetworkType: networkType, GossipInterval: gossipInterval} }
func newServer() (*server, error) { // Determine hostname in case it hasn't been specified in -rpc or -http. host, err := os.Hostname() if err != nil { host = "127.0.0.1" } // Resolve if strings.HasPrefix(*rpcAddr, ":") { *rpcAddr = host + *rpcAddr } _, err = net.ResolveTCPAddr("tcp", *rpcAddr) if err != nil { return nil, util.Errorf("unable to resolve RPC address %q: %v", *rpcAddr, err) } var tlsConfig *rpc.TLSConfig if *certDir == "" { tlsConfig = rpc.LoadInsecureTLSConfig() } else { var err error if tlsConfig, err = rpc.LoadTLSConfig(*certDir); err != nil { return nil, util.Errorf("unable to load TLS config: %v", err) } } s := &server{ host: host, mux: http.NewServeMux(), clock: hlc.NewClock(hlc.UnixNano), rpc: rpc.NewServer(util.MakeRawAddr("tcp", *rpcAddr), tlsConfig), } s.clock.SetMaxDrift(*maxDrift) s.gossip = gossip.New(tlsConfig) s.kvDB = kv.NewDB(kv.NewDistKV(s.gossip), s.clock) s.kvREST = rest.NewRESTServer(s.kvDB) s.node = NewNode(s.kvDB, s.gossip) s.admin = newAdminServer(s.kvDB) s.status = newStatusServer(s.kvDB) s.structuredDB = structured.NewDB(s.kvDB) s.structuredREST = structured.NewRESTServer(s.structuredDB) return s, nil }
// This is an example for using the Call() method to Put and then Get // a value for a given key. func ExampleKV_Call() { // Using built-in test server for this example code. serv := StartTestServer(nil) defer serv.Stop() // Replace with actual host:port address string (ex "localhost:8080") for server cluster. serverAddress := serv.HTTPAddr // Key Value Client initialization. sender := client.NewHTTPSender(serverAddress, &http.Transport{ TLSClientConfig: rpc.LoadInsecureTLSConfig().Config(), }) kvClient := client.NewKV(sender, nil) kvClient.User = storage.UserRoot defer kvClient.Close() key := proto.Key("a") value := []byte{1, 2, 3, 4} // Store test value. putResp := &proto.PutResponse{} if err := kvClient.Call(proto.Put, proto.PutArgs(key, value), putResp); err != nil { log.Fatal(err) } // Retrieve test value using same key. getResp := &proto.GetResponse{} if err := kvClient.Call(proto.Get, proto.GetArgs(key), getResp); err != nil { log.Fatal(err) } // Data validation. if getResp.Value == nil { log.Fatal("No value returned.") } if !bytes.Equal(value, getResp.Value.Bytes) { log.Fatal("Data mismatch on retrieved value.") } fmt.Println("Client example done.") // Output: Client example done. }
// startGossip creates local and remote gossip instances. // The remote gossip instance launches its gossip service. func startGossip(t *testing.T) (local, remote *Gossip, lserver, rserver *rpc.Server) { tlsConfig := rpc.LoadInsecureTLSConfig() laddr := util.CreateTestAddr("unix") lserver = rpc.NewServer(laddr, tlsConfig) if err := lserver.Start(); err != nil { t.Fatal(err) } local = New(tlsConfig) raddr := util.CreateTestAddr("unix") rserver = rpc.NewServer(raddr, tlsConfig) if err := rserver.Start(); err != nil { t.Fatal(err) } remote = New(tlsConfig) local.start(lserver) remote.start(rserver) time.Sleep(time.Millisecond) return }
// TestGossipInfoStore verifies operation of gossip instance infostore. func TestGossipInfoStore(t *testing.T) { g := New(rpc.LoadInsecureTLSConfig()) g.AddInfo("i", int64(1), time.Hour) if val, err := g.GetInfo("i"); val.(int64) != int64(1) || err != nil { t.Errorf("error fetching int64: %v", err) } if _, err := g.GetInfo("i2"); err == nil { t.Errorf("expected error fetching nonexistent key \"i2\"") } g.AddInfo("f", float64(3.14), time.Hour) if val, err := g.GetInfo("f"); val.(float64) != float64(3.14) || err != nil { t.Errorf("error fetching float64: %v", err) } if _, err := g.GetInfo("f2"); err == nil { t.Errorf("expected error fetching nonexistent key \"f2\"") } g.AddInfo("s", "b", time.Hour) if val, err := g.GetInfo("s"); val.(string) != "b" || err != nil { t.Errorf("error fetching string: %v", err) } if _, err := g.GetInfo("s2"); err == nil { t.Errorf("expected error fetching nonexistent key \"s2\"") } }
// createTestClient creates a new KV client which connects using // an HTTP sender to the server at addr. func createTestClient(addr string) *client.KV { sender := newNotifyingSender(client.NewHTTPSender(addr, &http.Transport{ TLSClientConfig: rpc.LoadInsecureTLSConfig().Config(), })) return client.NewKV(sender, nil) }
// This is an example for using the RunTransaction() method to submit // multiple Key Value API operations inside a transaction. func ExampleKV_RunTransaction() { // Using built-in test server for this example code. serv := StartTestServer(nil) defer serv.Stop() // Replace with actual host:port address string (ex "localhost:8080") for server cluster. serverAddress := serv.HTTPAddr // Key Value Client initialization. sender := client.NewHTTPSender(serverAddress, &http.Transport{ TLSClientConfig: rpc.LoadInsecureTLSConfig().Config(), }) kvClient := client.NewKV(sender, nil) kvClient.User = storage.UserRoot defer kvClient.Close() // Create test data. numKVPairs := 10 keys := make([]string, numKVPairs) values := make([][]byte, numKVPairs) for i := 0; i < numKVPairs; i++ { keys[i] = fmt.Sprintf("testkey-%03d", i) values[i] = []byte(fmt.Sprintf("testvalue-%03d", i)) } // Insert all KV pairs inside a transaction. putOpts := client.TransactionOptions{Name: "example put"} err := kvClient.RunTransaction(&putOpts, func(txn *client.KV) error { for i := 0; i < numKVPairs; i++ { txn.Prepare(proto.Put, proto.PutArgs(proto.Key(keys[i]), values[i]), &proto.PutResponse{}) } // Note that the KV client is flushed automatically on transaction // commit. Invoking Flush after individual API methods is only // required if the result needs to be received to take conditional // action. return nil }) if err != nil { log.Fatal(err) } // Read back KV pairs inside a transaction. getResponses := make([]proto.GetResponse, numKVPairs) getOpts := client.TransactionOptions{Name: "example get"} err = kvClient.RunTransaction(&getOpts, func(txn *client.KV) error { for i := 0; i < numKVPairs; i++ { txn.Prepare(proto.Get, proto.GetArgs(proto.Key(keys[i])), &getResponses[i]) } return nil }) if err != nil { log.Fatal(err) } // Check results. for i, getResp := range getResponses { if getResp.Value == nil { log.Fatal("No value returned for ", keys[i]) } else { if !bytes.Equal(values[i], getResp.Value.Bytes) { log.Fatal("Data mismatch for ", keys[i], ", got: ", getResp.Value.Bytes) } } } fmt.Println("Transaction example done.") // Output: Transaction example done. }
// This is an example for using the Prepare() method to submit // multiple Key Value API operations to be run in parallel. Flush() is // then used to begin execution of all the prepared operations. func ExampleKV_Prepare() { // Using built-in test server for this example code. serv := StartTestServer(nil) defer serv.Stop() // Replace with actual host:port address string (ex "localhost:8080") for server cluster. serverAddress := serv.HTTPAddr // Key Value Client initialization. sender := client.NewHTTPSender(serverAddress, &http.Transport{ TLSClientConfig: rpc.LoadInsecureTLSConfig().Config(), }) kvClient := client.NewKV(sender, nil) kvClient.User = storage.UserRoot defer kvClient.Close() // Insert test data. batchSize := 12 keys := make([]string, batchSize) values := make([][]byte, batchSize) for i := 0; i < batchSize; i++ { keys[i] = fmt.Sprintf("key-%03d", i) values[i] = []byte(fmt.Sprintf("value-%03d", i)) putReq := proto.PutArgs(proto.Key(keys[i]), values[i]) putResp := &proto.PutResponse{} kvClient.Prepare(proto.Put, putReq, putResp) } // Flush all puts for parallel execution. if err := kvClient.Flush(); err != nil { log.Fatal(err) } // Scan for the newly inserted rows in parallel. numScans := 3 rowsPerScan := batchSize / numScans scanResponses := make([]proto.ScanResponse, numScans) for i := 0; i < numScans; i++ { firstKey := proto.Key(keys[i*rowsPerScan]) lastKey := proto.Key(keys[((i+1)*rowsPerScan)-1]) kvClient.Prepare(proto.Scan, proto.ScanArgs(firstKey, lastKey.Next(), int64(rowsPerScan)), &scanResponses[i]) } // Flush all scans for parallel execution. if err := kvClient.Flush(); err != nil { log.Fatal(err) } // Check results which may be returned out-of-order from creation. var matchCount int for i := 0; i < numScans; i++ { for _, keyVal := range scanResponses[i].Rows { currKey := keyVal.Key currValue := keyVal.Value.Bytes for j, origKey := range keys { if bytes.Equal(currKey, proto.Key(origKey)) && bytes.Equal(currValue, values[j]) { matchCount++ } } } } if matchCount != batchSize { log.Fatal("Data mismatch.") } fmt.Println("Prepare Flush example done.") // Output: Prepare Flush example done. }