// BootstrapCluster bootstraps a multiple stores using the provided engines and // cluster ID. The first bootstrapped store contains a single range spanning // all keys. Initial range lookup metadata is populated for the range. // // Returns a KV client for unittest purposes. Caller should close the returned // client. func BootstrapCluster(clusterID string, engines []engine.Engine, stopper *stop.Stopper) (*client.DB, error) { ctx := storage.StoreContext{} ctx.ScanInterval = 10 * time.Minute ctx.Clock = hlc.NewClock(hlc.UnixNano) // Create a KV DB with a local sender. lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, ctx.Clock, false, nil, stopper) ctx.DB = client.NewDB(sender) ctx.Transport = multiraft.NewLocalRPCTransport(stopper) for i, eng := range engines { sIdent := roachpb.StoreIdent{ ClusterID: clusterID, NodeID: 1, StoreID: roachpb.StoreID(i + 1), } // The bootstrapping store will not connect to other nodes so its // StoreConfig doesn't really matter. s := storage.NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1}) // Verify the store isn't already part of a cluster. if len(s.Ident.ClusterID) > 0 { return nil, util.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID) } // Bootstrap store to persist the store ident. if err := s.Bootstrap(sIdent, stopper); err != nil { return nil, err } // Create first range, writing directly to engine. Note this does // not create the range, just its data. Only do this if this is the // first store. if i == 0 { // TODO(marc): this is better than having storage/ import sql, but still // not great. Find a better place to keep those. initialValues := sql.GetInitialSystemValues() if err := s.BootstrapRange(initialValues); err != nil { return nil, err } } if err := s.Start(stopper); err != nil { return nil, err } lSender.AddStore(s) // Initialize node and store ids. Only initialize the node once. if i == 0 { if nodeID, err := allocateNodeID(ctx.DB); nodeID != sIdent.NodeID || err != nil { return nil, util.Errorf("expected to initialize node id allocator to %d, got %d: %s", sIdent.NodeID, nodeID, err) } } if storeID, err := allocateStoreIDs(sIdent.NodeID, 1, ctx.DB); storeID != sIdent.StoreID || err != nil { return nil, util.Errorf("expected to initialize store id allocator to %d, got %d: %s", sIdent.StoreID, storeID, err) } } return ctx.DB, nil }
// Test a normal transaction. This and the other metrics tests below use real KV operations, // because it took far too much mucking with TxnCoordSender internals to mock out the sender // function as other tests do. func TestTxnCommit(t *testing.T) { defer leaktest.AfterTest(t)() _, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() value := []byte("value") db := client.NewDB(sender) // Test normal commit. if err := db.Txn(func(txn *client.Txn) error { key := []byte("key-commit") if err := txn.SetIsolation(roachpb.SNAPSHOT); err != nil { return err } if err := txn.Put(key, value); err != nil { return err } if err := txn.CommitOrCleanup(); err != nil { return err } return nil }); err != nil { t.Fatal(err) } teardownHeartbeats(sender) checkTxnMetrics(t, sender, "commit txn", 1, 0 /* not 1PC */, 0, 0, 0) }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool, context *storage.StoreContext) (*storage.Store, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) if context == nil { // make a copy ctx := storage.TestStoreContext context = &ctx } context.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) lSender := kv.NewLocalSender() sender := kv.NewTxnCoordSender(lSender, clock, false, nil, stopper) context.Clock = clock context.DB = client.NewDB(sender) context.Transport = multiraft.NewLocalRPCTransport(stopper) // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(*context, eng, &proto.NodeDescriptor{NodeID: 1}) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } } lSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(nil); err != nil { t.Fatal(err) } } if err := store.Start(stopper); err != nil { t.Fatal(err) } return store, stopper }
func TestTxnAbortCount(t *testing.T) { defer leaktest.AfterTest(t)() _, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() value := []byte("value") db := client.NewDB(sender) intentionalErrText := "intentional error to cause abort" // Test aborted transaction. if err := db.Txn(func(txn *client.Txn) error { key := []byte("key-abort") if err := txn.SetIsolation(roachpb.SNAPSHOT); err != nil { return err } if err := txn.Put(key, value); err != nil { t.Fatal(err) } return errors.New(intentionalErrText) }); !testutils.IsError(err, intentionalErrText) { t.Fatalf("unexpected error: %s", err) } teardownHeartbeats(sender) checkTxnMetrics(t, sender, "abort txn", 0, 0, 0, 1, 0) }
func TestTxnAbandonCount(t *testing.T) { defer leaktest.AfterTest(t)() manual, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() value := []byte("value") db := client.NewDB(sender) // Test abandoned transaction by making the client timeout ridiculously short. We also set // the sender to heartbeat very frequently, because the heartbeat detects and tears down // abandoned transactions. sender.heartbeatInterval = 2 * time.Millisecond sender.clientTimeout = 1 * time.Millisecond if err := db.Txn(func(txn *client.Txn) error { key := []byte("key-abandon") if err := txn.SetIsolation(roachpb.SNAPSHOT); err != nil { return err } if err := txn.Put(key, value); err != nil { return err } manual.Increment(int64(sender.clientTimeout + sender.heartbeatInterval*2)) checkTxnMetrics(t, sender, "abandon txn", 0, 0, 1, 0, 0) return nil }); !testutils.IsError(err, "writing transaction timed out") { t.Fatalf("unexpected error: %s", err) } }
// Start starts the test cluster by bootstrapping an in-memory store // (defaults to maximum of 50M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.Addr after Start() for client connections. Use Stop() // to shutdown the server after the test completes. func (ltc *LocalTestCluster) Start(t util.Tester) { ltc.Manual = hlc.NewManualClock(0) ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano) ltc.Stopper = stop.NewStopper() rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper) ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20) ltc.lSender = newRetryableLocalSender(NewLocalSender()) ltc.Sender = NewTxnCoordSender(ltc.lSender, ltc.Clock, false, nil, ltc.Stopper) ltc.DB = client.NewDB(ltc.Sender) transport := multiraft.NewLocalRPCTransport(ltc.Stopper) ltc.Stopper.AddCloser(transport) ctx := storage.TestStoreContext ctx.Clock = ltc.Clock ctx.DB = ltc.DB ctx.Gossip = ltc.Gossip ctx.Transport = transport ltc.Store = storage.NewStore(ctx, ltc.Eng, &proto.NodeDescriptor{NodeID: 1}) if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.lSender.AddStore(ltc.Store) if err := ltc.Store.BootstrapRange(nil); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } if err := ltc.Store.Start(ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } }
// createTestStoreWithoutStart creates a test store using an in-memory // engine without starting the store. It returns the store, the store // clock's manual unix nanos time and a stopper. The caller is // responsible for stopping the stopper upon completion. func createTestStoreWithoutStart(t *testing.T) (*Store, *hlc.ManualClock, *stop.Stopper) { stopper := stop.NewStopper() // Setup fake zone config handler. config.TestingSetupZoneConfigHook(stopper) rpcContext := rpc.NewContext(&base.Context{}, hlc.NewClock(hlc.UnixNano), stopper) ctx := TestStoreContext ctx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) ctx.StorePool = NewStorePool(ctx.Gossip, TestTimeUntilStoreDeadOff, stopper) manual := hlc.NewManualClock(0) ctx.Clock = hlc.NewClock(manual.UnixNano) eng := engine.NewInMem(roachpb.Attributes{}, 10<<20, stopper) ctx.Transport = multiraft.NewLocalRPCTransport(stopper) stopper.AddCloser(ctx.Transport) sender := &testSender{} ctx.DB = client.NewDB(sender) store := NewStore(ctx, eng, &roachpb.NodeDescriptor{NodeID: 1}) sender.store = store if err := store.Bootstrap(roachpb.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } if err := store.BootstrapRange(nil); err != nil { t.Fatal(err) } return store, manual, stopper }
// createTestStoreWithEngine creates a test store using the given engine and clock. // The caller is responsible for closing the store on exit. func createTestStoreWithEngine(t *testing.T, eng engine.Engine, clock *hlc.Clock, bootstrap bool, sCtx *storage.StoreContext) (*storage.Store, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{}, clock, stopper) if sCtx == nil { // make a copy ctx := storage.TestStoreContext sCtx = &ctx } nodeDesc := &proto.NodeDescriptor{NodeID: 1} sCtx.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) localSender := kv.NewLocalSender() rpcSend := func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { call := proto.Call{ Args: getArgs(nil /* net.Addr */).(proto.Request), Reply: getReply().(proto.Response), } localSender.Send(context.Background(), call) return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError() } // Mostly makes sure that we don't see a warning per request. { if err := sCtx.Gossip.AddInfoProto(gossip.MakeNodeIDKey(nodeDesc.NodeID), nodeDesc, time.Hour); err != nil { t.Fatal(err) } if err := sCtx.Gossip.SetNodeDescriptor(nodeDesc); err != nil { t.Fatal(err) } } distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: clock, RPCSend: rpcSend, // defined above RangeDescriptorDB: localSender, // for descriptor lookup }, sCtx.Gossip) sender := kv.NewTxnCoordSender(distSender, clock, false, nil, stopper) sCtx.Clock = clock sCtx.DB = client.NewDB(sender) sCtx.Transport = multiraft.NewLocalRPCTransport(stopper) // TODO(bdarnell): arrange to have the transport closed. store := storage.NewStore(*sCtx, eng, nodeDesc) if bootstrap { if err := store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, stopper); err != nil { t.Fatal(err) } } localSender.AddStore(store) if bootstrap { if err := store.BootstrapRange(sql.GetInitialSystemValues()); err != nil { t.Fatal(err) } } if err := store.Start(stopper); err != nil { t.Fatal(err) } return store, stopper }
func newKVNative(b *testing.B) kvInterface { enableTracing := tracing.Disable() s := server.StartTestServer(b) // TestServer.DB() returns the TxnCoordSender wrapped client. But that isn't // a fair comparison with SQL as we want these client requests to be sent // over the network. sender, err := client.NewSender( rpc.NewContext(&base.Context{ User: security.NodeUser, SSLCA: filepath.Join(security.EmbeddedCertsDir, security.EmbeddedCACert), SSLCert: filepath.Join(security.EmbeddedCertsDir, "node.crt"), SSLCertKey: filepath.Join(security.EmbeddedCertsDir, "node.key"), }, nil, s.Stopper()), s.ServingAddr()) if err != nil { b.Fatal(err) } return &kvNative{ db: client.NewDB(sender), doneFn: func() { s.Stop() enableTracing() }, } }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) ( *rpc.Server, *hlc.Clock, *Node, *stop.Stopper) { ctx := storage.StoreContext{} stopper := stop.NewStopper() ctx.Clock = hlc.NewClock(hlc.UnixNano) nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper) ctx.ScanInterval = 10 * time.Hour rpcServer := rpc.NewServer(addr, nodeRPCContext) if err := rpcServer.Start(); err != nil { t.Fatal(err) } g := gossip.New(nodeRPCContext, testContext.GossipInterval, testContext.GossipBootstrapResolvers) if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS == addr { gossipBS = rpcServer.Addr() } g.SetResolvers([]resolver.Resolver{resolver.NewResolverFromAddress(gossipBS)}) g.Start(rpcServer, stopper) } ctx.Gossip = g sender := kv.NewDistSender(&kv.DistSenderContext{Clock: ctx.Clock}, g) ctx.DB = client.NewDB(sender) // TODO(bdarnell): arrange to have the transport closed. // (or attach LocalRPCTransport.Close to the stopper) ctx.Transport = multiraft.NewLocalRPCTransport(stopper) ctx.EventFeed = util.NewFeed(stopper) node := NewNode(ctx) return rpcServer, ctx.Clock, node, stopper }
func TestTxnRestartCount(t *testing.T) { defer leaktest.AfterTest(t)() _, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() key := []byte("key-restart") value := []byte("value") db := client.NewDB(sender) // Start a transaction and do a GET. This forces a timestamp to be chosen for the transaction. txn := client.NewTxn(*db) if _, err := txn.Get(key); err != nil { t.Fatal(err) } // Outside of the transaction, read the same key as was read within the transaction. This // means that future attempts to write will increase the timestamp. if _, err := db.Get(key); err != nil { t.Fatal(err) } // This put increases the candidate timestamp, which causes an immediate restart. if err := txn.Put(key, value); err == nil { t.Fatalf("unexpected success") } teardownHeartbeats(sender) checkTxnMetrics(t, sender, "restart txn", 0, 1, 0, 1) }
// Start starts the test cluster by bootstrapping an in-memory store // (defaults to maximum of 50M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.Addr after Start() for client connections. Use Stop() // to shutdown the server after the test completes. func (ltc *LocalTestCluster) Start(t util.Tester) { nodeDesc := &proto.NodeDescriptor{NodeID: 1} ltc.tester = t ltc.Manual = hlc.NewManualClock(0) ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano) ltc.Stopper = stop.NewStopper() rpcContext := rpc.NewContext(testutils.NewNodeTestBaseContext(), ltc.Clock, ltc.Stopper) ltc.Gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) ltc.Eng = engine.NewInMem(proto.Attributes{}, 50<<20, ltc.Stopper) ltc.localSender = NewLocalSender() var rpcSend rpcSendFn = func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { // TODO(tschottdorf): remove getReply(). br, pErr := ltc.localSender.Send(context.Background(), *getArgs(nil).(*proto.BatchRequest)) if br == nil { br = &proto.BatchResponse{} } if br.Error != nil { panic(proto.ErrorUnexpectedlySet(ltc.localSender, br)) } br.Error = pErr return []gogoproto.Message{br}, nil } ltc.distSender = NewDistSender(&DistSenderContext{ Clock: ltc.Clock, RangeDescriptorCacheSize: defaultRangeDescriptorCacheSize, RangeLookupMaxRanges: defaultRangeLookupMaxRanges, LeaderCacheSize: defaultLeaderCacheSize, RPCRetryOptions: &defaultRPCRetryOptions, nodeDescriptor: nodeDesc, RPCSend: rpcSend, // defined above RangeDescriptorDB: ltc.localSender, // for descriptor lookup }, ltc.Gossip) ltc.Sender = NewTxnCoordSender(ltc.distSender, ltc.Clock, false /* !linearizable */, nil /* tracer */, ltc.Stopper) ltc.DB = client.NewDB(ltc.Sender) transport := multiraft.NewLocalRPCTransport(ltc.Stopper) ltc.Stopper.AddCloser(transport) ctx := storage.TestStoreContext ctx.Clock = ltc.Clock ctx.DB = ltc.DB ctx.Gossip = ltc.Gossip ctx.Transport = transport ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc) if err := ltc.Store.Bootstrap(proto.StoreIdent{NodeID: 1, StoreID: 1}, ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.localSender.AddStore(ltc.Store) if err := ltc.Store.BootstrapRange(nil); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } if err := ltc.Store.Start(ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } }
func (m *multiTestContext) Start(t *testing.T, numStores int) { m.t = t if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(&base.Context{}, m.clock, nil) m.gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) } if m.clientStopper == nil { m.clientStopper = stop.NewStopper() } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport(m.clientStopper) } if m.storePool == nil { if m.timeUntilStoreDead == 0 { m.timeUntilStoreDead = storage.TestTimeUntilStoreDeadOff } m.storePool = storage.NewStorePool(m.gossip, m.timeUntilStoreDead, m.clientStopper) } // Always create the first sender. m.senders = append(m.senders, kv.NewLocalSender()) rpcSend := func(_ rpc.Options, _ string, _ []net.Addr, getArgs func(addr net.Addr) gogoproto.Message, getReply func() gogoproto.Message, _ *rpc.Context) ([]gogoproto.Message, error) { call := proto.Call{ Args: getArgs(nil /* net.Addr */).(proto.Request), Reply: getReply().(proto.Response), } m.senders[0].Send(context.Background(), call) return []gogoproto.Message{call.Reply}, call.Reply.Header().GoError() } if m.db == nil { distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: m.clock, RangeDescriptorDB: m.senders[0], RPCSend: rpcSend, }, m.gossip) sender := kv.NewTxnCoordSender(distSender, m.clock, false, nil, m.clientStopper) m.db = client.NewDB(sender) } for i := 0; i < numStores; i++ { m.addStore() } if m.transportStopper == nil { m.transportStopper = stop.NewStopper() } m.transportStopper.AddCloser(m.transport) }
// TestInconsistentReads tests that the methods that generate inconsistent reads // generate outgoing requests with an INCONSISTENT read consistency. func TestInconsistentReads(t *testing.T) { defer leaktest.AfterTest(t)() // Mock out DistSender's sender function to check the read consistency for // outgoing BatchRequests and return an empty reply. var senderFn client.SenderFunc senderFn = func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { if ba.ReadConsistency != roachpb.INCONSISTENT { return nil, roachpb.NewErrorf("BatchRequest has unexpected ReadConsistency %s", ba.ReadConsistency) } return ba.CreateReply(), nil } db := client.NewDB(senderFn) prepInconsistent := func() *client.Batch { var b client.Batch b.Header.ReadConsistency = roachpb.INCONSISTENT return &b } // Perform inconsistent reads through the mocked sender function. { key := roachpb.Key([]byte("key")) b := prepInconsistent() b.Get(key) if err := db.Run(b); err != nil { t.Fatal(err) } } { b := prepInconsistent() key1 := roachpb.Key([]byte("key1")) key2 := roachpb.Key([]byte("key2")) const dontCareMaxRows = 1000 b.Scan(key1, key2, dontCareMaxRows) if err := db.Run(b); err != nil { t.Fatal(err) } } { key := roachpb.Key([]byte("key")) b := db.NewBatch() b.Header.ReadConsistency = roachpb.INCONSISTENT b.Get(key) if err := db.Run(b); err != nil { t.Fatal(err) } } }
func createTestClientForUser(t *testing.T, stopper *stop.Stopper, addr, user string) *client.DB { var ctx base.Context ctx.InitDefaults() ctx.User = user ctx.SSLCA = filepath.Join(security.EmbeddedCertsDir, security.EmbeddedCACert) ctx.SSLCert = filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.crt", user)) ctx.SSLCertKey = filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.key", user)) sender, err := client.NewSender(rpc.NewContext(&ctx, nil, stopper), addr) if err != nil { t.Fatal(err) } return client.NewDB(sender) }
func createTestClientForUser(t *testing.T, stopper *stop.Stopper, addr, user string) *client.DB { rpcContext := rpc.NewContext(&base.Context{ User: user, SSLCA: filepath.Join(security.EmbeddedCertsDir, security.EmbeddedCACert), SSLCert: filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.crt", user)), SSLCertKey: filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.key", user)), }, nil, stopper) sender, err := client.NewSender(rpcContext, addr) if err != nil { t.Fatal(err) } return client.NewDB(sender) }
// NewClient implements the Cluster interface. func (l *LocalCluster) NewClient(t *testing.T, i int) (*roachClient.DB, *stop.Stopper) { stopper := stop.NewStopper() rpcContext := rpc.NewContext(&base.Context{ User: security.NodeUser, SSLCA: filepath.Join(l.CertsDir, security.EmbeddedCACert), SSLCert: filepath.Join(l.CertsDir, security.EmbeddedNodeCert), SSLCertKey: filepath.Join(l.CertsDir, security.EmbeddedNodeKey), }, nil, stopper) sender, err := roachClient.NewSender(rpcContext, l.Nodes[i].Addr(DefaultTCP).String()) if err != nil { t.Fatal(err) } return roachClient.NewDB(sender), stopper }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) ( *rpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) { ctx := storage.StoreContext{} stopper := stop.NewStopper() ctx.Clock = hlc.NewClock(hlc.UnixNano) nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper) ctx.ScanInterval = 10 * time.Hour rpcServer := rpc.NewServer(nodeRPCContext) grpcServer := grpc.NewServer() tlsConfig, err := nodeRPCContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } ln, err := util.ListenAndServe(stopper, grpcutil.GRPCHandlerFunc(grpcServer, rpcServer), addr, tlsConfig) if err != nil { t.Fatal(err) } g := gossip.New(nodeRPCContext, testContext.GossipBootstrapResolvers, stopper) if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS.Network() == addr.Network() && gossipBS.String() == addr.String() { gossipBS = ln.Addr() } r, err := resolver.NewResolverFromAddress(gossipBS) if err != nil { t.Fatalf("bad gossip address %s: %s", gossipBS, err) } g.SetResolvers([]resolver.Resolver{r}) g.Start(grpcServer, ln.Addr()) } ctx.Gossip = g retryOpts := kv.GetDefaultDistSenderRetryOptions() retryOpts.Closer = stopper.ShouldDrain() distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: ctx.Clock, RPCContext: nodeRPCContext, RPCRetryOptions: &retryOpts, }, g) tracer := tracing.NewTracer() sender := kv.NewTxnCoordSender(distSender, ctx.Clock, false, tracer, stopper) ctx.DB = client.NewDB(sender) // TODO(bdarnell): arrange to have the transport closed. // (or attach LocalRPCTransport.Close to the stopper) ctx.Transport = storage.NewLocalRPCTransport(stopper) ctx.EventFeed = util.NewFeed(stopper) ctx.Tracer = tracer node := NewNode(ctx, metric.NewRegistry(), stopper, nil) return rpcServer, ln.Addr(), ctx.Clock, node, stopper }
func makeDBClient() (*client.DB, *stop.Stopper) { stopper := stop.NewStopper() context := &base.Context{ User: security.NodeUser, SSLCA: cliContext.SSLCA, SSLCert: cliContext.SSLCert, SSLCertKey: cliContext.SSLCertKey, Insecure: cliContext.Insecure, } sender, err := client.NewSender(rpc.NewContext(context, nil, stopper), cliContext.Addr) if err != nil { stopper.Stop() panicf("failed to initialize KV client: %s", err) } return client.NewDB(sender), stopper }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) ( *grpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) { ctx := storage.StoreContext{} stopper := stop.NewStopper() ctx.Clock = hlc.NewClock(hlc.UnixNano) nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper) ctx.ScanInterval = 10 * time.Hour ctx.ConsistencyCheckInterval = 10 * time.Hour grpcServer := rpc.NewServer(nodeRPCContext) ln, err := util.ListenAndServeGRPC(stopper, grpcServer, addr) if err != nil { t.Fatal(err) } serverCtx := NewTestContext() g := gossip.New(nodeRPCContext, serverCtx.GossipBootstrapResolvers, stopper) if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS.Network() == addr.Network() && gossipBS.String() == addr.String() { gossipBS = ln.Addr() } r, err := resolver.NewResolverFromAddress(gossipBS) if err != nil { t.Fatalf("bad gossip address %s: %s", gossipBS, err) } g.SetResolvers([]resolver.Resolver{r}) g.Start(grpcServer, ln.Addr()) } ctx.Gossip = g retryOpts := kv.GetDefaultDistSenderRetryOptions() retryOpts.Closer = stopper.ShouldDrain() distSender := kv.NewDistSender(&kv.DistSenderContext{ Clock: ctx.Clock, RPCContext: nodeRPCContext, RPCRetryOptions: &retryOpts, }, g) tracer := tracing.NewTracer() sender := kv.NewTxnCoordSender(distSender, ctx.Clock, false, tracer, stopper, kv.NewTxnMetrics(metric.NewRegistry())) ctx.DB = client.NewDB(sender) ctx.Transport = storage.NewDummyRaftTransport() ctx.Tracer = tracer node := NewNode(ctx, status.NewMetricsRecorder(ctx.Clock), stopper, kv.NewTxnMetrics(metric.NewRegistry()), sql.MakeEventLogger(nil)) roachpb.RegisterInternalServer(grpcServer, node) return grpcServer, ln.Addr(), ctx.Clock, node, stopper }
// TestTxnOnePhaseCommit verifies that 1PC metric tracking works. func TestTxnOnePhaseCommit(t *testing.T) { defer leaktest.AfterTest(t)() _, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() value := []byte("value") db := client.NewDB(sender) if err := db.Txn(func(txn *client.Txn) error { key := []byte("key-commit") b := txn.NewBatch() b.Put(key, value) return txn.CommitInBatch(b) }); err != nil { t.Fatal(err) } teardownHeartbeats(sender) checkTxnMetrics(t, sender, "commit 1PC txn", 1, 1 /* 1PC */, 0, 0, 0) }
// TestTxnReadAfterAbandon checks the fix for the condition in issue #4787: // after a transaction is abandoned we do a read as part of that transaction // which should fail. func TestTxnReadAfterAbandon(t *testing.T) { defer leaktest.AfterTest(t)() manual, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() value := []byte("value") db := client.NewDB(sender) // Test abandoned transaction by making the client timeout ridiculously short. We also set // the sender to heartbeat very frequently, because the heartbeat detects and tears down // abandoned transactions. sender.heartbeatInterval = 2 * time.Millisecond sender.clientTimeout = 1 * time.Millisecond pErr := db.Txn(func(txn *client.Txn) *roachpb.Error { key := []byte("key-abandon") if err := txn.SetIsolation(roachpb.SNAPSHOT); err != nil { t.Fatal(err) } if pErr := txn.Put(key, value); pErr != nil { t.Fatal(pErr) } manual.Increment(int64(sender.clientTimeout + sender.heartbeatInterval*2)) checkTxnMetrics(t, sender, "abandon txn", 0, 1, 0, 0) _, pErr := txn.Get(key) if pErr == nil { t.Fatalf("Get succeeded on abandoned txn") } else if !testutils.IsPError(pErr, "writing transaction timed out") { t.Fatalf("unexpected error from Get on abandoned txn: %s", pErr) } return pErr }) if pErr == nil { t.Fatalf("abandoned txn didn't fail") } }
func TestTxnDurations(t *testing.T) { defer leaktest.AfterTest(t)() manual, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() db := client.NewDB(sender) const puts = 10 const incr int64 = 1000 for i := 0; i < puts; i++ { key := roachpb.Key(fmt.Sprintf("key-txn-durations-%d", i)) if err := db.Txn(func(txn *client.Txn) error { if err := txn.SetIsolation(roachpb.SNAPSHOT); err != nil { return err } if err := txn.Put(key, []byte("val")); err != nil { return err } manual.Increment(incr) return nil }); err != nil { t.Fatal(err) } } teardownHeartbeats(sender) checkTxnMetrics(t, sender, "txn durations", puts, 0, 0, 0, 0) hist := sender.metrics.Durations[metric.Scale1M].Current() // The clock is a bit odd in these tests, so I can't test the mean without introducing // spurious errors or being overly lax. // TODO(cdo): look into cause of variance. if a, e := hist.TotalCount(), int64(puts); a != e { t.Fatalf("durations %d != expected %d", a, e) } if min := hist.Min(); min < incr { t.Fatalf("min %d < %d", min, incr) } }
func (m *multiTestContext) Start(t *testing.T, numStores int) { m.t = t if m.manualClock == nil { m.manualClock = hlc.NewManualClock(0) } if m.clock == nil { m.clock = hlc.NewClock(m.manualClock.UnixNano) } if m.gossip == nil { rpcContext := rpc.NewContext(&base.Context{}, m.clock, nil) m.gossip = gossip.New(rpcContext, gossip.TestInterval, gossip.TestBootstrap) } if m.clientStopper == nil { m.clientStopper = stop.NewStopper() } if m.transport == nil { m.transport = multiraft.NewLocalRPCTransport(m.clientStopper) } if m.storePool == nil { if m.timeUntilStoreDead == 0 { m.timeUntilStoreDead = storage.TestTimeUntilStoreDeadOff } m.storePool = storage.NewStorePool(m.gossip, m.timeUntilStoreDead, m.clientStopper) } // Always create the first sender. m.senders = append(m.senders, kv.NewLocalSender()) if m.db == nil { sender := kv.NewTxnCoordSender(m, m.clock, false, nil, m.clientStopper) m.db = client.NewDB(sender) } for i := 0; i < numStores; i++ { m.addStore() } if m.transportStopper == nil { m.transportStopper = stop.NewStopper() } m.transportStopper.AddCloser(m.transport) }
// createTestNode creates an rpc server using the specified address, // gossip instance, KV database and a node using the specified slice // of engines. The server, clock and node are returned. If gossipBS is // not nil, the gossip bootstrap address is set to gossipBS. func createTestNode(addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T) ( *rpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) { ctx := storage.StoreContext{} stopper := stop.NewStopper() ctx.Clock = hlc.NewClock(hlc.UnixNano) nodeRPCContext := rpc.NewContext(nodeTestBaseContext, ctx.Clock, stopper) ctx.ScanInterval = 10 * time.Hour rpcServer := rpc.NewServer(nodeRPCContext) tlsConfig, err := nodeRPCContext.GetServerTLSConfig() if err != nil { t.Fatal(err) } ln, err := util.ListenAndServe(stopper, rpcServer, addr, tlsConfig) if err != nil { t.Fatal(err) } g := gossip.New(nodeRPCContext, testContext.GossipBootstrapResolvers) if gossipBS != nil { // Handle possibility of a :0 port specification. if gossipBS == addr { gossipBS = ln.Addr() } r, err := resolver.NewResolverFromAddress(gossipBS) if err != nil { t.Fatalf("bad gossip address %s: %s", gossipBS, err) } g.SetResolvers([]resolver.Resolver{r}) g.Start(rpcServer, ln.Addr(), stopper) } ctx.Gossip = g sender := kv.NewDistSender(&kv.DistSenderContext{Clock: ctx.Clock, RPCContext: nodeRPCContext}, g) ctx.DB = client.NewDB(sender) // TODO(bdarnell): arrange to have the transport closed. // (or attach LocalRPCTransport.Close to the stopper) ctx.Transport = storage.NewLocalRPCTransport(stopper) ctx.EventFeed = util.NewFeed(stopper) node := NewNode(ctx, metric.NewRegistry(), stopper) return rpcServer, ln.Addr(), ctx.Clock, node, stopper }
// Start starts the test cluster by bootstrapping an in-memory store // (defaults to maximum of 50M). The server is started, launching the // node RPC server and all HTTP endpoints. Use the value of // TestServer.Addr after Start() for client connections. Use Stop() // to shutdown the server after the test completes. func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Context, initSender InitSenderFn) { nodeID := roachpb.NodeID(1) nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID} tracer := tracing.NewTracer() ltc.tester = t ltc.Manual = hlc.NewManualClock(0) ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano) ltc.Stopper = stop.NewStopper() rpcContext := rpc.NewContext(baseCtx, ltc.Clock, ltc.Stopper) ltc.Gossip = gossip.New(rpcContext, nil, ltc.Stopper) ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20, ltc.Stopper) ltc.Stores = storage.NewStores(ltc.Clock) ltc.Sender = initSender(nodeDesc, tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper, ltc.Gossip) ltc.DB = client.NewDB(ltc.Sender) transport := storage.NewDummyRaftTransport() ctx := storage.TestStoreContext() ctx.Clock = ltc.Clock ctx.DB = ltc.DB ctx.Gossip = ltc.Gossip ctx.Transport = transport ctx.Tracer = tracer ltc.Store = storage.NewStore(ctx, ltc.Eng, nodeDesc) if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}, ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.Stores.AddStore(ltc.Store) if err := ltc.Store.BootstrapRange(nil); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } if err := ltc.Store.Start(ltc.Stopper); err != nil { t.Fatalf("unable to start local test cluster: %s", err) } ltc.Gossip.SetNodeID(nodeDesc.NodeID) if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil { t.Fatalf("unable to set node descriptor: %s", err) } }
func TestTxnRestartCount(t *testing.T) { defer leaktest.AfterTest(t)() _, sender, cleanupFn := setupMetricsTest(t) defer cleanupFn() key := []byte("key-restart") value := []byte("value") db := client.NewDB(sender) // Start a transaction and do a GET. This forces a timestamp to be chosen for the transaction. txn := client.NewTxn(*db) if _, err := txn.Get(key); err != nil { t.Fatal(err) } // Outside of the transaction, read the same key as was read within the transaction. This // means that future attempts to write will increase the timestamp. if _, err := db.Get(key); err != nil { t.Fatal(err) } // This put will lay down an intent, txn timestamp will increase beyond original. if err := txn.Put(key, value); err != nil { t.Fatal(err) } if !txn.Proto.OrigTimestamp.Less(txn.Proto.Timestamp) { t.Errorf("expected timestamp to increase: %s", txn.Proto) } // Commit (should cause restart metric to increase). pErr := txn.Commit() if _, ok := pErr.GetDetail().(*roachpb.TransactionRetryError); !ok { t.Errorf("expected transaction retry err; got %s", pErr) } teardownHeartbeats(sender) checkTxnMetrics(t, sender, "restart txn", 0, 0, 1, 1) }
// NewServer creates a Server from a server.Context. func NewServer(ctx *Context, stopper *stop.Stopper) (*Server, error) { if ctx == nil { return nil, util.Errorf("ctx must not be null") } if _, err := net.ResolveTCPAddr("tcp", ctx.Addr); err != nil { return nil, util.Errorf("unable to resolve RPC address %q: %v", ctx.Addr, err) } if ctx.Insecure { log.Warning("running in insecure mode, this is strongly discouraged. See --insecure and --certs.") } // Try loading the TLS configs before anything else. if _, err := ctx.GetServerTLSConfig(); err != nil { return nil, err } if _, err := ctx.GetClientTLSConfig(); err != nil { return nil, err } s := &Server{ ctx: ctx, mux: http.NewServeMux(), clock: hlc.NewClock(hlc.UnixNano), stopper: stopper, } s.clock.SetMaxOffset(ctx.MaxOffset) s.rpcContext = crpc.NewContext(&ctx.Context, s.clock, stopper) stopper.RunWorker(func() { s.rpcContext.RemoteClocks.MonitorRemoteOffsets(stopper) }) s.rpc = crpc.NewServer(s.rpcContext) s.gossip = gossip.New(s.rpcContext, s.ctx.GossipBootstrapResolvers) s.storePool = storage.NewStorePool(s.gossip, s.clock, ctx.TimeUntilStoreDead, stopper) feed := util.NewFeed(stopper) tracer := tracer.NewTracer(feed, ctx.Addr) ds := kv.NewDistSender(&kv.DistSenderContext{Clock: s.clock, RPCContext: s.rpcContext}, s.gossip) sender := kv.NewTxnCoordSender(ds, s.clock, ctx.Linearizable, tracer, s.stopper) s.db = client.NewDB(sender) var err error s.raftTransport, err = newRPCTransport(s.gossip, s.rpc, s.rpcContext) if err != nil { return nil, err } s.stopper.AddCloser(s.raftTransport) s.kvDB = kv.NewDBServer(&s.ctx.Context, sender) if err := s.kvDB.RegisterRPC(s.rpc); err != nil { return nil, err } leaseMgr := sql.NewLeaseManager(0, *s.db, s.clock) leaseMgr.RefreshLeases(s.stopper, s.db, s.gossip) s.sqlServer = sql.MakeServer(&s.ctx.Context, *s.db, s.gossip, leaseMgr) if err := s.sqlServer.RegisterRPC(s.rpc); err != nil { return nil, err } s.pgServer = pgwire.NewServer(&pgwire.Context{ Context: &s.ctx.Context, Executor: s.sqlServer.Executor, Stopper: stopper, }) // TODO(bdarnell): make StoreConfig configurable. nCtx := storage.StoreContext{ Clock: s.clock, DB: s.db, Gossip: s.gossip, Transport: s.raftTransport, ScanInterval: s.ctx.ScanInterval, ScanMaxIdleTime: s.ctx.ScanMaxIdleTime, EventFeed: feed, Tracer: tracer, StorePool: s.storePool, AllocatorOptions: storage.AllocatorOptions{ AllowRebalance: true, Mode: s.ctx.BalanceMode, }, } s.node = NewNode(nCtx) s.admin = newAdminServer(s.db, s.stopper) s.status = newStatusServer(s.db, s.gossip, ctx) s.tsDB = ts.NewDB(s.db) s.tsServer = ts.NewServer(s.tsDB) return s, nil }
// TestTxnCoordSenderTxnUpdatedOnError verifies that errors adjust the // response transaction's timestamp and priority as appropriate. func TestTxnCoordSenderTxnUpdatedOnError(t *testing.T) { defer leaktest.AfterTest(t) origTS := makeTS(123, 0) testCases := []struct { pErr *roachpb.Error expEpoch uint32 expPri int32 expTS, expOrigTS roachpb.Timestamp nodeSeen bool }{ { // No error, so nothing interesting either. pErr: nil, expEpoch: 0, expPri: 1, expTS: origTS, expOrigTS: origTS, }, { // On uncertainty error, new epoch begins and node is seen. // Timestamp moves ahead of the existing write. pErr: roachpb.NewError(&roachpb.ReadWithinUncertaintyIntervalError{ NodeID: 1, ExistingTimestamp: origTS.Add(10, 10), }), expEpoch: 1, expPri: 1, expTS: origTS.Add(10, 11), expOrigTS: origTS.Add(10, 11), nodeSeen: true, }, { // On abort, nothing changes but we get a new priority to use for // the next attempt. pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionAbortedError{}, &roachpb.Transaction{ Timestamp: origTS.Add(20, 10), Priority: 10, }), expPri: 10, }, { // On failed push, new epoch begins just past the pushed timestamp. // Additionally, priority ratchets up to just below the pusher's. pErr: roachpb.NewError(&roachpb.TransactionPushError{ PusheeTxn: roachpb.Transaction{ Timestamp: origTS.Add(10, 10), Priority: int32(10)}, }), expEpoch: 1, expPri: 9, expTS: origTS.Add(10, 11), expOrigTS: origTS.Add(10, 11), }, { // On retry, restart with new epoch, timestamp and priority. pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionRetryError{}, &roachpb.Transaction{ Timestamp: origTS.Add(10, 10), Priority: int32(10)}), expEpoch: 1, expPri: 10, expTS: origTS.Add(10, 10), expOrigTS: origTS.Add(10, 10), }, } for i, test := range testCases { stopper := stop.NewStopper() manual := hlc.NewManualClock(origTS.WallTime) clock := hlc.NewClock(manual.UnixNano) clock.SetMaxOffset(20) ts := NewTxnCoordSender(senderFn(func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) { var reply *roachpb.BatchResponse if test.pErr == nil { reply = ba.CreateReply() } return reply, test.pErr }), clock, false, nil, stopper) db := client.NewDB(ts) txn := client.NewTxn(*db) txn.InternalSetPriority(1) txn.Proto.Name = "test txn" key := roachpb.Key("test-key") _, pErr := txn.Get(key) teardownHeartbeats(ts) stopper.Stop() if reflect.TypeOf(test.pErr) != reflect.TypeOf(pErr) { t.Fatalf("%d: expected %T; got %T: %v", i, test.pErr, pErr, pErr) } if txn.Proto.Epoch != test.expEpoch { t.Errorf("%d: expected epoch = %d; got %d", i, test.expEpoch, txn.Proto.Epoch) } if txn.Proto.Priority != test.expPri { t.Errorf("%d: expected priority = %d; got %d", i, test.expPri, txn.Proto.Priority) } if !txn.Proto.Timestamp.Equal(test.expTS) { t.Errorf("%d: expected timestamp to be %s; got %s", i, test.expTS, txn.Proto.Timestamp) } if !txn.Proto.OrigTimestamp.Equal(test.expOrigTS) { t.Errorf("%d: expected orig timestamp to be %s + 1; got %s", i, test.expOrigTS, txn.Proto.OrigTimestamp) } if nodes := txn.Proto.CertainNodes.Nodes; (len(nodes) != 0) != test.nodeSeen { t.Errorf("%d: expected nodeSeen=%t, but list of hosts is %v", i, test.nodeSeen, nodes) } } }
// TestTxnCoordSenderNoDuplicateIntents verifies that TxnCoordSender does not // generate duplicate intents and that it merges intents for overlapping ranges. func TestTxnCoordSenderNoDuplicateIntents(t *testing.T) { defer leaktest.AfterTest(t)() stopper := stop.NewStopper() manual := hlc.NewManualClock(0) clock := hlc.NewClock(manual.UnixNano) var expectedIntents []roachpb.Span senderFunc := func(_ context.Context, ba roachpb.BatchRequest) ( *roachpb.BatchResponse, *roachpb.Error) { if rArgs, ok := ba.GetArg(roachpb.EndTransaction); ok { et := rArgs.(*roachpb.EndTransactionRequest) if !reflect.DeepEqual(et.IntentSpans, expectedIntents) { t.Errorf("Invalid intents: %+v; expected %+v", et.IntentSpans, expectedIntents) } } br := ba.CreateReply() txnClone := ba.Txn.Clone() br.Txn = &txnClone br.Txn.Writing = true return br, nil } ts := NewTxnCoordSender(senderFn(senderFunc), clock, false, tracing.NewTracer(), stopper, NewTxnMetrics(metric.NewRegistry())) defer stopper.Stop() defer teardownHeartbeats(ts) db := client.NewDB(ts) txn := client.NewTxn(context.Background(), *db) // Write to a, b, u-w before the final batch. pErr := txn.Put(roachpb.Key("a"), []byte("value")) if pErr != nil { t.Fatal(pErr) } pErr = txn.Put(roachpb.Key("b"), []byte("value")) if pErr != nil { t.Fatal(pErr) } pErr = txn.DelRange(roachpb.Key("u"), roachpb.Key("w")) if pErr != nil { t.Fatal(pErr) } // The final batch overwrites key a and overlaps part of the u-w range. b := txn.NewBatch() b.Put(roachpb.Key("b"), []byte("value")) b.Put(roachpb.Key("c"), []byte("value")) b.DelRange(roachpb.Key("v"), roachpb.Key("z"), false) // The expected intents are a, b, c, and u-z. expectedIntents = []roachpb.Span{ {Key: roachpb.Key("a"), EndKey: nil}, {Key: roachpb.Key("b"), EndKey: nil}, {Key: roachpb.Key("c"), EndKey: nil}, {Key: roachpb.Key("u"), EndKey: roachpb.Key("z")}, } pErr = txn.CommitInBatch(b) if pErr != nil { t.Fatal(pErr) } }