// MakeDBClient creates a kv client for use in cli tools.
func MakeDBClient() (*client.DB, *stop.Stopper, error) {
conn, stopper, err := getGRPCConn()
if err != nil {
return nil, nil, err
}
return client.NewDB(client.NewSender(conn)), stopper, nil
}
func TestTxnAbandonCount(t *testing.T) {
defer leaktest.AfterTest(t)()
manual, sender, cleanupFn := setupMetricsTest(t)
defer cleanupFn()
value := []byte("value")
db := client.NewDB(sender)
// Test abandoned transaction by making the client timeout ridiculously short. We also set
// the sender to heartbeat very frequently, because the heartbeat detects and tears down
// abandoned transactions.
sender.heartbeatInterval = 2 * time.Millisecond
sender.clientTimeout = 1 * time.Millisecond
if err := db.Txn(context.TODO(), func(txn *client.Txn) error {
key := []byte("key-abandon")
if err := txn.SetIsolation(enginepb.SNAPSHOT); err != nil {
return err
}
if err := txn.Put(key, value); err != nil {
return err
}
manual.Increment(int64(sender.clientTimeout + sender.heartbeatInterval*2))
checkTxnMetrics(t, sender, "abandon txn", 0, 0, 1, 0, 0)
return nil
}); !testutils.IsError(err, "writing transaction timed out") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestTxnAbortCount(t *testing.T) {
defer leaktest.AfterTest(t)()
_, sender, cleanupFn := setupMetricsTest(t)
defer cleanupFn()
value := []byte("value")
db := client.NewDB(sender)
intentionalErrText := "intentional error to cause abort"
// Test aborted transaction.
if err := db.Txn(context.TODO(), func(txn *client.Txn) error {
key := []byte("key-abort")
if err := txn.SetIsolation(enginepb.SNAPSHOT); err != nil {
return err
}
if err := txn.Put(key, value); err != nil {
t.Fatal(err)
}
return errors.New(intentionalErrText)
}); !testutils.IsError(err, intentionalErrText) {
t.Fatalf("unexpected error: %v", err)
}
teardownHeartbeats(sender)
checkTxnMetrics(t, sender, "abort txn", 0, 0, 0, 1, 0)
}
// NewClient implements the Cluster interface.
func (f *Farmer) NewClient(ctx context.Context, t testing.TB, i int) *client.DB {
conn, err := f.RPCContext.GRPCDial(f.Addr(ctx, i, base.DefaultPort))
if err != nil {
t.Fatal(err)
}
return client.NewDB(client.NewSender(conn))
}
// createTestNode creates an rpc server using the specified address,
// gossip instance, KV database and a node using the specified slice
// of engines. The server, clock and node are returned. If gossipBS is
// not nil, the gossip bootstrap address is set to gossipBS.
func createTestNode(
addr net.Addr, engines []engine.Engine, gossipBS net.Addr, t *testing.T,
) (*grpc.Server, net.Addr, *hlc.Clock, *Node, *stop.Stopper) {
cfg := storage.StoreConfig{}
stopper := stop.NewStopper()
cfg.Clock = hlc.NewClock(hlc.UnixNano)
nodeRPCContext := rpc.NewContext(log.AmbientContext{}, nodeTestBaseContext, cfg.Clock, stopper)
cfg.ScanInterval = 10 * time.Hour
cfg.ConsistencyCheckInterval = 10 * time.Hour
grpcServer := rpc.NewServer(nodeRPCContext)
serverCfg := makeTestConfig()
cfg.Gossip = gossip.NewTest(
0,
nodeRPCContext,
grpcServer,
serverCfg.GossipBootstrapResolvers,
stopper,
metric.NewRegistry(),
)
ln, err := netutil.ListenAndServeGRPC(stopper, grpcServer, addr)
if err != nil {
t.Fatal(err)
}
if gossipBS != nil {
// Handle possibility of a :0 port specification.
if gossipBS.Network() == addr.Network() && gossipBS.String() == addr.String() {
gossipBS = ln.Addr()
}
r, err := resolver.NewResolverFromAddress(gossipBS)
if err != nil {
t.Fatalf("bad gossip address %s: %s", gossipBS, err)
}
cfg.Gossip.SetResolvers([]resolver.Resolver{r})
cfg.Gossip.Start(ln.Addr())
}
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = stopper.ShouldQuiesce()
distSender := kv.NewDistSender(kv.DistSenderConfig{
Clock: cfg.Clock,
RPCContext: nodeRPCContext,
RPCRetryOptions: &retryOpts,
}, cfg.Gossip)
cfg.AmbientCtx.Tracer = tracing.NewTracer()
sender := kv.NewTxnCoordSender(
cfg.AmbientCtx,
distSender,
cfg.Clock,
false,
stopper,
kv.MakeTxnMetrics(metric.TestSampleInterval),
)
cfg.DB = client.NewDB(sender)
cfg.Transport = storage.NewDummyRaftTransport()
cfg.MetricsSampleInterval = metric.TestSampleInterval
node := NewNode(cfg, status.NewMetricsRecorder(cfg.Clock), metric.NewRegistry(), stopper,
kv.MakeTxnMetrics(metric.TestSampleInterval), sql.MakeEventLogger(nil))
roachpb.RegisterInternalServer(grpcServer, node)
return grpcServer, ln.Addr(), cfg.Clock, node, stopper
}
// NewClient implements the Cluster interface.
func (f *Farmer) NewClient(ctx context.Context, i int) (*client.DB, error) {
conn, err := f.RPCContext.GRPCDial(f.Addr(ctx, i, base.DefaultPort))
if err != nil {
return nil, err
}
return client.NewDB(client.NewSender(conn)), nil
}
// Test a normal transaction. This and the other metrics tests below use real KV operations,
// because it took far too much mucking with TxnCoordSender internals to mock out the sender
// function as other tests do.
func TestTxnCommit(t *testing.T) {
defer leaktest.AfterTest(t)()
_, sender, cleanupFn := setupMetricsTest(t)
defer cleanupFn()
value := []byte("value")
db := client.NewDB(sender)
// Test normal commit.
if err := db.Txn(context.TODO(), func(txn *client.Txn) error {
key := []byte("key-commit")
if err := txn.SetIsolation(enginepb.SNAPSHOT); err != nil {
return err
}
if err := txn.Put(key, value); err != nil {
return err
}
if err := txn.CommitOrCleanup(); err != nil {
return err
}
return nil
}); err != nil {
t.Fatal(err)
}
teardownHeartbeats(sender)
checkTxnMetrics(t, sender, "commit txn", 1, 0 /* not 1PC */, 0, 0, 0)
}
func newKVNative(b *testing.B) kvInterface {
enableTracing := tracing.Disable()
s, _, _ := serverutils.StartServer(b, base.TestServerArgs{})
// TestServer.DB() returns the TxnCoordSender wrapped client. But that isn't
// a fair comparison with SQL as we want these client requests to be sent
// over the network.
sender, err := client.NewSender(
rpc.NewContext(log.AmbientContext{}, &base.Config{
User: security.NodeUser,
SSLCA: filepath.Join(security.EmbeddedCertsDir, security.EmbeddedCACert),
SSLCert: filepath.Join(security.EmbeddedCertsDir, "node.crt"),
SSLCertKey: filepath.Join(security.EmbeddedCertsDir, "node.key"),
}, nil, s.Stopper()),
s.ServingAddr())
if err != nil {
b.Fatal(err)
}
return &kvNative{
db: client.NewDB(sender),
doneFn: func() {
s.Stopper().Stop()
enableTracing()
},
}
}
func (c *Cluster) makeClient(nodeIdx int) *client.DB {
sender, err := client.NewSender(c.rpcCtx, c.RPCAddr(nodeIdx))
if err != nil {
log.Fatalf(context.Background(), "failed to initialize KV client: %s", err)
}
return client.NewDB(sender)
}
// NewClient implements the Cluster interface.
func (f *Farmer) NewClient(t *testing.T, i int) (*client.DB, *stop.Stopper) {
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{
Insecure: true,
User: security.NodeUser,
}, nil, stopper)
sender, err := client.NewSender(rpcContext, f.Addr(i, base.DefaultPort))
if err != nil {
t.Fatal(err)
}
return client.NewDB(sender), stopper
}
// TestInconsistentReads tests that the methods that generate inconsistent reads
// generate outgoing requests with an INCONSISTENT read consistency.
func TestInconsistentReads(t *testing.T) {
defer leaktest.AfterTest(t)()
// Mock out DistSender's sender function to check the read consistency for
// outgoing BatchRequests and return an empty reply.
var senderFn client.SenderFunc
senderFn = func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
if ba.ReadConsistency != roachpb.INCONSISTENT {
return nil, roachpb.NewErrorf("BatchRequest has unexpected ReadConsistency %s",
ba.ReadConsistency)
}
return ba.CreateReply(), nil
}
db := client.NewDB(senderFn)
ctx := context.TODO()
prepInconsistent := func() *client.Batch {
b := &client.Batch{}
b.Header.ReadConsistency = roachpb.INCONSISTENT
return b
}
// Perform inconsistent reads through the mocked sender function.
{
key := roachpb.Key([]byte("key"))
b := prepInconsistent()
b.Get(key)
if err := db.Run(ctx, b); err != nil {
t.Fatal(err)
}
}
{
b := prepInconsistent()
key1 := roachpb.Key([]byte("key1"))
key2 := roachpb.Key([]byte("key2"))
b.Scan(key1, key2)
if err := db.Run(ctx, b); err != nil {
t.Fatal(err)
}
}
{
key := roachpb.Key([]byte("key"))
b := &client.Batch{}
b.Header.ReadConsistency = roachpb.INCONSISTENT
b.Get(key)
if err := db.Run(ctx, b); err != nil {
t.Fatal(err)
}
}
}
// NewClient implements the Cluster interface.
func (l *LocalCluster) NewClient(ctx context.Context, i int) (*roachClient.DB, error) {
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{
User: security.NodeUser,
SSLCA: filepath.Join(l.CertsDir, security.EmbeddedCACert),
SSLCert: filepath.Join(l.CertsDir, security.EmbeddedNodeCert),
SSLCertKey: filepath.Join(l.CertsDir, security.EmbeddedNodeKey),
}, hlc.NewClock(hlc.UnixNano, 0), l.stopper)
conn, err := rpcContext.GRPCDial(l.Nodes[i].Addr(ctx, DefaultTCP).String())
if err != nil {
return nil, err
}
return roachClient.NewDB(roachClient.NewSender(conn)), nil
}
func createTestClientForUser(t *testing.T, stopper *stop.Stopper, addr, user string) *client.DB {
var ctx base.Config
ctx.InitDefaults()
ctx.User = user
ctx.SSLCA = filepath.Join(security.EmbeddedCertsDir, security.EmbeddedCACert)
ctx.SSLCert = filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.crt", user))
ctx.SSLCertKey = filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.key", user))
sender, err := client.NewSender(rpc.NewContext(log.AmbientContext{}, &ctx, nil, stopper), addr)
if err != nil {
t.Fatal(err)
}
return client.NewDB(sender)
}
// NewClient implements the Cluster interface.
func (l *LocalCluster) NewClient(t *testing.T, i int) (*roachClient.DB, *stop.Stopper) {
stopper := stop.NewStopper()
rpcContext := rpc.NewContext(log.AmbientContext{}, &base.Config{
User: security.NodeUser,
SSLCA: filepath.Join(l.CertsDir, security.EmbeddedCACert),
SSLCert: filepath.Join(l.CertsDir, security.EmbeddedNodeCert),
SSLCertKey: filepath.Join(l.CertsDir, security.EmbeddedNodeKey),
}, hlc.NewClock(hlc.UnixNano, 0), stopper)
conn, err := rpcContext.GRPCDial(l.Nodes[i].Addr(DefaultTCP).String())
if err != nil {
t.Fatal(err)
}
return roachClient.NewDB(roachClient.NewSender(conn)), stopper
}
func createTestClientForUser(
t *testing.T, s serverutils.TestServerInterface, user string,
) *client.DB {
var ctx base.Config
ctx.InitDefaults()
ctx.User = user
ctx.SSLCA = filepath.Join(security.EmbeddedCertsDir, security.EmbeddedCACert)
ctx.SSLCert = filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.crt", user))
ctx.SSLCertKey = filepath.Join(security.EmbeddedCertsDir, fmt.Sprintf("%s.key", user))
conn, err := rpc.NewContext(log.AmbientContext{}, &ctx, s.Clock(), s.Stopper()).GRPCDial(s.ServingAddr())
if err != nil {
t.Fatal(err)
}
return client.NewDB(client.NewSender(conn))
}
// TestTxnOnePhaseCommit verifies that 1PC metric tracking works.
func TestTxnOnePhaseCommit(t *testing.T) {
defer leaktest.AfterTest(t)()
_, sender, cleanupFn := setupMetricsTest(t)
defer cleanupFn()
value := []byte("value")
db := client.NewDB(sender)
if err := db.Txn(context.TODO(), func(txn *client.Txn) error {
key := []byte("key-commit")
b := txn.NewBatch()
b.Put(key, value)
return txn.CommitInBatch(b)
}); err != nil {
t.Fatal(err)
}
teardownHeartbeats(sender)
checkTxnMetrics(t, sender, "commit 1PC txn", 1, 1 /* 1PC */, 0, 0, 0)
}
func TestTxnDurations(t *testing.T) {
defer leaktest.AfterTest(t)()
manual, sender, cleanupFn := setupMetricsTest(t)
defer cleanupFn()
db := client.NewDB(sender)
const puts = 10
const incr int64 = 1000
for i := 0; i < puts; i++ {
key := roachpb.Key(fmt.Sprintf("key-txn-durations-%d", i))
if err := db.Txn(context.TODO(), func(txn *client.Txn) error {
if err := txn.SetIsolation(enginepb.SNAPSHOT); err != nil {
return err
}
if err := txn.Put(key, []byte("val")); err != nil {
return err
}
manual.Increment(incr)
return nil
}); err != nil {
t.Fatal(err)
}
}
teardownHeartbeats(sender)
checkTxnMetrics(t, sender, "txn durations", puts, 0, 0, 0, 0)
hist := sender.metrics.Durations
// The clock is a bit odd in these tests, so I can't test the mean without
// introducing spurious errors or being overly lax.
//
// TODO(cdo): look into cause of variance.
if a, e := hist.TotalCount(), int64(puts); a != e {
t.Fatalf("durations %d != expected %d", a, e)
}
// Metrics lose fidelity, so we can't compare incr directly.
if min, thresh := hist.Min(), incr-10; min < thresh {
t.Fatalf("min %d < %d", min, thresh)
}
}
func newKVNative(b *testing.B) kvInterface {
enableTracing := tracing.Disable()
s, _, _ := serverutils.StartServer(b, base.TestServerArgs{})
// TestServer.KVClient() returns the TxnCoordSender wrapped client. But that
// isn't a fair comparison with SQL as we want these client requests to be
// sent over the network.
rpcContext := s.RPCContext()
conn, err := rpcContext.GRPCDial(s.ServingAddr())
if err != nil {
b.Fatal(err)
}
return &kvNative{
db: client.NewDB(client.NewSender(conn)),
doneFn: func() {
s.Stopper().Stop()
enableTracing()
},
}
}
func makeDBClient() (*client.DB, *stop.Stopper, error) {
stopper := stop.NewStopper()
cfg := &base.Config{
User: security.NodeUser,
SSLCA: baseCfg.SSLCA,
SSLCert: baseCfg.SSLCert,
SSLCertKey: baseCfg.SSLCertKey,
Insecure: baseCfg.Insecure,
}
addr, err := addrWithDefaultHost(baseCfg.Addr)
if err != nil {
return nil, nil, err
}
sender, err := client.NewSender(
rpc.NewContext(log.AmbientContext{}, cfg, nil, stopper),
addr,
)
if err != nil {
stopper.Stop()
return nil, nil, errors.Wrap(err, "failed to initialize KV client")
}
return client.NewDB(sender), stopper, nil
}
func TestTxnRestartCount(t *testing.T) {
defer leaktest.AfterTest(t)()
_, sender, cleanupFn := setupMetricsTest(t)
defer cleanupFn()
key := []byte("key-restart")
value := []byte("value")
db := client.NewDB(sender)
// Start a transaction and do a GET. This forces a timestamp to be chosen for the transaction.
txn := client.NewTxn(context.Background(), *db)
if _, err := txn.Get(key); err != nil {
t.Fatal(err)
}
// Outside of the transaction, read the same key as was read within the transaction. This
// means that future attempts to write will increase the timestamp.
if _, err := db.Get(context.TODO(), key); err != nil {
t.Fatal(err)
}
// This put will lay down an intent, txn timestamp will increase beyond original.
if err := txn.Put(key, value); err != nil {
t.Fatal(err)
}
if !txn.Proto.OrigTimestamp.Less(txn.Proto.Timestamp) {
t.Errorf("expected timestamp to increase: %s", txn.Proto)
}
// Commit (should cause restart metric to increase).
err := txn.CommitOrCleanup()
assertTransactionRetryError(t, err)
teardownHeartbeats(sender)
checkTxnMetrics(t, sender, "restart txn", 0, 0, 0, 1, 1)
}
// bootstrapCluster bootstraps a multiple stores using the provided
// engines and cluster ID. The first bootstrapped store contains a
// single range spanning all keys. Initial range lookup metadata is
// populated for the range. Returns the cluster ID.
func bootstrapCluster(engines []engine.Engine, txnMetrics kv.TxnMetrics) (uuid.UUID, error) {
clusterID := uuid.MakeV4()
stopper := stop.NewStopper()
defer stopper.Stop()
cfg := storage.StoreConfig{}
cfg.ScanInterval = 10 * time.Minute
cfg.MetricsSampleInterval = time.Duration(math.MaxInt64)
cfg.ConsistencyCheckInterval = 10 * time.Minute
cfg.Clock = hlc.NewClock(hlc.UnixNano)
cfg.AmbientCtx.Tracer = tracing.NewTracer()
// Create a KV DB with a local sender.
stores := storage.NewStores(cfg.AmbientCtx, cfg.Clock)
sender := kv.NewTxnCoordSender(cfg.AmbientCtx, stores, cfg.Clock, false, stopper, txnMetrics)
cfg.DB = client.NewDB(sender)
cfg.Transport = storage.NewDummyRaftTransport()
for i, eng := range engines {
sIdent := roachpb.StoreIdent{
ClusterID: clusterID,
NodeID: FirstNodeID,
StoreID: roachpb.StoreID(i + 1),
}
// The bootstrapping store will not connect to other nodes so its
// StoreConfig doesn't really matter.
s := storage.NewStore(cfg, eng, &roachpb.NodeDescriptor{NodeID: FirstNodeID})
// Verify the store isn't already part of a cluster.
if s.Ident.ClusterID != *uuid.EmptyUUID {
return uuid.UUID{}, errors.Errorf("storage engine already belongs to a cluster (%s)", s.Ident.ClusterID)
}
// Bootstrap store to persist the store ident.
if err := s.Bootstrap(sIdent); err != nil {
return uuid.UUID{}, err
}
// Create first range, writing directly to engine. Note this does
// not create the range, just its data. Only do this if this is the
// first store.
if i == 0 {
initialValues := GetBootstrapSchema().GetInitialValues()
if err := s.BootstrapRange(initialValues); err != nil {
return uuid.UUID{}, err
}
}
if err := s.Start(context.Background(), stopper); err != nil {
return uuid.UUID{}, err
}
stores.AddStore(s)
ctx := context.TODO()
// Initialize node and store ids. Only initialize the node once.
if i == 0 {
if nodeID, err := allocateNodeID(ctx, cfg.DB); nodeID != sIdent.NodeID || err != nil {
return uuid.UUID{}, errors.Errorf("expected to initialize node id allocator to %d, got %d: %s",
sIdent.NodeID, nodeID, err)
}
}
if storeID, err := allocateStoreIDs(ctx, sIdent.NodeID, 1, cfg.DB); storeID != sIdent.StoreID || err != nil {
return uuid.UUID{}, errors.Errorf("expected to initialize store id allocator to %d, got %d: %s",
sIdent.StoreID, storeID, err)
}
}
return clusterID, nil
}
// TestTxnCoordSenderNoDuplicateIntents verifies that TxnCoordSender does not
// generate duplicate intents and that it merges intents for overlapping ranges.
func TestTxnCoordSenderNoDuplicateIntents(t *testing.T) {
defer leaktest.AfterTest(t)()
stopper := stop.NewStopper()
manual := hlc.NewManualClock(123)
clock := hlc.NewClock(manual.UnixNano, time.Nanosecond)
var expectedIntents []roachpb.Span
senderFunc := func(_ context.Context, ba roachpb.BatchRequest) (
*roachpb.BatchResponse, *roachpb.Error) {
if rArgs, ok := ba.GetArg(roachpb.EndTransaction); ok {
et := rArgs.(*roachpb.EndTransactionRequest)
if !reflect.DeepEqual(et.IntentSpans, expectedIntents) {
t.Errorf("Invalid intents: %+v; expected %+v", et.IntentSpans, expectedIntents)
}
}
br := ba.CreateReply()
txnClone := ba.Txn.Clone()
br.Txn = &txnClone
br.Txn.Writing = true
return br, nil
}
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
ts := NewTxnCoordSender(
ambient,
senderFn(senderFunc),
clock,
false,
stopper,
MakeTxnMetrics(metric.TestSampleInterval),
)
defer stopper.Stop()
defer teardownHeartbeats(ts)
db := client.NewDB(ts)
txn := client.NewTxn(context.Background(), *db)
// Write to a, b, u-w before the final batch.
pErr := txn.Put(roachpb.Key("a"), []byte("value"))
if pErr != nil {
t.Fatal(pErr)
}
pErr = txn.Put(roachpb.Key("b"), []byte("value"))
if pErr != nil {
t.Fatal(pErr)
}
pErr = txn.DelRange(roachpb.Key("u"), roachpb.Key("w"))
if pErr != nil {
t.Fatal(pErr)
}
// The final batch overwrites key a and overlaps part of the u-w range.
b := txn.NewBatch()
b.Put(roachpb.Key("b"), []byte("value"))
b.Put(roachpb.Key("c"), []byte("value"))
b.DelRange(roachpb.Key("v"), roachpb.Key("z"), false)
// The expected intents are a, b, c, and u-z.
expectedIntents = []roachpb.Span{
{Key: roachpb.Key("a"), EndKey: nil},
{Key: roachpb.Key("b"), EndKey: nil},
{Key: roachpb.Key("c"), EndKey: nil},
{Key: roachpb.Key("u"), EndKey: roachpb.Key("z")},
}
pErr = txn.CommitInBatch(b)
if pErr != nil {
t.Fatal(pErr)
}
}
// TestTxnCoordSenderTxnUpdatedOnError verifies that errors adjust the
// response transaction's timestamp and priority as appropriate.
func TestTxnCoordSenderTxnUpdatedOnError(t *testing.T) {
defer leaktest.AfterTest(t)()
origTS := makeTS(123, 0)
plus10 := origTS.Add(10, 10)
plus20 := plus10.Add(10, 0)
testCases := []struct {
pErr *roachpb.Error
expEpoch uint32
expPri int32
expTS, expOrigTS hlc.Timestamp
nodeSeen bool
}{
{
// No error, so nothing interesting either.
pErr: nil,
expEpoch: 0,
expPri: 1,
expTS: origTS,
expOrigTS: origTS,
},
{
// On uncertainty error, new epoch begins and node is seen.
// Timestamp moves ahead of the existing write.
pErr: func() *roachpb.Error {
pErr := roachpb.NewErrorWithTxn(
roachpb.NewReadWithinUncertaintyIntervalError(hlc.ZeroTimestamp, hlc.ZeroTimestamp),
&roachpb.Transaction{})
const nodeID = 1
pErr.GetTxn().UpdateObservedTimestamp(nodeID, plus10)
pErr.OriginNode = nodeID
return pErr
}(),
expEpoch: 1,
expPri: 1,
expTS: plus10,
expOrigTS: plus10,
nodeSeen: true,
},
{
// On abort, nothing changes but we get a new priority to use for
// the next attempt.
pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionAbortedError{},
&roachpb.Transaction{
TxnMeta: enginepb.TxnMeta{Timestamp: plus20, Priority: 10},
}),
expPri: 10,
},
{
// On failed push, new epoch begins just past the pushed timestamp.
// Additionally, priority ratchets up to just below the pusher's.
pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionPushError{
PusheeTxn: roachpb.Transaction{
TxnMeta: enginepb.TxnMeta{Timestamp: plus10, Priority: int32(10)},
},
},
&roachpb.Transaction{}),
expEpoch: 1,
expPri: 9,
expTS: plus10,
expOrigTS: plus10,
},
{
// On retry, restart with new epoch, timestamp and priority.
pErr: roachpb.NewErrorWithTxn(&roachpb.TransactionRetryError{},
&roachpb.Transaction{
TxnMeta: enginepb.TxnMeta{Timestamp: plus10, Priority: int32(10)},
},
),
expEpoch: 1,
expPri: 10,
expTS: plus10,
expOrigTS: plus10,
},
}
for i, test := range testCases {
stopper := stop.NewStopper()
manual := hlc.NewManualClock(origTS.WallTime)
clock := hlc.NewClock(manual.UnixNano, 20*time.Nanosecond)
senderFunc := func(_ context.Context, ba roachpb.BatchRequest) (*roachpb.BatchResponse, *roachpb.Error) {
var reply *roachpb.BatchResponse
if test.pErr == nil {
reply = ba.CreateReply()
}
return reply, test.pErr
}
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
ts := NewTxnCoordSender(
ambient,
senderFn(senderFunc),
clock,
false,
stopper,
MakeTxnMetrics(metric.TestSampleInterval),
)
db := client.NewDB(ts)
txn := client.NewTxn(context.Background(), *db)
txn.InternalSetPriority(1)
txn.Proto.Name = "test txn"
key := roachpb.Key("test-key")
_, err := txn.Get(key)
teardownHeartbeats(ts)
stopper.Stop()
if test.pErr != nil && err == nil {
t.Fatalf("expected an error")
}
if txn.Proto.Epoch != test.expEpoch {
t.Errorf("%d: expected epoch = %d; got %d",
i, test.expEpoch, txn.Proto.Epoch)
}
if txn.Proto.Priority != test.expPri {
t.Errorf("%d: expected priority = %d; got %d",
i, test.expPri, txn.Proto.Priority)
}
if !txn.Proto.Timestamp.Equal(test.expTS) {
t.Errorf("%d: expected timestamp to be %s; got %s",
i, test.expTS, txn.Proto.Timestamp)
}
if !txn.Proto.OrigTimestamp.Equal(test.expOrigTS) {
t.Errorf("%d: expected orig timestamp to be %s; got %s",
i, test.expOrigTS, txn.Proto.OrigTimestamp)
}
if ns := txn.Proto.ObservedTimestamps; (len(ns) != 0) != test.nodeSeen {
t.Errorf("%d: expected nodeSeen=%t, but list of hosts is %v",
i, test.nodeSeen, ns)
}
}
}
// NewServer creates a Server from a server.Context.
func NewServer(cfg Config, stopper *stop.Stopper) (*Server, error) {
if _, err := net.ResolveTCPAddr("tcp", cfg.AdvertiseAddr); err != nil {
return nil, errors.Errorf("unable to resolve RPC address %q: %v", cfg.AdvertiseAddr, err)
}
if cfg.AmbientCtx.Tracer == nil {
cfg.AmbientCtx.Tracer = tracing.NewTracer()
}
// Try loading the TLS configs before anything else.
if _, err := cfg.GetServerTLSConfig(); err != nil {
return nil, err
}
if _, err := cfg.GetClientTLSConfig(); err != nil {
return nil, err
}
s := &Server{
mux: http.NewServeMux(),
clock: hlc.NewClock(hlc.UnixNano, cfg.MaxOffset),
stopper: stopper,
cfg: cfg,
}
// Add a dynamic log tag value for the node ID.
//
// We need to pass an ambient context to the various server components, but we
// won't know the node ID until we Start(). At that point it's too late to
// change the ambient contexts in the components (various background processes
// will have already started using them).
//
// NodeIDContainer allows us to add the log tag to the context now and update
// the value asynchronously. It's not significantly more expensive than a
// regular tag since it's just doing an (atomic) load when a log/trace message
// is constructed. The node ID is set by the Store if this host was
// bootstrapped; otherwise a new one is allocated in Node.
s.cfg.AmbientCtx.AddLogTag("n", &s.nodeIDContainer)
ctx := s.AnnotateCtx(context.Background())
if s.cfg.Insecure {
log.Warning(ctx, "running in insecure mode, this is strongly discouraged. See --insecure.")
}
s.rpcContext = rpc.NewContext(s.cfg.AmbientCtx, s.cfg.Config, s.clock, s.stopper)
s.rpcContext.HeartbeatCB = func() {
if err := s.rpcContext.RemoteClocks.VerifyClockOffset(); err != nil {
log.Fatal(ctx, err)
}
}
s.grpc = rpc.NewServer(s.rpcContext)
s.registry = metric.NewRegistry()
s.gossip = gossip.New(
s.cfg.AmbientCtx,
&s.nodeIDContainer,
s.rpcContext,
s.grpc,
s.cfg.GossipBootstrapResolvers,
s.stopper,
s.registry,
)
s.storePool = storage.NewStorePool(
s.cfg.AmbientCtx,
s.gossip,
s.clock,
s.rpcContext,
s.cfg.TimeUntilStoreDead,
s.stopper,
/* deterministic */ false,
)
// A custom RetryOptions is created which uses stopper.ShouldQuiesce() as
// the Closer. This prevents infinite retry loops from occurring during
// graceful server shutdown
//
// Such a loop loop occurs with the DistSender attempts a connection to the
// local server during shutdown, and receives an internal server error (HTTP
// Code 5xx). This is the correct error for a server to return when it is
// shutting down, and is normally retryable in a cluster environment.
// However, on a single-node setup (such as a test), retries will never
// succeed because the only server has been shut down; thus, thus the
// DistSender needs to know that it should not retry in this situation.
retryOpts := base.DefaultRetryOptions()
retryOpts.Closer = s.stopper.ShouldQuiesce()
distSenderCfg := kv.DistSenderConfig{
AmbientCtx: s.cfg.AmbientCtx,
Clock: s.clock,
RPCContext: s.rpcContext,
RPCRetryOptions: &retryOpts,
}
s.distSender = kv.NewDistSender(distSenderCfg, s.gossip)
txnMetrics := kv.MakeTxnMetrics(s.cfg.MetricsSampleInterval)
s.registry.AddMetricStruct(txnMetrics)
s.txnCoordSender = kv.NewTxnCoordSender(
s.cfg.AmbientCtx,
s.distSender,
s.clock,
s.cfg.Linearizable,
s.stopper,
txnMetrics,
)
s.db = client.NewDB(s.txnCoordSender)
// Use the range lease expiration and renewal durations as the node
// liveness expiration and heartbeat interval.
active, renewal := storage.RangeLeaseDurations(
storage.RaftElectionTimeout(s.cfg.RaftTickInterval, s.cfg.RaftElectionTimeoutTicks))
s.nodeLiveness = storage.NewNodeLiveness(
s.cfg.AmbientCtx, s.clock, s.db, s.gossip, active, renewal,
)
s.registry.AddMetricStruct(s.nodeLiveness.Metrics())
s.raftTransport = storage.NewRaftTransport(
s.cfg.AmbientCtx, storage.GossipAddressResolver(s.gossip), s.grpc, s.rpcContext,
)
s.kvDB = kv.NewDBServer(s.cfg.Config, s.txnCoordSender, s.stopper)
roachpb.RegisterExternalServer(s.grpc, s.kvDB)
// Set up internal memory metrics for use by internal SQL executors.
s.internalMemMetrics = sql.MakeMemMetrics("internal")
s.registry.AddMetricStruct(s.internalMemMetrics)
// Set up Lease Manager
var lmKnobs sql.LeaseManagerTestingKnobs
if cfg.TestingKnobs.SQLLeaseManager != nil {
lmKnobs = *s.cfg.TestingKnobs.SQLLeaseManager.(*sql.LeaseManagerTestingKnobs)
}
s.leaseMgr = sql.NewLeaseManager(&s.nodeIDContainer, *s.db, s.clock, lmKnobs,
s.stopper, &s.internalMemMetrics)
s.leaseMgr.RefreshLeases(s.stopper, s.db, s.gossip)
// Set up the DistSQL server
distSQLCfg := distsql.ServerConfig{
AmbientContext: s.cfg.AmbientCtx,
DB: s.db,
RPCContext: s.rpcContext,
Stopper: s.stopper,
}
s.distSQLServer = distsql.NewServer(distSQLCfg)
distsql.RegisterDistSQLServer(s.grpc, s.distSQLServer)
// Set up admin memory metrics for use by admin SQL executors.
s.adminMemMetrics = sql.MakeMemMetrics("admin")
s.registry.AddMetricStruct(s.adminMemMetrics)
// Set up Executor
execCfg := sql.ExecutorConfig{
AmbientCtx: s.cfg.AmbientCtx,
NodeID: &s.nodeIDContainer,
DB: s.db,
Gossip: s.gossip,
LeaseManager: s.leaseMgr,
Clock: s.clock,
DistSQLSrv: s.distSQLServer,
MetricsSampleInterval: s.cfg.MetricsSampleInterval,
}
if s.cfg.TestingKnobs.SQLExecutor != nil {
execCfg.TestingKnobs = s.cfg.TestingKnobs.SQLExecutor.(*sql.ExecutorTestingKnobs)
} else {
execCfg.TestingKnobs = &sql.ExecutorTestingKnobs{}
}
if s.cfg.TestingKnobs.SQLSchemaChanger != nil {
execCfg.SchemaChangerTestingKnobs =
s.cfg.TestingKnobs.SQLSchemaChanger.(*sql.SchemaChangerTestingKnobs)
} else {
execCfg.SchemaChangerTestingKnobs = &sql.SchemaChangerTestingKnobs{}
}
s.sqlExecutor = sql.NewExecutor(execCfg, s.stopper, &s.adminMemMetrics)
s.registry.AddMetricStruct(s.sqlExecutor)
s.pgServer = pgwire.MakeServer(
s.cfg.AmbientCtx, s.cfg.Config, s.sqlExecutor, &s.internalMemMetrics, s.cfg.SQLMemoryPoolSize,
)
s.registry.AddMetricStruct(s.pgServer.Metrics())
s.tsDB = ts.NewDB(s.db)
s.tsServer = ts.MakeServer(s.cfg.AmbientCtx, s.tsDB, s.cfg.TimeSeriesServerConfig, s.stopper)
// TODO(bdarnell): make StoreConfig configurable.
storeCfg := storage.StoreConfig{
AmbientCtx: s.cfg.AmbientCtx,
Clock: s.clock,
DB: s.db,
Gossip: s.gossip,
NodeLiveness: s.nodeLiveness,
Transport: s.raftTransport,
RaftTickInterval: s.cfg.RaftTickInterval,
ScanInterval: s.cfg.ScanInterval,
ScanMaxIdleTime: s.cfg.ScanMaxIdleTime,
ConsistencyCheckInterval: s.cfg.ConsistencyCheckInterval,
ConsistencyCheckPanicOnFailure: s.cfg.ConsistencyCheckPanicOnFailure,
MetricsSampleInterval: s.cfg.MetricsSampleInterval,
StorePool: s.storePool,
SQLExecutor: sql.InternalExecutor{
LeaseManager: s.leaseMgr,
},
LogRangeEvents: s.cfg.EventLogEnabled,
AllocatorOptions: storage.AllocatorOptions{
AllowRebalance: true,
},
RangeLeaseActiveDuration: active,
RangeLeaseRenewalDuration: renewal,
TimeSeriesDataStore: s.tsDB,
}
if s.cfg.TestingKnobs.Store != nil {
storeCfg.TestingKnobs = *s.cfg.TestingKnobs.Store.(*storage.StoreTestingKnobs)
}
s.recorder = status.NewMetricsRecorder(s.clock)
s.registry.AddMetricStruct(s.rpcContext.RemoteClocks.Metrics())
s.runtime = status.MakeRuntimeStatSampler(s.clock)
s.registry.AddMetricStruct(s.runtime)
s.node = NewNode(storeCfg, s.recorder, s.registry, s.stopper, txnMetrics, sql.MakeEventLogger(s.leaseMgr))
roachpb.RegisterInternalServer(s.grpc, s.node)
storage.RegisterConsistencyServer(s.grpc, s.node.storesServer)
storage.RegisterFreezeServer(s.grpc, s.node.storesServer)
s.admin = newAdminServer(s)
s.status = newStatusServer(
s.cfg.AmbientCtx, s.db, s.gossip, s.recorder, s.rpcContext, s.node.stores,
)
for _, gw := range []grpcGatewayServer{s.admin, s.status, &s.tsServer} {
gw.RegisterService(s.grpc)
}
return s, nil
}