// TODO(tschottdorf): this method is somewhat awkward but unless we want to
// give this error back to the client, our options are limited. We'll have to
// run the whole thing for them, or any restart will still end up at the client
// which will not be prepared to be handed a Txn.
func (tc *TxnCoordSender) resendWithTxn(
ba roachpb.BatchRequest,
) (*roachpb.BatchResponse, *roachpb.Error) {
ctx := tc.AnnotateCtx(context.TODO())
// Run a one-off transaction with that single command.
if log.V(1) {
log.Infof(ctx, "%s: auto-wrapping in txn and re-executing: ", ba)
}
// TODO(bdarnell): need to be able to pass other parts of DBContext
// through here.
dbCtx := client.DefaultDBContext()
dbCtx.UserPriority = ba.UserPriority
tmpDB := client.NewDBWithContext(tc, dbCtx)
var br *roachpb.BatchResponse
err := tmpDB.Txn(ctx, func(txn *client.Txn) error {
txn.SetDebugName("auto-wrap", 0)
b := txn.NewBatch()
b.Header = ba.Header
for _, arg := range ba.Requests {
req := arg.GetInner()
b.AddRawRequest(req)
}
err := txn.CommitInBatch(b)
br = b.RawResponse()
return err
})
if err != nil {
return nil, roachpb.NewError(err)
}
br.Txn = nil // hide the evidence
return br, nil
}
// TestClientRunTransaction verifies some simple transaction isolation
// semantics.
func TestClientRunTransaction(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
dbCtx := client.DefaultDBContext()
dbCtx.TxnRetryOptions.InitialBackoff = 1 * time.Millisecond
db := createTestClientForUser(t, s, security.NodeUser, dbCtx)
for _, commit := range []bool{true, false} {
value := []byte("value")
key := []byte(fmt.Sprintf("%s/key-%t", testUser, commit))
// Use snapshot isolation so non-transactional read can always push.
err := db.Txn(context.TODO(), func(txn *client.Txn) error {
if err := txn.SetIsolation(enginepb.SNAPSHOT); err != nil {
return err
}
// Put transactional value.
if err := txn.Put(key, value); err != nil {
return err
}
// Attempt to read outside of txn.
if gr, err := db.Get(context.TODO(), key); err != nil {
return err
} else if gr.Value != nil {
return errors.Errorf("expected nil value; got %+v", gr.Value)
}
// Read within the transaction.
if gr, err := txn.Get(key); err != nil {
return err
} else if gr.Value == nil || !bytes.Equal(gr.ValueBytes(), value) {
return errors.Errorf("expected value %q; got %q", value, gr.ValueBytes())
}
if !commit {
return errors.Errorf("purposefully failing transaction")
}
return nil
})
if commit != (err == nil) {
t.Errorf("expected success? %t; got %s", commit, err)
} else if !commit && !testutils.IsError(err, "purposefully failing transaction") {
t.Errorf("unexpected failure with !commit: %v", err)
}
// Verify the value is now visible on commit == true, and not visible otherwise.
gr, err := db.Get(context.TODO(), key)
if commit {
if err != nil || gr.Value == nil || !bytes.Equal(gr.ValueBytes(), value) {
t.Errorf("expected success reading value: %+v, %s", gr.Value, err)
}
} else {
if err != nil || gr.Value != nil {
t.Errorf("expected success and nil value: %+v, %s", gr.Value, err)
}
}
}
}
// TestClientPermissions verifies permission enforcement.
func TestClientPermissions(t *testing.T) {
defer leaktest.AfterTest(t)()
s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
defer s.Stopper().Stop()
// NodeUser certs are required for all KV operations.
// RootUser has no KV privileges whatsoever.
nodeClient := createTestClientForUser(t, s.Stopper(), s.ServingAddr(),
security.NodeUser, client.DefaultDBContext())
rootClient := createTestClientForUser(t, s.Stopper(), s.ServingAddr(),
security.RootUser, client.DefaultDBContext())
testCases := []struct {
path string
client *client.DB
allowed bool
}{
{"foo", rootClient, false},
{"foo", nodeClient, true},
{testUser + "/foo", rootClient, false},
{testUser + "/foo", nodeClient, true},
{testUser + "foo", rootClient, false},
{testUser + "foo", nodeClient, true},
{testUser, rootClient, false},
{testUser, nodeClient, true},
{"unknown/foo", rootClient, false},
{"unknown/foo", nodeClient, true},
}
value := []byte("value")
const matchErr = "is not allowed"
for tcNum, tc := range testCases {
err := tc.client.Put(context.TODO(), tc.path, value)
if (err == nil) != tc.allowed || (!tc.allowed && !testutils.IsError(err, matchErr)) {
t.Errorf("#%d: expected allowed=%t, got err=%v", tcNum, tc.allowed, err)
}
_, err = tc.client.Get(context.TODO(), tc.path)
if (err == nil) != tc.allowed || (!tc.allowed && !testutils.IsError(err, matchErr)) {
t.Errorf("#%d: expected allowed=%t, got err=%v", tcNum, tc.allowed, err)
}
}
}
// createTestNotifyClient creates a new client which connects using an HTTP
// sender to the server at addr. It contains a waitgroup to allow waiting.
func createTestNotifyClient(
t *testing.T, s serverutils.TestServerInterface, priority roachpb.UserPriority,
) (*client.DB, *notifyingSender) {
db := createTestClient(t, s)
sender := ¬ifyingSender{wrapped: db.GetSender()}
dbCtx := client.DefaultDBContext()
dbCtx.UserPriority = priority
return client.NewDBWithContext(sender, dbCtx), sender
}
// createTestNotifyClient creates a new client which connects using an HTTP
// sender to the server at addr. It contains a waitgroup to allow waiting.
func createTestNotifyClient(
t *testing.T, stopper *stop.Stopper, addr string, priority roachpb.UserPriority,
) (*client.DB, *notifyingSender) {
db := createTestClient(t, stopper, addr)
sender := ¬ifyingSender{wrapped: db.GetSender()}
dbCtx := client.DefaultDBContext()
dbCtx.UserPriority = priority
return client.NewDBWithContext(sender, dbCtx), sender
}
// Start starts the test cluster by bootstrapping an in-memory store
// (defaults to maximum of 50M). The server is started, launching the
// node RPC server and all HTTP endpoints. Use the value of
// TestServer.Addr after Start() for client connections. Use Stop()
// to shutdown the server after the test completes.
func (ltc *LocalTestCluster) Start(t util.Tester, baseCtx *base.Config, initSender InitSenderFn) {
ambient := log.AmbientContext{Tracer: tracing.NewTracer()}
nc := &base.NodeIDContainer{}
ambient.AddLogTag("n", nc)
nodeID := roachpb.NodeID(1)
nodeDesc := &roachpb.NodeDescriptor{NodeID: nodeID}
ltc.tester = t
ltc.Manual = hlc.NewManualClock(0)
ltc.Clock = hlc.NewClock(ltc.Manual.UnixNano)
ltc.Stopper = stop.NewStopper()
rpcContext := rpc.NewContext(ambient, baseCtx, ltc.Clock, ltc.Stopper)
server := rpc.NewServer(rpcContext) // never started
ltc.Gossip = gossip.New(ambient, nc, rpcContext, server, nil, ltc.Stopper, metric.NewRegistry())
ltc.Eng = engine.NewInMem(roachpb.Attributes{}, 50<<20)
ltc.Stopper.AddCloser(ltc.Eng)
ltc.Stores = storage.NewStores(ambient, ltc.Clock)
ltc.Sender = initSender(nodeDesc, ambient.Tracer, ltc.Clock, ltc.Latency, ltc.Stores, ltc.Stopper,
ltc.Gossip)
if ltc.DBContext == nil {
dbCtx := client.DefaultDBContext()
ltc.DBContext = &dbCtx
}
ltc.DB = client.NewDBWithContext(ltc.Sender, *ltc.DBContext)
transport := storage.NewDummyRaftTransport()
cfg := storage.TestStoreConfig()
if ltc.RangeRetryOptions != nil {
cfg.RangeRetryOptions = *ltc.RangeRetryOptions
}
cfg.AmbientCtx = ambient
cfg.Clock = ltc.Clock
cfg.DB = ltc.DB
cfg.Gossip = ltc.Gossip
cfg.Transport = transport
cfg.MetricsSampleInterval = metric.TestSampleInterval
ltc.Store = storage.NewStore(cfg, ltc.Eng, nodeDesc)
if err := ltc.Store.Bootstrap(roachpb.StoreIdent{NodeID: nodeID, StoreID: 1}); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
ltc.Stores.AddStore(ltc.Store)
if err := ltc.Store.BootstrapRange(nil); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
if err := ltc.Store.Start(context.Background(), ltc.Stopper); err != nil {
t.Fatalf("unable to start local test cluster: %s", err)
}
nc.Set(context.TODO(), nodeDesc.NodeID)
if err := ltc.Gossip.SetNodeDescriptor(nodeDesc); err != nil {
t.Fatalf("unable to set node descriptor: %s", err)
}
}
// TestRangeSplitsWithWritePressure sets the zone config max bytes for
// a range to 256K and writes data until there are five ranges.
func TestRangeSplitsWithWritePressure(t *testing.T) {
defer leaktest.AfterTest(t)()
// Override default zone config.
cfg := config.DefaultZoneConfig()
cfg.RangeMaxBytes = 1 << 18
defer config.TestingSetDefaultZoneConfig(cfg)()
dbCtx := client.DefaultDBContext()
dbCtx.TxnRetryOptions = retry.Options{
InitialBackoff: 1 * time.Millisecond,
MaxBackoff: 10 * time.Millisecond,
Multiplier: 2,
}
s, _ := createTestDBWithContext(t, dbCtx)
// This is purely to silence log spam.
config.TestingSetupZoneConfigHook(s.Stopper)
defer s.Stop()
// Start test writer write about a 32K/key so there aren't too many writes necessary to split 64K range.
done := make(chan struct{})
var wg sync.WaitGroup
wg.Add(1)
go startTestWriter(s.DB, int64(0), 1<<15, &wg, nil, nil, done, t)
// Check that we split 5 times in allotted time.
testutils.SucceedsSoon(t, func() error {
// Scan the txn records.
rows, err := s.DB.Scan(context.TODO(), keys.Meta2Prefix, keys.MetaMax, 0)
if err != nil {
return errors.Errorf("failed to scan meta2 keys: %s", err)
}
if lr := len(rows); lr < 5 {
return errors.Errorf("expected >= 5 scans; got %d", lr)
}
return nil
})
close(done)
wg.Wait()
// This write pressure test often causes splits while resolve
// intents are in flight, causing them to fail with range key
// mismatch errors. However, LocalSender should retry in these
// cases. Check here via MVCC scan that there are no dangling write
// intents. We do this using a SucceedsSoon construct to account
// for timing of finishing the test writer and a possibly-ongoing
// asynchronous split.
testutils.SucceedsSoon(t, func() error {
if _, _, _, err := engine.MVCCScan(context.Background(), s.Eng, keys.LocalMax, roachpb.KeyMax, math.MaxInt64, hlc.MaxTimestamp, true, nil); err != nil {
return errors.Errorf("failed to verify no dangling intents: %s", err)
}
return nil
})
}
// TestUncertaintyRestart verifies that a transaction which finds a write in
// its near future will restart exactly once, meaning that it's made a note of
// that node's clock for its new timestamp.
func TestUncertaintyRestart(t *testing.T) {
defer leaktest.AfterTest(t)()
const maxOffset = 250 * time.Millisecond
dbCtx := client.DefaultDBContext()
s := &localtestcluster.LocalTestCluster{
Clock: hlc.NewClock(hlc.UnixNano, maxOffset),
DBContext: &dbCtx,
}
s.Start(t, testutils.NewNodeTestBaseContext(), InitSenderForLocalTestCluster)
defer s.Stop()
if err := disableOwnNodeCertain(s); err != nil {
t.Fatal(err)
}
s.Manual.Increment(s.Clock.MaxOffset().Nanoseconds() + 1)
var key = roachpb.Key("a")
errChan := make(chan error)
start := make(chan struct{})
go func() {
<-start
errChan <- s.DB.Txn(context.TODO(), func(txn *client.Txn) error {
return txn.Put(key, "hi")
})
}()
if err := s.DB.Txn(context.TODO(), func(txn *client.Txn) error {
if txn.Proto.Epoch > 2 {
t.Fatal("expected only one restart")
}
// Issue a read to pick a timestamp.
if _, err := txn.Get(key.Next()); err != nil {
t.Fatal(err)
}
if txn.Proto.Epoch == 0 {
close(start) // let someone write into our future
// when they're done, try to read
if err := <-errChan; err != nil {
t.Fatal(err)
}
}
if _, err := txn.Get(key.Next()); err != nil {
if _, ok := err.(*roachpb.ReadWithinUncertaintyIntervalError); !ok {
t.Fatalf("unexpected error: %T: %s", err, err)
}
}
return nil
}); err != nil {
t.Fatal(err)
}
}
// checkConcurrency creates a history verifier, starts a new database
// and runs the verifier.
func checkConcurrency(
name string, isolations []enginepb.IsolationType, txns []string, verify *verifier, t *testing.T,
) {
verifier := newHistoryVerifier(name, txns, verify, t)
dbCtx := client.DefaultDBContext()
dbCtx.TxnRetryOptions = correctnessTestRetryOptions
s := &localtestcluster.LocalTestCluster{
DBContext: &dbCtx,
RangeRetryOptions: &correctnessTestRetryOptions,
}
s.Start(t, testutils.NewNodeTestBaseContext(), InitSenderForLocalTestCluster)
defer s.Stop()
verifier.run(isolations, s.DB, t)
}
// createTestDB creates a local test server and starts it. The caller
// is responsible for stopping the test server.
func createTestDB(t testing.TB) (*localtestcluster.LocalTestCluster, *TxnCoordSender) {
return createTestDBWithContext(t, client.DefaultDBContext())
}
// TestUncertaintyObservedTimestampForwarding checks that when receiving an
// uncertainty restart on a node, the next attempt to read (at the increased
// timestamp) is free from uncertainty. See roachpb.Transaction for details.
func TestUncertaintyMaxTimestampForwarding(t *testing.T) {
defer leaktest.AfterTest(t)()
dbCtx := client.DefaultDBContext()
s := &localtestcluster.LocalTestCluster{
// Large offset so that any value in the future is an uncertain read. Also
// makes sure that the values we write in the future below don't actually
// wind up in the past.
Clock: hlc.NewClock(hlc.UnixNano, 50*time.Second),
DBContext: &dbCtx,
}
s.Start(t, testutils.NewNodeTestBaseContext(), InitSenderForLocalTestCluster)
defer s.Stop()
disableOwnNodeCertain(t, s)
offsetNS := int64(100)
keySlow := roachpb.Key("slow")
keyFast := roachpb.Key("fast")
valSlow := []byte("wols")
valFast := []byte("tsaf")
// Write keySlow at now+offset, keyFast at now+2*offset
futureTS := s.Clock.Now()
futureTS.WallTime += offsetNS
val := roachpb.MakeValueFromBytes(valSlow)
if err := engine.MVCCPut(context.Background(), s.Eng, nil, keySlow, futureTS, val, nil); err != nil {
t.Fatal(err)
}
futureTS.WallTime += offsetNS
val.SetBytes(valFast)
if err := engine.MVCCPut(context.Background(), s.Eng, nil, keyFast, futureTS, val, nil); err != nil {
t.Fatal(err)
}
i := 0
if tErr := s.DB.Txn(context.TODO(), func(txn *client.Txn) error {
i++
// The first command serves to start a Txn, fixing the timestamps.
// There will be a restart, but this is idempotent.
if _, err := txn.Scan("t", roachpb.Key("t").Next(), 0); err != nil {
t.Fatal(err)
}
// This is a bit of a hack for the sake of this test: By visiting the
// node above, we've made a note of its clock, which allows us to
// prevent the restart. But we want to catch the restart, so reset the
// observed timestamps.
txn.Proto.ResetObservedTimestamps()
// The server's clock suddenly jumps ahead of keyFast's timestamp.
s.Manual.Increment(2*offsetNS + 1)
// Now read slowKey first. It should read at 0, catch an uncertainty error,
// and get keySlow's timestamp in that error, but upgrade it to the larger
// node clock (which is ahead of keyFast as well). If the last part does
// not happen, the read of keyFast should fail (i.e. read nothing).
// There will be exactly one restart here.
if gr, err := txn.Get(keySlow); err != nil {
if i != 1 {
t.Fatalf("unexpected transaction error: %s", err)
}
return err
} else if !gr.Exists() || !bytes.Equal(gr.ValueBytes(), valSlow) {
t.Fatalf("read of %q returned %v, wanted value %q", keySlow, gr.Value, valSlow)
}
// The node should already be certain, so we expect no restart here
// and to read the correct key.
if gr, err := txn.Get(keyFast); err != nil {
t.Fatalf("second Get failed with %s", err)
} else if !gr.Exists() || !bytes.Equal(gr.ValueBytes(), valFast) {
t.Fatalf("read of %q returned %v, wanted value %q", keyFast, gr.Value, valFast)
}
return nil
}); tErr != nil {
t.Fatal(tErr)
}
}
func createTestClient(t *testing.T, s serverutils.TestServerInterface) *client.DB {
return createTestClientForUser(t, s, security.NodeUser, client.DefaultDBContext())
}
func createTestClient(t *testing.T, stopper *stop.Stopper, addr string) *client.DB {
return createTestClientForUser(t, stopper, addr, security.NodeUser, client.DefaultDBContext())
}