func runMVCCConditionalPut(valueSize int, createFirst bool, b *testing.B) {
rng, _ := randutil.NewPseudoRand()
value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize))
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
stopper := stop.NewStopper()
defer stopper.Stop()
rocksdb := NewInMem(roachpb.Attributes{}, testCacheSize, stopper)
b.SetBytes(int64(valueSize))
var expected *roachpb.Value
if createFirst {
for i := 0; i < b.N; i++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCPut(rocksdb, nil, key, ts, value, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
expected = &value
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCConditionalPut(rocksdb, nil, key, ts, value, expected, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
b.StopTimer()
}
// TestTxnCoordSenderGC verifies that the coordinator cleans up extant
// transactions after the lastUpdateNanos exceeds the timeout.
func TestTxnCoordSenderGC(t *testing.T) {
defer leaktest.AfterTest(t)
s := createTestDB(t)
defer s.Stop()
// Set heartbeat interval to 1ms for testing.
s.Sender.heartbeatInterval = 1 * time.Millisecond
txn := newTxn(s.Clock, roachpb.Key("a"))
put, h := createPutRequest(roachpb.Key("a"), []byte("value"), txn)
if _, err := client.SendWrappedWith(s.Sender, nil, h, put); err != nil {
t.Fatal(err)
}
// Now, advance clock past the default client timeout.
// Locking the TxnCoordSender to prevent a data race.
s.Sender.Lock()
s.Manual.Set(defaultClientTimeout.Nanoseconds() + 1)
s.Sender.Unlock()
if err := util.IsTrueWithin(func() bool {
// Locking the TxnCoordSender to prevent a data race.
s.Sender.Lock()
_, ok := s.Sender.txns[string(txn.ID)]
s.Sender.Unlock()
return !ok
}, 50*time.Millisecond); err != nil {
t.Error("expected garbage collection")
}
}
// TestRangeLookupOptionOnReverseScan verifies that a lookup triggered by a
// ReverseScan request has the useReverseScan specified.
func TestRangeLookupOptionOnReverseScan(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, _ func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
return []proto.Message{getArgs(nil).(*roachpb.BatchRequest).CreateReply()}, nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(k roachpb.RKey, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
if len(k) > 0 && !useReverseScan {
t.Fatalf("expected UseReverseScan to be set")
}
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
rScan := &roachpb.ReverseScanRequest{
Span: roachpb.Span{Key: roachpb.Key("a"), EndKey: roachpb.Key("b")},
}
if _, err := client.SendWrapped(ds, nil, rScan); err != nil {
t.Fatal(err)
}
}
// TestTimestampCacheReadVsWrite verifies that the timestamp cache
// can differentiate between read and write timestamp.
func TestTimestampCacheReadVsWrite(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
tc := NewTimestampCache(clock)
// Add read-only non-txn entry at current time.
ts1 := clock.Now()
tc.Add(roachpb.Key("a"), roachpb.Key("b"), ts1, nil, true)
// Add two successive txn entries; one read-only and one read-write.
txn1ID := uuid.NewV4()
txn2ID := uuid.NewV4()
ts2 := clock.Now()
tc.Add(roachpb.Key("a"), nil, ts2, txn1ID, true)
ts3 := clock.Now()
tc.Add(roachpb.Key("a"), nil, ts3, txn2ID, false)
// Fetching with no transaction gets latest values.
if rTS, wTS := tc.GetMaxRead(roachpb.Key("a"), nil, nil), tc.GetMaxWrite(roachpb.Key("a"), nil, nil); !rTS.Equal(ts2) || !wTS.Equal(ts3) {
t.Errorf("expected %s %s; got %s %s", ts2, ts3, rTS, wTS)
}
// Fetching with txn ID "1" gets low water mark for read and most recent for write.
if rTS, wTS := tc.GetMaxRead(roachpb.Key("a"), nil, txn1ID), tc.GetMaxWrite(roachpb.Key("a"), nil, txn1ID); !rTS.Equal(tc.lowWater) || !wTS.Equal(ts3) {
t.Errorf("expected %s %s; got %s %s", ts1, ts3, rTS, wTS)
}
// Fetching with txn ID "2" gets ts2 for read and low water mark for write.
if rTS, wTS := tc.GetMaxRead(roachpb.Key("a"), nil, txn2ID), tc.GetMaxWrite(roachpb.Key("a"), nil, txn2ID); !rTS.Equal(ts2) || !wTS.Equal(tc.lowWater) {
t.Errorf("expected %s %s; got %s %s", ts2, tc.lowWater, rTS, wTS)
}
}
func runMVCCConditionalPut(emk engineMaker, valueSize int, createFirst bool, b *testing.B) {
rng, _ := randutil.NewPseudoRand()
value := roachpb.MakeValueFromBytes(randutil.RandBytes(rng, valueSize))
keyBuf := append(make([]byte, 0, 64), []byte("key-")...)
eng, stopper := emk(b, fmt.Sprintf("cput_%d", valueSize))
defer stopper.Stop()
b.SetBytes(int64(valueSize))
var expected *roachpb.Value
if createFirst {
for i := 0; i < b.N; i++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCPut(context.Background(), eng, nil, key, ts, value, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
expected = &value
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
key := roachpb.Key(encoding.EncodeUvarintAscending(keyBuf[:4], uint64(i)))
ts := makeTS(timeutil.Now().UnixNano(), 0)
if err := MVCCConditionalPut(context.Background(), eng, nil, key, ts, value, expected, nil); err != nil {
b.Fatalf("failed put: %s", err)
}
}
b.StopTimer()
}
// TestRangeSplitMeta executes various splits (including at meta addressing)
// and checks that all created intents are resolved. This includes both intents
// which are resolved synchronously with EndTransaction and via RPC.
func TestRangeSplitMeta(t *testing.T) {
defer leaktest.AfterTest(t)
s := createTestDB(t)
defer s.Stop()
splitKeys := []roachpb.Key{roachpb.Key("G"), keys.RangeMetaKey(roachpb.Key("F")),
keys.RangeMetaKey(roachpb.Key("K")), keys.RangeMetaKey(roachpb.Key("H"))}
// Execute the consecutive splits.
for _, splitKey := range splitKeys {
log.Infof("starting split at key %q...", splitKey)
if err := s.DB.AdminSplit(splitKey); err != nil {
t.Fatal(err)
}
log.Infof("split at key %q complete", splitKey)
}
if err := util.IsTrueWithin(func() bool {
if _, _, err := engine.MVCCScan(s.Eng, keys.LocalMax, roachpb.KeyMax, 0, roachpb.MaxTimestamp, true, nil); err != nil {
log.Infof("mvcc scan should be clean: %s", err)
return false
}
return true
}, 500*time.Millisecond); err != nil {
t.Error("failed to verify no dangling intents within 500ms")
}
}
// TestTimestampCacheNoEviction verifies that even after
// the MinTSCacheWindow interval, if the cache has not hit
// its size threshold, it will not evict entries.
func TestTimestampCacheNoEviction(t *testing.T) {
defer leaktest.AfterTest(t)()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
clock.SetMaxOffset(maxClockOffset)
tc := newTimestampCache(clock)
// Increment time to the maxClockOffset low water mark + 1.
manual.Set(maxClockOffset.Nanoseconds() + 1)
aTS := clock.Now()
tc.add(roachpb.Key("a"), nil, aTS, nil, true)
tc.AddRequest(cacheRequest{
reads: []roachpb.Span{{Key: roachpb.Key("c")}},
timestamp: aTS,
})
// Increment time by the MinTSCacheWindow and add another key.
manual.Increment(MinTSCacheWindow.Nanoseconds())
tc.add(roachpb.Key("b"), nil, clock.Now(), nil, true)
tc.AddRequest(cacheRequest{
reads: []roachpb.Span{{Key: roachpb.Key("d")}},
timestamp: clock.Now(),
})
// Verify that the cache still has 4 entries in it
if l, want := tc.len(), 4; l != want {
t.Errorf("expected %d entries to remain, got %d", want, l)
}
}
// newTestRangeSet creates a new range set that has the count number of ranges.
func newTestRangeSet(count int, t *testing.T) *testRangeSet {
rs := &testRangeSet{rangesByKey: btree.New(64 /* degree */)}
for i := 0; i < count; i++ {
desc := &roachpb.RangeDescriptor{
RangeID: roachpb.RangeID(i),
StartKey: roachpb.Key(fmt.Sprintf("%03d", i)),
EndKey: roachpb.Key(fmt.Sprintf("%03d", i+1)),
}
// Initialize the range stat so the scanner can use it.
rng := &Replica{
stats: &rangeStats{
rangeID: desc.RangeID,
MVCCStats: engine.MVCCStats{
KeyBytes: 1,
ValBytes: 2,
KeyCount: 1,
LiveCount: 1,
},
},
}
if err := rng.setDesc(desc); err != nil {
t.Fatal(err)
}
if exRngItem := rs.rangesByKey.ReplaceOrInsert(rng); exRngItem != nil {
t.Fatalf("failed to insert range %s", rng)
}
}
return rs
}
// TestMultiRangeScanReverseScanInconsistent verifies that a Scan/ReverseScan
// across ranges that doesn't require read consistency will set a timestamp
// using the clock local to the distributed sender.
func TestMultiRangeScanReverseScanInconsistent(t *testing.T) {
defer leaktest.AfterTest(t)
s, db := setupMultipleRanges(t, "b")
defer s.Stop()
// Write keys "a" and "b", the latter of which is the first key in the
// second range.
keys := []string{"a", "b"}
ts := []time.Time{}
for i, key := range keys {
b := &client.Batch{}
b.Put(key, "value")
if err := db.Run(b); err != nil {
t.Fatal(err)
}
ts = append(ts, b.Results[0].Rows[0].Timestamp())
log.Infof("%d: %s", i, b.Results[0].Rows[0].Timestamp())
}
// Do an inconsistent Scan/ReverseScan from a new DistSender and verify
// it does the read at its local clock and doesn't receive an
// OpRequiresTxnError. We set the local clock to the timestamp of
// the first key to verify it's used to read only key "a".
manual := hlc.NewManualClock(ts[1].UnixNano() - 1)
clock := hlc.NewClock(manual.UnixNano)
ds := kv.NewDistSender(&kv.DistSenderContext{Clock: clock}, s.Gossip())
// Scan.
sa := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ScanRequest)
reply, err := client.SendWrappedWith(ds, nil, roachpb.BatchRequest_Header{
ReadConsistency: roachpb.INCONSISTENT,
}, sa)
if err != nil {
t.Fatal(err)
}
sr := reply.(*roachpb.ScanResponse)
if l := len(sr.Rows); l != 1 {
t.Fatalf("expected 1 row; got %d", l)
}
if key := string(sr.Rows[0].Key); keys[0] != key {
t.Errorf("expected key %q; got %q", keys[0], key)
}
// ReverseScan.
rsa := roachpb.NewReverseScan(roachpb.Key("a"), roachpb.Key("c"), 0).(*roachpb.ReverseScanRequest)
reply, err = client.SendWrappedWith(ds, nil, roachpb.BatchRequest_Header{
ReadConsistency: roachpb.INCONSISTENT,
}, rsa)
if err != nil {
t.Fatal(err)
}
rsr := reply.(*roachpb.ReverseScanResponse)
if l := len(rsr.Rows); l != 1 {
t.Fatalf("expected 1 row; got %d", l)
}
if key := string(rsr.Rows[0].Key); keys[0] != key {
t.Errorf("expected key %q; got %q", keys[0], key)
}
}
// TestRangeLookupOptionOnReverseScan verifies that a lookup triggered by a
// ReverseScan request has the useReverseScan specified.
func TestRangeLookupOptionOnReverseScan(t *testing.T) {
defer leaktest.AfterTest(t)()
g, s := makeTestGossip(t)
defer s()
var testFn rpcSendFn = func(_ SendOptions, _ ReplicaSlice,
args roachpb.BatchRequest, _ *rpc.Context) (*roachpb.BatchResponse, error) {
return args.CreateReply(), nil
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(k roachpb.RKey, considerIntents, useReverseScan bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
if len(k) > 0 && !useReverseScan {
t.Fatalf("expected UseReverseScan to be set")
}
return []roachpb.RangeDescriptor{testRangeDescriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
rScan := &roachpb.ReverseScanRequest{
Span: roachpb.Span{Key: roachpb.Key("a"), EndKey: roachpb.Key("b")},
}
if _, err := client.SendWrapped(ds, nil, rScan); err != nil {
t.Fatal(err)
}
}
// CopyInto copies all the cached results from this response cache
// into the destRangeID response cache. Failures decoding individual
// cache entries return an error.
func (rc *ResponseCache) CopyInto(e engine.Engine, destRangeID roachpb.RangeID) error {
start := engine.MVCCEncodeKey(
keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMin))
end := engine.MVCCEncodeKey(
keys.ResponseCacheKey(rc.rangeID, roachpb.KeyMax))
return e.Iterate(start, end, func(kv engine.MVCCKeyValue) (bool, error) {
// Decode the key into a cmd, skipping on error. Otherwise,
// write it to the corresponding key in the new cache.
family, err := rc.decodeResponseCacheKey(kv.Key)
if err != nil {
return false, util.Errorf("could not decode a response cache key %s: %s",
roachpb.Key(kv.Key), err)
}
key := keys.ResponseCacheKey(destRangeID, family)
encKey := engine.MVCCEncodeKey(key)
// Decode the value, update the checksum and re-encode.
meta := &engine.MVCCMetadata{}
if err := proto.Unmarshal(kv.Value, meta); err != nil {
return false, util.Errorf("could not decode response cache value %s [% x]: %s",
roachpb.Key(kv.Key), kv.Value, err)
}
meta.Value.Checksum = nil
meta.Value.InitChecksum(key)
_, _, err = engine.PutProto(e, encKey, meta)
return false, err
})
}
func TestCommandQueueCovering(t *testing.T) {
defer leaktest.AfterTest(t)()
cq := NewCommandQueue()
a := roachpb.Span{Key: roachpb.Key("a")}
b := roachpb.Span{Key: roachpb.Key("b")}
c := roachpb.Span{Key: roachpb.Key("c")}
{
// Test adding a covering entry and then not expanding it.
wk := cq.add(false, a, b)
var wg sync.WaitGroup
cq.getWait(false, &wg, c)
wg.Wait()
cq.remove(wk)
}
{
// Test adding a covering entry and expanding it.
wk := cq.add(false, a, b)
var wg sync.WaitGroup
cq.getWait(false, &wg, a)
cq.remove(wk)
wg.Wait()
}
}
func TestCommandQueueMultiplePendingCommands(t *testing.T) {
defer leaktest.AfterTest(t)
cq := NewCommandQueue()
wg1 := sync.WaitGroup{}
wg2 := sync.WaitGroup{}
wg3 := sync.WaitGroup{}
// Add a command which will overlap all commands.
wk := add(cq, roachpb.Key("a"), roachpb.Key("d"), false)
getWait(cq, roachpb.Key("a"), nil, false, &wg1)
getWait(cq, roachpb.Key("b"), nil, false, &wg2)
getWait(cq, roachpb.Key("c"), nil, false, &wg3)
cmdDone1 := waitForCmd(&wg1)
cmdDone2 := waitForCmd(&wg2)
cmdDone3 := waitForCmd(&wg3)
if testCmdDone(cmdDone1, 1*time.Millisecond) ||
testCmdDone(cmdDone2, 1*time.Millisecond) ||
testCmdDone(cmdDone3, 1*time.Millisecond) {
t.Fatal("no commands should finish with command outstanding")
}
cq.Remove([]interface{}{wk})
if !testCmdDone(cmdDone1, 5*time.Millisecond) ||
!testCmdDone(cmdDone2, 5*time.Millisecond) ||
!testCmdDone(cmdDone3, 5*time.Millisecond) {
t.Fatal("commands should finish with no commands outstanding")
}
}
// runClientScan first creates test data (and resets the benchmarking
// timer). It then performs b.N client scans in increments of numRows
// keys over all of the data, restarting at the beginning of the
// keyspace, as many times as necessary.
func runClientScan(useSSL bool, numRows, numVersions int, b *testing.B) {
const numKeys = 100000
s, db := setupClientBenchData(useSSL, numVersions, numKeys, b)
defer s.Stop()
b.SetBytes(int64(numRows * valueSize))
b.ResetTimer()
b.RunParallel(func(pb *testing.PB) {
startKeyBuf := append(make([]byte, 0, 64), []byte("key-")...)
endKeyBuf := append(make([]byte, 0, 64), []byte("key-")...)
for pb.Next() {
// Choose a random key to start scan.
keyIdx := rand.Int31n(int32(numKeys - numRows))
startKey := roachpb.Key(encoding.EncodeUvarintAscending(
startKeyBuf, uint64(keyIdx)))
endKey := roachpb.Key(encoding.EncodeUvarintAscending(
endKeyBuf, uint64(keyIdx)+uint64(numRows)))
rows, pErr := db.Scan(startKey, endKey, int64(numRows))
if pErr != nil {
b.Fatalf("failed scan: %s", pErr)
}
if len(rows) != numRows {
b.Fatalf("failed to scan: %d != %d", len(rows), numRows)
}
}
})
b.StopTimer()
}
// TestBatchError verifies that Range returns an error if a request has an invalid range.
func TestBatchError(t *testing.T) {
testCases := []struct {
req [2]string
errMsg string
}{
{
req: [2]string{"\xff\xff\xff\xff", "a"},
errMsg: "must be less than KeyMax",
},
{
req: [2]string{"a", "\xff\xff\xff\xff"},
errMsg: "must be less than or equal to KeyMax",
},
}
for i, c := range testCases {
var ba roachpb.BatchRequest
ba.Add(&roachpb.ScanRequest{Span: roachpb.Span{Key: roachpb.Key(c.req[0]), EndKey: roachpb.Key(c.req[1])}})
if _, err := Range(ba); !testutils.IsError(err, c.errMsg) {
t.Errorf("%d: unexpected error %v", i, err)
}
}
// Test a case where a non-range request has an end key.
var ba roachpb.BatchRequest
ba.Add(&roachpb.GetRequest{Span: roachpb.Span{Key: roachpb.Key("a"), EndKey: roachpb.Key("b")}})
if _, err := Range(ba); !testutils.IsError(err, "end key specified for non-range operation") {
t.Errorf("unexpected error %v", err)
}
}
// deleteRow adds to the batch the kv operations necessary to delete a table row
// with the given values.
func (rd *rowDeleter) deleteRow(b *client.Batch, values []parser.Datum) error {
if err := rd.fks.checkAll(values); err != nil {
return err
}
primaryIndexKey, secondaryIndexEntries, err := rd.helper.encodeIndexes(rd.fetchColIDtoRowIndex, values)
if err != nil {
return err
}
for _, secondaryIndexEntry := range secondaryIndexEntries {
if log.V(2) {
log.Infof("Del %s", secondaryIndexEntry.Key)
}
b.Del(secondaryIndexEntry.Key)
}
// Delete the row.
rd.startKey = roachpb.Key(primaryIndexKey)
rd.endKey = roachpb.Key(encoding.EncodeNotNullDescending(primaryIndexKey))
if log.V(2) {
log.Infof("DelRange %s - %s", rd.startKey, rd.endKey)
}
b.DelRange(&rd.startKey, &rd.endKey, false)
rd.startKey, rd.endKey = nil, nil
return nil
}
func copySeqCache(e engine.Engine, srcID, dstID roachpb.RangeID, keyMin, keyMax engine.MVCCKey) error {
var scratch [64]byte
return e.Iterate(keyMin, keyMax,
func(kv engine.MVCCKeyValue) (bool, error) {
// Decode the key into a cmd, skipping on error. Otherwise,
// write it to the corresponding key in the new cache.
id, epoch, seq, err := decodeSequenceCacheMVCCKey(kv.Key, scratch[:0])
if err != nil {
return false, util.Errorf("could not decode a sequence cache key %s: %s",
roachpb.Key(kv.Key), err)
}
key := keys.SequenceCacheKey(dstID, id, epoch, seq)
encKey := engine.MVCCEncodeKey(key)
// Decode the value, update the checksum and re-encode.
meta := &engine.MVCCMetadata{}
if err := proto.Unmarshal(kv.Value, meta); err != nil {
return false, util.Errorf("could not decode sequence cache value %s [% x]: %s",
roachpb.Key(kv.Key), kv.Value, err)
}
meta.Value.Checksum = nil
meta.Value.InitChecksum(key)
_, _, err = engine.PutProto(e, encKey, meta)
return false, err
})
}
// TestLeaderAfterSplit verifies that a raft group created by a split
// elects a leader without waiting for an election timeout.
func TestLeaderAfterSplit(t *testing.T) {
defer leaktest.AfterTest(t)
storeContext := storage.TestStoreContext
storeContext.RaftElectionTimeoutTicks = 1000000
mtc := &multiTestContext{
storeContext: &storeContext,
}
mtc.Start(t, 3)
defer mtc.Stop()
mtc.replicateRange(1, 0, 1, 2)
leftKey := roachpb.Key("a")
splitKey := roachpb.Key("m")
rightKey := roachpb.Key("z")
splitArgs := adminSplitArgs(roachpb.KeyMin, splitKey)
if _, err := client.SendWrapped(mtc.distSender, nil, &splitArgs); err != nil {
t.Fatal(err)
}
incArgs := incrementArgs(leftKey, 1)
if _, err := client.SendWrapped(mtc.distSender, nil, &incArgs); err != nil {
t.Fatal(err)
}
incArgs = incrementArgs(rightKey, 2)
if _, err := client.SendWrapped(mtc.distSender, nil, &incArgs); err != nil {
t.Fatal(err)
}
}
// TestStoreRangeUpReplicate verifies that the replication queue will notice
// under-replicated ranges and replicate them.
func TestStoreRangeUpReplicate(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 3)
defer mtc.Stop()
// Initialize the gossip network.
var wg sync.WaitGroup
wg.Add(len(mtc.stores))
key := gossip.MakePrefixPattern(gossip.KeyStorePrefix)
mtc.stores[0].Gossip().RegisterCallback(key, func(_ string, _ []byte) { wg.Done() })
for _, s := range mtc.stores {
s.GossipStore()
}
wg.Wait()
// Once we know our peers, trigger a scan.
mtc.stores[0].ForceReplicationScan(t)
// The range should become available on every node.
if err := util.IsTrueWithin(func() bool {
for _, s := range mtc.stores {
r := s.LookupReplica(roachpb.Key("a"), roachpb.Key("b"))
if r == nil {
return false
}
}
return true
}, 1*time.Second); err != nil {
t.Fatal(err)
}
}
// TestSendRPCRetry verifies that sendRPC failed on first address but succeed on
// second address, the second reply should be successfully returned back.
func TestSendRPCRetry(t *testing.T) {
defer leaktest.AfterTest(t)
g, s := makeTestGossip(t)
defer s()
g.SetNodeID(1)
if err := g.SetNodeDescriptor(&roachpb.NodeDescriptor{NodeID: 1}); err != nil {
t.Fatal(err)
}
// Fill RangeDescriptor with 2 replicas
var descriptor = roachpb.RangeDescriptor{
RangeID: 1,
StartKey: roachpb.RKey("a"),
EndKey: roachpb.RKey("z"),
}
for i := 1; i <= 2; i++ {
addr := util.MakeUnresolvedAddr("tcp", fmt.Sprintf("node%d", i))
nd := &roachpb.NodeDescriptor{
NodeID: roachpb.NodeID(i),
Address: util.MakeUnresolvedAddr(addr.Network(), addr.String()),
}
if err := g.AddInfoProto(gossip.MakeNodeIDKey(roachpb.NodeID(i)), nd, time.Hour); err != nil {
t.Fatal(err)
}
descriptor.Replicas = append(descriptor.Replicas, roachpb.ReplicaDescriptor{
NodeID: roachpb.NodeID(i),
StoreID: roachpb.StoreID(i),
})
}
// Define our rpcSend stub which returns success on the second address.
var testFn rpcSendFn = func(_ rpc.Options, method string, addrs []net.Addr, getArgs func(addr net.Addr) proto.Message, getReply func() proto.Message, _ *rpc.Context) ([]proto.Message, error) {
if method == "Node.Batch" {
// reply from first address failed
_ = getReply()
// reply from second address succeed
batchReply := getReply().(*roachpb.BatchResponse)
reply := &roachpb.ScanResponse{}
batchReply.Add(reply)
reply.Rows = append([]roachpb.KeyValue{}, roachpb.KeyValue{Key: roachpb.Key("b"), Value: roachpb.Value{}})
return []proto.Message{batchReply}, nil
}
return nil, util.Errorf("unexpected method %v", method)
}
ctx := &DistSenderContext{
RPCSend: testFn,
RangeDescriptorDB: mockRangeDescriptorDB(func(_ roachpb.RKey, _, _ bool) ([]roachpb.RangeDescriptor, *roachpb.Error) {
return []roachpb.RangeDescriptor{descriptor}, nil
}),
}
ds := NewDistSender(ctx, g)
scan := roachpb.NewScan(roachpb.Key("a"), roachpb.Key("d"), 1)
sr, err := client.SendWrapped(ds, nil, scan)
if err != nil {
t.Fatal(err)
}
if l := len(sr.(*roachpb.ScanResponse).Rows); l != 1 {
t.Fatalf("expected 1 row; got %d", l)
}
}
// TestUpdateRangeAddressingSplitMeta1 verifies that it's an error to
// attempt to update range addressing records that would allow a split
// of meta1 records.
func TestUpdateRangeAddressingSplitMeta1(t *testing.T) {
defer leaktest.AfterTest(t)
left := &roachpb.RangeDescriptor{StartKey: roachpb.KeyMin, EndKey: meta1Key(roachpb.Key("a"))}
right := &roachpb.RangeDescriptor{StartKey: meta1Key(roachpb.Key("a")), EndKey: roachpb.KeyMax}
if err := splitRangeAddressing(&client.Batch{}, left, right); err == nil {
t.Error("expected failure trying to update addressing records for meta1 split")
}
}
// TestRocksDBCompaction verifies that a garbage collector can be
// installed on a RocksDB engine and will properly compact response
// cache and transaction entries.
func TestRocksDBCompaction(t *testing.T) {
defer leaktest.AfterTest(t)
stopper := stop.NewStopper()
defer stopper.Stop()
rocksdb := newMemRocksDB(roachpb.Attributes{Attrs: []string{"ssd"}}, testCacheSize, stopper)
err := rocksdb.Open()
if err != nil {
t.Fatalf("could not create new in-memory rocksdb db instance: %v", err)
}
rocksdb.SetGCTimeouts(1, 2)
cmdID := &roachpb.ClientCmdID{WallTime: 1, Random: 1}
// Write two transaction values and two response cache values such
// that exactly one of each should be GC'd based on our GC timeouts.
kvs := []roachpb.KeyValue{
{
Key: keys.ResponseCacheKey(1, cmdID),
Value: roachpb.MakeValueFromBytes(encodePutResponse(makeTS(2, 0), t)),
},
{
Key: keys.ResponseCacheKey(2, cmdID),
Value: roachpb.MakeValueFromBytes(encodePutResponse(makeTS(3, 0), t)),
},
{
Key: keys.TransactionKey(roachpb.Key("a"), roachpb.Key(uuid.NewUUID4())),
Value: roachpb.MakeValueFromBytes(encodeTransaction(makeTS(1, 0), t)),
},
{
Key: keys.TransactionKey(roachpb.Key("b"), roachpb.Key(uuid.NewUUID4())),
Value: roachpb.MakeValueFromBytes(encodeTransaction(makeTS(2, 0), t)),
},
}
for _, kv := range kvs {
if err := MVCCPut(rocksdb, nil, kv.Key, roachpb.ZeroTimestamp, kv.Value, nil); err != nil {
t.Fatal(err)
}
}
// Compact range and scan remaining values to compare.
rocksdb.CompactRange(nil, nil)
actualKVs, _, err := MVCCScan(rocksdb, keyMin, keyMax, 0, roachpb.ZeroTimestamp, true, nil)
if err != nil {
t.Fatalf("could not run scan: %v", err)
}
var keys []roachpb.Key
for _, kv := range actualKVs {
keys = append(keys, kv.Key)
}
expKeys := []roachpb.Key{
kvs[1].Key,
kvs[3].Key,
}
if !reflect.DeepEqual(expKeys, keys) {
t.Errorf("expected keys %+v, got keys %+v", expKeys, keys)
}
}
// TestCommandQueueSelfOverlap makes sure that GetWait adds all of the
// key ranges simultaneously. If that weren't the case, all but the first
// span would wind up waiting on overlapping previous spans, resulting
// in deadlock.
func TestCommandQueueSelfOverlap(t *testing.T) {
defer leaktest.AfterTest(t)()
cq := NewCommandQueue()
a := roachpb.Key("a")
k := add(cq, a, roachpb.Key("b"), false)
chans := cq.getWait(false, []roachpb.Span{{Key: a}, {Key: a}, {Key: a}}...)
cq.remove(k)
waitCmdDone(chans)
}
// TestCommandQueueExclusiveEnd verifies that an end key is treated as
// an exclusive end when GetWait calculates overlapping commands. Test
// it by calling GetWait with a command whose start key is equal to
// the end key of a previous command.
func TestCommandQueueExclusiveEnd(t *testing.T) {
defer leaktest.AfterTest(t)()
cq := NewCommandQueue()
add(cq, roachpb.Key("a"), roachpb.Key("b"), false)
// Verify no wait on the second writer command on "b" since
// it does not overlap with the first command on ["a", "b").
waitCmdDone(getWait(cq, roachpb.Key("b"), nil, false))
}
// TestChangeReplicasDuplicateError tests that a replica change aborts if
// another change has been made to the RangeDescriptor since it was initiated.
func TestChangeReplicasDescriptorInvariant(t *testing.T) {
defer leaktest.AfterTest(t)
mtc := startMultiTestContext(t, 3)
defer mtc.Stop()
repl, err := mtc.stores[0].GetReplica(1)
if err != nil {
t.Fatal(err)
}
addReplica := func(storeNum int, desc *roachpb.RangeDescriptor) error {
return repl.ChangeReplicas(roachpb.ADD_REPLICA,
roachpb.ReplicaDescriptor{
NodeID: mtc.stores[storeNum].Ident.NodeID,
StoreID: mtc.stores[storeNum].Ident.StoreID,
},
desc)
}
// Retain the descriptor for the range at this point.
origDesc := repl.Desc()
// Add replica to the second store, which should succeed.
if err := addReplica(1, origDesc); err != nil {
t.Fatal(err)
}
if err := util.IsTrueWithin(func() bool {
r := mtc.stores[1].LookupReplica(roachpb.Key("a"), roachpb.Key("b"))
if r == nil {
return false
}
return true
}, time.Second); err != nil {
t.Fatal(err)
}
// Attempt to add replica to the third store with the original descriptor.
// This should fail because the descriptor is stale.
if err := addReplica(2, origDesc); err == nil {
t.Fatal("Expected error calling ChangeReplicas with stale RangeDescriptor")
}
// Add to third store with fresh descriptor.
if err := addReplica(2, repl.Desc()); err != nil {
t.Fatal(err)
}
if err := util.IsTrueWithin(func() bool {
r := mtc.stores[2].LookupReplica(roachpb.Key("a"), roachpb.Key("b"))
if r == nil {
return false
}
return true
}, time.Second); err != nil {
t.Fatal(err)
}
}
// TestCommandQueueSelfOverlap makes sure that GetWait adds all of the
// key ranges simultaneously. If that weren't the case, all but the first
// span would wind up waiting on overlapping previous spans, resulting
// in deadlock.
func TestCommandQueueSelfOverlap(t *testing.T) {
defer leaktest.AfterTest(t)()
cq := NewCommandQueue()
a := roachpb.Key("a")
k := add(cq, a, roachpb.Key("b"), false)
var wg sync.WaitGroup
cq.getWait(false, &wg, []roachpb.Span{{Key: a}, {Key: a}, {Key: a}}...)
cq.remove(k)
wg.Wait()
}
// TestCommandQueueSelfOverlap makes sure that GetWait adds all of the
// key ranges simultaneously. If that weren't the case, all but the first
// span would wind up waiting on overlapping previous spans, resulting
// in deadlock.
func TestCommandQueueSelfOverlap(t *testing.T) {
defer leaktest.AfterTest(t)
cq := NewCommandQueue()
a := roachpb.Key("a")
k := add(cq, a, roachpb.Key("b"), false)
var wg sync.WaitGroup
cq.GetWait(false, &wg, []keys.Span{{Start: a}, {Start: a}, {Start: a}}...)
cq.Remove([]interface{}{k})
wg.Wait()
}
func TestMakeKey(t *testing.T) {
if !bytes.Equal(makeKey(roachpb.Key("A"), roachpb.Key("B")), roachpb.Key("AB")) ||
!bytes.Equal(makeKey(roachpb.Key("A")), roachpb.Key("A")) ||
!bytes.Equal(makeKey(roachpb.Key("A"), roachpb.Key("B"), roachpb.Key("C")), roachpb.Key("ABC")) {
t.Fatalf("MakeKey is broken")
}
}
// TestStoreRecoverWithErrors verifies that even commands that fail are marked as
// applied so they are not retried after recovery.
func TestStoreRecoverWithErrors(t *testing.T) {
defer leaktest.AfterTest(t)
defer func() { storage.TestingCommandFilter = nil }()
manual := hlc.NewManualClock(0)
clock := hlc.NewClock(manual.UnixNano)
engineStopper := stop.NewStopper()
defer engineStopper.Stop()
eng := engine.NewInMem(roachpb.Attributes{}, 1<<20, engineStopper)
numIncrements := 0
storage.TestingCommandFilter = func(_ roachpb.StoreID, args roachpb.Request, _ roachpb.Header) error {
if _, ok := args.(*roachpb.IncrementRequest); ok && args.Header().Key.Equal(roachpb.Key("a")) {
numIncrements++
}
return nil
}
func() {
stopper := stop.NewStopper()
defer stopper.Stop()
store := createTestStoreWithEngine(t, eng, clock, true, nil, stopper)
// Write a bytes value so the increment will fail.
putArgs := putArgs(roachpb.Key("a"), []byte("asdf"))
if _, err := client.SendWrapped(rg1(store), nil, &putArgs); err != nil {
t.Fatal(err)
}
// Try and fail to increment the key. It is important for this test that the
// failure be the last thing in the raft log when the store is stopped.
incArgs := incrementArgs(roachpb.Key("a"), 42)
if _, err := client.SendWrapped(rg1(store), nil, &incArgs); err == nil {
t.Fatal("did not get expected error")
}
}()
if numIncrements != 1 {
t.Fatalf("expected 1 increments; was %d", numIncrements)
}
// Recover from the engine.
store := createTestStoreWithEngine(t, eng, clock, false, nil, engineStopper)
// Issue a no-op write to lazily initialize raft on the range.
incArgs := incrementArgs(roachpb.Key("b"), 0)
if _, err := client.SendWrapped(rg1(store), nil, &incArgs); err != nil {
t.Fatal(err)
}
// No additional increments were performed on key A during recovery.
if numIncrements != 1 {
t.Fatalf("expected 1 increments; was %d", numIncrements)
}
}
func marshalKey(k interface{}) (roachpb.Key, error) {
switch t := k.(type) {
case string:
return roachpb.Key(t), nil
case roachpb.Key:
return t, nil
case []byte:
return roachpb.Key(t), nil
}
return nil, fmt.Errorf("unable to marshal key: %T %q", k, k)
}
