// TestGCQueueProcess creates test data in the range over various time
// scales and verifies that scan queue process properly GCs test data.
func TestGCQueueProcess(t *testing.T) {
defer leaktest.AfterTest(t)()
tc := testContext{}
tc.Start(t)
defer tc.Stop()
const now int64 = 48 * 60 * 60 * 1E9 // 2d past the epoch
tc.manualClock.Set(now)
ts1 := makeTS(now-2*24*60*60*1E9+1, 0) // 2d old (add one nanosecond so we're not using zero timestamp)
ts2 := makeTS(now-25*60*60*1E9, 0) // GC will occur at time=25 hours
ts2m1 := ts2.Prev() // ts2 - 1 so we have something not right at the GC time
ts3 := makeTS(now-intentAgeThreshold.Nanoseconds(), 0) // 2h old
ts4 := makeTS(now-(intentAgeThreshold.Nanoseconds()-1), 0) // 2h-1ns old
ts5 := makeTS(now-1E9, 0) // 1s old
key1 := roachpb.Key("a")
key2 := roachpb.Key("b")
key3 := roachpb.Key("c")
key4 := roachpb.Key("d")
key5 := roachpb.Key("e")
key6 := roachpb.Key("f")
key7 := roachpb.Key("g")
key8 := roachpb.Key("h")
key9 := roachpb.Key("i")
key10 := roachpb.Key("j")
key11 := roachpb.Key("k")
data := []struct {
key roachpb.Key
ts hlc.Timestamp
del bool
txn bool
}{
// For key1, we expect first value to GC.
{key1, ts1, false, false},
{key1, ts2, false, false},
{key1, ts5, false, false},
// For key2, we expect values to GC, even though most recent is deletion.
{key2, ts1, false, false},
{key2, ts2m1, false, false}, // use a value < the GC time to verify it's kept
{key2, ts5, true, false},
// For key3, we expect just ts1 to GC, because most recent deletion is intent.
{key3, ts1, false, false},
{key3, ts2, false, false},
{key3, ts5, true, true},
// For key4, expect oldest value to GC.
{key4, ts1, false, false},
{key4, ts2, false, false},
// For key5, expect all values to GC (most recent value deleted).
{key5, ts1, false, false},
{key5, ts2, true, false}, // deleted, so GC
// For key6, expect no values to GC because most recent value is intent.
{key6, ts1, false, false},
{key6, ts5, false, true},
// For key7, expect no values to GC because intent is exactly 2h old.
{key7, ts2, false, false},
{key7, ts4, false, true},
// For key8, expect most recent value to resolve by aborting, which will clean it up.
{key8, ts2, false, false},
{key8, ts3, true, true},
// For key9, resolve naked intent with no remaining values.
{key9, ts3, false, true},
// For key10, GC ts1 because it's a delete but not ts3 because it's above the threshold.
{key10, ts1, true, false},
{key10, ts3, true, false},
{key10, ts4, false, false},
{key10, ts5, false, false},
// For key11, we can't GC anything because ts1 isn't a delete.
{key11, ts1, false, false},
{key11, ts3, true, false},
{key11, ts4, true, false},
{key11, ts5, true, false},
}
for i, datum := range data {
if datum.del {
dArgs := deleteArgs(datum.key)
var txn *roachpb.Transaction
if datum.txn {
txn = newTransaction("test", datum.key, 1, enginepb.SERIALIZABLE, tc.clock)
txn.OrigTimestamp = datum.ts
txn.Timestamp = datum.ts
}
if _, err := tc.SendWrappedWith(roachpb.Header{
Timestamp: datum.ts,
Txn: txn,
}, &dArgs); err != nil {
t.Fatalf("%d: could not delete data: %s", i, err)
}
} else {
pArgs := putArgs(datum.key, []byte("value"))
var txn *roachpb.Transaction
if datum.txn {
txn = newTransaction("test", datum.key, 1, enginepb.SERIALIZABLE, tc.clock)
txn.OrigTimestamp = datum.ts
txn.Timestamp = datum.ts
}
if _, err := tc.SendWrappedWith(roachpb.Header{
Timestamp: datum.ts,
Txn: txn,
}, &pArgs); err != nil {
t.Fatalf("%d: could not put data: %s", i, err)
}
}
}
cfg, ok := tc.gossip.GetSystemConfig()
if !ok {
t.Fatal("config not set")
}
// Process through a scan queue.
gcQ := newGCQueue(tc.gossip)
if err := gcQ.process(tc.clock.Now(), tc.rng, cfg); err != nil {
t.Fatal(err)
}
expKVs := []struct {
key roachpb.Key
ts hlc.Timestamp
}{
{key1, ts5},
{key1, ts2},
{key2, ts5},
{key2, ts2m1},
{key3, hlc.ZeroTimestamp},
{key3, ts5},
{key3, ts2},
{key4, ts2},
{key6, hlc.ZeroTimestamp},
{key6, ts5},
{key6, ts1},
{key7, hlc.ZeroTimestamp},
{key7, ts4},
{key7, ts2},
{key8, ts2},
{key10, ts5},
{key10, ts4},
{key10, ts3},
{key11, ts5},
{key11, ts4},
{key11, ts3},
{key11, ts1},
}
// Read data directly from engine to avoid intent errors from MVCC.
kvs, err := engine.Scan(tc.store.Engine(), engine.MakeMVCCMetadataKey(key1),
engine.MakeMVCCMetadataKey(keys.MaxKey), 0)
if err != nil {
t.Fatal(err)
}
for i, kv := range kvs {
if log.V(1) {
log.Infof("%d: %s", i, kv.Key)
}
}
if len(kvs) != len(expKVs) {
t.Fatalf("expected length %d; got %d", len(expKVs), len(kvs))
}
for i, kv := range kvs {
if !kv.Key.Key.Equal(expKVs[i].key) {
t.Errorf("%d: expected key %q; got %q", i, expKVs[i].key, kv.Key.Key)
}
if !kv.Key.Timestamp.Equal(expKVs[i].ts) {
t.Errorf("%d: expected ts=%s; got %s", i, expKVs[i].ts, kv.Key.Timestamp)
}
if log.V(1) {
log.Infof("%d: %s", i, kv.Key)
}
}
// Verify that the last verification timestamp was updated as whole range was scanned.
if _, err := tc.rng.getLastVerificationTimestamp(); err != nil {
t.Fatal(err)
}
}
// TestGCQueueProcess creates test data in the range over various time
// scales and verifies that scan queue process properly GCs test data.
func TestGCQueueProcess(t *testing.T) {
defer leaktest.AfterTest(t)
tc := testContext{}
tc.Start(t)
defer tc.Stop()
const now int64 = 48 * 60 * 60 * 1E9 // 2d past the epoch
tc.manualClock.Set(now)
ts1 := makeTS(now-2*24*60*60*1E9+1, 0) // 2d old (add one nanosecond so we're not using zero timestamp)
ts2 := makeTS(now-25*60*60*1E9, 0) // GC will occur at time=25 hours
ts3 := makeTS(now-intentAgeThreshold.Nanoseconds(), 0) // 2h old
ts4 := makeTS(now-(intentAgeThreshold.Nanoseconds()-1), 0) // 2h-1ns old
ts5 := makeTS(now-1E9, 0) // 1s old
key1 := roachpb.Key("a")
key2 := roachpb.Key("b")
key3 := roachpb.Key("c")
key4 := roachpb.Key("d")
key5 := roachpb.Key("e")
key6 := roachpb.Key("f")
key7 := roachpb.Key("g")
key8 := roachpb.Key("h")
key9 := roachpb.Key("i")
data := []struct {
key roachpb.Key
ts roachpb.Timestamp
del bool
txn bool
}{
// For key1, we expect first two values to GC.
{key1, ts1, false, false},
{key1, ts2, false, false},
{key1, ts5, false, false},
// For key2, we expect all values to GC, because most recent is deletion.
{key2, ts1, false, false},
{key2, ts2, false, false},
{key2, ts5, true, false},
// For key3, we expect just ts1 to GC, because most recent deletion is intent.
{key3, ts1, false, false},
{key3, ts2, false, false},
{key3, ts5, true, true},
// For key4, expect oldest value to GC.
{key4, ts1, false, false},
{key4, ts2, false, false},
// For key5, expect all values to GC (most recent value deleted).
{key5, ts1, false, false},
{key5, ts2, true, false},
// For key6, expect no values to GC because most recent value is intent.
{key6, ts1, false, false},
{key6, ts5, false, true},
// For key7, expect no values to GC because intent is exactly 2h old.
{key7, ts2, false, false},
{key7, ts4, false, true},
// For key8, expect most recent value to resolve by aborting, which will clean it up.
{key8, ts2, false, false},
{key8, ts3, true, true},
// For key9, resolve naked intent with no remaining values.
{key9, ts3, true, false},
}
for i, datum := range data {
if datum.del {
dArgs := deleteArgs(datum.key)
var txn *roachpb.Transaction
if datum.txn {
txn = newTransaction("test", datum.key, 1, roachpb.SERIALIZABLE, tc.clock)
txn.OrigTimestamp = datum.ts
txn.Timestamp = datum.ts
}
if _, err := client.SendWrappedWith(tc.Sender(), tc.rng.context(), roachpb.BatchRequest_Header{
Timestamp: datum.ts,
Txn: txn,
}, &dArgs); err != nil {
t.Fatalf("%d: could not delete data: %s", i, err)
}
} else {
pArgs := putArgs(datum.key, []byte("value"))
var txn *roachpb.Transaction
if datum.txn {
txn = newTransaction("test", datum.key, 1, roachpb.SERIALIZABLE, tc.clock)
txn.OrigTimestamp = datum.ts
txn.Timestamp = datum.ts
}
if _, err := client.SendWrappedWith(tc.Sender(), tc.rng.context(), roachpb.BatchRequest_Header{
Timestamp: datum.ts,
Txn: txn,
}, &pArgs); err != nil {
t.Fatalf("%d: could not put data: %s", i, err)
}
}
}
cfg := tc.gossip.GetSystemConfig()
if cfg == nil {
t.Fatal("nil config")
}
// Process through a scan queue.
gcQ := newGCQueue(tc.gossip)
if err := gcQ.process(tc.clock.Now(), tc.rng, cfg); err != nil {
t.Fatal(err)
}
expKVs := []struct {
key roachpb.Key
ts roachpb.Timestamp
}{
{key1, roachpb.ZeroTimestamp},
{key1, ts5},
{key3, roachpb.ZeroTimestamp},
{key3, ts5},
{key3, ts2},
{key4, roachpb.ZeroTimestamp},
{key4, ts2},
{key6, roachpb.ZeroTimestamp},
{key6, ts5},
{key6, ts1},
{key7, roachpb.ZeroTimestamp},
{key7, ts4},
{key7, ts2},
{key8, roachpb.ZeroTimestamp},
{key8, ts2},
}
// Read data directly from engine to avoid intent errors from MVCC.
kvs, err := engine.Scan(tc.store.Engine(), engine.MVCCEncodeKey(key1), engine.MVCCEncodeKey(keys.TableDataPrefix), 0)
if err != nil {
t.Fatal(err)
}
for i, kv := range kvs {
if key, ts, isValue, err := engine.MVCCDecodeKey(kv.Key); isValue {
if err != nil {
t.Fatal(err)
}
if log.V(1) {
log.Infof("%d: %q, ts=%s", i, key, ts)
}
} else {
if log.V(1) {
log.Infof("%d: %q meta", i, key)
}
}
}
if len(kvs) != len(expKVs) {
t.Fatalf("expected length %d; got %d", len(expKVs), len(kvs))
}
for i, kv := range kvs {
key, ts, isValue, err := engine.MVCCDecodeKey(kv.Key)
if err != nil {
t.Fatal(err)
}
if !key.Equal(expKVs[i].key) {
t.Errorf("%d: expected key %q; got %q", i, expKVs[i].key, key)
}
if !ts.Equal(expKVs[i].ts) {
t.Errorf("%d: expected ts=%s; got %s", i, expKVs[i].ts, ts)
}
if isValue {
if log.V(1) {
log.Infof("%d: %q, ts=%s", i, key, ts)
}
} else {
if log.V(1) {
log.Infof("%d: %q meta", i, key)
}
}
}
// Verify the oldest extant intent age.
gcMeta, err := tc.rng.GetGCMetadata()
if err != nil {
t.Fatal(err)
}
if gcMeta.LastScanNanos != now {
t.Errorf("expected last scan nanos=%d; got %d", now, gcMeta.LastScanNanos)
}
if *gcMeta.OldestIntentNanos != ts4.WallTime {
t.Errorf("expected oldest intent nanos=%d; got %d", ts4.WallTime, gcMeta.OldestIntentNanos)
}
// Verify that the last verification timestamp was updated as whole range was scanned.
ts, err := tc.rng.GetLastVerificationTimestamp()
if err != nil {
t.Fatal(err)
}
if gcMeta.LastScanNanos != ts.WallTime {
t.Errorf("expected walltime nanos %d; got %d", gcMeta.LastScanNanos, ts.WallTime)
}
}