// TestReadTimestampCacheLayeredIntervals verifies the maximum
// timestamp is chosen if previous reads have ranges which are
// layered over each other.
func TestReadTimestampCacheLayeredIntervals(t *testing.T) {
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
clock.SetMaxDrift(maxClockSkew)
rtc := NewReadTimestampCache(clock)
manual = hlc.ManualClock(maxClockSkew.Nanoseconds() + 1)
adTS := clock.Now()
rtc.Add(engine.Key("a"), engine.Key("d"), adTS)
beTS := clock.Now()
rtc.Add(engine.Key("b"), engine.Key("e"), beTS)
cTS := clock.Now()
rtc.Add(engine.Key("c"), nil, cTS)
// Try different sub ranges.
if rtc.GetMax(engine.Key("a"), nil) != adTS {
t.Error("expected \"a\" to have adTS timestamp")
}
if rtc.GetMax(engine.Key("b"), nil) != beTS {
t.Error("expected \"b\" to have beTS timestamp")
}
if rtc.GetMax(engine.Key("c"), nil) != cTS {
t.Error("expected \"b\" to have cTS timestamp")
}
if rtc.GetMax(engine.Key("d"), nil) != beTS {
t.Error("expected \"d\" to have beTS timestamp")
}
if rtc.GetMax(engine.Key("a"), engine.Key("b")) != adTS {
t.Error("expected \"a\"-\"b\" to have adTS timestamp")
}
if rtc.GetMax(engine.Key("a"), engine.Key("c")) != beTS {
t.Error("expected \"a\"-\"c\" to have beTS timestamp")
}
if rtc.GetMax(engine.Key("a"), engine.Key("d")) != cTS {
t.Error("expected \"a\"-\"d\" to have cTS timestamp")
}
if rtc.GetMax(engine.Key("b"), engine.Key("d")) != cTS {
t.Error("expected \"b\"-\"d\" to have cTS timestamp")
}
if rtc.GetMax(engine.Key("c"), engine.Key("d")) != cTS {
t.Error("expected \"c\"-\"d\" to have cTS timestamp")
}
if rtc.GetMax(engine.Key("c0"), engine.Key("d")) != beTS {
t.Error("expected \"c0\"-\"d\" to have beTS timestamp")
}
}
// createTestRange creates a range using a blocking engine. Returns
// the range clock's manual unix nanos time and the range.
func createTestRangeWithClock(t *testing.T) (*Range, *hlc.ManualClock, *blockingEngine) {
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
engine := newBlockingEngine()
rng := NewRange(RangeMetadata{}, clock, engine, nil, nil)
rng.Start()
return rng, &manual, engine
}
// TestReadTimestampCacheEviction verifies the eviction of
// read timestamp cache entries after minCacheWindow interval.
func TestReadTimestampCacheEviction(t *testing.T) {
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
clock.SetMaxDrift(maxClockSkew)
rtc := NewReadTimestampCache(clock)
// Increment time to the maxClockSkew high water mark + 1.
manual = hlc.ManualClock(maxClockSkew.Nanoseconds() + 1)
aTS := clock.Now()
rtc.Add(engine.Key("a"), nil, aTS)
// Increment time by the minCacheWindow and add another key.
manual = hlc.ManualClock(int64(manual) + minCacheWindow.Nanoseconds())
rtc.Add(engine.Key("b"), nil, clock.Now())
// Verify looking up key "c" returns the new high water mark ("a"'s timestamp).
if rtc.GetMax(engine.Key("c"), nil) != aTS {
t.Error("expected high water mark %+v, got %+v", aTS, rtc.GetMax(engine.Key("c"), nil))
}
}
// createTestStore creates a test store using an in-memory
// engine. Returns the store clock's manual unix nanos time and the
// store. A single range from key "a" to key "z" is setup in the store
// with a default replica descriptor (i.e. StoreID = 0, RangeID = 1,
// etc.). The caller is responsible for closing the store on exit.
func createTestStore(t *testing.T) (*Store, *hlc.ManualClock) {
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
engine := NewInMem(Attributes{}, 1<<20)
store := NewStore(clock, engine, nil)
replica := Replica{RangeID: 1}
_, err := store.CreateRange(Key("a"), Key("z"), []Replica{replica})
if err != nil {
t.Fatal(err)
}
return store, &manual
}
func TestReadTimestampCacheClear(t *testing.T) {
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
clock.SetMaxDrift(maxClockSkew)
rtc := NewReadTimestampCache(clock)
// Increment time to the maxClockSkew high water mark + 1.
manual = hlc.ManualClock(maxClockSkew.Nanoseconds() + 1)
ts := clock.Now()
rtc.Add(engine.Key("a"), nil, ts)
// Clear the cache, which will reset the high water mark to
// the current time + maxClockSkew.
rtc.Clear()
// Fetching any keys should give current time + maxClockSkew
expTS := clock.Timestamp()
expTS.WallTime += maxClockSkew.Nanoseconds()
if rtc.GetMax(engine.Key("a"), nil) != expTS {
t.Error("expected \"a\" to have cleared timestamp")
}
}
// TestStoreExecuteCmdWithClockDrift verifies that if the request
// specifies a timestamp further into the future than the node's
// maximum allowed clock drift, the cmd fails with an error.
func TestStoreExecuteCmdWithClockDrift(t *testing.T) {
store, mc := createTestStore(t)
defer store.Close()
args, reply := getArgs("a", 1)
// Set clock to time 1.
*mc = hlc.ManualClock(1)
// Set clock max drift to 250ms.
maxDrift := 250 * time.Millisecond
store.clock.SetMaxDrift(maxDrift)
// Set args timestamp to exceed max drift.
args.Timestamp = store.clock.Now()
args.Timestamp.WallTime += maxDrift.Nanoseconds() + 1
err := store.ExecuteCmd("Get", &args.RequestHeader, args, reply)
if err == nil {
t.Error("expected max drift clock error")
}
}
// TestStoreExecuteCmdWithZeroTime verifies that no timestamp causes
// the command to assume the node's wall time.
func TestStoreExecuteCmdWithZeroTime(t *testing.T) {
store, mc := createTestStore(t)
defer store.Close()
args, reply := getArgs("a", 1)
// Set clock to time 1.
*mc = hlc.ManualClock(1)
err := store.ExecuteCmd("Get", &args.RequestHeader, args, reply)
if err != nil {
t.Fatal(err)
}
// The Logical time will increase over the course of the command
// execution so we can only rely on comparing the WallTime.
if reply.Timestamp.WallTime != store.clock.Timestamp().WallTime {
t.Errorf("expected reply to have store clock time %+v; got %+v",
store.clock.Timestamp(), reply.Timestamp)
}
}
// TestRangeUseTSCache verifies that write timestamps are upgraded
// based on the read timestamp cache.
func TestRangeUseTSCache(t *testing.T) {
rng, mc, _ := createTestRangeWithClock(t)
defer rng.Stop()
// Set clock to time 1s and do the read.
t0 := 1 * time.Second
*mc = hlc.ManualClock(t0.Nanoseconds())
args, reply := getArgs("a", 0)
args.Timestamp = rng.tsCache.clock.Now()
err := rng.ReadOnlyCmd("Get", args, reply)
if err != nil {
t.Error(err)
}
pArgs, pReply := putArgs("a", "value", 0)
err = rng.ReadWriteCmd("Put", pArgs, pReply)
if err != nil {
t.Fatal(err)
}
if pReply.Timestamp.WallTime != rng.tsCache.clock.Timestamp().WallTime {
t.Errorf("expected write timestamp to upgrade to 1s; got %+v", pReply.Timestamp)
}
}
// TestStoreInitAndBootstrap verifies store initialization and
// bootstrap.
func TestStoreInitAndBootstrap(t *testing.T) {
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
engine := NewInMem(Attributes{}, 1<<20)
store := NewStore(clock, engine, nil)
defer store.Close()
// Can't init as haven't bootstrapped.
if err := store.Init(); err == nil {
t.Error("expected failure init'ing un-bootstrapped store")
}
// Bootstrap with a fake ident.
if err := store.Bootstrap(testIdent); err != nil {
t.Errorf("error bootstrapping store: %v", err)
}
// Try to get 1st range--non-existent.
if _, err := store.GetRange(1); err == nil {
t.Error("expected error fetching non-existent range")
}
// Create range and fetch.
if _, err := store.CreateRange(KeyMin, KeyMax, []Replica{}); err != nil {
t.Errorf("failure to create first range: %v", err)
}
if _, err := store.GetRange(1); err != nil {
t.Errorf("failure fetching 1st range: %v", err)
}
// Now, attempt to initialize a store with a now-bootstrapped engine.
store = NewStore(clock, engine, nil)
if err := store.Init(); err != nil {
t.Errorf("failure initializing bootstrapped store: %v", err)
}
// 1st range should be available.
if _, err := store.GetRange(1); err != nil {
t.Errorf("failure fetching 1st range: %v", err)
}
}
// TestBootstrapOfNonEmptyStore verifies bootstrap failure if engine
// is not empty.
func TestBootstrapOfNonEmptyStore(t *testing.T) {
engine := NewInMem(Attributes{}, 1<<20)
// Put some random garbage into the engine.
if err := engine.put(Key("foo"), []byte("bar")); err != nil {
t.Errorf("failure putting key foo into engine: %v", err)
}
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
store := NewStore(clock, engine, nil)
defer store.Close()
// Can't init as haven't bootstrapped.
if err := store.Init(); err == nil {
t.Error("expected failure init'ing un-bootstrapped store")
}
// Bootstrap should fail on non-empty engine.
if err := store.Bootstrap(testIdent); err == nil {
t.Error("expected bootstrap error on non-empty store")
}
}
// TestRangeUpdateTSCache verifies that reads update the read
// timestamp cache.
func TestRangeUpdateTSCache(t *testing.T) {
rng, mc, _ := createTestRangeWithClock(t)
defer rng.Stop()
// Set clock to time 1s and do the read.
t0 := 1 * time.Second
*mc = hlc.ManualClock(t0.Nanoseconds())
args, reply := getArgs("a", 0)
args.Timestamp = rng.tsCache.clock.Now()
err := rng.ReadOnlyCmd("Get", args, reply)
if err != nil {
t.Error(err)
}
// Verify the read timestamp cache has 1sec for "a".
ts := rng.tsCache.GetMax(engine.Key("a"), nil)
if ts.WallTime != t0.Nanoseconds() {
t.Errorf("expected wall time to have 1s, but got %+v", ts)
}
// Verify another key ("b") has 0sec in timestamp cache.
ts = rng.tsCache.GetMax(engine.Key("b"), nil)
if ts.WallTime != 0 {
t.Errorf("expected wall time to have 0s, but got %+v", ts)
}
}
func TestReadTimestampCache(t *testing.T) {
manual := hlc.ManualClock(0)
clock := hlc.NewHLClock(manual.UnixNano)
clock.SetMaxDrift(maxClockSkew)
rtc := NewReadTimestampCache(clock)
// First simulate a read of just "a" at time 0.
rtc.Add(engine.Key("a"), nil, clock.Now())
// Verify GetMax returns the highWater mark which is maxClockSkew.
if rtc.GetMax(engine.Key("a"), nil).WallTime != maxClockSkew.Nanoseconds() {
t.Error("expected maxClockSkew for key \"a\"")
}
if rtc.GetMax(engine.Key("notincache"), nil).WallTime != maxClockSkew.Nanoseconds() {
t.Error("expected maxClockSkew for key \"notincache\"")
}
// Advance the clock and verify same high water mark.
manual = hlc.ManualClock(maxClockSkew.Nanoseconds() + 1)
if rtc.GetMax(engine.Key("a"), nil).WallTime != maxClockSkew.Nanoseconds() {
t.Error("expected maxClockSkew for key \"a\"")
}
if rtc.GetMax(engine.Key("notincache"), nil).WallTime != maxClockSkew.Nanoseconds() {
t.Error("expected maxClockSkew for key \"notincache\"")
}
// Sim a read of "b"-"c" at time maxClockSkew + 1.
ts := clock.Now()
rtc.Add(engine.Key("b"), engine.Key("c"), ts)
// Verify all permutations of direct and range access.
if rtc.GetMax(engine.Key("b"), nil) != ts {
t.Error("expected current time for key \"b\"; got %+v", rtc.GetMax(engine.Key("b"), nil))
}
if rtc.GetMax(engine.Key("bb"), nil) != ts {
t.Error("expected current time for key \"bb\"")
}
if rtc.GetMax(engine.Key("c"), nil).WallTime != maxClockSkew.Nanoseconds() {
t.Error("expected maxClockSkew for key \"c\"")
}
if rtc.GetMax(engine.Key("b"), engine.Key("c")) != ts {
t.Error("expected current time for key \"b\"-\"c\"")
}
if rtc.GetMax(engine.Key("bb"), engine.Key("bz")) != ts {
t.Error("expected current time for key \"bb\"-\"bz\"")
}
if rtc.GetMax(engine.Key("a"), engine.Key("b")).WallTime != maxClockSkew.Nanoseconds() {
t.Error("expected maxClockSkew for key \"a\"-\"b\"")
}
if rtc.GetMax(engine.Key("a"), engine.Key("bb")) != ts {
t.Error("expected current time for key \"a\"-\"bb\"")
}
if rtc.GetMax(engine.Key("a"), engine.Key("d")) != ts {
t.Error("expected current time for key \"a\"-\"d\"")
}
if rtc.GetMax(engine.Key("bz"), engine.Key("c")) != ts {
t.Error("expected current time for key \"bz\"-\"c\"")
}
if rtc.GetMax(engine.Key("bz"), engine.Key("d")) != ts {
t.Error("expected current time for key \"bz\"-\"d\"")
}
if rtc.GetMax(engine.Key("c"), engine.Key("d")).WallTime != maxClockSkew.Nanoseconds() {
t.Error("expected maxClockSkew for key \"c\"-\"d\"")
}
}