func TestDSCache(t *testing.T) {
t.Parallel()
zeroTime, err := time.Parse("2006-01-02T15:04:05.999999999Z", "2006-01-02T15:04:05.999999999Z")
if err != nil {
panic(err)
}
Convey("Test dscache", t, func() {
c := mathrand.Set(context.Background(), rand.New(rand.NewSource(1)))
clk := testclock.New(zeroTime)
c = clock.Set(c, clk)
c = memory.Use(c)
dsUnder := datastore.Get(c)
mc := memcache.Get(c)
shardsForKey := func(k *datastore.Key) int {
last := k.LastTok()
if last.Kind == "shardObj" {
return int(last.IntID)
}
if last.Kind == "noCacheObj" {
return 0
}
return DefaultShards
}
numMemcacheItems := func() uint64 {
stats, err := mc.Stats()
So(err, ShouldBeNil)
return stats.Items
}
Convey("enabled cases", func() {
c = FilterRDS(c, shardsForKey)
ds := datastore.Get(c)
So(dsUnder, ShouldNotBeNil)
So(ds, ShouldNotBeNil)
So(mc, ShouldNotBeNil)
Convey("basically works", func() {
pm := datastore.PropertyMap{
"BigData": {datastore.MkProperty([]byte(""))},
"Value": {datastore.MkProperty("hi")},
}
encoded := append([]byte{0}, serialize.ToBytes(pm)...)
o := object{ID: 1, Value: "hi"}
So(ds.Put(&o), ShouldBeNil)
o = object{ID: 1}
So(dsUnder.Get(&o), ShouldBeNil)
So(o.Value, ShouldEqual, "hi")
itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
So(err, ShouldEqual, memcache.ErrCacheMiss)
o = object{ID: 1}
So(ds.Get(&o), ShouldBeNil)
So(o.Value, ShouldEqual, "hi")
itm, err = mc.Get(itm.Key())
So(err, ShouldBeNil)
So(itm.Value(), ShouldResemble, encoded)
Convey("now we don't need the datastore!", func() {
o := object{ID: 1}
// delete it, bypassing the cache filter. Don't do this in production
// unless you want a crappy cache.
So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)
itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
So(err, ShouldBeNil)
So(itm.Value(), ShouldResemble, encoded)
So(ds.Get(&o), ShouldBeNil)
So(o.Value, ShouldEqual, "hi")
})
Convey("deleting it properly records that fact, however", func() {
o := object{ID: 1}
So(ds.Delete(ds.KeyForObj(&o)), ShouldBeNil)
itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
So(err, ShouldEqual, memcache.ErrCacheMiss)
So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)
itm, err = mc.Get(itm.Key())
So(err, ShouldBeNil)
So(itm.Value(), ShouldResemble, []byte{})
// this one hits memcache
So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)
})
})
Convey("compression works", func() {
o := object{ID: 2, Value: `¯\_(ツ)_/¯`}
data := make([]byte, 4000)
for i := range data {
const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()"
data[i] = alpha[i%len(alpha)]
}
o.BigData = data
So(ds.Put(&o), ShouldBeNil)
So(ds.Get(&o), ShouldBeNil)
itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
So(err, ShouldBeNil)
So(itm.Value()[0], ShouldEqual, ZlibCompression)
So(len(itm.Value()), ShouldEqual, 653) // a bit smaller than 4k
// ensure the next Get comes from the cache
So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)
o = object{ID: 2}
So(ds.Get(&o), ShouldBeNil)
So(o.Value, ShouldEqual, `¯\_(ツ)_/¯`)
So(o.BigData, ShouldResemble, data)
})
Convey("transactions", func() {
Convey("work", func() {
// populate an object @ ID1
So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
So(ds.Get(&object{ID: 1}), ShouldBeNil)
So(ds.Put(&object{ID: 2, Value: "nurbs"}), ShouldBeNil)
So(ds.Get(&object{ID: 2}), ShouldBeNil)
// memcache now has the wrong value (simulated race)
So(dsUnder.Put(&object{ID: 1, Value: "else"}), ShouldBeNil)
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
o := &object{ID: 1}
So(ds.Get(o), ShouldBeNil)
So(o.Value, ShouldEqual, "else")
o.Value = "txn"
So(ds.Put(o), ShouldBeNil)
So(ds.Delete(ds.KeyForObj(&object{ID: 2})), ShouldBeNil)
return nil
}, &datastore.TransactionOptions{XG: true}), ShouldBeNil)
_, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 1})))
So(err, ShouldEqual, memcache.ErrCacheMiss)
_, err = mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 2})))
So(err, ShouldEqual, memcache.ErrCacheMiss)
o := &object{ID: 1}
So(ds.Get(o), ShouldBeNil)
So(o.Value, ShouldEqual, "txn")
})
Convey("errors don't invalidate", func() {
// populate an object @ ID1
So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
So(ds.Get(&object{ID: 1}), ShouldBeNil)
So(numMemcacheItems(), ShouldEqual, 1)
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
o := &object{ID: 1}
So(ds.Get(o), ShouldBeNil)
So(o.Value, ShouldEqual, "something")
o.Value = "txn"
So(ds.Put(o), ShouldBeNil)
return errors.New("OH NOES")
}, nil).Error(), ShouldContainSubstring, "OH NOES")
// memcache still has the original
So(numMemcacheItems(), ShouldEqual, 1)
So(dsUnder.Delete(ds.KeyForObj(&object{ID: 1})), ShouldBeNil)
o := &object{ID: 1}
So(ds.Get(o), ShouldBeNil)
So(o.Value, ShouldEqual, "something")
})
})
Convey("control", func() {
Convey("per-model bypass", func() {
type model struct {
ID string `gae:"$id"`
UseDSCache datastore.Toggle `gae:"$dscache.enable,false"`
Value string
}
itms := []model{
{ID: "hi", Value: "something"},
{ID: "there", Value: "else", UseDSCache: datastore.On},
}
So(ds.PutMulti(itms), ShouldBeNil)
So(ds.GetMulti(itms), ShouldBeNil)
So(numMemcacheItems(), ShouldEqual, 1)
})
Convey("per-key shard count", func() {
s := &shardObj{ID: 4, Value: "hi"}
So(ds.Put(s), ShouldBeNil)
So(ds.Get(s), ShouldBeNil)
So(numMemcacheItems(), ShouldEqual, 1)
for i := 0; i < 20; i++ {
So(ds.Get(s), ShouldBeNil)
}
So(numMemcacheItems(), ShouldEqual, 4)
})
Convey("per-key cache disablement", func() {
n := &noCacheObj{ID: "nurbs", Value: true}
So(ds.Put(n), ShouldBeNil)
So(ds.Get(n), ShouldBeNil)
So(numMemcacheItems(), ShouldEqual, 0)
})
Convey("per-model expiration", func() {
type model struct {
ID int64 `gae:"$id"`
DSCacheExp int64 `gae:"$dscache.expiration,7"`
Value string
}
So(ds.Put(&model{ID: 1, Value: "mooo"}), ShouldBeNil)
So(ds.Get(&model{ID: 1}), ShouldBeNil)
itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&model{ID: 1})))
So(err, ShouldBeNil)
clk.Add(10 * time.Second)
_, err = mc.Get(itm.Key())
So(err, ShouldEqual, memcache.ErrCacheMiss)
})
})
Convey("screw cases", func() {
Convey("memcache contains bogus value (simulated failed AddMulti)", func() {
o := &object{ID: 1, Value: "spleen"}
So(ds.Put(o), ShouldBeNil)
sekret := []byte("I am a banana")
itm := mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).SetValue(sekret)
So(mc.Set(itm), ShouldBeNil)
o = &object{ID: 1}
So(ds.Get(o), ShouldBeNil)
So(o.Value, ShouldEqual, "spleen")
itm, err := mc.Get(itm.Key())
So(err, ShouldBeNil)
So(itm.Flags(), ShouldEqual, ItemUKNONWN)
So(itm.Value(), ShouldResemble, sekret)
})
Convey("memcache contains bogus value (corrupt entry)", func() {
o := &object{ID: 1, Value: "spleen"}
So(ds.Put(o), ShouldBeNil)
sekret := []byte("I am a banana")
itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
SetValue(sekret).
SetFlags(uint32(ItemHasData)))
So(mc.Set(itm), ShouldBeNil)
o = &object{ID: 1}
So(ds.Get(o), ShouldBeNil)
So(o.Value, ShouldEqual, "spleen")
itm, err := mc.Get(itm.Key())
So(err, ShouldBeNil)
So(itm.Flags(), ShouldEqual, ItemHasData)
So(itm.Value(), ShouldResemble, sekret)
})
Convey("other entity has the lock", func() {
o := &object{ID: 1, Value: "spleen"}
So(ds.Put(o), ShouldBeNil)
sekret := []byte("r@vmarod!#)%9T")
itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
SetValue(sekret).
SetFlags(uint32(ItemHasLock)))
So(mc.Set(itm), ShouldBeNil)
o = &object{ID: 1}
So(ds.Get(o), ShouldBeNil)
So(o.Value, ShouldEqual, "spleen")
itm, err := mc.Get(itm.Key())
So(err, ShouldBeNil)
So(itm.Flags(), ShouldEqual, ItemHasLock)
So(itm.Value(), ShouldResemble, sekret)
})
Convey("massive entities can't be cached", func() {
o := &object{ID: 1, Value: "spleen"}
mr := mathrand.Get(c)
numRounds := (internalValueSizeLimit / 8) * 2
buf := bytes.Buffer{}
for i := 0; i < numRounds; i++ {
So(binary.Write(&buf, binary.LittleEndian, mr.Int63()), ShouldBeNil)
}
o.BigData = buf.Bytes()
So(ds.Put(o), ShouldBeNil)
o.BigData = nil
So(ds.Get(o), ShouldBeNil)
itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(o)))
So(err, ShouldBeNil)
// Is locked until the next put, forcing all access to the datastore.
So(itm.Value(), ShouldResemble, []byte{})
So(itm.Flags(), ShouldEqual, ItemHasLock)
o.BigData = []byte("hi :)")
So(ds.Put(o), ShouldBeNil)
So(ds.Get(o), ShouldBeNil)
itm, err = mc.Get(itm.Key())
So(err, ShouldBeNil)
So(itm.Flags(), ShouldEqual, ItemHasData)
})
Convey("failure on Setting memcache locks is a hard stop", func() {
c, fb := featureBreaker.FilterMC(c, nil)
fb.BreakFeatures(nil, "SetMulti")
ds := datastore.Get(c)
So(ds.Put(&object{ID: 1}).Error(), ShouldContainSubstring, "SetMulti")
})
Convey("failure on Setting memcache locks in a transaction is a hard stop", func() {
c, fb := featureBreaker.FilterMC(c, nil)
fb.BreakFeatures(nil, "SetMulti")
ds := datastore.Get(c)
So(ds.RunInTransaction(func(c context.Context) error {
So(datastore.Get(c).Put(&object{ID: 1}), ShouldBeNil)
// no problems here... memcache operations happen after the function
// body quits.
return nil
}, nil).Error(), ShouldContainSubstring, "SetMulti")
})
})
Convey("misc", func() {
Convey("verify numShards caps at MaxShards", func() {
sc := supportContext{shardsForKey: shardsForKey}
So(sc.numShards(ds.KeyForObj(&shardObj{ID: 9001})), ShouldEqual, MaxShards)
})
Convey("CompressionType.String", func() {
So(NoCompression.String(), ShouldEqual, "NoCompression")
So(ZlibCompression.String(), ShouldEqual, "ZlibCompression")
So(CompressionType(100).String(), ShouldEqual, "UNKNOWN_CompressionType(100)")
})
})
})
Convey("disabled cases", func() {
defer func() {
globalEnabled = true
}()
So(IsGloballyEnabled(c), ShouldBeTrue)
So(SetGlobalEnable(c, false), ShouldBeNil)
// twice is a nop
So(SetGlobalEnable(c, false), ShouldBeNil)
// but it takes 5 minutes to kick in
So(IsGloballyEnabled(c), ShouldBeTrue)
clk.Add(time.Minute*5 + time.Second)
So(IsGloballyEnabled(c), ShouldBeFalse)
So(mc.Set(mc.NewItem("test").SetValue([]byte("hi"))), ShouldBeNil)
So(numMemcacheItems(), ShouldEqual, 1)
So(SetGlobalEnable(c, true), ShouldBeNil)
// memcache gets flushed as a side effect
So(numMemcacheItems(), ShouldEqual, 0)
// Still takes 5 minutes to kick in
So(IsGloballyEnabled(c), ShouldBeFalse)
clk.Add(time.Minute*5 + time.Second)
So(IsGloballyEnabled(c), ShouldBeTrue)
})
})
}
func TestPushBuffer(t *testing.T) {
Convey("Noop run", t, func() {
buf := NewPushBuffer(PushBufferOptions{})
So(buf.Stop(nil), ShouldBeNil)
})
Convey("Send one, then stop", t, func() {
cl := testclock.New(time.Time{})
client := &fakeClient{clock: cl}
buf := NewPushBuffer(PushBufferOptions{Client: client, Clock: cl})
buf.Add(Entry{})
So(buf.Stop(nil), ShouldBeNil)
So(len(client.getCalls()), ShouldEqual, 1)
})
Convey("Send a big chunk to trigger immediate flush, then stop.", t, func() {
cl := testclock.New(time.Time{})
client := &fakeClient{clock: cl}
buf := NewPushBuffer(PushBufferOptions{
Client: client,
Clock: cl,
FlushThreshold: 2,
})
buf.Add(Entry{}, Entry{}, Entry{}, Entry{})
cl.Add(time.Second)
So(buf.Stop(nil), ShouldBeNil)
So(len(client.getCalls()), ShouldEqual, 1)
So(client.getCalls()[0].ts, ShouldResemble, time.Time{}) // i.e. before Stop
})
Convey("Send a bunch of entries, wait for flush", t, func() {
cl := testclock.New(time.Time{})
client := &fakeClient{clock: cl, ch: make(chan pushEntriesCall)}
buf := NewPushBuffer(PushBufferOptions{Client: client, Clock: cl})
buf.Add(Entry{})
cl.Add(time.Second)
buf.Add(Entry{})
// Spin time until flush is called.
done := false
for !done {
select {
case <-client.ch:
done = true
default:
cl.Add(time.Second)
}
}
So(buf.Stop(nil), ShouldBeNil)
So(len(client.getCalls()), ShouldEqual, 1)
})
Convey("Retry works", t, func() {
cl := testclock.New(time.Time{})
cl.SetTimerCallback(func(clock.Timer) { cl.Add(1 * time.Second) })
client := &fakeClient{brokenReplies: 4, clock: cl}
buf := NewPushBuffer(PushBufferOptions{
Client: client,
Clock: cl,
MaxPushAttempts: 10,
PushRetryDelay: 1 * time.Second,
})
buf.Add(Entry{})
So(buf.Stop(nil), ShouldBeNil)
So(len(client.getCalls()), ShouldEqual, 5) // 4 failures, 1 success
})
Convey("Gives up retrying", t, func() {
cl := testclock.New(time.Time{})
cl.SetTimerCallback(func(clock.Timer) { cl.Add(1 * time.Second) })
client := &fakeClient{brokenReplies: 10000, clock: cl}
buf := NewPushBuffer(PushBufferOptions{
Client: client,
Clock: cl,
MaxPushAttempts: 5,
PushRetryDelay: 1 * time.Second,
})
buf.Add(Entry{})
So(buf.Stop(nil), ShouldNotBeNil)
So(len(client.calls), ShouldEqual, 5)
})
Convey("Stop timeout works", t, func() {
startTs := time.Time{}
cl := testclock.New(startTs)
cl.SetTimerCallback(func(clock.Timer) { cl.Add(1 * time.Second) })
client := &fakeClient{brokenReplies: 10000, clock: cl}
buf := NewPushBuffer(PushBufferOptions{
Client: client,
Clock: cl,
MaxPushAttempts: 5,
PushRetryDelay: 100 * time.Second,
StopTimeout: 1 * time.Second,
})
buf.Add(Entry{})
So(buf.Stop(nil), ShouldNotBeNil)
So(len(client.calls), ShouldEqual, 1)
So(cl.Now().Sub(startTs), ShouldBeLessThan, 100*time.Second)
})
}