// SetGlobalEnable is a convenience function for manipulating the GlobalConfig. // // It's meant to be called from admin handlers on your app to turn dscache // functionality on or off in emergencies. func SetGlobalEnable(c context.Context, memcacheEnabled bool) error { // always go to the default namespace c, err := info.Get(c).Namespace("") if err != nil { return err } return datastore.Get(c).RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) cfg := &GlobalConfig{Enable: true} if err := ds.Get(cfg); err != nil && err != datastore.ErrNoSuchEntity { return err } if cfg.Enable == memcacheEnabled { return nil } cfg.Enable = memcacheEnabled if memcacheEnabled { // when going false -> true, wipe memcache. if err := memcache.Get(c).Flush(); err != nil { return err } } return ds.Put(cfg) }, nil) }
// AlwaysFilterRDS installs a caching RawDatastore filter in the context. // // Unlike FilterRDS it doesn't check GlobalConfig via IsGloballyEnabled call, // assuming caller already knows whether filter should be applied or not. func AlwaysFilterRDS(c context.Context, shardsForKey func(*ds.Key) int) context.Context { return ds.AddRawFilters(c, func(c context.Context, ds ds.RawInterface) ds.RawInterface { i := info.Get(c) sc := &supportContext{ i.AppID(), i.GetNamespace(), c, mc.Get(c), mathrand.Get(c), shardsForKey, } v := c.Value(dsTxnCacheKey) if v == nil { return &dsCache{ds, sc} } return &dsTxnCache{ds, v.(*dsTxnState), sc} }) }
func TestContextAccess(t *testing.T) { t.Parallel() // p is a function which recovers an error and then immediately panics with // the contained string. It's defer'd in each test so that we can use the // ShouldPanicWith assertion (which does an == comparison and not // a reflect.DeepEquals comparison). p := func() { panic(recover().(error).Error()) } Convey("Context Access", t, func() { c := context.Background() Convey("blank", func() { So(dsS.GetRaw(c), ShouldBeNil) So(mcS.GetRaw(c), ShouldBeNil) So(tqS.GetRaw(c), ShouldBeNil) So(infoS.Get(c), ShouldBeNil) }) // needed for everything else c = infoS.Set(c, Info()) Convey("Info", func() { So(infoS.Get(c), ShouldNotBeNil) So(func() { defer p() infoS.Get(c).Datacenter() }, ShouldPanicWith, "dummy: method Info.Datacenter is not implemented") }) Convey("Datastore", func() { c = dsS.SetRaw(c, Datastore()) So(dsS.Get(c), ShouldNotBeNil) So(func() { defer p() _, _ = dsS.Get(c).DecodeCursor("wut") }, ShouldPanicWith, "dummy: method Datastore.DecodeCursor is not implemented") }) Convey("Memcache", func() { c = mcS.SetRaw(c, Memcache()) So(mcS.Get(c), ShouldNotBeNil) So(func() { defer p() _ = mcS.Get(c).Add(nil) }, ShouldPanicWith, "dummy: method Memcache.AddMulti is not implemented") }) Convey("TaskQueue", func() { c = tqS.SetRaw(c, TaskQueue()) So(tqS.Get(c), ShouldNotBeNil) So(func() { defer p() _ = tqS.Get(c).Purge("") }, ShouldPanicWith, "dummy: method TaskQueue.Purge is not implemented") }) Convey("User", func() { c = userS.Set(c, User()) So(userS.Get(c), ShouldNotBeNil) So(func() { defer p() _ = userS.Get(c).IsAdmin() }, ShouldPanicWith, "dummy: method User.IsAdmin is not implemented") }) Convey("Mail", func() { c = mailS.Set(c, Mail()) So(mailS.Get(c), ShouldNotBeNil) So(func() { defer p() _ = mailS.Get(c).Send(nil) }, ShouldPanicWith, "dummy: method Mail.Send is not implemented") }) Convey("Module", func() { c = modS.Set(c, Module()) So(modS.Get(c), ShouldNotBeNil) So(func() { defer p() modS.Get(c).List() }, ShouldPanicWith, "dummy: method Module.List is not implemented") }) }) }
func TestCount(t *testing.T) { t.Parallel() Convey("Test Count filter", t, func() { c, fb := featureBreaker.FilterRDS(memory.Use(context.Background()), nil) c, ctr := FilterRDS(c) So(c, ShouldNotBeNil) So(ctr, ShouldNotBeNil) ds := datastore.Get(c) vals := []datastore.PropertyMap{{ "Val": {datastore.MkProperty(100)}, "$key": {datastore.MkPropertyNI(ds.NewKey("Kind", "", 1, nil))}, }} Convey("Calling a ds function should reflect in counter", func() { So(ds.PutMulti(vals), ShouldBeNil) So(ctr.PutMulti.Successes(), ShouldEqual, 1) Convey("effects are cumulative", func() { So(ds.PutMulti(vals), ShouldBeNil) So(ctr.PutMulti.Successes(), ShouldEqual, 2) Convey("even within transactions", func() { die(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) So(ds.PutMulti(append(vals, vals[0])), ShouldBeNil) return nil }, nil)) }) }) }) Convey("errors count against errors", func() { fb.BreakFeatures(nil, "GetMulti") So(ds.GetMulti(vals), ShouldErrLike, `"GetMulti" is broken`) So(ctr.GetMulti.Errors(), ShouldEqual, 1) fb.UnbreakFeatures("GetMulti") So(ds.PutMulti(vals), ShouldBeNil) die(ds.GetMulti(vals)) So(ctr.GetMulti.Errors(), ShouldEqual, 1) So(ctr.GetMulti.Successes(), ShouldEqual, 1) So(ctr.GetMulti.Total(), ShouldEqual, 2) }) }) Convey("works for memcache", t, func() { c, ctr := FilterMC(memory.Use(context.Background())) So(c, ShouldNotBeNil) So(ctr, ShouldNotBeNil) mc := memcache.Get(c) die(mc.Set(mc.NewItem("hello").SetValue([]byte("sup")))) _, err := mc.Get("Wat") So(err, ShouldNotBeNil) _, err = mc.Get("hello") die(err) So(ctr.SetMulti, shouldHaveSuccessesAndErrors, 1, 0) So(ctr.GetMulti, shouldHaveSuccessesAndErrors, 2, 0) So(ctr.NewItem, shouldHaveSuccessesAndErrors, 3, 0) }) Convey("works for taskqueue", t, func() { c, ctr := FilterTQ(memory.Use(context.Background())) So(c, ShouldNotBeNil) So(ctr, ShouldNotBeNil) tq := taskqueue.Get(c) die(tq.Add(&taskqueue.Task{Name: "wat"}, "")) So(tq.Add(&taskqueue.Task{Name: "wat"}, "DNE_QUEUE"), ShouldErrLike, "UNKNOWN_QUEUE") So(ctr.AddMulti, shouldHaveSuccessesAndErrors, 1, 1) }) Convey("works for global info", t, func() { c, fb := featureBreaker.FilterGI(memory.Use(context.Background()), nil) c, ctr := FilterGI(c) So(c, ShouldNotBeNil) So(ctr, ShouldNotBeNil) gi := info.Get(c) _, err := gi.Namespace("foo") die(err) fb.BreakFeatures(nil, "Namespace") _, err = gi.Namespace("boom") So(err, ShouldErrLike, `"Namespace" is broken`) So(ctr.Namespace, shouldHaveSuccessesAndErrors, 1, 1) }) Convey("works for user", t, func() { c, fb := featureBreaker.FilterUser(memory.Use(context.Background()), nil) c, ctr := FilterUser(c) So(c, ShouldNotBeNil) So(ctr, ShouldNotBeNil) u := user.Get(c) _, err := u.CurrentOAuth("foo") die(err) fb.BreakFeatures(nil, "CurrentOAuth") _, err = u.CurrentOAuth("foo") So(err, ShouldErrLike, `"CurrentOAuth" is broken`) So(ctr.CurrentOAuth, shouldHaveSuccessesAndErrors, 1, 1) }) Convey("works for mail", t, func() { c, fb := featureBreaker.FilterMail(memory.Use(context.Background()), nil) c, ctr := FilterMail(c) So(c, ShouldNotBeNil) So(ctr, ShouldNotBeNil) m := mail.Get(c) err := m.Send(&mail.Message{ Sender: "*****@*****.**", To: []string{"*****@*****.**"}, Body: "hi", }) die(err) fb.BreakFeatures(nil, "Send") err = m.Send(&mail.Message{ Sender: "*****@*****.**", To: []string{"*****@*****.**"}, Body: "hi", }) So(err, ShouldErrLike, `"Send" is broken`) So(ctr.Send, shouldHaveSuccessesAndErrors, 1, 1) }) }
func TestDSCache(t *testing.T) { t.Parallel() zeroTime, err := time.Parse("2006-01-02T15:04:05.999999999Z", "2006-01-02T15:04:05.999999999Z") if err != nil { panic(err) } Convey("Test dscache", t, func() { c := mathrand.Set(context.Background(), rand.New(rand.NewSource(1))) clk := testclock.New(zeroTime) c = clock.Set(c, clk) c = memory.Use(c) dsUnder := datastore.Get(c) mc := memcache.Get(c) shardsForKey := func(k *datastore.Key) int { last := k.LastTok() if last.Kind == "shardObj" { return int(last.IntID) } if last.Kind == "noCacheObj" { return 0 } return DefaultShards } numMemcacheItems := func() uint64 { stats, err := mc.Stats() So(err, ShouldBeNil) return stats.Items } Convey("enabled cases", func() { c = FilterRDS(c, shardsForKey) ds := datastore.Get(c) So(dsUnder, ShouldNotBeNil) So(ds, ShouldNotBeNil) So(mc, ShouldNotBeNil) Convey("basically works", func() { pm := datastore.PropertyMap{ "BigData": {datastore.MkProperty([]byte(""))}, "Value": {datastore.MkProperty("hi")}, } encoded := append([]byte{0}, serialize.ToBytes(pm)...) o := object{ID: 1, Value: "hi"} So(ds.Put(&o), ShouldBeNil) o = object{ID: 1} So(dsUnder.Get(&o), ShouldBeNil) So(o.Value, ShouldEqual, "hi") itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o))) So(err, ShouldEqual, memcache.ErrCacheMiss) o = object{ID: 1} So(ds.Get(&o), ShouldBeNil) So(o.Value, ShouldEqual, "hi") itm, err = mc.Get(itm.Key()) So(err, ShouldBeNil) So(itm.Value(), ShouldResemble, encoded) Convey("now we don't need the datastore!", func() { o := object{ID: 1} // delete it, bypassing the cache filter. Don't do this in production // unless you want a crappy cache. So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil) itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o))) So(err, ShouldBeNil) So(itm.Value(), ShouldResemble, encoded) So(ds.Get(&o), ShouldBeNil) So(o.Value, ShouldEqual, "hi") }) Convey("deleting it properly records that fact, however", func() { o := object{ID: 1} So(ds.Delete(ds.KeyForObj(&o)), ShouldBeNil) itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o))) So(err, ShouldEqual, memcache.ErrCacheMiss) So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity) itm, err = mc.Get(itm.Key()) So(err, ShouldBeNil) So(itm.Value(), ShouldResemble, []byte{}) // this one hits memcache So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity) }) }) Convey("compression works", func() { o := object{ID: 2, Value: `¯\_(ツ)_/¯`} data := make([]byte, 4000) for i := range data { const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()" data[i] = alpha[i%len(alpha)] } o.BigData = data So(ds.Put(&o), ShouldBeNil) So(ds.Get(&o), ShouldBeNil) itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o))) So(err, ShouldBeNil) So(itm.Value()[0], ShouldEqual, ZlibCompression) So(len(itm.Value()), ShouldEqual, 653) // a bit smaller than 4k // ensure the next Get comes from the cache So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil) o = object{ID: 2} So(ds.Get(&o), ShouldBeNil) So(o.Value, ShouldEqual, `¯\_(ツ)_/¯`) So(o.BigData, ShouldResemble, data) }) Convey("transactions", func() { Convey("work", func() { // populate an object @ ID1 So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil) So(ds.Get(&object{ID: 1}), ShouldBeNil) So(ds.Put(&object{ID: 2, Value: "nurbs"}), ShouldBeNil) So(ds.Get(&object{ID: 2}), ShouldBeNil) // memcache now has the wrong value (simulated race) So(dsUnder.Put(&object{ID: 1, Value: "else"}), ShouldBeNil) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) o := &object{ID: 1} So(ds.Get(o), ShouldBeNil) So(o.Value, ShouldEqual, "else") o.Value = "txn" So(ds.Put(o), ShouldBeNil) So(ds.Delete(ds.KeyForObj(&object{ID: 2})), ShouldBeNil) return nil }, &datastore.TransactionOptions{XG: true}), ShouldBeNil) _, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 1}))) So(err, ShouldEqual, memcache.ErrCacheMiss) _, err = mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 2}))) So(err, ShouldEqual, memcache.ErrCacheMiss) o := &object{ID: 1} So(ds.Get(o), ShouldBeNil) So(o.Value, ShouldEqual, "txn") }) Convey("errors don't invalidate", func() { // populate an object @ ID1 So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil) So(ds.Get(&object{ID: 1}), ShouldBeNil) So(numMemcacheItems(), ShouldEqual, 1) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) o := &object{ID: 1} So(ds.Get(o), ShouldBeNil) So(o.Value, ShouldEqual, "something") o.Value = "txn" So(ds.Put(o), ShouldBeNil) return errors.New("OH NOES") }, nil).Error(), ShouldContainSubstring, "OH NOES") // memcache still has the original So(numMemcacheItems(), ShouldEqual, 1) So(dsUnder.Delete(ds.KeyForObj(&object{ID: 1})), ShouldBeNil) o := &object{ID: 1} So(ds.Get(o), ShouldBeNil) So(o.Value, ShouldEqual, "something") }) }) Convey("control", func() { Convey("per-model bypass", func() { type model struct { ID string `gae:"$id"` UseDSCache datastore.Toggle `gae:"$dscache.enable,false"` Value string } itms := []model{ {ID: "hi", Value: "something"}, {ID: "there", Value: "else", UseDSCache: datastore.On}, } So(ds.PutMulti(itms), ShouldBeNil) So(ds.GetMulti(itms), ShouldBeNil) So(numMemcacheItems(), ShouldEqual, 1) }) Convey("per-key shard count", func() { s := &shardObj{ID: 4, Value: "hi"} So(ds.Put(s), ShouldBeNil) So(ds.Get(s), ShouldBeNil) So(numMemcacheItems(), ShouldEqual, 1) for i := 0; i < 20; i++ { So(ds.Get(s), ShouldBeNil) } So(numMemcacheItems(), ShouldEqual, 4) }) Convey("per-key cache disablement", func() { n := &noCacheObj{ID: "nurbs", Value: true} So(ds.Put(n), ShouldBeNil) So(ds.Get(n), ShouldBeNil) So(numMemcacheItems(), ShouldEqual, 0) }) Convey("per-model expiration", func() { type model struct { ID int64 `gae:"$id"` DSCacheExp int64 `gae:"$dscache.expiration,7"` Value string } So(ds.Put(&model{ID: 1, Value: "mooo"}), ShouldBeNil) So(ds.Get(&model{ID: 1}), ShouldBeNil) itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&model{ID: 1}))) So(err, ShouldBeNil) clk.Add(10 * time.Second) _, err = mc.Get(itm.Key()) So(err, ShouldEqual, memcache.ErrCacheMiss) }) }) Convey("screw cases", func() { Convey("memcache contains bogus value (simulated failed AddMulti)", func() { o := &object{ID: 1, Value: "spleen"} So(ds.Put(o), ShouldBeNil) sekret := []byte("I am a banana") itm := mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).SetValue(sekret) So(mc.Set(itm), ShouldBeNil) o = &object{ID: 1} So(ds.Get(o), ShouldBeNil) So(o.Value, ShouldEqual, "spleen") itm, err := mc.Get(itm.Key()) So(err, ShouldBeNil) So(itm.Flags(), ShouldEqual, ItemUKNONWN) So(itm.Value(), ShouldResemble, sekret) }) Convey("memcache contains bogus value (corrupt entry)", func() { o := &object{ID: 1, Value: "spleen"} So(ds.Put(o), ShouldBeNil) sekret := []byte("I am a banana") itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))). SetValue(sekret). SetFlags(uint32(ItemHasData))) So(mc.Set(itm), ShouldBeNil) o = &object{ID: 1} So(ds.Get(o), ShouldBeNil) So(o.Value, ShouldEqual, "spleen") itm, err := mc.Get(itm.Key()) So(err, ShouldBeNil) So(itm.Flags(), ShouldEqual, ItemHasData) So(itm.Value(), ShouldResemble, sekret) }) Convey("other entity has the lock", func() { o := &object{ID: 1, Value: "spleen"} So(ds.Put(o), ShouldBeNil) sekret := []byte("r@vmarod!#)%9T") itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))). SetValue(sekret). SetFlags(uint32(ItemHasLock))) So(mc.Set(itm), ShouldBeNil) o = &object{ID: 1} So(ds.Get(o), ShouldBeNil) So(o.Value, ShouldEqual, "spleen") itm, err := mc.Get(itm.Key()) So(err, ShouldBeNil) So(itm.Flags(), ShouldEqual, ItemHasLock) So(itm.Value(), ShouldResemble, sekret) }) Convey("massive entities can't be cached", func() { o := &object{ID: 1, Value: "spleen"} mr := mathrand.Get(c) numRounds := (internalValueSizeLimit / 8) * 2 buf := bytes.Buffer{} for i := 0; i < numRounds; i++ { So(binary.Write(&buf, binary.LittleEndian, mr.Int63()), ShouldBeNil) } o.BigData = buf.Bytes() So(ds.Put(o), ShouldBeNil) o.BigData = nil So(ds.Get(o), ShouldBeNil) itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(o))) So(err, ShouldBeNil) // Is locked until the next put, forcing all access to the datastore. So(itm.Value(), ShouldResemble, []byte{}) So(itm.Flags(), ShouldEqual, ItemHasLock) o.BigData = []byte("hi :)") So(ds.Put(o), ShouldBeNil) So(ds.Get(o), ShouldBeNil) itm, err = mc.Get(itm.Key()) So(err, ShouldBeNil) So(itm.Flags(), ShouldEqual, ItemHasData) }) Convey("failure on Setting memcache locks is a hard stop", func() { c, fb := featureBreaker.FilterMC(c, nil) fb.BreakFeatures(nil, "SetMulti") ds := datastore.Get(c) So(ds.Put(&object{ID: 1}).Error(), ShouldContainSubstring, "SetMulti") }) Convey("failure on Setting memcache locks in a transaction is a hard stop", func() { c, fb := featureBreaker.FilterMC(c, nil) fb.BreakFeatures(nil, "SetMulti") ds := datastore.Get(c) So(ds.RunInTransaction(func(c context.Context) error { So(datastore.Get(c).Put(&object{ID: 1}), ShouldBeNil) // no problems here... memcache operations happen after the function // body quits. return nil }, nil).Error(), ShouldContainSubstring, "SetMulti") }) }) Convey("misc", func() { Convey("verify numShards caps at MaxShards", func() { sc := supportContext{shardsForKey: shardsForKey} So(sc.numShards(ds.KeyForObj(&shardObj{ID: 9001})), ShouldEqual, MaxShards) }) Convey("CompressionType.String", func() { So(NoCompression.String(), ShouldEqual, "NoCompression") So(ZlibCompression.String(), ShouldEqual, "ZlibCompression") So(CompressionType(100).String(), ShouldEqual, "UNKNOWN_CompressionType(100)") }) }) }) Convey("disabled cases", func() { defer func() { globalEnabled = true }() So(IsGloballyEnabled(c), ShouldBeTrue) So(SetGlobalEnable(c, false), ShouldBeNil) // twice is a nop So(SetGlobalEnable(c, false), ShouldBeNil) // but it takes 5 minutes to kick in So(IsGloballyEnabled(c), ShouldBeTrue) clk.Add(time.Minute*5 + time.Second) So(IsGloballyEnabled(c), ShouldBeFalse) So(mc.Set(mc.NewItem("test").SetValue([]byte("hi"))), ShouldBeNil) So(numMemcacheItems(), ShouldEqual, 1) So(SetGlobalEnable(c, true), ShouldBeNil) // memcache gets flushed as a side effect So(numMemcacheItems(), ShouldEqual, 0) // Still takes 5 minutes to kick in So(IsGloballyEnabled(c), ShouldBeFalse) clk.Add(time.Minute*5 + time.Second) So(IsGloballyEnabled(c), ShouldBeTrue) }) }) }
func TestBasicDatastore(t *testing.T) { t.Parallel() Convey("basic", t, func() { inst, err := aetest.NewInstance(&aetest.Options{ StronglyConsistentDatastore: true, }) So(err, ShouldBeNil) defer inst.Close() req, err := inst.NewRequest("GET", "/", nil) So(err, ShouldBeNil) ctx := Use(context.Background(), req) ds := datastore.Get(ctx) mc := memcache.Get(ctx) inf := info.Get(ctx) Convey("logging allows you to tweak the level", func() { // You have to visually confirm that this actually happens in the stdout // of the test... yeah I know. logging.Debugf(ctx, "SHOULD NOT SEE") logging.Infof(ctx, "SHOULD SEE") ctx = logging.SetLevel(ctx, logging.Debug) logging.Debugf(ctx, "SHOULD SEE") logging.Infof(ctx, "SHOULD SEE (2)") }) Convey("Can probe/change Namespace", func() { So(inf.GetNamespace(), ShouldEqual, "") ctx, err = inf.Namespace("wat") So(err, ShouldBeNil) inf = info.Get(ctx) So(inf.GetNamespace(), ShouldEqual, "wat") ds = datastore.Get(ctx) So(ds.MakeKey("Hello", "world").Namespace(), ShouldEqual, "wat") }) Convey("Can get non-transactional context", func() { ctx, err := inf.Namespace("foo") So(err, ShouldBeNil) ds = datastore.Get(ctx) inf = info.Get(ctx) ds.RunInTransaction(func(ctx context.Context) error { So(ds.MakeKey("Foo", "bar").Namespace(), ShouldEqual, "foo") So(ds.Put(&TestStruct{ValueI: []int64{100}}), ShouldBeNil) err = datastore.GetNoTxn(ctx).RunInTransaction(func(ctx context.Context) error { ds = datastore.Get(ctx) So(ds.MakeKey("Foo", "bar").Namespace(), ShouldEqual, "foo") So(ds.Put(&TestStruct{ValueI: []int64{100}}), ShouldBeNil) return nil }, nil) So(err, ShouldBeNil) return nil }, nil) }) Convey("Can Put/Get", func() { orig := TestStruct{ ValueI: []int64{1, 7, 946688461000000, 996688461000000}, ValueB: []bool{true, false}, ValueS: []string{"hello", "world"}, ValueF: []float64{1.0, 7.0, 946688461000000.0, 996688461000000.0}, ValueBS: [][]byte{ []byte("allo"), []byte("hello"), []byte("world"), []byte("zurple"), }, ValueK: []*datastore.Key{ ds.NewKey("Something", "Cool", 0, nil), ds.NewKey("Something", "", 1, nil), ds.NewKey("Something", "Recursive", 0, ds.NewKey("Parent", "", 2, nil)), }, ValueBK: []blobstore.Key{"bellow", "hello"}, ValueGP: []datastore.GeoPoint{ {Lat: 120.7, Lng: 95.5}, }, } So(ds.Put(&orig), ShouldBeNil) ret := TestStruct{ID: orig.ID} So(ds.Get(&ret), ShouldBeNil) So(ret, ShouldResemble, orig) // can't be sure the indexes have caught up... so sleep time.Sleep(time.Second) Convey("Can query", func() { q := datastore.NewQuery("TestStruct") ds.Run(q, func(ts *TestStruct) { So(*ts, ShouldResemble, orig) }) count, err := ds.Count(q) So(err, ShouldBeNil) So(count, ShouldEqual, 1) }) Convey("Can project", func() { q := datastore.NewQuery("TestStruct").Project("ValueS") rslts := []datastore.PropertyMap{} So(ds.GetAll(q, &rslts), ShouldBeNil) So(rslts, ShouldResemble, []datastore.PropertyMap{ { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueS": {mp("hello")}, }, { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueS": {mp("world")}, }, }) q = datastore.NewQuery("TestStruct").Project("ValueBS") rslts = []datastore.PropertyMap{} So(ds.GetAll(q, &rslts), ShouldBeNil) So(rslts, ShouldResemble, []datastore.PropertyMap{ { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueBS": {mp("allo")}, }, { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueBS": {mp("hello")}, }, { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueBS": {mp("world")}, }, { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueBS": {mp("zurple")}, }, }) count, err := ds.Count(q) So(err, ShouldBeNil) So(count, ShouldEqual, 4) q = datastore.NewQuery("TestStruct").Lte("ValueI", 7).Project("ValueS").Distinct(true) rslts = []datastore.PropertyMap{} So(ds.GetAll(q, &rslts), ShouldBeNil) So(rslts, ShouldResemble, []datastore.PropertyMap{ { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueI": {mp(1)}, "ValueS": {mp("hello")}, }, { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueI": {mp(1)}, "ValueS": {mp("world")}, }, { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueI": {mp(7)}, "ValueS": {mp("hello")}, }, { "$key": {mpNI(ds.KeyForObj(&orig))}, "ValueI": {mp(7)}, "ValueS": {mp("world")}, }, }) count, err = ds.Count(q) So(err, ShouldBeNil) So(count, ShouldEqual, 4) }) }) Convey("Can Put/Get (time)", func() { // time comparisons in Go are wonky, so this is pulled out pm := datastore.PropertyMap{ "$key": {mpNI(ds.NewKey("Something", "value", 0, nil))}, "Time": { mp(time.Date(1938, time.January, 1, 1, 1, 1, 1, time.UTC)), mp(time.Time{}), }, } So(ds.Put(&pm), ShouldBeNil) rslt := datastore.PropertyMap{} rslt.SetMeta("key", ds.KeyForObj(pm)) So(ds.Get(&rslt), ShouldBeNil) So(pm["Time"][0].Value(), ShouldResemble, rslt["Time"][0].Value()) q := datastore.NewQuery("Something").Project("Time") all := []datastore.PropertyMap{} So(ds.GetAll(q, &all), ShouldBeNil) So(len(all), ShouldEqual, 2) prop := all[0]["Time"][0] So(prop.Type(), ShouldEqual, datastore.PTInt) tval, err := prop.Project(datastore.PTTime) So(err, ShouldBeNil) So(tval, ShouldResemble, time.Time{}.UTC()) tval, err = all[1]["Time"][0].Project(datastore.PTTime) So(err, ShouldBeNil) So(tval, ShouldResemble, pm["Time"][0].Value()) ent := datastore.PropertyMap{ "$key": {mpNI(ds.MakeKey("Something", "value"))}, } So(ds.Get(&ent), ShouldBeNil) So(ent["Time"], ShouldResemble, pm["Time"]) }) Convey("memcache: Set (nil) is the same as Set ([]byte{})", func() { So(mc.Set(mc.NewItem("bob")), ShouldBeNil) // normally would panic because Value is nil bob, err := mc.Get("bob") So(err, ShouldBeNil) So(bob.Value(), ShouldResemble, []byte{}) }) }) }
func TestMemcache(t *testing.T) { t.Parallel() Convey("memcache", t, func() { now := time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC) c, tc := testclock.UseTime(context.Background(), now) c = Use(c) mc := mcS.Get(c) Convey("implements MCSingleReadWriter", func() { Convey("Add", func() { itm := (mc.NewItem("sup"). SetValue([]byte("cool")). SetExpiration(time.Second)) So(mc.Add(itm), ShouldBeNil) Convey("which rejects objects already there", func() { So(mc.Add(itm), ShouldEqual, mcS.ErrNotStored) }) }) Convey("Get", func() { itm := &mcItem{ key: "sup", value: []byte("cool"), expiration: time.Second, } So(mc.Add(itm), ShouldBeNil) testItem := &mcItem{ key: "sup", value: []byte("cool"), CasID: 1, } getItm, err := mc.Get("sup") So(err, ShouldBeNil) So(getItm, ShouldResemble, testItem) Convey("which can expire", func() { tc.Add(time.Second * 4) getItm, err := mc.Get("sup") So(err, ShouldEqual, mcS.ErrCacheMiss) So(getItm, ShouldResemble, &mcItem{key: "sup"}) }) }) Convey("Delete", func() { Convey("works if it's there", func() { itm := &mcItem{ key: "sup", value: []byte("cool"), expiration: time.Second, } So(mc.Add(itm), ShouldBeNil) So(mc.Delete("sup"), ShouldBeNil) _, err := mc.Get("sup") So(err, ShouldEqual, mcS.ErrCacheMiss) }) Convey("but not if it's not there", func() { So(mc.Delete("sup"), ShouldEqual, mcS.ErrCacheMiss) }) }) Convey("Set", func() { itm := &mcItem{ key: "sup", value: []byte("cool"), expiration: time.Second, } So(mc.Add(itm), ShouldBeNil) itm.SetValue([]byte("newp")) So(mc.Set(itm), ShouldBeNil) testItem := &mcItem{ key: "sup", value: []byte("newp"), CasID: 2, } getItm, err := mc.Get("sup") So(err, ShouldBeNil) So(getItm, ShouldResemble, testItem) Convey("Flush works too", func() { So(mc.Flush(), ShouldBeNil) _, err := mc.Get("sup") So(err, ShouldEqual, mcS.ErrCacheMiss) }) }) Convey("Set (nil) is equivalent to Set([]byte{})", func() { So(mc.Set(mc.NewItem("bob")), ShouldBeNil) bob, err := mc.Get("bob") So(err, ShouldBeNil) So(bob.Value(), ShouldResemble, []byte{}) }) Convey("Increment", func() { val, err := mc.Increment("num", 7, 2) So(err, ShouldBeNil) So(val, ShouldEqual, 9) Convey("IncrementExisting", func() { val, err := mc.IncrementExisting("num", -2) So(err, ShouldBeNil) So(val, ShouldEqual, 7) val, err = mc.IncrementExisting("num", -100) So(err, ShouldBeNil) So(val, ShouldEqual, 0) _, err = mc.IncrementExisting("noexist", 2) So(err, ShouldEqual, mcS.ErrCacheMiss) So(mc.Set(mc.NewItem("text").SetValue([]byte("hello world, hooman!"))), ShouldBeNil) _, err = mc.IncrementExisting("text", 2) So(err.Error(), ShouldContainSubstring, "got invalid current value") }) }) Convey("CompareAndSwap", func() { itm := mcS.Item(&mcItem{ key: "sup", value: []byte("cool"), expiration: time.Second * 2, }) So(mc.Add(itm), ShouldBeNil) Convey("works after a Get", func() { itm, err := mc.Get("sup") So(err, ShouldBeNil) So(itm.(*mcItem).CasID, ShouldEqual, 1) itm.SetValue([]byte("newp")) So(mc.CompareAndSwap(itm), ShouldBeNil) }) Convey("but fails if you don't", func() { itm.SetValue([]byte("newp")) So(mc.CompareAndSwap(itm), ShouldEqual, mcS.ErrCASConflict) }) Convey("and fails if the item is expired/gone", func() { tc.Add(3 * time.Second) itm.SetValue([]byte("newp")) So(mc.CompareAndSwap(itm), ShouldEqual, mcS.ErrNotStored) }) }) }) Convey("check that the internal implementation is sane", func() { curTime := now err := mc.Add(&mcItem{ key: "sup", value: []byte("cool"), expiration: time.Second * 2, }) for i := 0; i < 4; i++ { _, err := mc.Get("sup") So(err, ShouldBeNil) } _, err = mc.Get("wot") So(err, ShouldErrLike, mcS.ErrCacheMiss) mci := mc.Raw().(*memcacheImpl) stats, err := mc.Stats() So(err, ShouldBeNil) So(stats.Items, ShouldEqual, 1) So(stats.Bytes, ShouldEqual, 4) So(stats.Hits, ShouldEqual, 4) So(stats.Misses, ShouldEqual, 1) So(stats.ByteHits, ShouldEqual, 4*4) So(mci.data.casID, ShouldEqual, 1) So(mci.data.items["sup"], ShouldResemble, &mcDataItem{ value: []byte("cool"), expiration: curTime.Add(time.Second * 2).Truncate(time.Second), casID: 1, }) getItm, err := mc.Get("sup") So(err, ShouldBeNil) So(len(mci.data.items), ShouldEqual, 1) So(mci.data.casID, ShouldEqual, 1) testItem := &mcItem{ key: "sup", value: []byte("cool"), CasID: 1, } So(getItm, ShouldResemble, testItem) }) }) }