Beispiel #1
0
func TestTaskQueue(t *testing.T) {
	t.Parallel()

	Convey("TaskQueue", t, func() {
		now := time.Date(2000, time.January, 1, 1, 1, 1, 1, time.UTC)
		c, tc := testclock.UseTime(context.Background(), now)
		c = mathrand.Set(c, rand.New(rand.NewSource(clock.Now(c).UnixNano())))
		c = Use(c)

		tq := tqS.Get(c)
		tqt := tq.Testable()
		So(tqt, ShouldNotBeNil)

		So(tq, ShouldNotBeNil)

		Convey("implements TQMultiReadWriter", func() {
			Convey("Add", func() {
				t := tq.NewTask("/hello/world")

				Convey("works", func() {
					t.Delay = 4 * time.Second
					t.Header = http.Header{}
					t.Header.Add("Cat", "tabby")
					t.Payload = []byte("watwatwat")
					t.RetryOptions = &tqS.RetryOptions{AgeLimit: 7 * time.Second}
					So(tq.Add(t, ""), ShouldBeNil)

					name := "Z_UjshxM9ecyMQfGbZmUGOEcgxWU0_5CGLl_-RntudwAw2DqQ5-58bzJiWQN4OKzeuUb9O4JrPkUw2rOvk2Ax46THojnQ6avBQgZdrKcJmrwQ6o4qKfJdiyUbGXvy691yRfzLeQhs6cBhWrgf3wH-VPMcA4SC-zlbJ2U8An7I0zJQA5nBFnMNoMgT-2peGoay3rCSbj4z9VFFm9kS_i6JCaQH518ujLDSNCYdjTq6B6lcWrZAh0U_q3a1S2nXEwrKiw_t9MTNQFgAQZWyGBbvZQPmeRYtu8SPaWzTfd25v_YWgBuVL2rRSPSMvlDwE04nNdtvVzE8vNNiA1zRimmdzKeqATQF9_ReUvj4D7U8dcS703DZWfKMBLgBffY9jqCassOOOw77V72Oq5EVauUw3Qw0L6bBsfM9FtahTKUdabzRZjXUoze3EK4KXPt3-wdidau-8JrVf2XFocjjZbwHoxcGvbtT3b4nGLDlgwdC00bwaFBZWff"
					So(tqt.GetScheduledTasks()["default"][name], ShouldResemble, &tqS.Task{
						ETA:          now.Add(4 * time.Second),
						Header:       http.Header{"Cat": []string{"tabby"}},
						Method:       "POST",
						Name:         name,
						Path:         "/hello/world",
						Payload:      []byte("watwatwat"),
						RetryOptions: &tqS.RetryOptions{AgeLimit: 7 * time.Second},
					})
				})

				Convey("picks up namespace", func() {
					c, err := info.Get(c).Namespace("coolNamespace")
					So(err, ShouldBeNil)
					tq = tqS.Get(c)

					t := tq.NewTask("")
					So(tq.Add(t, ""), ShouldBeNil)
					So(t.Header, ShouldResemble, http.Header{
						"X-Appengine-Current-Namespace": {"coolNamespace"},
					})

				})

				Convey("cannot add to bad queues", func() {
					So(tq.Add(nil, "waaat").Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")

					Convey("but you can add Queues when testing", func() {
						tqt.CreateQueue("waaat")
						So(tq.Add(t, "waaat"), ShouldBeNil)

						Convey("you just can't add them twice", func() {
							So(func() { tqt.CreateQueue("waaat") }, ShouldPanic)
						})
					})
				})

				Convey("supplies a URL if it's missing", func() {
					t.Path = ""
					So(tq.Add(t, ""), ShouldBeNil)
					So(t.Path, ShouldEqual, "/_ah/queue/default")
				})

				Convey("cannot add twice", func() {
					t.Name = "bob"
					So(tq.Add(t, ""), ShouldBeNil)

					// can't add the same one twice!
					So(tq.Add(t, ""), ShouldEqual, tqS.ErrTaskAlreadyAdded)
				})

				Convey("cannot add deleted task", func() {
					t.Name = "bob"
					So(tq.Add(t, ""), ShouldBeNil)

					So(tq.Delete(t, ""), ShouldBeNil)

					// can't add a deleted task!
					So(tq.Add(t, ""), ShouldEqual, tqS.ErrTaskAlreadyAdded)
				})

				Convey("cannot set ETA+Delay", func() {
					t.ETA = clock.Now(c).Add(time.Hour)
					tc.Add(time.Second)
					t.Delay = time.Hour
					So(func() {
						So(tq.Add(t, ""), ShouldBeNil)
					}, ShouldPanic)
				})

				Convey("must use a reasonable method", func() {
					t.Method = "Crystal"
					So(tq.Add(t, "").Error(), ShouldContainSubstring, "bad method")
				})

				Convey("payload gets dumped for non POST/PUT methods", func() {
					t.Method = "HEAD"
					t.Payload = []byte("coool")
					So(tq.Add(t, ""), ShouldBeNil)
					So(t.Payload, ShouldBeNil)
				})

				Convey("invalid names are rejected", func() {
					t.Name = "happy times"
					So(tq.Add(t, "").Error(), ShouldContainSubstring, "INVALID_TASK_NAME")
				})

				Convey("AddMulti also works", func() {
					t2 := t.Duplicate()
					t2.Path = "/hi/city"

					expect := []*tqS.Task{t, t2}

					So(tq.AddMulti(expect, "default"), ShouldBeNil)
					So(len(expect), ShouldEqual, 2)
					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 2)

					for i := range expect {
						Convey(fmt.Sprintf("task %d: %s", i, expect[i].Path), func() {
							So(expect[i].Method, ShouldEqual, "POST")
							So(expect[i].ETA, ShouldHappenOnOrBefore, now)
							So(len(expect[i].Name), ShouldEqual, 500)
						})
					}

					Convey("stats work too", func() {
						delay := -time.Second * 400

						t := tq.NewTask("/somewhere")
						t.Delay = delay
						So(tq.Add(t, ""), ShouldBeNil)

						stats, err := tq.Stats("")
						So(err, ShouldBeNil)
						So(stats[0].Tasks, ShouldEqual, 3)
						So(stats[0].OldestETA, ShouldHappenOnOrBefore, clock.Now(c).Add(delay))

						_, err = tq.Stats("noexist")
						So(err.Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")
					})

					Convey("can purge all tasks", func() {
						So(tq.Add(&tqS.Task{Path: "/wut/nerbs"}, ""), ShouldBeNil)
						So(tq.Purge(""), ShouldBeNil)

						So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
						So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 0)
						So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)

						Convey("purging a queue which DNE fails", func() {
							So(tq.Purge("noexist").Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")
						})
					})

				})
			})

			Convey("Delete", func() {
				t := &tqS.Task{Path: "/hello/world"}
				So(tq.Add(t, ""), ShouldBeNil)

				Convey("works", func() {
					err := tq.Delete(t, "")
					So(err, ShouldBeNil)
					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
					So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 1)
					So(tqt.GetTombstonedTasks()["default"][t.Name], ShouldResemble, t)
				})

				Convey("cannot delete a task twice", func() {
					So(tq.Delete(t, ""), ShouldBeNil)

					So(tq.Delete(t, "").Error(), ShouldContainSubstring, "TOMBSTONED_TASK")

					Convey("but you can if you do a reset", func() {
						tqt.ResetTasks()

						So(tq.Add(t, ""), ShouldBeNil)
						So(tq.Delete(t, ""), ShouldBeNil)
					})
				})

				Convey("cannot delete from bogus queues", func() {
					err := tq.Delete(t, "wat")
					So(err.Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")
				})

				Convey("cannot delete a missing task", func() {
					t.Name = "tarntioarenstyw"
					err := tq.Delete(t, "")
					So(err.Error(), ShouldContainSubstring, "UNKNOWN_TASK")
				})

				Convey("DeleteMulti also works", func() {
					t2 := t.Duplicate()
					t2.Name = ""
					t2.Path = "/hi/city"
					So(tq.Add(t2, ""), ShouldBeNil)

					Convey("usually works", func() {
						So(tq.DeleteMulti([]*tqS.Task{t, t2}, ""), ShouldBeNil)
						So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
						So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 2)
					})
				})
			})
		})

		Convey("works with transactions", func() {
			t := &tqS.Task{Path: "/hello/world"}
			So(tq.Add(t, ""), ShouldBeNil)

			t2 := &tqS.Task{Path: "/hi/city"}
			So(tq.Add(t2, ""), ShouldBeNil)

			So(tq.Delete(t2, ""), ShouldBeNil)

			Convey("can view regular tasks", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					tqt := tqS.GetRaw(c).Testable()

					So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
					So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
					So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("can add a new task", func() {
				t3 := &tqS.Task{Path: "/sandwitch/victory"}

				err := dsS.Get(c).RunInTransaction(func(c context.Context) error {
					tq := tqS.Get(c)
					tqt := tq.Testable()

					So(tq.Add(t3, ""), ShouldBeNil)
					So(t3.Name, ShouldEqual, "")

					So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
					So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
					So(tqt.GetTransactionTasks()["default"][0], ShouldResemble, t3)
					return nil
				}, nil)
				So(err, ShouldBeNil)

				for _, tsk := range tqt.GetScheduledTasks()["default"] {
					if tsk.Name == t.Name {
						So(tsk, ShouldResemble, t)
					} else {
						tsk.Name = ""
						So(tsk, ShouldResemble, t3)
					}
				}

				So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
				So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
			})

			Convey("can add a new task (but reset the state in a test)", func() {
				t3 := &tqS.Task{Path: "/sandwitch/victory"}

				ttq := tqS.Interface(nil)

				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					ttq = tqS.Get(c)
					tqt := ttq.Testable()

					So(ttq.Add(t3, ""), ShouldBeNil)

					So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
					So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
					So(tqt.GetTransactionTasks()["default"][0], ShouldResemble, t3)

					tqt.ResetTasks()

					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
					So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 0)
					So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)

					return nil
				}, nil), ShouldBeNil)

				So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 0)
				So(len(tqt.GetTombstonedTasks()["default"]), ShouldEqual, 0)
				So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)

				Convey("and reusing a closed context is bad times", func() {
					So(ttq.Add(nil, "").Error(), ShouldContainSubstring, "expired")
				})
			})

			Convey("you can AddMulti as well", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					tq := tqS.Get(c)
					tqt := tq.Testable()

					t.Name = ""
					tasks := []*tqS.Task{t.Duplicate(), t.Duplicate(), t.Duplicate()}
					So(tq.AddMulti(tasks, ""), ShouldBeNil)
					So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 1)
					So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 3)
					return nil
				}, nil), ShouldBeNil)
				So(len(tqt.GetScheduledTasks()["default"]), ShouldEqual, 4)
				So(len(tqt.GetTransactionTasks()["default"]), ShouldEqual, 0)
			})

			Convey("unless you add too many things", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					for i := 0; i < 5; i++ {
						So(tqS.Get(c).Add(t.Duplicate(), ""), ShouldBeNil)
					}
					So(tqS.Get(c).Add(t, "").Error(), ShouldContainSubstring, "BAD_REQUEST")
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("unless you Add to a bad queue", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					So(tqS.Get(c).Add(t, "meat").Error(), ShouldContainSubstring, "UNKNOWN_QUEUE")

					Convey("unless you add it!", func() {
						tqS.GetRaw(c).Testable().CreateQueue("meat")
						So(tqS.Get(c).Add(t, "meat"), ShouldBeNil)
					})

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("No other features are available, however", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					So(tqS.Get(c).Delete(t, "").Error(), ShouldContainSubstring, "cannot DeleteMulti from a transaction")
					So(tqS.Get(c).Purge("").Error(), ShouldContainSubstring, "cannot Purge from a transaction")
					_, err := tqS.Get(c).Stats("")
					So(err.Error(), ShouldContainSubstring, "cannot Stats from a transaction")
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("can get the non-transactional taskqueue context though", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					So(tqS.GetNoTxn(c).Delete(t, ""), ShouldBeNil)
					So(tqS.GetNoTxn(c).Purge(""), ShouldBeNil)
					_, err := tqS.GetNoTxn(c).Stats("")
					So(err, ShouldBeNil)
					return nil
				}, nil), ShouldBeNil)
			})

			Convey("adding a new task only happens if we don't errout", func() {
				So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
					t3 := tq.NewTask("/sandwitch/victory")
					So(tqS.Get(c).Add(t3, ""), ShouldBeNil)
					return fmt.Errorf("nooooo")
				}, nil), ShouldErrLike, "nooooo")

				So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
				So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
				So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
			})

			Convey("likewise, a panic doesn't schedule anything", func() {
				func() {
					defer func() { _ = recover() }()
					So(dsS.Get(c).RunInTransaction(func(c context.Context) error {
						tq := tqS.Get(c)

						So(tq.Add(tq.NewTask("/sandwitch/victory"), ""), ShouldBeNil)

						panic(fmt.Errorf("nooooo"))
					}, nil), ShouldBeNil)
				}()

				So(tqt.GetScheduledTasks()["default"][t.Name], ShouldResemble, t)
				So(tqt.GetTombstonedTasks()["default"][t2.Name], ShouldResemble, t2)
				So(tqt.GetTransactionTasks()["default"], ShouldBeNil)
			})

		})
	})
}
Beispiel #2
0
func TestDSCache(t *testing.T) {
	t.Parallel()

	zeroTime, err := time.Parse("2006-01-02T15:04:05.999999999Z", "2006-01-02T15:04:05.999999999Z")
	if err != nil {
		panic(err)
	}

	Convey("Test dscache", t, func() {
		c := mathrand.Set(context.Background(), rand.New(rand.NewSource(1)))
		clk := testclock.New(zeroTime)
		c = clock.Set(c, clk)
		c = memory.Use(c)

		dsUnder := datastore.Get(c)
		mc := memcache.Get(c)

		shardsForKey := func(k *datastore.Key) int {
			last := k.LastTok()
			if last.Kind == "shardObj" {
				return int(last.IntID)
			}
			if last.Kind == "noCacheObj" {
				return 0
			}
			return DefaultShards
		}

		numMemcacheItems := func() uint64 {
			stats, err := mc.Stats()
			So(err, ShouldBeNil)
			return stats.Items
		}

		Convey("enabled cases", func() {
			c = FilterRDS(c, shardsForKey)
			ds := datastore.Get(c)
			So(dsUnder, ShouldNotBeNil)
			So(ds, ShouldNotBeNil)
			So(mc, ShouldNotBeNil)

			Convey("basically works", func() {
				pm := datastore.PropertyMap{
					"BigData": {datastore.MkProperty([]byte(""))},
					"Value":   {datastore.MkProperty("hi")},
				}
				encoded := append([]byte{0}, serialize.ToBytes(pm)...)

				o := object{ID: 1, Value: "hi"}
				So(ds.Put(&o), ShouldBeNil)

				o = object{ID: 1}
				So(dsUnder.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, "hi")

				itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
				So(err, ShouldEqual, memcache.ErrCacheMiss)

				o = object{ID: 1}
				So(ds.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, "hi")

				itm, err = mc.Get(itm.Key())
				So(err, ShouldBeNil)
				So(itm.Value(), ShouldResemble, encoded)

				Convey("now we don't need the datastore!", func() {
					o := object{ID: 1}

					// delete it, bypassing the cache filter. Don't do this in production
					// unless you want a crappy cache.
					So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
					So(err, ShouldBeNil)
					So(itm.Value(), ShouldResemble, encoded)

					So(ds.Get(&o), ShouldBeNil)
					So(o.Value, ShouldEqual, "hi")
				})

				Convey("deleting it properly records that fact, however", func() {
					o := object{ID: 1}
					So(ds.Delete(ds.KeyForObj(&o)), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)

					itm, err = mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Value(), ShouldResemble, []byte{})

					// this one hits memcache
					So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)
				})
			})

			Convey("compression works", func() {
				o := object{ID: 2, Value: `¯\_(ツ)_/¯`}
				data := make([]byte, 4000)
				for i := range data {
					const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()"
					data[i] = alpha[i%len(alpha)]
				}
				o.BigData = data

				So(ds.Put(&o), ShouldBeNil)
				So(ds.Get(&o), ShouldBeNil)

				itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
				So(err, ShouldBeNil)

				So(itm.Value()[0], ShouldEqual, ZlibCompression)
				So(len(itm.Value()), ShouldEqual, 653) // a bit smaller than 4k

				// ensure the next Get comes from the cache
				So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)

				o = object{ID: 2}
				So(ds.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, `¯\_(ツ)_/¯`)
				So(o.BigData, ShouldResemble, data)
			})

			Convey("transactions", func() {
				Convey("work", func() {
					// populate an object @ ID1
					So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
					So(ds.Get(&object{ID: 1}), ShouldBeNil)

					So(ds.Put(&object{ID: 2, Value: "nurbs"}), ShouldBeNil)
					So(ds.Get(&object{ID: 2}), ShouldBeNil)

					// memcache now has the wrong value (simulated race)
					So(dsUnder.Put(&object{ID: 1, Value: "else"}), ShouldBeNil)
					So(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						o := &object{ID: 1}
						So(ds.Get(o), ShouldBeNil)
						So(o.Value, ShouldEqual, "else")
						o.Value = "txn"
						So(ds.Put(o), ShouldBeNil)

						So(ds.Delete(ds.KeyForObj(&object{ID: 2})), ShouldBeNil)
						return nil
					}, &datastore.TransactionOptions{XG: true}), ShouldBeNil)

					_, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 1})))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					_, err = mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 2})))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					o := &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "txn")
				})

				Convey("errors don't invalidate", func() {
					// populate an object @ ID1
					So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
					So(ds.Get(&object{ID: 1}), ShouldBeNil)
					So(numMemcacheItems(), ShouldEqual, 1)

					So(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						o := &object{ID: 1}
						So(ds.Get(o), ShouldBeNil)
						So(o.Value, ShouldEqual, "something")
						o.Value = "txn"
						So(ds.Put(o), ShouldBeNil)
						return errors.New("OH NOES")
					}, nil).Error(), ShouldContainSubstring, "OH NOES")

					// memcache still has the original
					So(numMemcacheItems(), ShouldEqual, 1)
					So(dsUnder.Delete(ds.KeyForObj(&object{ID: 1})), ShouldBeNil)
					o := &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "something")
				})
			})

			Convey("control", func() {
				Convey("per-model bypass", func() {
					type model struct {
						ID         string           `gae:"$id"`
						UseDSCache datastore.Toggle `gae:"$dscache.enable,false"`

						Value string
					}

					itms := []model{
						{ID: "hi", Value: "something"},
						{ID: "there", Value: "else", UseDSCache: datastore.On},
					}

					So(ds.PutMulti(itms), ShouldBeNil)
					So(ds.GetMulti(itms), ShouldBeNil)

					So(numMemcacheItems(), ShouldEqual, 1)
				})

				Convey("per-key shard count", func() {
					s := &shardObj{ID: 4, Value: "hi"}
					So(ds.Put(s), ShouldBeNil)
					So(ds.Get(s), ShouldBeNil)

					So(numMemcacheItems(), ShouldEqual, 1)
					for i := 0; i < 20; i++ {
						So(ds.Get(s), ShouldBeNil)
					}
					So(numMemcacheItems(), ShouldEqual, 4)
				})

				Convey("per-key cache disablement", func() {
					n := &noCacheObj{ID: "nurbs", Value: true}
					So(ds.Put(n), ShouldBeNil)
					So(ds.Get(n), ShouldBeNil)
					So(numMemcacheItems(), ShouldEqual, 0)
				})

				Convey("per-model expiration", func() {
					type model struct {
						ID         int64 `gae:"$id"`
						DSCacheExp int64 `gae:"$dscache.expiration,7"`

						Value string
					}

					So(ds.Put(&model{ID: 1, Value: "mooo"}), ShouldBeNil)
					So(ds.Get(&model{ID: 1}), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&model{ID: 1})))
					So(err, ShouldBeNil)

					clk.Add(10 * time.Second)
					_, err = mc.Get(itm.Key())
					So(err, ShouldEqual, memcache.ErrCacheMiss)
				})
			})

			Convey("screw cases", func() {
				Convey("memcache contains bogus value (simulated failed AddMulti)", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("I am a banana")
					itm := mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).SetValue(sekret)
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemUKNONWN)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("memcache contains bogus value (corrupt entry)", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("I am a banana")
					itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
						SetValue(sekret).
						SetFlags(uint32(ItemHasData)))
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasData)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("other entity has the lock", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("r@vmarod!#)%9T")
					itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
						SetValue(sekret).
						SetFlags(uint32(ItemHasLock)))
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasLock)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("massive entities can't be cached", func() {
					o := &object{ID: 1, Value: "spleen"}
					mr := mathrand.Get(c)
					numRounds := (internalValueSizeLimit / 8) * 2
					buf := bytes.Buffer{}
					for i := 0; i < numRounds; i++ {
						So(binary.Write(&buf, binary.LittleEndian, mr.Int63()), ShouldBeNil)
					}
					o.BigData = buf.Bytes()
					So(ds.Put(o), ShouldBeNil)

					o.BigData = nil
					So(ds.Get(o), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(o)))
					So(err, ShouldBeNil)

					// Is locked until the next put, forcing all access to the datastore.
					So(itm.Value(), ShouldResemble, []byte{})
					So(itm.Flags(), ShouldEqual, ItemHasLock)

					o.BigData = []byte("hi :)")
					So(ds.Put(o), ShouldBeNil)
					So(ds.Get(o), ShouldBeNil)

					itm, err = mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasData)
				})

				Convey("failure on Setting memcache locks is a hard stop", func() {
					c, fb := featureBreaker.FilterMC(c, nil)
					fb.BreakFeatures(nil, "SetMulti")
					ds := datastore.Get(c)
					So(ds.Put(&object{ID: 1}).Error(), ShouldContainSubstring, "SetMulti")
				})

				Convey("failure on Setting memcache locks in a transaction is a hard stop", func() {
					c, fb := featureBreaker.FilterMC(c, nil)
					fb.BreakFeatures(nil, "SetMulti")
					ds := datastore.Get(c)
					So(ds.RunInTransaction(func(c context.Context) error {
						So(datastore.Get(c).Put(&object{ID: 1}), ShouldBeNil)
						// no problems here... memcache operations happen after the function
						// body quits.
						return nil
					}, nil).Error(), ShouldContainSubstring, "SetMulti")
				})

			})

			Convey("misc", func() {
				Convey("verify numShards caps at MaxShards", func() {
					sc := supportContext{shardsForKey: shardsForKey}
					So(sc.numShards(ds.KeyForObj(&shardObj{ID: 9001})), ShouldEqual, MaxShards)
				})

				Convey("CompressionType.String", func() {
					So(NoCompression.String(), ShouldEqual, "NoCompression")
					So(ZlibCompression.String(), ShouldEqual, "ZlibCompression")
					So(CompressionType(100).String(), ShouldEqual, "UNKNOWN_CompressionType(100)")
				})
			})
		})

		Convey("disabled cases", func() {
			defer func() {
				globalEnabled = true
			}()

			So(IsGloballyEnabled(c), ShouldBeTrue)

			So(SetGlobalEnable(c, false), ShouldBeNil)
			// twice is a nop
			So(SetGlobalEnable(c, false), ShouldBeNil)

			// but it takes 5 minutes to kick in
			So(IsGloballyEnabled(c), ShouldBeTrue)
			clk.Add(time.Minute*5 + time.Second)
			So(IsGloballyEnabled(c), ShouldBeFalse)

			So(mc.Set(mc.NewItem("test").SetValue([]byte("hi"))), ShouldBeNil)
			So(numMemcacheItems(), ShouldEqual, 1)
			So(SetGlobalEnable(c, true), ShouldBeNil)
			// memcache gets flushed as a side effect
			So(numMemcacheItems(), ShouldEqual, 0)

			// Still takes 5 minutes to kick in
			So(IsGloballyEnabled(c), ShouldBeFalse)
			clk.Add(time.Minute*5 + time.Second)
			So(IsGloballyEnabled(c), ShouldBeTrue)
		})
	})
}