Пример #1
0
// SetGlobalEnable is a convenience function for manipulating the GlobalConfig.
//
// It's meant to be called from admin handlers on your app to turn dscache
// functionality on or off in emergencies.
func SetGlobalEnable(c context.Context, memcacheEnabled bool) error {
	// always go to the default namespace
	c, err := info.Get(c).Namespace("")
	if err != nil {
		return err
	}
	return datastore.Get(c).RunInTransaction(func(c context.Context) error {
		ds := datastore.Get(c)
		cfg := &GlobalConfig{Enable: true}
		if err := ds.Get(cfg); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		if cfg.Enable == memcacheEnabled {
			return nil
		}
		cfg.Enable = memcacheEnabled
		if memcacheEnabled {
			// when going false -> true, wipe memcache.
			if err := memcache.Get(c).Flush(); err != nil {
				return err
			}
		}
		return ds.Put(cfg)
	}, nil)
}
Пример #2
0
// AlwaysFilterRDS installs a caching RawDatastore filter in the context.
//
// Unlike FilterRDS it doesn't check GlobalConfig via IsGloballyEnabled call,
// assuming caller already knows whether filter should be applied or not.
func AlwaysFilterRDS(c context.Context, shardsForKey func(*ds.Key) int) context.Context {
	return ds.AddRawFilters(c, func(c context.Context, ds ds.RawInterface) ds.RawInterface {
		i := info.Get(c)

		sc := &supportContext{
			i.AppID(),
			i.GetNamespace(),
			c,
			mc.Get(c),
			mathrand.Get(c),
			shardsForKey,
		}

		v := c.Value(dsTxnCacheKey)
		if v == nil {
			return &dsCache{ds, sc}
		}
		return &dsTxnCache{ds, v.(*dsTxnState), sc}
	})
}
Пример #3
0
func TestCount(t *testing.T) {
	t.Parallel()

	Convey("Test Count filter", t, func() {
		c, fb := featureBreaker.FilterRDS(memory.Use(context.Background()), nil)
		c, ctr := FilterRDS(c)

		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		ds := datastore.Get(c)
		vals := []datastore.PropertyMap{{
			"Val":  {datastore.MkProperty(100)},
			"$key": {datastore.MkPropertyNI(ds.NewKey("Kind", "", 1, nil))},
		}}

		Convey("Calling a ds function should reflect in counter", func() {
			So(ds.PutMulti(vals), ShouldBeNil)
			So(ctr.PutMulti.Successes(), ShouldEqual, 1)

			Convey("effects are cumulative", func() {
				So(ds.PutMulti(vals), ShouldBeNil)
				So(ctr.PutMulti.Successes(), ShouldEqual, 2)

				Convey("even within transactions", func() {
					die(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						So(ds.PutMulti(append(vals, vals[0])), ShouldBeNil)
						return nil
					}, nil))
				})
			})
		})

		Convey("errors count against errors", func() {
			fb.BreakFeatures(nil, "GetMulti")

			So(ds.GetMulti(vals), ShouldErrLike, `"GetMulti" is broken`)
			So(ctr.GetMulti.Errors(), ShouldEqual, 1)

			fb.UnbreakFeatures("GetMulti")

			So(ds.PutMulti(vals), ShouldBeNil)

			die(ds.GetMulti(vals))
			So(ctr.GetMulti.Errors(), ShouldEqual, 1)
			So(ctr.GetMulti.Successes(), ShouldEqual, 1)
			So(ctr.GetMulti.Total(), ShouldEqual, 2)
		})
	})

	Convey("works for memcache", t, func() {
		c, ctr := FilterMC(memory.Use(context.Background()))
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)
		mc := memcache.Get(c)

		die(mc.Set(mc.NewItem("hello").SetValue([]byte("sup"))))

		_, err := mc.Get("Wat")
		So(err, ShouldNotBeNil)

		_, err = mc.Get("hello")
		die(err)

		So(ctr.SetMulti, shouldHaveSuccessesAndErrors, 1, 0)
		So(ctr.GetMulti, shouldHaveSuccessesAndErrors, 2, 0)
		So(ctr.NewItem, shouldHaveSuccessesAndErrors, 3, 0)
	})

	Convey("works for taskqueue", t, func() {
		c, ctr := FilterTQ(memory.Use(context.Background()))
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)
		tq := taskqueue.Get(c)

		die(tq.Add(&taskqueue.Task{Name: "wat"}, ""))
		So(tq.Add(&taskqueue.Task{Name: "wat"}, "DNE_QUEUE"),
			ShouldErrLike, "UNKNOWN_QUEUE")

		So(ctr.AddMulti, shouldHaveSuccessesAndErrors, 1, 1)
	})

	Convey("works for global info", t, func() {
		c, fb := featureBreaker.FilterGI(memory.Use(context.Background()), nil)
		c, ctr := FilterGI(c)
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		gi := info.Get(c)

		_, err := gi.Namespace("foo")
		die(err)
		fb.BreakFeatures(nil, "Namespace")
		_, err = gi.Namespace("boom")
		So(err, ShouldErrLike, `"Namespace" is broken`)

		So(ctr.Namespace, shouldHaveSuccessesAndErrors, 1, 1)
	})

	Convey("works for user", t, func() {
		c, fb := featureBreaker.FilterUser(memory.Use(context.Background()), nil)
		c, ctr := FilterUser(c)
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		u := user.Get(c)

		_, err := u.CurrentOAuth("foo")
		die(err)
		fb.BreakFeatures(nil, "CurrentOAuth")
		_, err = u.CurrentOAuth("foo")
		So(err, ShouldErrLike, `"CurrentOAuth" is broken`)

		So(ctr.CurrentOAuth, shouldHaveSuccessesAndErrors, 1, 1)
	})

	Convey("works for mail", t, func() {
		c, fb := featureBreaker.FilterMail(memory.Use(context.Background()), nil)
		c, ctr := FilterMail(c)
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		m := mail.Get(c)

		err := m.Send(&mail.Message{
			Sender: "*****@*****.**",
			To:     []string{"*****@*****.**"},
			Body:   "hi",
		})
		die(err)

		fb.BreakFeatures(nil, "Send")
		err = m.Send(&mail.Message{
			Sender: "*****@*****.**",
			To:     []string{"*****@*****.**"},
			Body:   "hi",
		})
		So(err, ShouldErrLike, `"Send" is broken`)

		So(ctr.Send, shouldHaveSuccessesAndErrors, 1, 1)
	})
}
Пример #4
0
func TestSimple(t *testing.T) {
	// TODO(riannucci): Mock time.After so that we don't have to delay for real.

	const key = memlockKeyPrefix + "testkey"

	Convey("basic locking", t, func() {
		start := time.Date(1986, time.October, 26, 1, 20, 00, 00, time.UTC)
		ctx, clk := testclock.UseTime(context.Background(), start)
		blocker := make(chan struct{})
		clk.SetTimerCallback(func(clock.Timer) {
			clk.Add(delay)
			select {
			case blocker <- struct{}{}:
			default:
			}
		})
		waitFalse := func(ctx context.Context) {
		loop:
			for {
				select {
				case <-blocker:
					continue
				case <-ctx.Done():
					break loop
				}
			}
		}

		ctx, fb := featureBreaker.FilterMC(memory.Use(ctx), nil)
		mc := memcache.Get(ctx)

		Convey("fails to acquire when memcache is down", func() {
			fb.BreakFeatures(nil, "Add")
			err := TryWithLock(ctx, "testkey", "id", func(context.Context) error {
				// should never reach here
				So(false, ShouldBeTrue)
				return nil
			})
			So(err, ShouldEqual, ErrFailedToLock)
		})

		Convey("returns the inner error", func() {
			toRet := fmt.Errorf("sup")
			err := TryWithLock(ctx, "testkey", "id", func(context.Context) error {
				return toRet
			})
			So(err, ShouldEqual, toRet)
		})

		Convey("returns the error", func() {
			toRet := fmt.Errorf("sup")
			err := TryWithLock(ctx, "testkey", "id", func(context.Context) error {
				return toRet
			})
			So(err, ShouldEqual, toRet)
		})

		Convey("can acquire when empty", func() {
			err := TryWithLock(ctx, "testkey", "id", func(ctx context.Context) error {
				isDone := func() bool {
					select {
					case <-ctx.Done():
						return true
					default:
						return false
					}
				}

				So(isDone(), ShouldBeFalse)

				Convey("waiting for a while keeps refreshing the lock", func() {
					// simulate waiting for 64*delay time, and ensuring that checkLoop
					// runs that many times.
					for i := 0; i < 64; i++ {
						<-blocker
						clk.Add(delay)
					}
					So(isDone(), ShouldBeFalse)
				})

				Convey("but sometimes we might lose it", func() {
					Convey("because it was evicted", func() {
						mc.Delete(key)
						clk.Add(memcacheLockTime)
						waitFalse(ctx)
					})

					Convey("or because of service issues", func() {
						fb.BreakFeatures(nil, "CompareAndSwap")
						waitFalse(ctx)
					})
				})
				return nil
			})
			So(err, ShouldBeNil)
		})

		Convey("can lose it when it gets stolen", func() {
			gbf := &getBlockerFilter{}
			ctx = memcache.AddFilters(ctx, func(_ context.Context, mc memcache.Interface) memcache.Interface {
				gbf.Interface = mc
				return gbf
			})
			mc = memcache.Get(ctx)
			err := TryWithLock(ctx, "testkey", "id", func(ctx context.Context) error {
				// simulate waiting for 64*delay time, and ensuring that checkLoop
				// runs that many times.
				for i := 0; i < 64; i++ {
					<-blocker
					clk.Add(delay)
				}
				gbf.Lock()
				mc.Set(mc.NewItem(key).SetValue([]byte("wat")))
				gbf.Unlock()
				waitFalse(ctx)
				return nil
			})
			So(err, ShouldBeNil)
		})

		Convey("an empty context id is an error", func() {
			So(TryWithLock(ctx, "testkey", "", nil), ShouldEqual, ErrEmptyClientID)
		})
	})
}
Пример #5
0
func TestMemcache(t *testing.T) {
	t.Parallel()

	Convey("memcache", t, func() {
		now := time.Date(2015, 1, 1, 0, 0, 0, 0, time.UTC)
		c, tc := testclock.UseTime(context.Background(), now)
		c = Use(c)
		mc := mcS.Get(c)

		Convey("implements MCSingleReadWriter", func() {
			Convey("Add", func() {
				itm := (mc.NewItem("sup").
					SetValue([]byte("cool")).
					SetExpiration(time.Second))
				So(mc.Add(itm), ShouldBeNil)
				Convey("which rejects objects already there", func() {
					So(mc.Add(itm), ShouldEqual, mcS.ErrNotStored)
				})
			})

			Convey("Get", func() {
				itm := &mcItem{
					key:        "sup",
					value:      []byte("cool"),
					expiration: time.Second,
				}
				So(mc.Add(itm), ShouldBeNil)

				testItem := &mcItem{
					key:   "sup",
					value: []byte("cool"),
					CasID: 1,
				}
				getItm, err := mc.Get("sup")
				So(err, ShouldBeNil)
				So(getItm, ShouldResemble, testItem)

				Convey("which can expire", func() {
					tc.Add(time.Second * 4)
					getItm, err := mc.Get("sup")
					So(err, ShouldEqual, mcS.ErrCacheMiss)
					So(getItm, ShouldResemble, &mcItem{key: "sup"})
				})
			})

			Convey("Delete", func() {
				Convey("works if it's there", func() {
					itm := &mcItem{
						key:        "sup",
						value:      []byte("cool"),
						expiration: time.Second,
					}
					So(mc.Add(itm), ShouldBeNil)

					So(mc.Delete("sup"), ShouldBeNil)

					_, err := mc.Get("sup")
					So(err, ShouldEqual, mcS.ErrCacheMiss)
				})

				Convey("but not if it's not there", func() {
					So(mc.Delete("sup"), ShouldEqual, mcS.ErrCacheMiss)
				})
			})

			Convey("Set", func() {
				itm := &mcItem{
					key:        "sup",
					value:      []byte("cool"),
					expiration: time.Second,
				}
				So(mc.Add(itm), ShouldBeNil)

				itm.SetValue([]byte("newp"))
				So(mc.Set(itm), ShouldBeNil)

				testItem := &mcItem{
					key:   "sup",
					value: []byte("newp"),
					CasID: 2,
				}
				getItm, err := mc.Get("sup")
				So(err, ShouldBeNil)
				So(getItm, ShouldResemble, testItem)

				Convey("Flush works too", func() {
					So(mc.Flush(), ShouldBeNil)
					_, err := mc.Get("sup")
					So(err, ShouldEqual, mcS.ErrCacheMiss)
				})
			})

			Convey("Set (nil) is equivalent to Set([]byte{})", func() {
				So(mc.Set(mc.NewItem("bob")), ShouldBeNil)

				bob, err := mc.Get("bob")
				So(err, ShouldBeNil)
				So(bob.Value(), ShouldResemble, []byte{})
			})

			Convey("Increment", func() {
				val, err := mc.Increment("num", 7, 2)
				So(err, ShouldBeNil)
				So(val, ShouldEqual, 9)

				Convey("IncrementExisting", func() {
					val, err := mc.IncrementExisting("num", -2)
					So(err, ShouldBeNil)
					So(val, ShouldEqual, 7)

					val, err = mc.IncrementExisting("num", -100)
					So(err, ShouldBeNil)
					So(val, ShouldEqual, 0)

					_, err = mc.IncrementExisting("noexist", 2)
					So(err, ShouldEqual, mcS.ErrCacheMiss)

					So(mc.Set(mc.NewItem("text").SetValue([]byte("hello world, hooman!"))), ShouldBeNil)

					_, err = mc.IncrementExisting("text", 2)
					So(err.Error(), ShouldContainSubstring, "got invalid current value")
				})
			})

			Convey("CompareAndSwap", func() {
				itm := mcS.Item(&mcItem{
					key:        "sup",
					value:      []byte("cool"),
					expiration: time.Second * 2,
				})
				So(mc.Add(itm), ShouldBeNil)

				Convey("works after a Get", func() {
					itm, err := mc.Get("sup")
					So(err, ShouldBeNil)
					So(itm.(*mcItem).CasID, ShouldEqual, 1)

					itm.SetValue([]byte("newp"))
					So(mc.CompareAndSwap(itm), ShouldBeNil)
				})

				Convey("but fails if you don't", func() {
					itm.SetValue([]byte("newp"))
					So(mc.CompareAndSwap(itm), ShouldEqual, mcS.ErrCASConflict)
				})

				Convey("and fails if the item is expired/gone", func() {
					tc.Add(3 * time.Second)
					itm.SetValue([]byte("newp"))
					So(mc.CompareAndSwap(itm), ShouldEqual, mcS.ErrNotStored)
				})
			})
		})

		Convey("check that the internal implementation is sane", func() {
			curTime := now
			err := mc.Add(&mcItem{
				key:        "sup",
				value:      []byte("cool"),
				expiration: time.Second * 2,
			})

			for i := 0; i < 4; i++ {
				_, err := mc.Get("sup")
				So(err, ShouldBeNil)
			}
			_, err = mc.Get("wot")
			So(err, ShouldErrLike, mcS.ErrCacheMiss)

			mci := mc.Raw().(*memcacheImpl)

			stats, err := mc.Stats()
			So(err, ShouldBeNil)
			So(stats.Items, ShouldEqual, 1)
			So(stats.Bytes, ShouldEqual, 4)
			So(stats.Hits, ShouldEqual, 4)
			So(stats.Misses, ShouldEqual, 1)
			So(stats.ByteHits, ShouldEqual, 4*4)
			So(mci.data.casID, ShouldEqual, 1)
			So(mci.data.items["sup"], ShouldResemble, &mcDataItem{
				value:      []byte("cool"),
				expiration: curTime.Add(time.Second * 2).Truncate(time.Second),
				casID:      1,
			})

			getItm, err := mc.Get("sup")
			So(err, ShouldBeNil)
			So(len(mci.data.items), ShouldEqual, 1)
			So(mci.data.casID, ShouldEqual, 1)

			testItem := &mcItem{
				key:   "sup",
				value: []byte("cool"),
				CasID: 1,
			}
			So(getItm, ShouldResemble, testItem)
		})

	})
}
Пример #6
0
func TestContextAccess(t *testing.T) {
	t.Parallel()

	// p is a function which recovers an error and then immediately panics with
	// the contained string. It's defer'd in each test so that we can use the
	// ShouldPanicWith assertion (which does an == comparison and not
	// a reflect.DeepEquals comparison).
	p := func() { panic(recover().(error).Error()) }

	Convey("Context Access", t, func() {
		c := context.Background()

		Convey("blank", func() {
			So(dsS.GetRaw(c), ShouldBeNil)
			So(mcS.GetRaw(c), ShouldBeNil)
			So(tqS.GetRaw(c), ShouldBeNil)
			So(infoS.Get(c), ShouldBeNil)
		})

		// needed for everything else
		c = infoS.Set(c, Info())

		Convey("Info", func() {
			So(infoS.Get(c), ShouldNotBeNil)
			So(func() {
				defer p()
				infoS.Get(c).Datacenter()
			}, ShouldPanicWith, "dummy: method Info.Datacenter is not implemented")
		})

		Convey("Datastore", func() {
			c = dsS.SetRaw(c, Datastore())
			So(dsS.Get(c), ShouldNotBeNil)
			So(func() {
				defer p()
				_, _ = dsS.Get(c).DecodeCursor("wut")
			}, ShouldPanicWith, "dummy: method Datastore.DecodeCursor is not implemented")
		})

		Convey("Memcache", func() {
			c = mcS.SetRaw(c, Memcache())
			So(mcS.Get(c), ShouldNotBeNil)
			So(func() {
				defer p()
				_ = mcS.Get(c).Add(nil)
			}, ShouldPanicWith, "dummy: method Memcache.AddMulti is not implemented")
		})

		Convey("TaskQueue", func() {
			c = tqS.SetRaw(c, TaskQueue())
			So(tqS.Get(c), ShouldNotBeNil)
			So(func() {
				defer p()
				_ = tqS.Get(c).Purge("")
			}, ShouldPanicWith, "dummy: method TaskQueue.Purge is not implemented")
		})

		Convey("User", func() {
			c = userS.Set(c, User())
			So(userS.Get(c), ShouldNotBeNil)
			So(func() {
				defer p()
				_ = userS.Get(c).IsAdmin()
			}, ShouldPanicWith, "dummy: method User.IsAdmin is not implemented")
		})

		Convey("Mail", func() {
			c = mailS.Set(c, Mail())
			So(mailS.Get(c), ShouldNotBeNil)
			So(func() {
				defer p()
				_ = mailS.Get(c).Send(nil)
			}, ShouldPanicWith, "dummy: method Mail.Send is not implemented")
		})

	})
}
Пример #7
0
func TestBasicDatastore(t *testing.T) {
	t.Parallel()

	Convey("basic", t, func() {
		inst, err := aetest.NewInstance(&aetest.Options{
			StronglyConsistentDatastore: true,
		})
		So(err, ShouldBeNil)
		defer inst.Close()

		req, err := inst.NewRequest("GET", "/", nil)
		So(err, ShouldBeNil)

		ctx := Use(context.Background(), req)
		ds := datastore.Get(ctx)
		mc := memcache.Get(ctx)
		inf := info.Get(ctx)

		Convey("logging allows you to tweak the level", func() {
			// You have to visually confirm that this actually happens in the stdout
			// of the test... yeah I know.
			logging.Debugf(ctx, "SHOULD NOT SEE")
			logging.Infof(ctx, "SHOULD SEE")

			ctx = logging.SetLevel(ctx, logging.Debug)
			logging.Debugf(ctx, "SHOULD SEE")
			logging.Infof(ctx, "SHOULD SEE (2)")
		})

		Convey("Can probe/change Namespace", func() {
			So(inf.GetNamespace(), ShouldEqual, "")
			ctx, err = inf.Namespace("wat")
			So(err, ShouldBeNil)
			inf = info.Get(ctx)
			So(inf.GetNamespace(), ShouldEqual, "wat")
			ds = datastore.Get(ctx)
			So(ds.MakeKey("Hello", "world").Namespace(), ShouldEqual, "wat")
		})

		Convey("Can get non-transactional context", func() {
			ctx, err := inf.Namespace("foo")
			So(err, ShouldBeNil)
			ds = datastore.Get(ctx)
			inf = info.Get(ctx)

			ds.RunInTransaction(func(ctx context.Context) error {
				So(ds.MakeKey("Foo", "bar").Namespace(), ShouldEqual, "foo")

				So(ds.Put(&TestStruct{ValueI: []int64{100}}), ShouldBeNil)

				err = datastore.GetNoTxn(ctx).RunInTransaction(func(ctx context.Context) error {
					ds = datastore.Get(ctx)
					So(ds.MakeKey("Foo", "bar").Namespace(), ShouldEqual, "foo")
					So(ds.Put(&TestStruct{ValueI: []int64{100}}), ShouldBeNil)
					return nil
				}, nil)
				So(err, ShouldBeNil)

				return nil
			}, nil)
		})

		Convey("Can Put/Get", func() {
			orig := TestStruct{
				ValueI: []int64{1, 7, 946688461000000, 996688461000000},
				ValueB: []bool{true, false},
				ValueS: []string{"hello", "world"},
				ValueF: []float64{1.0, 7.0, 946688461000000.0, 996688461000000.0},
				ValueBS: [][]byte{
					[]byte("allo"),
					[]byte("hello"),
					[]byte("world"),
					[]byte("zurple"),
				},
				ValueK: []*datastore.Key{
					ds.NewKey("Something", "Cool", 0, nil),
					ds.NewKey("Something", "", 1, nil),
					ds.NewKey("Something", "Recursive", 0,
						ds.NewKey("Parent", "", 2, nil)),
				},
				ValueBK: []blobstore.Key{"bellow", "hello"},
				ValueGP: []datastore.GeoPoint{
					{Lat: 120.7, Lng: 95.5},
				},
			}
			So(ds.Put(&orig), ShouldBeNil)

			ret := TestStruct{ID: orig.ID}
			So(ds.Get(&ret), ShouldBeNil)
			So(ret, ShouldResemble, orig)

			// can't be sure the indexes have caught up... so sleep
			time.Sleep(time.Second)

			Convey("Can query", func() {
				q := datastore.NewQuery("TestStruct")
				ds.Run(q, func(ts *TestStruct) {
					So(*ts, ShouldResemble, orig)
				})
				count, err := ds.Count(q)
				So(err, ShouldBeNil)
				So(count, ShouldEqual, 1)
			})

			Convey("Can project", func() {
				q := datastore.NewQuery("TestStruct").Project("ValueS")
				rslts := []datastore.PropertyMap{}
				So(ds.GetAll(q, &rslts), ShouldBeNil)
				So(rslts, ShouldResemble, []datastore.PropertyMap{
					{
						"$key":   {mpNI(ds.KeyForObj(&orig))},
						"ValueS": {mp("hello")},
					},
					{
						"$key":   {mpNI(ds.KeyForObj(&orig))},
						"ValueS": {mp("world")},
					},
				})

				q = datastore.NewQuery("TestStruct").Project("ValueBS")
				rslts = []datastore.PropertyMap{}
				So(ds.GetAll(q, &rslts), ShouldBeNil)
				So(rslts, ShouldResemble, []datastore.PropertyMap{
					{
						"$key":    {mpNI(ds.KeyForObj(&orig))},
						"ValueBS": {mp("allo")},
					},
					{
						"$key":    {mpNI(ds.KeyForObj(&orig))},
						"ValueBS": {mp("hello")},
					},
					{
						"$key":    {mpNI(ds.KeyForObj(&orig))},
						"ValueBS": {mp("world")},
					},
					{
						"$key":    {mpNI(ds.KeyForObj(&orig))},
						"ValueBS": {mp("zurple")},
					},
				})

				count, err := ds.Count(q)
				So(err, ShouldBeNil)
				So(count, ShouldEqual, 4)
			})
		})

		Convey("Can Put/Get (time)", func() {
			// time comparisons in Go are wonky, so this is pulled out
			pm := datastore.PropertyMap{
				"$key": {mpNI(ds.NewKey("Something", "value", 0, nil))},
				"Time": {
					mp(time.Date(1938, time.January, 1, 1, 1, 1, 1, time.UTC)),
					mp(time.Time{}),
				},
			}
			So(ds.Put(&pm), ShouldBeNil)

			rslt := datastore.PropertyMap{}
			rslt.SetMeta("key", ds.KeyForObj(pm))
			So(ds.Get(&rslt), ShouldBeNil)

			So(pm["Time"][0].Value(), ShouldResemble, rslt["Time"][0].Value())

			q := datastore.NewQuery("Something").Project("Time")
			all := []datastore.PropertyMap{}
			So(ds.GetAll(q, &all), ShouldBeNil)
			So(len(all), ShouldEqual, 2)
			prop := all[0]["Time"][0]
			So(prop.Type(), ShouldEqual, datastore.PTInt)

			tval, err := prop.Project(datastore.PTTime)
			So(err, ShouldBeNil)
			So(tval, ShouldResemble, time.Time{})

			tval, err = all[1]["Time"][0].Project(datastore.PTTime)
			So(err, ShouldBeNil)
			So(tval, ShouldResemble, pm["Time"][0].Value())

			ent := datastore.PropertyMap{
				"$key": {mpNI(ds.MakeKey("Something", "value"))},
			}
			So(ds.Get(&ent), ShouldBeNil)
			So(ent["Time"], ShouldResemble, pm["Time"])
		})

		Convey("memcache: Set (nil) is the same as Set ([]byte{})", func() {
			So(mc.Set(mc.NewItem("bob")), ShouldBeNil) // normally would panic because Value is nil

			bob, err := mc.Get("bob")
			So(err, ShouldBeNil)
			So(bob.Value(), ShouldResemble, []byte{})
		})
	})
}
Пример #8
0
func TestDSCache(t *testing.T) {
	t.Parallel()

	zeroTime, err := time.Parse("2006-01-02T15:04:05.999999999Z", "2006-01-02T15:04:05.999999999Z")
	if err != nil {
		panic(err)
	}

	Convey("Test dscache", t, func() {
		c := mathrand.Set(context.Background(), rand.New(rand.NewSource(1)))
		clk := testclock.New(zeroTime)
		c = clock.Set(c, clk)
		c = memory.Use(c)

		dsUnder := datastore.Get(c)
		mc := memcache.Get(c)

		shardsForKey := func(k *datastore.Key) int {
			last := k.LastTok()
			if last.Kind == "shardObj" {
				return int(last.IntID)
			}
			if last.Kind == "noCacheObj" {
				return 0
			}
			return DefaultShards
		}

		numMemcacheItems := func() uint64 {
			stats, err := mc.Stats()
			So(err, ShouldBeNil)
			return stats.Items
		}

		Convey("enabled cases", func() {
			c = FilterRDS(c, shardsForKey)
			ds := datastore.Get(c)
			So(dsUnder, ShouldNotBeNil)
			So(ds, ShouldNotBeNil)
			So(mc, ShouldNotBeNil)

			Convey("basically works", func() {
				pm := datastore.PropertyMap{
					"BigData": {datastore.MkProperty([]byte(""))},
					"Value":   {datastore.MkProperty("hi")},
				}
				encoded := append([]byte{0}, serialize.ToBytes(pm)...)

				o := object{ID: 1, Value: "hi"}
				So(ds.Put(&o), ShouldBeNil)

				o = object{ID: 1}
				So(dsUnder.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, "hi")

				itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
				So(err, ShouldEqual, memcache.ErrCacheMiss)

				o = object{ID: 1}
				So(ds.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, "hi")

				itm, err = mc.Get(itm.Key())
				So(err, ShouldBeNil)
				So(itm.Value(), ShouldResemble, encoded)

				Convey("now we don't need the datastore!", func() {
					o := object{ID: 1}

					// delete it, bypassing the cache filter. Don't do this in production
					// unless you want a crappy cache.
					So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
					So(err, ShouldBeNil)
					So(itm.Value(), ShouldResemble, encoded)

					So(ds.Get(&o), ShouldBeNil)
					So(o.Value, ShouldEqual, "hi")
				})

				Convey("deleting it properly records that fact, however", func() {
					o := object{ID: 1}
					So(ds.Delete(ds.KeyForObj(&o)), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)

					itm, err = mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Value(), ShouldResemble, []byte{})

					// this one hits memcache
					So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)
				})
			})

			Convey("compression works", func() {
				o := object{ID: 2, Value: `¯\_(ツ)_/¯`}
				data := make([]byte, 4000)
				for i := range data {
					const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()"
					data[i] = alpha[i%len(alpha)]
				}
				o.BigData = data

				So(ds.Put(&o), ShouldBeNil)
				So(ds.Get(&o), ShouldBeNil)

				itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
				So(err, ShouldBeNil)

				So(itm.Value()[0], ShouldEqual, ZlibCompression)
				So(len(itm.Value()), ShouldEqual, 653) // a bit smaller than 4k

				// ensure the next Get comes from the cache
				So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)

				o = object{ID: 2}
				So(ds.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, `¯\_(ツ)_/¯`)
				So(o.BigData, ShouldResemble, data)
			})

			Convey("transactions", func() {
				Convey("work", func() {
					// populate an object @ ID1
					So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
					So(ds.Get(&object{ID: 1}), ShouldBeNil)

					So(ds.Put(&object{ID: 2, Value: "nurbs"}), ShouldBeNil)
					So(ds.Get(&object{ID: 2}), ShouldBeNil)

					// memcache now has the wrong value (simulated race)
					So(dsUnder.Put(&object{ID: 1, Value: "else"}), ShouldBeNil)
					So(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						o := &object{ID: 1}
						So(ds.Get(o), ShouldBeNil)
						So(o.Value, ShouldEqual, "else")
						o.Value = "txn"
						So(ds.Put(o), ShouldBeNil)

						So(ds.Delete(ds.KeyForObj(&object{ID: 2})), ShouldBeNil)
						return nil
					}, &datastore.TransactionOptions{XG: true}), ShouldBeNil)

					_, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 1})))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					_, err = mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 2})))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					o := &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "txn")
				})

				Convey("errors don't invalidate", func() {
					// populate an object @ ID1
					So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
					So(ds.Get(&object{ID: 1}), ShouldBeNil)
					So(numMemcacheItems(), ShouldEqual, 1)

					So(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						o := &object{ID: 1}
						So(ds.Get(o), ShouldBeNil)
						So(o.Value, ShouldEqual, "something")
						o.Value = "txn"
						So(ds.Put(o), ShouldBeNil)
						return errors.New("OH NOES")
					}, nil).Error(), ShouldContainSubstring, "OH NOES")

					// memcache still has the original
					So(numMemcacheItems(), ShouldEqual, 1)
					So(dsUnder.Delete(ds.KeyForObj(&object{ID: 1})), ShouldBeNil)
					o := &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "something")
				})
			})

			Convey("control", func() {
				Convey("per-model bypass", func() {
					type model struct {
						ID         string           `gae:"$id"`
						UseDSCache datastore.Toggle `gae:"$dscache.enable,false"`

						Value string
					}

					itms := []model{
						{ID: "hi", Value: "something"},
						{ID: "there", Value: "else", UseDSCache: datastore.On},
					}

					So(ds.PutMulti(itms), ShouldBeNil)
					So(ds.GetMulti(itms), ShouldBeNil)

					So(numMemcacheItems(), ShouldEqual, 1)
				})

				Convey("per-key shard count", func() {
					s := &shardObj{ID: 4, Value: "hi"}
					So(ds.Put(s), ShouldBeNil)
					So(ds.Get(s), ShouldBeNil)

					So(numMemcacheItems(), ShouldEqual, 1)
					for i := 0; i < 20; i++ {
						So(ds.Get(s), ShouldBeNil)
					}
					So(numMemcacheItems(), ShouldEqual, 4)
				})

				Convey("per-key cache disablement", func() {
					n := &noCacheObj{ID: "nurbs", Value: true}
					So(ds.Put(n), ShouldBeNil)
					So(ds.Get(n), ShouldBeNil)
					So(numMemcacheItems(), ShouldEqual, 0)
				})

				Convey("per-model expiration", func() {
					type model struct {
						ID         int64 `gae:"$id"`
						DSCacheExp int64 `gae:"$dscache.expiration,7"`

						Value string
					}

					So(ds.Put(&model{ID: 1, Value: "mooo"}), ShouldBeNil)
					So(ds.Get(&model{ID: 1}), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&model{ID: 1})))
					So(err, ShouldBeNil)

					clk.Add(10 * time.Second)
					_, err = mc.Get(itm.Key())
					So(err, ShouldEqual, memcache.ErrCacheMiss)
				})
			})

			Convey("screw cases", func() {
				Convey("memcache contains bogus value (simulated failed AddMulti)", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("I am a banana")
					itm := mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).SetValue(sekret)
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemUKNONWN)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("memcache contains bogus value (corrupt entry)", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("I am a banana")
					itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
						SetValue(sekret).
						SetFlags(uint32(ItemHasData)))
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasData)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("other entity has the lock", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("r@vmarod!#)%9T")
					itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
						SetValue(sekret).
						SetFlags(uint32(ItemHasLock)))
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasLock)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("massive entities can't be cached", func() {
					o := &object{ID: 1, Value: "spleen"}
					mr := mathrand.Get(c)
					numRounds := (internalValueSizeLimit / 8) * 2
					buf := bytes.Buffer{}
					for i := 0; i < numRounds; i++ {
						So(binary.Write(&buf, binary.LittleEndian, mr.Int63()), ShouldBeNil)
					}
					o.BigData = buf.Bytes()
					So(ds.Put(o), ShouldBeNil)

					o.BigData = nil
					So(ds.Get(o), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(o)))
					So(err, ShouldBeNil)

					// Is locked until the next put, forcing all access to the datastore.
					So(itm.Value(), ShouldResemble, []byte{})
					So(itm.Flags(), ShouldEqual, ItemHasLock)

					o.BigData = []byte("hi :)")
					So(ds.Put(o), ShouldBeNil)
					So(ds.Get(o), ShouldBeNil)

					itm, err = mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasData)
				})

				Convey("failure on Setting memcache locks is a hard stop", func() {
					c, fb := featureBreaker.FilterMC(c, nil)
					fb.BreakFeatures(nil, "SetMulti")
					ds := datastore.Get(c)
					So(ds.Put(&object{ID: 1}).Error(), ShouldContainSubstring, "SetMulti")
				})

				Convey("failure on Setting memcache locks in a transaction is a hard stop", func() {
					c, fb := featureBreaker.FilterMC(c, nil)
					fb.BreakFeatures(nil, "SetMulti")
					ds := datastore.Get(c)
					So(ds.RunInTransaction(func(c context.Context) error {
						So(datastore.Get(c).Put(&object{ID: 1}), ShouldBeNil)
						// no problems here... memcache operations happen after the function
						// body quits.
						return nil
					}, nil).Error(), ShouldContainSubstring, "SetMulti")
				})

			})

			Convey("misc", func() {
				Convey("verify numShards caps at MaxShards", func() {
					sc := supportContext{shardsForKey: shardsForKey}
					So(sc.numShards(ds.KeyForObj(&shardObj{ID: 9001})), ShouldEqual, MaxShards)
				})

				Convey("CompressionType.String", func() {
					So(NoCompression.String(), ShouldEqual, "NoCompression")
					So(ZlibCompression.String(), ShouldEqual, "ZlibCompression")
					So(CompressionType(100).String(), ShouldEqual, "UNKNOWN_CompressionType(100)")
				})
			})
		})

		Convey("disabled cases", func() {
			defer func() {
				globalEnabled = true
			}()

			So(IsGloballyEnabled(c), ShouldBeTrue)

			So(SetGlobalEnable(c, false), ShouldBeNil)
			// twice is a nop
			So(SetGlobalEnable(c, false), ShouldBeNil)

			// but it takes 5 minutes to kick in
			So(IsGloballyEnabled(c), ShouldBeTrue)
			clk.Add(time.Minute*5 + time.Second)
			So(IsGloballyEnabled(c), ShouldBeFalse)

			So(mc.Set(mc.NewItem("test").SetValue([]byte("hi"))), ShouldBeNil)
			So(numMemcacheItems(), ShouldEqual, 1)
			So(SetGlobalEnable(c, true), ShouldBeNil)
			// memcache gets flushed as a side effect
			So(numMemcacheItems(), ShouldEqual, 0)

			// Still takes 5 minutes to kick in
			So(IsGloballyEnabled(c), ShouldBeFalse)
			clk.Add(time.Minute*5 + time.Second)
			So(IsGloballyEnabled(c), ShouldBeTrue)
		})
	})
}
Пример #9
0
func TestCount(t *testing.T) {
	t.Parallel()

	Convey("Test Count filter", t, func() {
		c, fb := featureBreaker.FilterRDS(memory.Use(context.Background()), nil)
		c, ctr := FilterRDS(c)

		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		ds := datastore.Get(c)
		vals := []datastore.PropertyMap{{
			"Val":  {datastore.MkProperty(100)},
			"$key": {datastore.MkPropertyNI(ds.NewKey("Kind", "", 1, nil))},
		}}

		Convey("Calling a ds function should reflect in counter", func() {
			So(ds.PutMulti(vals), ShouldBeNil)
			So(ctr.NewKey.Successes(), ShouldEqual, 1)
			So(ctr.PutMulti.Successes(), ShouldEqual, 1)

			Convey("effects are cumulative", func() {
				So(ds.PutMulti(vals), ShouldBeNil)
				So(ctr.PutMulti.Successes(), ShouldEqual, 2)

				Convey("even within transactions", func() {
					ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						So(ds.PutMulti(append(vals, vals[0])), ShouldBeNil)
						return nil
					}, nil)
				})
			})
		})

		Convey("errors count against errors", func() {
			fb.BreakFeatures(nil, "GetMulti")

			ds.GetMulti(vals)
			So(ctr.GetMulti.Errors(), ShouldEqual, 1)

			fb.UnbreakFeatures("GetMulti")

			So(ds.PutMulti(vals), ShouldBeNil)

			ds.GetMulti(vals)
			So(ctr.GetMulti.Errors(), ShouldEqual, 1)
			So(ctr.GetMulti.Successes(), ShouldEqual, 1)
			So(ctr.GetMulti.Total(), ShouldEqual, 2)
		})
	})

	Convey("works for memcache", t, func() {
		c, ctr := FilterMC(memory.Use(context.Background()))
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)
		mc := memcache.Get(c)

		mc.Set(mc.NewItem("hello").SetValue([]byte("sup")))
		So(mc.Get(mc.NewItem("Wat")), ShouldNotBeNil)
		mc.Get(mc.NewItem("hello"))

		So(ctr.SetMulti, shouldHaveSuccessesAndErrors, 1, 0)
		So(ctr.GetMulti, shouldHaveSuccessesAndErrors, 2, 0)
		So(ctr.NewItem, shouldHaveSuccessesAndErrors, 3, 0)
	})

	Convey("works for taskqueue", t, func() {
		c, ctr := FilterTQ(memory.Use(context.Background()))
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)
		tq := taskqueue.Get(c)

		tq.Add(&taskqueue.Task{Name: "wat"}, "")
		tq.Add(&taskqueue.Task{Name: "wat"}, "DNE_QUEUE")

		So(ctr.AddMulti, shouldHaveSuccessesAndErrors, 1, 1)
	})

	Convey("works for global info", t, func() {
		c, fb := featureBreaker.FilterGI(memory.Use(context.Background()), nil)
		c, ctr := FilterGI(c)
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		gi := info.Get(c)

		gi.Namespace("foo")
		fb.BreakFeatures(nil, "Namespace")
		gi.Namespace("boom")

		So(ctr.Namespace, shouldHaveSuccessesAndErrors, 1, 1)
	})
}