Beispiel #1
0
func TestBrokenFeatures(t *testing.T) {
	t.Parallel()

	e := errors.New("default err")

	Convey("BrokenFeatures", t, func() {
		c := memory.Use(context.Background())

		Convey("Can break ds", func() {
			Convey("without a default", func() {
				c, bf := FilterRDS(c, nil)
				ds := datastore.Get(c)
				vals := []datastore.PropertyMap{{
					"$key": {datastore.MkPropertyNI(ds.NewKey("Wut", "", 1, nil))},
				}}

				Convey("by specifying an error", func() {
					bf.BreakFeatures(e, "GetMulti", "PutMulti")
					So(ds.GetMulti(vals), ShouldEqual, e)

					Convey("and you can unbreak them as well", func() {
						bf.UnbreakFeatures("GetMulti")

						So(errors.SingleError(ds.GetMulti(vals)), ShouldEqual, datastore.ErrNoSuchEntity)

						Convey("no broken features at all is a shortcut", func() {
							bf.UnbreakFeatures("PutMulti")
							So(errors.SingleError(ds.GetMulti(vals)), ShouldEqual, datastore.ErrNoSuchEntity)
						})
					})
				})

				Convey("Not specifying an error gets you a generic error", func() {
					bf.BreakFeatures(nil, "GetMulti")
					So(ds.GetMulti(vals).Error(), ShouldContainSubstring, `feature "GetMulti" is broken`)
				})
			})

			Convey("with a default", func() {
				c, bf := FilterRDS(c, e)
				ds := datastore.Get(c)
				vals := []datastore.PropertyMap{{
					"$key": {datastore.MkPropertyNI(ds.NewKey("Wut", "", 1, nil))},
				}}
				bf.BreakFeatures(nil, "GetMulti")
				So(ds.GetMulti(vals), ShouldEqual, e)
			})
		})
	})
}
Beispiel #2
0
func incrementLocked(ents *memCollection, key []byte, amt int) int64 {
	if amt <= 0 {
		panic(fmt.Errorf("incrementLocked called with bad `amt`: %d", amt))
	}
	ret := curVersion(ents, key) + 1
	ents.Set(key, serialize.ToBytes(ds.PropertyMap{
		"__version__": {ds.MkPropertyNI(ret + int64(amt-1))},
	}))
	return ret
}
Beispiel #3
0
func TestCount(t *testing.T) {
	t.Parallel()

	Convey("Test Count filter", t, func() {
		c, fb := featureBreaker.FilterRDS(memory.Use(context.Background()), nil)
		c, ctr := FilterRDS(c)

		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		ds := datastore.Get(c)
		vals := []datastore.PropertyMap{{
			"Val":  {datastore.MkProperty(100)},
			"$key": {datastore.MkPropertyNI(ds.NewKey("Kind", "", 1, nil))},
		}}

		Convey("Calling a ds function should reflect in counter", func() {
			So(ds.PutMulti(vals), ShouldBeNil)
			So(ctr.PutMulti.Successes(), ShouldEqual, 1)

			Convey("effects are cumulative", func() {
				So(ds.PutMulti(vals), ShouldBeNil)
				So(ctr.PutMulti.Successes(), ShouldEqual, 2)

				Convey("even within transactions", func() {
					die(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						So(ds.PutMulti(append(vals, vals[0])), ShouldBeNil)
						return nil
					}, nil))
				})
			})
		})

		Convey("errors count against errors", func() {
			fb.BreakFeatures(nil, "GetMulti")

			So(ds.GetMulti(vals), ShouldErrLike, `"GetMulti" is broken`)
			So(ctr.GetMulti.Errors(), ShouldEqual, 1)

			fb.UnbreakFeatures("GetMulti")

			So(ds.PutMulti(vals), ShouldBeNil)

			die(ds.GetMulti(vals))
			So(ctr.GetMulti.Errors(), ShouldEqual, 1)
			So(ctr.GetMulti.Successes(), ShouldEqual, 1)
			So(ctr.GetMulti.Total(), ShouldEqual, 2)
		})
	})

	Convey("works for memcache", t, func() {
		c, ctr := FilterMC(memory.Use(context.Background()))
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)
		mc := memcache.Get(c)

		die(mc.Set(mc.NewItem("hello").SetValue([]byte("sup"))))

		_, err := mc.Get("Wat")
		So(err, ShouldNotBeNil)

		_, err = mc.Get("hello")
		die(err)

		So(ctr.SetMulti, shouldHaveSuccessesAndErrors, 1, 0)
		So(ctr.GetMulti, shouldHaveSuccessesAndErrors, 2, 0)
		So(ctr.NewItem, shouldHaveSuccessesAndErrors, 3, 0)
	})

	Convey("works for taskqueue", t, func() {
		c, ctr := FilterTQ(memory.Use(context.Background()))
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)
		tq := taskqueue.Get(c)

		die(tq.Add(&taskqueue.Task{Name: "wat"}, ""))
		So(tq.Add(&taskqueue.Task{Name: "wat"}, "DNE_QUEUE"),
			ShouldErrLike, "UNKNOWN_QUEUE")

		So(ctr.AddMulti, shouldHaveSuccessesAndErrors, 1, 1)
	})

	Convey("works for global info", t, func() {
		c, fb := featureBreaker.FilterGI(memory.Use(context.Background()), nil)
		c, ctr := FilterGI(c)
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		gi := info.Get(c)

		_, err := gi.Namespace("foo")
		die(err)
		fb.BreakFeatures(nil, "Namespace")
		_, err = gi.Namespace("boom")
		So(err, ShouldErrLike, `"Namespace" is broken`)

		So(ctr.Namespace, shouldHaveSuccessesAndErrors, 1, 1)
	})

	Convey("works for user", t, func() {
		c, fb := featureBreaker.FilterUser(memory.Use(context.Background()), nil)
		c, ctr := FilterUser(c)
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		u := user.Get(c)

		_, err := u.CurrentOAuth("foo")
		die(err)
		fb.BreakFeatures(nil, "CurrentOAuth")
		_, err = u.CurrentOAuth("foo")
		So(err, ShouldErrLike, `"CurrentOAuth" is broken`)

		So(ctr.CurrentOAuth, shouldHaveSuccessesAndErrors, 1, 1)
	})

	Convey("works for mail", t, func() {
		c, fb := featureBreaker.FilterMail(memory.Use(context.Background()), nil)
		c, ctr := FilterMail(c)
		So(c, ShouldNotBeNil)
		So(ctr, ShouldNotBeNil)

		m := mail.Get(c)

		err := m.Send(&mail.Message{
			Sender: "*****@*****.**",
			To:     []string{"*****@*****.**"},
			Body:   "hi",
		})
		die(err)

		fb.BreakFeatures(nil, "Send")
		err = m.Send(&mail.Message{
			Sender: "*****@*****.**",
			To:     []string{"*****@*****.**"},
			Body:   "hi",
		})
		So(err, ShouldErrLike, `"Send" is broken`)

		So(ctr.Send, shouldHaveSuccessesAndErrors, 1, 1)
	})
}
Beispiel #4
0
func TestDatastoreSingleReadWriter(t *testing.T) {
	t.Parallel()

	Convey("Datastore single reads and writes", t, func() {
		c := Use(context.Background())
		ds := dsS.Get(c)
		So(ds, ShouldNotBeNil)

		Convey("getting objects that DNE is an error", func() {
			So(ds.Get(&Foo{ID: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
		})

		Convey("bad namespaces fail", func() {
			_, err := infoS.Get(c).Namespace("$$blzyall")
			So(err.Error(), ShouldContainSubstring, "namespace \"$$blzyall\" does not match")
		})

		Convey("Can Put stuff", func() {
			// with an incomplete key!
			f := &Foo{Val: 10}
			So(ds.Put(f), ShouldBeNil)
			k := ds.KeyForObj(f)
			So(k.String(), ShouldEqual, "dev~app::/Foo,1")

			Convey("and Get it back", func() {
				newFoo := &Foo{ID: 1}
				So(ds.Get(newFoo), ShouldBeNil)
				So(newFoo, ShouldResemble, f)

				Convey("but it's hidden from a different namespace", func() {
					c, err := infoS.Get(c).Namespace("whombat")
					So(err, ShouldBeNil)
					ds = dsS.Get(c)
					So(ds.Get(f), ShouldEqual, dsS.ErrNoSuchEntity)
				})

				Convey("and we can Delete it", func() {
					So(ds.Delete(k), ShouldBeNil)
					So(ds.Get(newFoo), ShouldEqual, dsS.ErrNoSuchEntity)
				})

			})
			Convey("Deleteing with a bogus key is bad", func() {
				So(ds.Delete(ds.NewKey("Foo", "wat", 100, nil)), ShouldEqual, dsS.ErrInvalidKey)
			})
			Convey("Deleteing a DNE entity is fine", func() {
				So(ds.Delete(ds.NewKey("Foo", "wat", 0, nil)), ShouldBeNil)
			})

			Convey("Deleting entities from a nonexistant namespace works", func() {
				aid := infoS.Get(c).FullyQualifiedAppID()
				keys := make([]*dsS.Key, 10)
				for i := range keys {
					keys[i] = ds.MakeKey(aid, "noexist", "Kind", i+1)
				}
				So(ds.DeleteMulti(keys), ShouldBeNil)
				count := 0
				So(ds.Raw().DeleteMulti(keys, func(err error) error {
					count++
					So(err, ShouldBeNil)
					return nil
				}), ShouldBeNil)
				So(count, ShouldEqual, len(keys))
			})

			Convey("with multiple puts", func() {
				So(testGetMeta(c, k), ShouldEqual, 1)

				foos := make([]Foo, 10)
				for i := range foos {
					foos[i].Val = 10
					foos[i].Parent = k
				}
				So(ds.PutMulti(foos), ShouldBeNil)
				So(testGetMeta(c, k), ShouldEqual, 11)

				keys := make([]*dsS.Key, len(foos))
				for i, f := range foos {
					keys[i] = ds.KeyForObj(&f)
				}

				Convey("ensure that group versions persist across deletes", func() {
					So(ds.DeleteMulti(append(keys, k)), ShouldBeNil)

					ds.Testable().CatchupIndexes()

					count := 0
					So(ds.Run(dsS.NewQuery(""), func(_ *dsS.Key) {
						count++
					}), ShouldBeNil)
					So(count, ShouldEqual, 3)

					So(testGetMeta(c, k), ShouldEqual, 22)

					So(ds.Put(&Foo{ID: 1}), ShouldBeNil)
					So(testGetMeta(c, k), ShouldEqual, 23)
				})

				Convey("can Get", func() {
					vals := make([]dsS.PropertyMap, len(keys))
					for i := range vals {
						vals[i] = dsS.PropertyMap{}
						So(vals[i].SetMeta("key", keys[i]), ShouldBeTrue)
					}
					So(ds.GetMulti(vals), ShouldBeNil)

					for i, val := range vals {
						So(val, ShouldResemble, dsS.PropertyMap{
							"Val":  {dsS.MkProperty(10)},
							"$key": {dsS.MkPropertyNI(keys[i])},
						})
					}
				})

			})

			Convey("allocating ids prevents their use", func() {
				start, err := ds.AllocateIDs(ds.MakeKey("Foo", 0), 100)
				So(err, ShouldBeNil)
				So(start, ShouldEqual, 2)

				f := &Foo{Val: 10}
				So(ds.Put(f), ShouldBeNil)
				k := ds.KeyForObj(f)
				So(k.String(), ShouldEqual, "dev~app::/Foo,102")
			})
		})

		Convey("implements DSTransactioner", func() {
			Convey("Put", func() {
				f := &Foo{Val: 10}
				So(ds.Put(f), ShouldBeNil)
				k := ds.KeyForObj(f)
				So(k.String(), ShouldEqual, "dev~app::/Foo,1")

				Convey("can Put new entity groups", func() {
					err := ds.RunInTransaction(func(c context.Context) error {
						ds := dsS.Get(c)

						f := &Foo{Val: 100}
						So(ds.Put(f), ShouldBeNil)
						So(f.ID, ShouldEqual, 2)

						f.ID = 0
						f.Val = 200
						So(ds.Put(f), ShouldBeNil)
						So(f.ID, ShouldEqual, 3)

						return nil
					}, &dsS.TransactionOptions{XG: true})
					So(err, ShouldBeNil)

					f := &Foo{ID: 2}
					So(ds.Get(f), ShouldBeNil)
					So(f.Val, ShouldEqual, 100)

					f.ID = 3
					So(ds.Get(f), ShouldBeNil)
					So(f.Val, ShouldEqual, 200)
				})

				Convey("can Put new entities in a current group", func() {
					err := ds.RunInTransaction(func(c context.Context) error {
						ds := dsS.Get(c)

						f := &Foo{Val: 100, Parent: k}
						So(ds.Put(f), ShouldBeNil)
						So(ds.KeyForObj(f).String(), ShouldEqual, "dev~app::/Foo,1/Foo,1")

						f.ID = 0
						f.Val = 200
						So(ds.Put(f), ShouldBeNil)
						So(ds.KeyForObj(f).String(), ShouldEqual, "dev~app::/Foo,1/Foo,2")

						return nil
					}, nil)
					So(err, ShouldBeNil)

					f := &Foo{ID: 1, Parent: k}
					So(ds.Get(f), ShouldBeNil)
					So(f.Val, ShouldEqual, 100)

					f.ID = 2
					So(ds.Get(f), ShouldBeNil)
					So(f.Val, ShouldEqual, 200)
				})

				Convey("Deletes work too", func() {
					err := ds.RunInTransaction(func(c context.Context) error {
						return dsS.Get(c).Delete(k)
					}, nil)
					So(err, ShouldBeNil)
					So(ds.Get(&Foo{ID: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
				})

				Convey("A Get counts against your group count", func() {
					err := ds.RunInTransaction(func(c context.Context) error {
						ds := dsS.Get(c)

						pm := dsS.PropertyMap{}
						So(pm.SetMeta("key", ds.NewKey("Foo", "", 20, nil)), ShouldBeTrue)
						So(ds.Get(pm), ShouldEqual, dsS.ErrNoSuchEntity)

						So(pm.SetMeta("key", k), ShouldBeTrue)
						So(ds.Get(pm).Error(), ShouldContainSubstring, "cross-group")
						return nil
					}, nil)
					So(err, ShouldBeNil)
				})

				Convey("Get takes a snapshot", func() {
					err := ds.RunInTransaction(func(c context.Context) error {
						ds := dsS.Get(c)

						So(ds.Get(f), ShouldBeNil)
						So(f.Val, ShouldEqual, 10)

						// Don't ever do this in a real program unless you want to guarantee
						// a failed transaction :)
						f.Val = 11
						So(dsS.GetNoTxn(c).Put(f), ShouldBeNil)

						So(ds.Get(f), ShouldBeNil)
						So(f.Val, ShouldEqual, 10)

						return nil
					}, nil)
					So(err, ShouldBeNil)

					f := &Foo{ID: 1}
					So(ds.Get(f), ShouldBeNil)
					So(f.Val, ShouldEqual, 11)
				})

				Convey("and snapshots are consistent even after Puts", func() {
					err := ds.RunInTransaction(func(c context.Context) error {
						ds := dsS.Get(c)

						f := &Foo{ID: 1}
						So(ds.Get(f), ShouldBeNil)
						So(f.Val, ShouldEqual, 10)

						// Don't ever do this in a real program unless you want to guarantee
						// a failed transaction :)
						f.Val = 11
						So(dsS.GetNoTxn(c).Put(f), ShouldBeNil)

						So(ds.Get(f), ShouldBeNil)
						So(f.Val, ShouldEqual, 10)

						f.Val = 20
						So(ds.Put(f), ShouldBeNil)

						So(ds.Get(f), ShouldBeNil)
						So(f.Val, ShouldEqual, 10) // still gets 10

						return nil
					}, &dsS.TransactionOptions{Attempts: 1})
					So(err.Error(), ShouldContainSubstring, "concurrent")

					f := &Foo{ID: 1}
					So(ds.Get(f), ShouldBeNil)
					So(f.Val, ShouldEqual, 11)
				})

				Convey("Reusing a transaction context is bad news", func() {
					txnDS := dsS.Interface(nil)
					err := ds.RunInTransaction(func(c context.Context) error {
						txnDS = dsS.Get(c)
						So(txnDS.Get(f), ShouldBeNil)
						return nil
					}, nil)
					So(err, ShouldBeNil)
					So(txnDS.Get(f).Error(), ShouldContainSubstring, "expired")
				})

				Convey("Nested transactions are rejected", func() {
					err := ds.RunInTransaction(func(c context.Context) error {
						err := dsS.Get(c).RunInTransaction(func(c context.Context) error {
							panic("noooo")
						}, nil)
						So(err.Error(), ShouldContainSubstring, "nested transactions")
						return nil
					}, nil)
					So(err, ShouldBeNil)
				})

				Convey("Concurrent transactions only accept one set of changes", func() {
					// Note: I think this implementation is actually /slightly/ wrong.
					// According to my read of the docs for appengine, when you open a
					// transaction it actually (essentially) holds a reference to the
					// entire datastore. Our implementation takes a snapshot of the
					// entity group as soon as something observes/affects it.
					//
					// That said... I'm not sure if there's really a semantic difference.
					err := ds.RunInTransaction(func(c context.Context) error {
						So(dsS.Get(c).Put(&Foo{ID: 1, Val: 21}), ShouldBeNil)

						err := dsS.GetNoTxn(c).RunInTransaction(func(c context.Context) error {
							So(dsS.Get(c).Put(&Foo{ID: 1, Val: 27}), ShouldBeNil)
							return nil
						}, nil)
						So(err, ShouldBeNil)

						return nil
					}, nil)
					So(err.Error(), ShouldContainSubstring, "concurrent")

					f := &Foo{ID: 1}
					So(ds.Get(f), ShouldBeNil)
					So(f.Val, ShouldEqual, 27)
				})

				Convey("XG", func() {
					Convey("Modifying two groups with XG=false is invalid", func() {
						err := ds.RunInTransaction(func(c context.Context) error {
							ds := dsS.Get(c)
							f := &Foo{ID: 1, Val: 200}
							So(ds.Put(f), ShouldBeNil)

							f.ID = 2
							err := ds.Put(f)
							So(err.Error(), ShouldContainSubstring, "cross-group")
							return err
						}, nil)
						So(err.Error(), ShouldContainSubstring, "cross-group")
					})

					Convey("Modifying >25 groups with XG=true is invald", func() {
						err := ds.RunInTransaction(func(c context.Context) error {
							ds := dsS.Get(c)
							foos := make([]Foo, 25)
							for i := int64(1); i < 26; i++ {
								foos[i-1].ID = i
								foos[i-1].Val = 200
							}
							So(ds.PutMulti(foos), ShouldBeNil)
							err := ds.Put(&Foo{ID: 26})
							So(err.Error(), ShouldContainSubstring, "too many entity groups")
							return err
						}, &dsS.TransactionOptions{XG: true})
						So(err.Error(), ShouldContainSubstring, "too many entity groups")
					})
				})

				Convey("Errors and panics", func() {
					Convey("returning an error aborts", func() {
						err := ds.RunInTransaction(func(c context.Context) error {
							ds := dsS.Get(c)
							So(ds.Put(&Foo{ID: 1, Val: 200}), ShouldBeNil)
							return fmt.Errorf("thingy")
						}, nil)
						So(err.Error(), ShouldEqual, "thingy")

						f := &Foo{ID: 1}
						So(ds.Get(f), ShouldBeNil)
						So(f.Val, ShouldEqual, 10)
					})

					Convey("panicing aborts", func() {
						So(func() {
							So(ds.RunInTransaction(func(c context.Context) error {
								ds := dsS.Get(c)
								So(ds.Put(&Foo{Val: 200}), ShouldBeNil)
								panic("wheeeeee")
							}, nil), ShouldBeNil)
						}, ShouldPanic)

						f := &Foo{ID: 1}
						So(ds.Get(f), ShouldBeNil)
						So(f.Val, ShouldEqual, 10)
					})
				})

				Convey("Transaction retries", func() {
					tst := ds.Testable()
					Reset(func() { tst.SetTransactionRetryCount(0) })

					Convey("SetTransactionRetryCount set to zero", func() {
						tst.SetTransactionRetryCount(0)
						calls := 0
						So(ds.RunInTransaction(func(c context.Context) error {
							calls++
							return nil
						}, nil), ShouldBeNil)
						So(calls, ShouldEqual, 1)
					})

					Convey("default TransactionOptions is 3 attempts", func() {
						tst.SetTransactionRetryCount(100) // more than 3
						calls := 0
						So(ds.RunInTransaction(func(c context.Context) error {
							calls++
							return nil
						}, nil), ShouldEqual, dsS.ErrConcurrentTransaction)
						So(calls, ShouldEqual, 3)
					})

					Convey("non-default TransactionOptions ", func() {
						tst.SetTransactionRetryCount(100) // more than 20
						calls := 0
						So(ds.RunInTransaction(func(c context.Context) error {
							calls++
							return nil
						}, &dsS.TransactionOptions{Attempts: 20}), ShouldEqual, dsS.ErrConcurrentTransaction)
						So(calls, ShouldEqual, 20)
					})

					Convey("SetTransactionRetryCount is respected", func() {
						tst.SetTransactionRetryCount(1) // less than 3
						calls := 0
						So(ds.RunInTransaction(func(c context.Context) error {
							calls++
							return nil
						}, nil), ShouldBeNil)
						So(calls, ShouldEqual, 2)
					})

					Convey("fatal errors are not retried", func() {
						tst.SetTransactionRetryCount(1)
						calls := 0
						So(ds.RunInTransaction(func(c context.Context) error {
							calls++
							return fmt.Errorf("omg")
						}, nil).Error(), ShouldEqual, "omg")
						So(calls, ShouldEqual, 1)
					})
				})
			})
		})

		Convey("Testable.Consistent", func() {
			Convey("false", func() {
				ds.Testable().Consistent(false) // the default
				for i := 0; i < 10; i++ {
					So(ds.Put(&Foo{ID: int64(i + 1), Val: i + 1}), ShouldBeNil)
				}
				q := dsS.NewQuery("Foo").Gt("Val", 3)
				count, err := ds.Count(q)
				So(err, ShouldBeNil)
				So(count, ShouldEqual, 0)

				So(ds.Delete(ds.MakeKey("Foo", 4)), ShouldBeNil)

				count, err = ds.Count(q)
				So(err, ShouldBeNil)
				So(count, ShouldEqual, 0)

				ds.Testable().Consistent(true)
				count, err = ds.Count(q)
				So(err, ShouldBeNil)
				So(count, ShouldEqual, 6)
			})

			Convey("true", func() {
				ds.Testable().Consistent(true)
				for i := 0; i < 10; i++ {
					So(ds.Put(&Foo{ID: int64(i + 1), Val: i + 1}), ShouldBeNil)
				}
				q := dsS.NewQuery("Foo").Gt("Val", 3)
				count, err := ds.Count(q)
				So(err, ShouldBeNil)
				So(count, ShouldEqual, 7)

				So(ds.Delete(ds.MakeKey("Foo", 4)), ShouldBeNil)

				count, err = ds.Count(q)
				So(err, ShouldBeNil)
				So(count, ShouldEqual, 6)
			})
		})

		Convey("Testable.DisableSpecialEntities", func() {
			ds.Testable().DisableSpecialEntities(true)

			So(ds.Put(&Foo{}), ShouldErrLike, "allocateIDs is disabled")

			So(ds.Put(&Foo{ID: 1}), ShouldBeNil)

			ds.Testable().CatchupIndexes()

			count, err := ds.Count(dsS.NewQuery(""))
			So(err, ShouldBeNil)
			So(count, ShouldEqual, 1) // normally this would include __entity_group__
		})
	})
}