// Remove safely deletes an account and all its associated information in the datastore. This includes
// any objects that are descendants of the Account (i.e., a cascading delete).
func Remove(ctx context.Context, account *Account) error {
return datastore.RunInTransaction(ctx, func(txCtx context.Context) error {
acctKey := account.Key(txCtx)
q := datastore.NewQuery("").
Ancestor(acctKey).
KeysOnly()
if changed, err := HasChanged(txCtx, account); err != nil {
return err
} else if changed {
return ErrConflict
}
keys, err := q.GetAll(txCtx, nil)
if err != nil {
return err
}
keys = append(keys, acctKey)
return nds.DeleteMulti(txCtx, keys)
}, nil)
}
func TestDeleteMemcacheFail(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "Entity", "", 1, nil)
keys := []*datastore.Key{key}
entities := make([]testEntity, 1)
entities[0].Val = 43
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
nds.SetMemcacheSetMulti(func(c context.Context,
items []*memcache.Item) error {
return errors.New("expected error")
})
defer func() {
nds.SetMemcacheSetMulti(memcache.SetMulti)
}()
if err := nds.DeleteMulti(c, keys); err == nil {
t.Fatal("expected DeleteMulti error")
}
}
func TestDeleteInTransaction(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "TestEntity", "", 1, nil)
if _, err := nds.Put(c, key, &testEntity{2}); err != nil {
t.Fatal(err)
}
// Prime cache.
if err := nds.Get(c, key, &testEntity{}); err != nil {
t.Fatal(err)
}
if err := nds.RunInTransaction(c, func(tc context.Context) error {
return nds.DeleteMulti(tc, []*datastore.Key{key})
}, nil); err != nil {
t.Fatal(err)
}
if err := nds.Get(c, key, &testEntity{}); err == nil {
t.Fatal("expected no entity")
} else if err != datastore.ErrNoSuchEntity {
t.Fatal(err)
}
}
// Delete removes all values for the counter from the datastore.
func (c *Counter) Delete(ctx context.Context) error {
// generate all the keys we need
if err := nds.DeleteMulti(ctx, c.Keys(ctx)); err == nil || err == datastore.ErrNoSuchEntity {
return nil
} else {
return err
}
}
func TestDeleteMulti(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type TestEntity struct {
Value int
}
for _, count := range []int{499, 500, 501} {
keys := make([]*datastore.Key, count)
entities := make([]TestEntity, count)
for i := range keys {
keys[i] = datastore.NewKey(c, "TestEntity", strconv.Itoa(i), 0, nil)
entities[i] = TestEntity{i}
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Prime cache.
entities = make([]TestEntity, count)
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
err := nds.GetMulti(c, keys, make([]TestEntity, count))
if err == nil {
t.Fatal("expect error")
}
me, ok := err.(appengine.MultiError)
if !ok {
t.Fatal("should be MultiError")
}
for _, e := range me {
if e != datastore.ErrNoSuchEntity {
t.Fatal("expected ErrNoSuchEntity")
}
}
}
}
// Creates and configures a store that stores entities in Google AppEngines memcache and datastore.
// github.com/qedus/nds is used for strongly consistent automatic caching.
func NewGaeStore(kind string, ctx context.Context, idf sus.IdFactory, vf sus.VersionFactory, ei sus.EntityInitializer) sus.Store {
getKey := func(ctx context.Context, id string) *datastore.Key {
return datastore.NewKey(ctx, kind, id, 0, nil)
}
getMulti := func(ids []string) (vs []sus.Version, err error) {
count := len(ids)
vs = make([]sus.Version, count, count)
ks := make([]*datastore.Key, count, count)
for i := 0; i < count; i++ {
vs[i] = vf()
ks[i] = getKey(ctx, ids[i])
}
err = nds.GetMulti(ctx, ks, vs)
return
}
putMulti := func(ids []string, vs []sus.Version) (err error) {
count := len(ids)
ks := make([]*datastore.Key, count, count)
for i := 0; i < count; i++ {
ks[i] = getKey(ctx, ids[i])
}
_, err = nds.PutMulti(ctx, ks, vs)
return
}
delMulti := func(ids []string) error {
count := len(ids)
ks := make([]*datastore.Key, count, count)
for i := 0; i < count; i++ {
ks[i] = getKey(ctx, ids[i])
}
return nds.DeleteMulti(ctx, ks)
}
isNonExtantError := func(err error) bool {
return err.Error() == datastore.ErrNoSuchEntity.Error()
}
rit := func(tran sus.Transaction) error {
return nds.RunInTransaction(ctx, func(ctx context.Context) error {
return tran()
}, &datastore.TransactionOptions{XG: true})
}
return sus.NewStore(getMulti, putMulti, delMulti, idf, vf, ei, isNonExtantError, rit)
}
func TestDelete(t *testing.T) {
c, closeFunc := NewContext(t, nil)
defer closeFunc()
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "Entity", "", 1, nil)
keys := []*datastore.Key{key}
entities := make([]testEntity, 1)
entities[0].Val = 43
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
entities = make([]testEntity, 1)
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
entity := entities[0]
if entity.Val != 43 {
t.Fatal("incorrect entity.Val", entity.Val)
}
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
keys = []*datastore.Key{key}
entities = make([]testEntity, 1)
err := nds.GetMulti(c, keys, entities)
if me, ok := err.(appengine.MultiError); ok {
if me[0] != datastore.ErrNoSuchEntity {
t.Fatal("entity should be deleted", entities)
}
} else {
t.Fatal("expected appengine.MultiError")
}
}
func TestGetMultiPaths(t *testing.T) {
expectedErr := errors.New("expected error")
type memcacheGetMultiFunc func(c context.Context,
keys []string) (map[string]*memcache.Item, error)
memcacheGetMultiFail := func(c context.Context,
keys []string) (map[string]*memcache.Item, error) {
return nil, expectedErr
}
type memcacheAddMultiFunc func(c context.Context,
items []*memcache.Item) error
memcacheAddMultiFail := func(c context.Context,
items []*memcache.Item) error {
return expectedErr
}
type memcacheCompareAndSwapMultiFunc func(c context.Context,
items []*memcache.Item) error
memcacheCompareAndSwapMultiFail := func(c context.Context,
items []*memcache.Item) error {
return expectedErr
}
type datastoreGetMultiFunc func(c context.Context,
keys []*datastore.Key, vals interface{}) error
datastoreGetMultiFail := func(c context.Context,
keys []*datastore.Key, vals interface{}) error {
return expectedErr
}
type marshalFunc func(pl datastore.PropertyList) ([]byte, error)
marshalFail := func(pl datastore.PropertyList) ([]byte, error) {
return nil, expectedErr
}
type unmarshalFunc func(data []byte, pl *datastore.PropertyList) error
/*
unmarshalFail := func(data []byte, pl *datastore.PropertyList) error {
return expectedErr
}
*/
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int64
}
keysVals := func(c context.Context, count int64) (
[]*datastore.Key, []testEntity) {
keys, vals := make([]*datastore.Key, count), make([]testEntity, count)
for i := int64(0); i < count; i++ {
keys[i] = datastore.NewKey(c, "Entity", "", i+1, nil)
vals[i] = testEntity{i + 1}
}
return keys, vals
}
tests := []struct {
description string
// Number of keys used to as GetMulti params.
keyCount int64
// Number of times GetMulti is called.
callCount int
// There are 2 memcacheGetMulti calls for every GetMulti call.
memcacheGetMultis []memcacheGetMultiFunc
memcacheAddMulti memcacheAddMultiFunc
memcacheCompareAndSwapMulti memcacheCompareAndSwapMultiFunc
datastoreGetMulti datastoreGetMultiFunc
marshal marshalFunc
// There are 2 unmarshal calls for every GetMultiCall.
unmarshals []unmarshalFunc
expectedErrs []error
}{
{
"no errors",
20,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"datastore unknown error",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastoreGetMultiFail,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{expectedErr},
},
{
"datastore unknown multierror",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
func(c context.Context,
keys []*datastore.Key, vals interface{}) error {
me := make(appengine.MultiError, len(keys))
for i := range me {
me[i] = expectedErr
}
return me
},
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{
appengine.MultiError{expectedErr, expectedErr},
},
},
{
"marshal error",
5,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
marshalFail,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"total memcache fail",
20,
1,
[]memcacheGetMultiFunc{
memcacheGetMultiFail,
memcacheGetMultiFail,
},
memcacheAddMultiFail,
memcacheCompareAndSwapMultiFail,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"lock memcache fail",
20,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcacheGetMultiFail,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"memcache corrupt",
2,
2,
[]memcacheGetMultiFunc{
// Charge memcache.
memcache.GetMulti,
memcache.GetMulti,
// Corrupt memcache.
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt items.
for _, item := range items {
item.Value = []byte("corrupt string")
}
return items, err
},
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil, nil},
},
{
"memcache flag corrupt",
2,
2,
[]memcacheGetMultiFunc{
// Charge memcache.
memcache.GetMulti,
memcache.GetMulti,
// Corrupt memcache flags.
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Flags = 56
}
return items, err
},
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil, nil},
},
{
"lock memcache value fail",
20,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Value = []byte("corrupt value")
}
return items, err
},
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"lock memcache value none item",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Flags = nds.NoneItem
}
return items, err
},
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{
appengine.MultiError{
datastore.ErrNoSuchEntity,
datastore.ErrNoSuchEntity,
},
},
},
{
"memcache get no entity unmarshal fail",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Flags = nds.EntityItem
}
return items, err
},
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
}
for _, test := range tests {
t.Log("Start", test.description)
keys, putVals := keysVals(c, test.keyCount)
if _, err := nds.PutMulti(c, keys, putVals); err != nil {
t.Fatal(err)
}
memcacheGetChan := make(chan memcacheGetMultiFunc,
len(test.memcacheGetMultis))
for _, fn := range test.memcacheGetMultis {
memcacheGetChan <- fn
}
nds.SetMemcacheGetMulti(func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
fn := <-memcacheGetChan
return fn(c, keys)
})
nds.SetMemcacheAddMulti(test.memcacheAddMulti)
nds.SetMemcacheCompareAndSwapMulti(test.memcacheCompareAndSwapMulti)
nds.SetDatastoreGetMulti(test.datastoreGetMulti)
nds.SetMarshal(test.marshal)
unmarshalChan := make(chan unmarshalFunc,
len(test.unmarshals))
for _, fn := range test.unmarshals {
unmarshalChan <- fn
}
nds.SetUnmarshal(func(data []byte, pl *datastore.PropertyList) error {
fn := <-unmarshalChan
return fn(data, pl)
})
for i := 0; i < test.callCount; i++ {
getVals := make([]testEntity, test.keyCount)
err := nds.GetMulti(c, keys, getVals)
expectedErr := test.expectedErrs[i]
if expectedErr == nil {
if err != nil {
t.Fatal(err)
}
for i := range getVals {
if getVals[i].IntVal != putVals[i].IntVal {
t.Fatal("incorrect IntVal")
}
}
continue
}
if err == nil {
t.Fatal("expected error")
}
expectedMultiErr, isMultiErr := expectedErr.(appengine.MultiError)
if isMultiErr {
me, ok := err.(appengine.MultiError)
if !ok {
t.Fatal("expected appengine.MultiError but got", err)
}
if len(me) != len(expectedMultiErr) {
t.Fatal("appengine.MultiError length incorrect")
}
for i, e := range me {
if e != expectedMultiErr[i] {
t.Fatal("non matching errors", e, expectedMultiErr[i])
}
if e == nil {
if getVals[i].IntVal != putVals[i].IntVal {
t.Fatal("incorrect IntVal")
}
}
}
}
}
// Reset App Engine API calls.
nds.SetMemcacheGetMulti(memcache.GetMulti)
nds.SetMemcacheAddMulti(memcache.AddMulti)
nds.SetMemcacheCompareAndSwapMulti(memcache.CompareAndSwapMulti)
nds.SetDatastoreGetMulti(datastore.GetMulti)
nds.SetMarshal(nds.MarshalPropertyList)
nds.SetUnmarshal(nds.UnmarshalPropertyList)
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
t.Log("End", test.description)
}
}
func TestInterfaces(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
incompleteKey := datastore.NewIncompleteKey(c, "Entity", nil)
incompleteKeys := []*datastore.Key{incompleteKey}
entities := []interface{}{&testEntity{43}}
keys, err := nds.PutMulti(c, incompleteKeys, entities)
if err != nil {
t.Fatal(err)
}
if len(keys) != 1 {
t.Fatal("len(keys) != 1")
}
if keys[0].Incomplete() {
t.Fatal("Key is incomplete")
}
entities = []interface{}{&testEntity{}}
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if entities[0].(*testEntity).Val != 43 {
t.Fatal("te.Val != 43")
}
// Get from cache.
entities = []interface{}{&testEntity{}}
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if entities[0].(*testEntity).Val != 43 {
t.Fatal("te.Val != 43")
}
// Change value.
entities = []interface{}{&testEntity{64}}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Get from nds with struct.
entities = []interface{}{&testEntity{}}
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if entities[0].(*testEntity).Val != 64 {
t.Fatal("te.Val != 64")
}
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
entities = []interface{}{testEntity{}}
err = nds.GetMulti(c, keys, entities)
if me, ok := err.(appengine.MultiError); ok {
if len(me) != 1 {
t.Fatal("expected 1 appengine.MultiError")
}
if me[0] != datastore.ErrNoSuchEntity {
t.Fatal("expected datastore.ErrNoSuchEntity")
}
} else {
t.Fatal("expected datastore.ErrNoSuchEntity", err)
}
}
