func TestRunInTransaction(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "Entity", "", 3, nil)
keys := []*datastore.Key{key}
entity := testEntity{42}
entities := []testEntity{entity}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
err := nds.RunInTransaction(c, func(tc context.Context) error {
entities := make([]testEntity, 1, 1)
if err := nds.GetMulti(tc, keys, entities); err != nil {
t.Fatal(err)
}
entity := entities[0]
if entity.Val != 42 {
t.Fatalf("entity.Val != 42: %d", entity.Val)
}
entities[0].Val = 43
putKeys, err := nds.PutMulti(tc, keys, entities)
if err != nil {
t.Fatal(err)
} else if len(putKeys) != 1 {
t.Fatal("putKeys should be len 1")
} else if !putKeys[0].Equal(key) {
t.Fatal("keys not equal")
}
return nil
}, nil)
if err != nil {
t.Fatal(err)
}
entities = make([]testEntity, 1, 1)
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
entity = entities[0]
if entity.Val != 43 {
t.Fatalf("entity.Val != 43: %d", entity.Val)
}
}
func TestPutMulti(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type TestEntity struct {
Value int
}
for _, count := range []int{499, 500, 501} {
keys := make([]*datastore.Key, count)
entities := make([]TestEntity, count)
for i := range keys {
keys[i] = datastore.NewKey(c, "TestEntity", strconv.Itoa(i), 0, nil)
entities[i] = TestEntity{i}
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
entities = make([]TestEntity, count)
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
}
}
func TestPutPropertyLoadSaver(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int
}
te := &testEntity{2}
pl, err := datastore.SaveStruct(te)
if err != nil {
t.Fatal(err)
}
keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}
pls := datastore.PropertyList(pl)
if _, err := nds.PutMulti(c, keys,
[]datastore.PropertyLoadSaver{&pls}); err != nil {
t.Fatal(err)
}
getPl := datastore.PropertyList{}
if err := nds.GetMulti(c,
keys, []datastore.PropertyLoadSaver{&getPl}); err != nil {
t.Fatal(err)
}
getTe := &testEntity{}
if err := datastore.LoadStruct(getTe, getPl); err != nil {
t.Fatal(err)
}
if te.IntVal != getTe.IntVal {
t.Fatal("expected same IntVal", getTe.IntVal)
}
}
func TestGetMultiLockReturnEntitySetValueFail(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int64
}
keys := []*datastore.Key{}
entities := []testEntity{}
for i := int64(1); i < 3; i++ {
keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
entities = append(entities, testEntity{i})
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Fail to unmarshal test.
memcacheGetChan := make(chan func(c context.Context, keys []string) (
map[string]*memcache.Item, error), 2)
memcacheGetChan <- memcache.GetMulti
memcacheGetChan <- func(c context.Context,
keys []string) (map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
if err != nil {
return nil, err
}
pl := datastore.PropertyList{
datastore.Property{"One", 1, false, false},
}
value, err := nds.MarshalPropertyList(pl)
if err != nil {
return nil, err
}
items[keys[0]].Flags = nds.EntityItem
items[keys[0]].Value = value
items[keys[1]].Flags = nds.EntityItem
items[keys[1]].Value = value
return items, nil
}
nds.SetMemcacheGetMulti(func(c context.Context,
keys []string) (map[string]*memcache.Item, error) {
f := <-memcacheGetChan
return f(c, keys)
})
response := make([]testEntity, len(keys))
if err := nds.GetMulti(c, keys, response); err != nil {
t.Fatal(err)
}
defer nds.SetMemcacheGetMulti(memcache.GetMulti)
for i := 0; i < len(keys); i++ {
if entities[i].IntVal != response[i].IntVal {
t.Fatal("IntVal not equal")
}
}
}
// this will generate some random data for the given day
// 24 * 12 on appengine (288)
// 24 only on development
func generateRandom(c context.Context, day time.Time) error {
var x int
if appengine.IsDevAppServer() {
x = 1
} else {
x = 12
}
keys := make([]*datastore.Key, 24*x)
photos := make([]*Photo, 24*x)
id := 0
for h := 0; h < 24; h++ {
taken := day.Add(time.Duration(h) * time.Hour)
for i := 0; i < x; i++ {
photographer := photographers[rand.Int31n(4)]
photos[id] = &Photo{
Photographer: photographer,
Uploaded: time.Now().UTC(),
Width: 8000,
Height: 6000,
Taken: taken,
TakenDay: day,
}
keys[id] = datastore.NewIncompleteKey(c, "photo", nil)
id++
}
}
nds.PutMulti(c, keys, photos)
return nil
}
func TestDeleteMemcacheFail(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "Entity", "", 1, nil)
keys := []*datastore.Key{key}
entities := make([]testEntity, 1)
entities[0].Val = 43
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
nds.SetMemcacheSetMulti(func(c context.Context,
items []*memcache.Item) error {
return errors.New("expected error")
})
defer func() {
nds.SetMemcacheSetMulti(memcache.SetMulti)
}()
if err := nds.DeleteMulti(c, keys); err == nil {
t.Fatal("expected DeleteMulti error")
}
}
func TestPutMultiZeroKeys(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
if _, err := nds.PutMulti(c, []*datastore.Key{},
[]interface{}{}); err != nil {
t.Fatal(err)
}
}
func TestGetMultiInterface(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int64
}
keys := []*datastore.Key{}
entities := []testEntity{}
for i := int64(1); i < 3; i++ {
keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
entities = append(entities, testEntity{i})
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Get from datastore.
response := make([]interface{}, len(keys))
for i := 0; i < len(response); i++ {
response[i] = &testEntity{}
}
if err := nds.GetMulti(c, keys, response); err != nil {
t.Fatal(err)
}
for i := int64(0); i < 2; i++ {
if te, ok := response[i].(*testEntity); ok {
if te.IntVal != i+1 {
t.Fatal("incorrect IntVal")
}
} else {
t.Fatal("incorrect type")
}
}
// Get from cache.
response = make([]interface{}, len(keys))
for i := 0; i < len(response); i++ {
response[i] = &testEntity{}
}
if err := nds.GetMulti(c, keys, response); err != nil {
t.Fatal(err)
}
for i := int64(0); i < 2; i++ {
if te, ok := response[i].(*testEntity); ok {
if te.IntVal != i+1 {
t.Fatal("incorrect IntVal")
}
} else {
t.Fatal("incorrect type")
}
}
}
func TestPutMultiNoPropertyList(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}
pl := datastore.PropertyList{datastore.Property{}}
if _, err := nds.PutMulti(c, keys, pl); err == nil {
t.Fatal("expecting no PropertyList error")
}
}
func TestGetMultiLockReturnUnknown(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int64
}
keys := []*datastore.Key{}
entities := []testEntity{}
for i := int64(1); i < 3; i++ {
keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
entities = append(entities, testEntity{i})
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
memcacheGetChan := make(chan func(c context.Context, keys []string) (
map[string]*memcache.Item, error), 2)
memcacheGetChan <- memcache.GetMulti
memcacheGetChan <- func(c context.Context,
keys []string) (map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
if err != nil {
return nil, err
}
// Unknown lock values.
items[keys[0]].Flags = 23
items[keys[1]].Flags = 24
return items, nil
}
nds.SetMemcacheGetMulti(func(c context.Context,
keys []string) (map[string]*memcache.Item, error) {
f := <-memcacheGetChan
return f(c, keys)
})
response := make([]testEntity, len(keys))
if err := nds.GetMulti(c, keys, response); err != nil {
t.Fatal(err)
}
defer nds.SetMemcacheGetMulti(memcache.GetMulti)
for i := 0; i < len(keys); i++ {
if entities[i].IntVal != response[i].IntVal {
t.Fatal("IntVal not equal")
}
}
}
func TestDeleteMulti(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type TestEntity struct {
Value int
}
for _, count := range []int{499, 500, 501} {
keys := make([]*datastore.Key, count)
entities := make([]TestEntity, count)
for i := range keys {
keys[i] = datastore.NewKey(c, "TestEntity", strconv.Itoa(i), 0, nil)
entities[i] = TestEntity{i}
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Prime cache.
entities = make([]TestEntity, count)
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
err := nds.GetMulti(c, keys, make([]TestEntity, count))
if err == nil {
t.Fatal("expect error")
}
me, ok := err.(appengine.MultiError)
if !ok {
t.Fatal("should be MultiError")
}
for _, e := range me {
if e != datastore.ErrNoSuchEntity {
t.Fatal("expected ErrNoSuchEntity")
}
}
}
}
// Creates and configures a store that stores entities in Google AppEngines memcache and datastore.
// github.com/qedus/nds is used for strongly consistent automatic caching.
func NewGaeStore(kind string, ctx context.Context, idf sus.IdFactory, vf sus.VersionFactory, ei sus.EntityInitializer) sus.Store {
getKey := func(ctx context.Context, id string) *datastore.Key {
return datastore.NewKey(ctx, kind, id, 0, nil)
}
getMulti := func(ids []string) (vs []sus.Version, err error) {
count := len(ids)
vs = make([]sus.Version, count, count)
ks := make([]*datastore.Key, count, count)
for i := 0; i < count; i++ {
vs[i] = vf()
ks[i] = getKey(ctx, ids[i])
}
err = nds.GetMulti(ctx, ks, vs)
return
}
putMulti := func(ids []string, vs []sus.Version) (err error) {
count := len(ids)
ks := make([]*datastore.Key, count, count)
for i := 0; i < count; i++ {
ks[i] = getKey(ctx, ids[i])
}
_, err = nds.PutMulti(ctx, ks, vs)
return
}
delMulti := func(ids []string) error {
count := len(ids)
ks := make([]*datastore.Key, count, count)
for i := 0; i < count; i++ {
ks[i] = getKey(ctx, ids[i])
}
return nds.DeleteMulti(ctx, ks)
}
isNonExtantError := func(err error) bool {
return err.Error() == datastore.ErrNoSuchEntity.Error()
}
rit := func(tran sus.Transaction) error {
return nds.RunInTransaction(ctx, func(ctx context.Context) error {
return tran()
}, &datastore.TransactionOptions{XG: true})
}
return sus.NewStore(getMulti, putMulti, delMulti, idf, vf, ei, isNonExtantError, rit)
}
func TestGetMultiInterfaceError(t *testing.T) {
c, closeFunc := NewContext(t, nil)
defer closeFunc()
type testEntity struct {
IntVal int64
}
keys := []*datastore.Key{}
entities := []testEntity{}
for i := int64(1); i < 3; i++ {
keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
entities = append(entities, testEntity{i})
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Get from datastore.
// No errors expected.
response := []interface{}{&testEntity{}, &testEntity{}}
if err := nds.GetMulti(c, keys, response); err != nil {
t.Fatal(err)
}
for i := int64(0); i < 2; i++ {
if te, ok := response[i].(*testEntity); ok {
if te.IntVal != i+1 {
t.Fatal("incorrect IntVal")
}
} else {
t.Fatal("incorrect type")
}
}
// Get from cache.
// Errors expected.
response = []interface{}{&testEntity{}, testEntity{}}
if err := nds.GetMulti(c, keys, response); err == nil {
t.Fatal("expected invalid entity type error")
}
}
func TestGetMultiNoErrors(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
for _, count := range []int{999, 1000, 1001} {
// Create entities.
keys := []*datastore.Key{}
entities := []*testEntity{}
for i := 0; i < count; i++ {
key := datastore.NewKey(c, "Test", strconv.Itoa(i), 0, nil)
keys = append(keys, key)
entities = append(entities, &testEntity{i})
}
// Save entities.
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
respEntities := []testEntity{}
for range keys {
respEntities = append(respEntities, testEntity{})
}
if err := nds.GetMulti(c, keys, respEntities); err != nil {
t.Fatal(err)
}
// Check respEntities are in order.
for i, re := range respEntities {
if re.Val != entities[i].Val {
t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
entities[i].Val)
}
}
}
}
func TestGetMultiStructPtrNil(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int64
}
keys := []*datastore.Key{}
entities := []testEntity{}
for i := int64(1); i < 3; i++ {
keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
entities = append(entities, testEntity{i})
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Get from datastore.
response := make([]*testEntity, len(keys))
if err := nds.GetMulti(c, keys, response); err != nil {
t.Fatal(err)
}
for i := int64(0); i < 2; i++ {
if response[i].IntVal != i+1 {
t.Fatal("incorrect IntVal")
}
}
// Get from cache.
response = make([]*testEntity, len(keys))
if err := nds.GetMulti(c, keys, response); err != nil {
t.Fatal(err)
}
for i := int64(0); i < 2; i++ {
if response[i].IntVal != i+1 {
t.Fatal("incorrect IntVal")
}
}
}
func TestDelete(t *testing.T) {
c, closeFunc := NewContext(t, nil)
defer closeFunc()
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "Entity", "", 1, nil)
keys := []*datastore.Key{key}
entities := make([]testEntity, 1)
entities[0].Val = 43
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
entities = make([]testEntity, 1)
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
entity := entities[0]
if entity.Val != 43 {
t.Fatal("incorrect entity.Val", entity.Val)
}
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
keys = []*datastore.Key{key}
entities = make([]testEntity, 1)
err := nds.GetMulti(c, keys, entities)
if me, ok := err.(appengine.MultiError); ok {
if me[0] != datastore.ErrNoSuchEntity {
t.Fatal("entity should be deleted", entities)
}
} else {
t.Fatal("expected appengine.MultiError")
}
}
func TestPutMultiError(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
expectedErrs := appengine.MultiError{
nil,
errors.New("expected error"),
}
nds.SetDatastorePutMulti(func(c context.Context,
keys []*datastore.Key, vals interface{}) ([]*datastore.Key, error) {
return keys, expectedErrs
})
defer nds.SetDatastorePutMulti(datastore.PutMulti)
keys := []*datastore.Key{
datastore.NewKey(c, "Test", "", 1, nil),
datastore.NewKey(c, "Test", "", 2, nil),
}
type TestEntity struct {
Value int
}
entities := []TestEntity{
{1},
{2},
}
_, err := nds.PutMulti(c, keys, entities)
me, ok := err.(appengine.MultiError)
if !ok {
t.Fatal("expected appengine.MultiError")
}
for i, e := range me {
if e != expectedErrs[i] {
t.Fatal("error incorrect")
}
}
}
// Make sure PutMulti still works if we have a memcache unlock failure.
func TestPutMultiUnlockMemcacheSuccess(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int
}
nds.SetMemcacheDeleteMulti(func(c context.Context, keys []string) error {
return errors.New("expected error")
})
defer func() {
nds.SetMemcacheDeleteMulti(memcache.DeleteMulti)
}()
keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}
vals := []testEntity{testEntity{42}}
if _, err := nds.PutMulti(c, keys, vals); err != nil {
t.Fatal(err)
}
}
func TestPutMultiLockFailure(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int
}
nds.SetMemcacheSetMulti(func(c context.Context,
items []*memcache.Item) error {
return errors.New("expected error")
})
defer func() {
nds.SetMemcacheSetMulti(memcache.SetMulti)
}()
keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}
vals := []testEntity{testEntity{42}}
if _, err := nds.PutMulti(c, keys, vals); err == nil {
t.Fatal("expected nds.PutMulti error")
}
}
func TestGetMultiPaths(t *testing.T) {
expectedErr := errors.New("expected error")
type memcacheGetMultiFunc func(c context.Context,
keys []string) (map[string]*memcache.Item, error)
memcacheGetMultiFail := func(c context.Context,
keys []string) (map[string]*memcache.Item, error) {
return nil, expectedErr
}
type memcacheAddMultiFunc func(c context.Context,
items []*memcache.Item) error
memcacheAddMultiFail := func(c context.Context,
items []*memcache.Item) error {
return expectedErr
}
type memcacheCompareAndSwapMultiFunc func(c context.Context,
items []*memcache.Item) error
memcacheCompareAndSwapMultiFail := func(c context.Context,
items []*memcache.Item) error {
return expectedErr
}
type datastoreGetMultiFunc func(c context.Context,
keys []*datastore.Key, vals interface{}) error
datastoreGetMultiFail := func(c context.Context,
keys []*datastore.Key, vals interface{}) error {
return expectedErr
}
type marshalFunc func(pl datastore.PropertyList) ([]byte, error)
marshalFail := func(pl datastore.PropertyList) ([]byte, error) {
return nil, expectedErr
}
type unmarshalFunc func(data []byte, pl *datastore.PropertyList) error
/*
unmarshalFail := func(data []byte, pl *datastore.PropertyList) error {
return expectedErr
}
*/
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int64
}
keysVals := func(c context.Context, count int64) (
[]*datastore.Key, []testEntity) {
keys, vals := make([]*datastore.Key, count), make([]testEntity, count)
for i := int64(0); i < count; i++ {
keys[i] = datastore.NewKey(c, "Entity", "", i+1, nil)
vals[i] = testEntity{i + 1}
}
return keys, vals
}
tests := []struct {
description string
// Number of keys used to as GetMulti params.
keyCount int64
// Number of times GetMulti is called.
callCount int
// There are 2 memcacheGetMulti calls for every GetMulti call.
memcacheGetMultis []memcacheGetMultiFunc
memcacheAddMulti memcacheAddMultiFunc
memcacheCompareAndSwapMulti memcacheCompareAndSwapMultiFunc
datastoreGetMulti datastoreGetMultiFunc
marshal marshalFunc
// There are 2 unmarshal calls for every GetMultiCall.
unmarshals []unmarshalFunc
expectedErrs []error
}{
{
"no errors",
20,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"datastore unknown error",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastoreGetMultiFail,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{expectedErr},
},
{
"datastore unknown multierror",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
func(c context.Context,
keys []*datastore.Key, vals interface{}) error {
me := make(appengine.MultiError, len(keys))
for i := range me {
me[i] = expectedErr
}
return me
},
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{
appengine.MultiError{expectedErr, expectedErr},
},
},
{
"marshal error",
5,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
marshalFail,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"total memcache fail",
20,
1,
[]memcacheGetMultiFunc{
memcacheGetMultiFail,
memcacheGetMultiFail,
},
memcacheAddMultiFail,
memcacheCompareAndSwapMultiFail,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"lock memcache fail",
20,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
memcacheGetMultiFail,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"memcache corrupt",
2,
2,
[]memcacheGetMultiFunc{
// Charge memcache.
memcache.GetMulti,
memcache.GetMulti,
// Corrupt memcache.
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt items.
for _, item := range items {
item.Value = []byte("corrupt string")
}
return items, err
},
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil, nil},
},
{
"memcache flag corrupt",
2,
2,
[]memcacheGetMultiFunc{
// Charge memcache.
memcache.GetMulti,
memcache.GetMulti,
// Corrupt memcache flags.
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Flags = 56
}
return items, err
},
memcache.GetMulti,
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil, nil},
},
{
"lock memcache value fail",
20,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Value = []byte("corrupt value")
}
return items, err
},
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
{
"lock memcache value none item",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Flags = nds.NoneItem
}
return items, err
},
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{
appengine.MultiError{
datastore.ErrNoSuchEntity,
datastore.ErrNoSuchEntity,
},
},
},
{
"memcache get no entity unmarshal fail",
2,
1,
[]memcacheGetMultiFunc{
memcache.GetMulti,
func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
items, err := memcache.GetMulti(c, keys)
// Corrupt flags with unknown number.
for _, item := range items {
item.Flags = nds.EntityItem
}
return items, err
},
},
memcache.AddMulti,
memcache.CompareAndSwapMulti,
datastore.GetMulti,
nds.MarshalPropertyList,
[]unmarshalFunc{
nds.UnmarshalPropertyList,
nds.UnmarshalPropertyList,
},
[]error{nil},
},
}
for _, test := range tests {
t.Log("Start", test.description)
keys, putVals := keysVals(c, test.keyCount)
if _, err := nds.PutMulti(c, keys, putVals); err != nil {
t.Fatal(err)
}
memcacheGetChan := make(chan memcacheGetMultiFunc,
len(test.memcacheGetMultis))
for _, fn := range test.memcacheGetMultis {
memcacheGetChan <- fn
}
nds.SetMemcacheGetMulti(func(c context.Context, keys []string) (
map[string]*memcache.Item, error) {
fn := <-memcacheGetChan
return fn(c, keys)
})
nds.SetMemcacheAddMulti(test.memcacheAddMulti)
nds.SetMemcacheCompareAndSwapMulti(test.memcacheCompareAndSwapMulti)
nds.SetDatastoreGetMulti(test.datastoreGetMulti)
nds.SetMarshal(test.marshal)
unmarshalChan := make(chan unmarshalFunc,
len(test.unmarshals))
for _, fn := range test.unmarshals {
unmarshalChan <- fn
}
nds.SetUnmarshal(func(data []byte, pl *datastore.PropertyList) error {
fn := <-unmarshalChan
return fn(data, pl)
})
for i := 0; i < test.callCount; i++ {
getVals := make([]testEntity, test.keyCount)
err := nds.GetMulti(c, keys, getVals)
expectedErr := test.expectedErrs[i]
if expectedErr == nil {
if err != nil {
t.Fatal(err)
}
for i := range getVals {
if getVals[i].IntVal != putVals[i].IntVal {
t.Fatal("incorrect IntVal")
}
}
continue
}
if err == nil {
t.Fatal("expected error")
}
expectedMultiErr, isMultiErr := expectedErr.(appengine.MultiError)
if isMultiErr {
me, ok := err.(appengine.MultiError)
if !ok {
t.Fatal("expected appengine.MultiError but got", err)
}
if len(me) != len(expectedMultiErr) {
t.Fatal("appengine.MultiError length incorrect")
}
for i, e := range me {
if e != expectedMultiErr[i] {
t.Fatal("non matching errors", e, expectedMultiErr[i])
}
if e == nil {
if getVals[i].IntVal != putVals[i].IntVal {
t.Fatal("incorrect IntVal")
}
}
}
}
}
// Reset App Engine API calls.
nds.SetMemcacheGetMulti(memcache.GetMulti)
nds.SetMemcacheAddMulti(memcache.AddMulti)
nds.SetMemcacheCompareAndSwapMulti(memcache.CompareAndSwapMulti)
nds.SetDatastoreGetMulti(datastore.GetMulti)
nds.SetMarshal(nds.MarshalPropertyList)
nds.SetUnmarshal(nds.UnmarshalPropertyList)
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
t.Log("End", test.description)
}
}
func TestGetMultiPropertyLoadSaver(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int
}
keys := []*datastore.Key{}
entities := []datastore.PropertyList{}
for i := 1; i < 3; i++ {
keys = append(keys, datastore.NewKey(c, "Entity", "", int64(i), nil))
pl, err := datastore.SaveStruct(&testEntity{i})
if err != nil {
t.Fatal(err)
}
entities = append(entities, pl)
}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Prime the cache.
uncachedEntities := make([]datastore.PropertyList, len(keys))
if err := nds.GetMulti(c, keys, uncachedEntities); err != nil {
t.Fatal(err)
}
for i, e := range entities {
if !reflect.DeepEqual(e, uncachedEntities[i]) {
t.Fatal("uncachedEntities not equal", e, uncachedEntities[i])
}
}
// Use cache.
cachedEntities := make([]datastore.PropertyList, len(keys))
if err := nds.GetMulti(c, keys, cachedEntities); err != nil {
t.Fatal(err)
}
for i, e := range entities {
if !reflect.DeepEqual(e, cachedEntities[i]) {
t.Fatal("cachedEntities not equal", e, cachedEntities[i])
}
}
// We know the datastore supports property load saver but we need to make
// sure that memcache does by ensuring memcache does not error when we
// change to fetching with structs.
// Do this by making sure the datastore is not called on this following
// GetMulti as memcache should have worked.
nds.SetDatastoreGetMulti(func(c context.Context,
keys []*datastore.Key, vals interface{}) error {
if len(keys) != 0 {
return errors.New("should not be called")
}
return nil
})
defer func() {
nds.SetDatastoreGetMulti(datastore.GetMulti)
}()
tes := make([]testEntity, len(entities))
if err := nds.GetMulti(c, keys, tes); err != nil {
t.Fatal(err)
}
}
func Process(ctx context.Context, r io.Reader) error {
decoder := json.NewDecoder(r)
// read off the opening object brace
token, err := decoder.Token()
if err != nil {
return err
}
switch t := token.(type) {
case json.Delim:
if t.String() != "{" {
return ErrInvalidStream
}
default:
return ErrInvalidStream
}
keys := make([]*datastore.Key, 0, 1000)
values := make([]datastore.PropertyList, 0, 1000)
for decoder.More() {
encodedEntityKey := ""
nextEntity := entity{}
nextPropertyList := []datastore.Property{}
token, err = decoder.Token()
if err != nil {
return err
}
switch t := token.(type) {
case string:
encodedEntityKey = t
default:
return ErrInvalidStream
}
if entityKey, err := decodeDatastoreKey(ctx, encodedEntityKey); err != nil {
return err
} else {
keys = append(keys, entityKey)
}
if err := decoder.Decode(&nextEntity); err != nil {
return err
} else if err := nextEntity.FetchProperties(ctx, &nextPropertyList); err != nil {
return err
} else {
values = append(values, nextPropertyList)
}
// flush if necessary
if len(keys) == cap(keys) {
if _, err := nds.PutMulti(ctx, keys, values); err != nil {
return err
}
keys = keys[:0]
values = values[:0]
}
}
// flush the buffer if there's anything left in it
if len(keys) > 0 {
_, err := nds.PutMulti(ctx, keys, []datastore.PropertyList(values))
return err
} else {
return nil
}
}
func TestInterfaces(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
incompleteKey := datastore.NewIncompleteKey(c, "Entity", nil)
incompleteKeys := []*datastore.Key{incompleteKey}
entities := []interface{}{&testEntity{43}}
keys, err := nds.PutMulti(c, incompleteKeys, entities)
if err != nil {
t.Fatal(err)
}
if len(keys) != 1 {
t.Fatal("len(keys) != 1")
}
if keys[0].Incomplete() {
t.Fatal("Key is incomplete")
}
entities = []interface{}{&testEntity{}}
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if entities[0].(*testEntity).Val != 43 {
t.Fatal("te.Val != 43")
}
// Get from cache.
entities = []interface{}{&testEntity{}}
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if entities[0].(*testEntity).Val != 43 {
t.Fatal("te.Val != 43")
}
// Change value.
entities = []interface{}{&testEntity{64}}
if _, err := nds.PutMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
// Get from nds with struct.
entities = []interface{}{&testEntity{}}
if err := nds.GetMulti(c, keys, entities); err != nil {
t.Fatal(err)
}
if entities[0].(*testEntity).Val != 64 {
t.Fatal("te.Val != 64")
}
if err := nds.DeleteMulti(c, keys); err != nil {
t.Fatal(err)
}
entities = []interface{}{testEntity{}}
err = nds.GetMulti(c, keys, entities)
if me, ok := err.(appengine.MultiError); ok {
if len(me) != 1 {
t.Fatal("expected 1 appengine.MultiError")
}
if me[0] != datastore.ErrNoSuchEntity {
t.Fatal("expected datastore.ErrNoSuchEntity")
}
} else {
t.Fatal("expected datastore.ErrNoSuchEntity", err)
}
}
func TestMultiCache(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
const entityCount = 88
// Create entities.
keys := []*datastore.Key{}
entities := []testEntity{}
for i := 0; i < entityCount; i++ {
key := datastore.NewKey(c, "Test", strconv.Itoa(i), 0, nil)
keys = append(keys, key)
entities = append(entities, testEntity{i})
}
// Save every other entity.
putKeys := []*datastore.Key{}
putEntities := []testEntity{}
for i, key := range keys {
if i%2 == 0 {
putKeys = append(putKeys, key)
putEntities = append(putEntities, entities[i])
}
}
if keys, err := nds.PutMulti(c, putKeys, putEntities); err != nil {
t.Fatal(err)
} else if len(keys) != len(putKeys) {
t.Fatal("incorrect key len")
}
// Get from nds.
respEntities := make([]testEntity, len(keys))
err := nds.GetMulti(c, keys, respEntities)
if err == nil {
t.Fatal("should be errors")
}
me, ok := err.(appengine.MultiError)
if !ok {
t.Fatalf("not an appengine.MultiError: %+T, %s", err, err)
}
// Check respEntities are in order.
for i, re := range respEntities {
if i%2 == 0 {
if re.Val != entities[i].Val {
t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
entities[i].Val)
}
if me[i] != nil {
t.Fatalf("should be nil error: %s", me[i])
}
} else {
if re.Val != 0 {
t.Fatal("entity not zeroed")
}
if me[i] != datastore.ErrNoSuchEntity {
t.Fatalf("incorrect error %+v, index %d, of %d",
me, i, entityCount)
}
}
}
// Get from local cache.
respEntities = make([]testEntity, len(keys))
err = nds.GetMulti(c, keys, respEntities)
if err == nil {
t.Fatal("should be errors")
}
me, ok = err.(appengine.MultiError)
if !ok {
t.Fatalf("not an appengine.MultiError: %s", err)
}
// Check respEntities are in order.
for i, re := range respEntities {
if i%2 == 0 {
if re.Val != entities[i].Val {
t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
entities[i].Val)
}
if me[i] != nil {
t.Fatal("should be nil error")
}
} else {
if re.Val != 0 {
t.Fatal("entity not zeroed")
}
if me[i] != datastore.ErrNoSuchEntity {
t.Fatalf("incorrect error %+v, index %d, of %d",
me, i, entityCount)
}
}
}
// Get from memcache.
respEntities = make([]testEntity, len(keys))
err = nds.GetMulti(c, keys, respEntities)
if err == nil {
t.Fatal("should be errors")
}
me, ok = err.(appengine.MultiError)
if !ok {
t.Fatalf("not an appengine.MultiError: %+T", me)
}
// Check respEntities are in order.
for i, re := range respEntities {
if i%2 == 0 {
if re.Val != entities[i].Val {
t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
entities[i].Val)
}
if me[i] != nil {
t.Fatal("should be nil error")
}
} else {
if re.Val != 0 {
t.Fatal("entity not zeroed")
}
if me[i] != datastore.ErrNoSuchEntity {
t.Fatalf("incorrect error %+v, index %d, of %d",
me, i, entityCount)
}
}
}
}
func TestGetMultiErrorMix(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
for _, count := range []int{999, 1000, 1001} {
// Create entities.
keys := []*datastore.Key{}
entities := []testEntity{}
for i := 0; i < count; i++ {
key := datastore.NewKey(c, "Test", strconv.Itoa(i), 0, nil)
keys = append(keys, key)
entities = append(entities, testEntity{i})
}
// Save every other entity.
putKeys := []*datastore.Key{}
putEntities := []testEntity{}
for i, key := range keys {
if i%2 == 0 {
putKeys = append(putKeys, key)
putEntities = append(putEntities, entities[i])
}
}
if _, err := nds.PutMulti(c, putKeys, putEntities); err != nil {
t.Fatal(err)
}
respEntities := make([]testEntity, len(keys))
err := nds.GetMulti(c, keys, respEntities)
if err == nil {
t.Fatal("should be errors")
}
if me, ok := err.(appengine.MultiError); !ok {
t.Fatal("not appengine.MultiError")
} else if len(me) != len(keys) {
t.Fatal("incorrect length appengine.MultiError")
}
// Check respEntities are in order.
for i, re := range respEntities {
if i%2 == 0 {
if re.Val != entities[i].Val {
t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
entities[i].Val)
}
} else if me, ok := err.(appengine.MultiError); ok {
if me[i] != datastore.ErrNoSuchEntity {
t.Fatalf("incorrect error %+v, index %d, of %d",
me, i, count)
}
} else {
t.Fatalf("incorrect error, index %d", i)
}
}
}
}