Пример #1
0
func TestRunInTransaction(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		Val int
	}

	key := datastore.NewKey(c, "Entity", "", 3, nil)
	keys := []*datastore.Key{key}
	entity := testEntity{42}
	entities := []testEntity{entity}

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	err := nds.RunInTransaction(c, func(tc context.Context) error {
		entities := make([]testEntity, 1, 1)
		if err := nds.GetMulti(tc, keys, entities); err != nil {
			t.Fatal(err)
		}
		entity := entities[0]

		if entity.Val != 42 {
			t.Fatalf("entity.Val != 42: %d", entity.Val)
		}

		entities[0].Val = 43

		putKeys, err := nds.PutMulti(tc, keys, entities)
		if err != nil {
			t.Fatal(err)
		} else if len(putKeys) != 1 {
			t.Fatal("putKeys should be len 1")
		} else if !putKeys[0].Equal(key) {
			t.Fatal("keys not equal")
		}
		return nil

	}, nil)
	if err != nil {
		t.Fatal(err)
	}

	entities = make([]testEntity, 1, 1)
	if err := nds.GetMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}
	entity = entities[0]
	if entity.Val != 43 {
		t.Fatalf("entity.Val != 43: %d", entity.Val)
	}
}
Пример #2
0
func TestPutMulti(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type TestEntity struct {
		Value int
	}

	for _, count := range []int{499, 500, 501} {
		keys := make([]*datastore.Key, count)
		entities := make([]TestEntity, count)

		for i := range keys {
			keys[i] = datastore.NewKey(c, "TestEntity", strconv.Itoa(i), 0, nil)
			entities[i] = TestEntity{i}
		}

		if _, err := nds.PutMulti(c, keys, entities); err != nil {
			t.Fatal(err)
		}

		entities = make([]TestEntity, count)
		if err := nds.GetMulti(c, keys, entities); err != nil {
			t.Fatal(err)
		}
	}
}
Пример #3
0
func TestPutPropertyLoadSaver(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int
	}

	te := &testEntity{2}
	pl, err := datastore.SaveStruct(te)
	if err != nil {
		t.Fatal(err)
	}

	keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}

	pls := datastore.PropertyList(pl)
	if _, err := nds.PutMulti(c, keys,
		[]datastore.PropertyLoadSaver{&pls}); err != nil {
		t.Fatal(err)
	}

	getPl := datastore.PropertyList{}
	if err := nds.GetMulti(c,
		keys, []datastore.PropertyLoadSaver{&getPl}); err != nil {
		t.Fatal(err)
	}
	getTe := &testEntity{}
	if err := datastore.LoadStruct(getTe, getPl); err != nil {
		t.Fatal(err)
	}
	if te.IntVal != getTe.IntVal {
		t.Fatal("expected same IntVal", getTe.IntVal)
	}
}
Пример #4
0
func TestGetMultiLockReturnEntitySetValueFail(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int64
	}

	keys := []*datastore.Key{}
	entities := []testEntity{}
	for i := int64(1); i < 3; i++ {
		keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
		entities = append(entities, testEntity{i})
	}

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	// Fail to unmarshal test.
	memcacheGetChan := make(chan func(c context.Context, keys []string) (
		map[string]*memcache.Item, error), 2)
	memcacheGetChan <- memcache.GetMulti
	memcacheGetChan <- func(c context.Context,
		keys []string) (map[string]*memcache.Item, error) {
		items, err := memcache.GetMulti(c, keys)
		if err != nil {
			return nil, err
		}
		pl := datastore.PropertyList{
			datastore.Property{"One", 1, false, false},
		}
		value, err := nds.MarshalPropertyList(pl)
		if err != nil {
			return nil, err
		}
		items[keys[0]].Flags = nds.EntityItem
		items[keys[0]].Value = value
		items[keys[1]].Flags = nds.EntityItem
		items[keys[1]].Value = value
		return items, nil
	}
	nds.SetMemcacheGetMulti(func(c context.Context,
		keys []string) (map[string]*memcache.Item, error) {
		f := <-memcacheGetChan
		return f(c, keys)
	})

	response := make([]testEntity, len(keys))
	if err := nds.GetMulti(c, keys, response); err != nil {
		t.Fatal(err)
	}
	defer nds.SetMemcacheGetMulti(memcache.GetMulti)

	for i := 0; i < len(keys); i++ {
		if entities[i].IntVal != response[i].IntVal {
			t.Fatal("IntVal not equal")
		}
	}
}
Пример #5
0
// this will generate some random data for the given day
// 24 * 12 on appengine (288)
// 24 only on development
func generateRandom(c context.Context, day time.Time) error {
	var x int
	if appengine.IsDevAppServer() {
		x = 1
	} else {
		x = 12
	}
	keys := make([]*datastore.Key, 24*x)
	photos := make([]*Photo, 24*x)

	id := 0
	for h := 0; h < 24; h++ {
		taken := day.Add(time.Duration(h) * time.Hour)
		for i := 0; i < x; i++ {
			photographer := photographers[rand.Int31n(4)]
			photos[id] = &Photo{
				Photographer: photographer,
				Uploaded:     time.Now().UTC(),
				Width:        8000,
				Height:       6000,
				Taken:        taken,
				TakenDay:     day,
			}
			keys[id] = datastore.NewIncompleteKey(c, "photo", nil)
			id++
		}
	}
	nds.PutMulti(c, keys, photos)
	return nil
}
Пример #6
0
func TestDeleteMemcacheFail(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		Val int
	}

	key := datastore.NewKey(c, "Entity", "", 1, nil)
	keys := []*datastore.Key{key}
	entities := make([]testEntity, 1)
	entities[0].Val = 43

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	nds.SetMemcacheSetMulti(func(c context.Context,
		items []*memcache.Item) error {
		return errors.New("expected error")
	})

	defer func() {
		nds.SetMemcacheSetMulti(memcache.SetMulti)
	}()

	if err := nds.DeleteMulti(c, keys); err == nil {
		t.Fatal("expected DeleteMulti error")
	}
}
Пример #7
0
func TestPutMultiZeroKeys(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	if _, err := nds.PutMulti(c, []*datastore.Key{},
		[]interface{}{}); err != nil {
		t.Fatal(err)
	}
}
Пример #8
0
func TestGetMultiInterface(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int64
	}

	keys := []*datastore.Key{}
	entities := []testEntity{}
	for i := int64(1); i < 3; i++ {
		keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
		entities = append(entities, testEntity{i})
	}

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	// Get from datastore.
	response := make([]interface{}, len(keys))
	for i := 0; i < len(response); i++ {
		response[i] = &testEntity{}
	}

	if err := nds.GetMulti(c, keys, response); err != nil {
		t.Fatal(err)
	}
	for i := int64(0); i < 2; i++ {
		if te, ok := response[i].(*testEntity); ok {
			if te.IntVal != i+1 {
				t.Fatal("incorrect IntVal")
			}
		} else {
			t.Fatal("incorrect type")
		}
	}

	// Get from cache.
	response = make([]interface{}, len(keys))
	for i := 0; i < len(response); i++ {
		response[i] = &testEntity{}
	}
	if err := nds.GetMulti(c, keys, response); err != nil {
		t.Fatal(err)
	}
	for i := int64(0); i < 2; i++ {
		if te, ok := response[i].(*testEntity); ok {
			if te.IntVal != i+1 {
				t.Fatal("incorrect IntVal")
			}
		} else {
			t.Fatal("incorrect type")
		}
	}
}
Пример #9
0
func TestPutMultiNoPropertyList(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}
	pl := datastore.PropertyList{datastore.Property{}}

	if _, err := nds.PutMulti(c, keys, pl); err == nil {
		t.Fatal("expecting no PropertyList error")
	}
}
Пример #10
0
func TestGetMultiLockReturnUnknown(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int64
	}

	keys := []*datastore.Key{}
	entities := []testEntity{}
	for i := int64(1); i < 3; i++ {
		keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
		entities = append(entities, testEntity{i})
	}

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	memcacheGetChan := make(chan func(c context.Context, keys []string) (
		map[string]*memcache.Item, error), 2)
	memcacheGetChan <- memcache.GetMulti
	memcacheGetChan <- func(c context.Context,
		keys []string) (map[string]*memcache.Item, error) {
		items, err := memcache.GetMulti(c, keys)
		if err != nil {
			return nil, err
		}

		// Unknown lock values.
		items[keys[0]].Flags = 23
		items[keys[1]].Flags = 24
		return items, nil
	}
	nds.SetMemcacheGetMulti(func(c context.Context,
		keys []string) (map[string]*memcache.Item, error) {
		f := <-memcacheGetChan
		return f(c, keys)
	})

	response := make([]testEntity, len(keys))
	if err := nds.GetMulti(c, keys, response); err != nil {
		t.Fatal(err)
	}
	defer nds.SetMemcacheGetMulti(memcache.GetMulti)

	for i := 0; i < len(keys); i++ {
		if entities[i].IntVal != response[i].IntVal {
			t.Fatal("IntVal not equal")
		}
	}
}
Пример #11
0
func TestDeleteMulti(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type TestEntity struct {
		Value int
	}

	for _, count := range []int{499, 500, 501} {
		keys := make([]*datastore.Key, count)
		entities := make([]TestEntity, count)

		for i := range keys {
			keys[i] = datastore.NewKey(c, "TestEntity", strconv.Itoa(i), 0, nil)
			entities[i] = TestEntity{i}
		}

		if _, err := nds.PutMulti(c, keys, entities); err != nil {
			t.Fatal(err)
		}

		// Prime cache.
		entities = make([]TestEntity, count)
		if err := nds.GetMulti(c, keys, entities); err != nil {
			t.Fatal(err)
		}

		if err := nds.DeleteMulti(c, keys); err != nil {
			t.Fatal(err)
		}

		err := nds.GetMulti(c, keys, make([]TestEntity, count))
		if err == nil {
			t.Fatal("expect error")
		}

		me, ok := err.(appengine.MultiError)
		if !ok {
			t.Fatal("should be MultiError")
		}

		for _, e := range me {
			if e != datastore.ErrNoSuchEntity {
				t.Fatal("expected ErrNoSuchEntity")
			}
		}
	}
}
Пример #12
0
Файл: gus.go Проект: robsix/rps
// Creates and configures a store that stores entities in Google AppEngines memcache and datastore.
// github.com/qedus/nds is used for strongly consistent automatic caching.
func NewGaeStore(kind string, ctx context.Context, idf sus.IdFactory, vf sus.VersionFactory, ei sus.EntityInitializer) sus.Store {
	getKey := func(ctx context.Context, id string) *datastore.Key {
		return datastore.NewKey(ctx, kind, id, 0, nil)
	}

	getMulti := func(ids []string) (vs []sus.Version, err error) {
		count := len(ids)
		vs = make([]sus.Version, count, count)
		ks := make([]*datastore.Key, count, count)
		for i := 0; i < count; i++ {
			vs[i] = vf()
			ks[i] = getKey(ctx, ids[i])
		}
		err = nds.GetMulti(ctx, ks, vs)
		return
	}

	putMulti := func(ids []string, vs []sus.Version) (err error) {
		count := len(ids)
		ks := make([]*datastore.Key, count, count)
		for i := 0; i < count; i++ {
			ks[i] = getKey(ctx, ids[i])
		}
		_, err = nds.PutMulti(ctx, ks, vs)
		return
	}

	delMulti := func(ids []string) error {
		count := len(ids)
		ks := make([]*datastore.Key, count, count)
		for i := 0; i < count; i++ {
			ks[i] = getKey(ctx, ids[i])
		}
		return nds.DeleteMulti(ctx, ks)
	}

	isNonExtantError := func(err error) bool {
		return err.Error() == datastore.ErrNoSuchEntity.Error()
	}

	rit := func(tran sus.Transaction) error {
		return nds.RunInTransaction(ctx, func(ctx context.Context) error {
			return tran()
		}, &datastore.TransactionOptions{XG: true})
	}

	return sus.NewStore(getMulti, putMulti, delMulti, idf, vf, ei, isNonExtantError, rit)
}
Пример #13
0
func TestGetMultiInterfaceError(t *testing.T) {
	c, closeFunc := NewContext(t, nil)
	defer closeFunc()

	type testEntity struct {
		IntVal int64
	}

	keys := []*datastore.Key{}
	entities := []testEntity{}
	for i := int64(1); i < 3; i++ {
		keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
		entities = append(entities, testEntity{i})
	}

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	// Get from datastore.
	// No errors expected.
	response := []interface{}{&testEntity{}, &testEntity{}}

	if err := nds.GetMulti(c, keys, response); err != nil {
		t.Fatal(err)
	}
	for i := int64(0); i < 2; i++ {
		if te, ok := response[i].(*testEntity); ok {
			if te.IntVal != i+1 {
				t.Fatal("incorrect IntVal")
			}
		} else {
			t.Fatal("incorrect type")
		}
	}

	// Get from cache.
	// Errors expected.
	response = []interface{}{&testEntity{}, testEntity{}}
	if err := nds.GetMulti(c, keys, response); err == nil {
		t.Fatal("expected invalid entity type error")
	}
}
Пример #14
0
func TestGetMultiNoErrors(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		Val int
	}

	for _, count := range []int{999, 1000, 1001} {

		// Create entities.
		keys := []*datastore.Key{}
		entities := []*testEntity{}
		for i := 0; i < count; i++ {
			key := datastore.NewKey(c, "Test", strconv.Itoa(i), 0, nil)
			keys = append(keys, key)
			entities = append(entities, &testEntity{i})
		}

		// Save entities.
		if _, err := nds.PutMulti(c, keys, entities); err != nil {
			t.Fatal(err)
		}

		respEntities := []testEntity{}
		for range keys {
			respEntities = append(respEntities, testEntity{})
		}

		if err := nds.GetMulti(c, keys, respEntities); err != nil {
			t.Fatal(err)
		}

		// Check respEntities are in order.
		for i, re := range respEntities {
			if re.Val != entities[i].Val {
				t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
					entities[i].Val)
			}
		}
	}
}
Пример #15
0
func TestGetMultiStructPtrNil(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int64
	}

	keys := []*datastore.Key{}
	entities := []testEntity{}
	for i := int64(1); i < 3; i++ {
		keys = append(keys, datastore.NewKey(c, "Entity", "", i, nil))
		entities = append(entities, testEntity{i})
	}

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	// Get from datastore.
	response := make([]*testEntity, len(keys))
	if err := nds.GetMulti(c, keys, response); err != nil {
		t.Fatal(err)
	}
	for i := int64(0); i < 2; i++ {
		if response[i].IntVal != i+1 {
			t.Fatal("incorrect IntVal")
		}
	}

	// Get from cache.
	response = make([]*testEntity, len(keys))
	if err := nds.GetMulti(c, keys, response); err != nil {
		t.Fatal(err)
	}
	for i := int64(0); i < 2; i++ {
		if response[i].IntVal != i+1 {
			t.Fatal("incorrect IntVal")
		}
	}
}
Пример #16
0
func TestDelete(t *testing.T) {
	c, closeFunc := NewContext(t, nil)
	defer closeFunc()

	type testEntity struct {
		Val int
	}

	key := datastore.NewKey(c, "Entity", "", 1, nil)
	keys := []*datastore.Key{key}
	entities := make([]testEntity, 1)
	entities[0].Val = 43

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	entities = make([]testEntity, 1)
	if err := nds.GetMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}
	entity := entities[0]
	if entity.Val != 43 {
		t.Fatal("incorrect entity.Val", entity.Val)
	}

	if err := nds.DeleteMulti(c, keys); err != nil {
		t.Fatal(err)
	}

	keys = []*datastore.Key{key}
	entities = make([]testEntity, 1)
	err := nds.GetMulti(c, keys, entities)
	if me, ok := err.(appengine.MultiError); ok {
		if me[0] != datastore.ErrNoSuchEntity {
			t.Fatal("entity should be deleted", entities)
		}
	} else {
		t.Fatal("expected appengine.MultiError")
	}
}
Пример #17
0
func TestPutMultiError(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	expectedErrs := appengine.MultiError{
		nil,
		errors.New("expected error"),
	}

	nds.SetDatastorePutMulti(func(c context.Context,
		keys []*datastore.Key, vals interface{}) ([]*datastore.Key, error) {
		return keys, expectedErrs
	})
	defer nds.SetDatastorePutMulti(datastore.PutMulti)

	keys := []*datastore.Key{
		datastore.NewKey(c, "Test", "", 1, nil),
		datastore.NewKey(c, "Test", "", 2, nil),
	}

	type TestEntity struct {
		Value int
	}
	entities := []TestEntity{
		{1},
		{2},
	}

	_, err := nds.PutMulti(c, keys, entities)
	me, ok := err.(appengine.MultiError)
	if !ok {
		t.Fatal("expected appengine.MultiError")
	}
	for i, e := range me {
		if e != expectedErrs[i] {
			t.Fatal("error incorrect")
		}
	}
}
Пример #18
0
// Make sure PutMulti still works if we have a memcache unlock failure.
func TestPutMultiUnlockMemcacheSuccess(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int
	}

	nds.SetMemcacheDeleteMulti(func(c context.Context, keys []string) error {
		return errors.New("expected error")
	})

	defer func() {
		nds.SetMemcacheDeleteMulti(memcache.DeleteMulti)
	}()

	keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}
	vals := []testEntity{testEntity{42}}

	if _, err := nds.PutMulti(c, keys, vals); err != nil {
		t.Fatal(err)
	}
}
Пример #19
0
func TestPutMultiLockFailure(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int
	}

	nds.SetMemcacheSetMulti(func(c context.Context,
		items []*memcache.Item) error {
		return errors.New("expected error")
	})

	defer func() {
		nds.SetMemcacheSetMulti(memcache.SetMulti)
	}()

	keys := []*datastore.Key{datastore.NewKey(c, "Test", "", 1, nil)}
	vals := []testEntity{testEntity{42}}

	if _, err := nds.PutMulti(c, keys, vals); err == nil {
		t.Fatal("expected nds.PutMulti error")
	}
}
Пример #20
0
func TestGetMultiPaths(t *testing.T) {
	expectedErr := errors.New("expected error")

	type memcacheGetMultiFunc func(c context.Context,
		keys []string) (map[string]*memcache.Item, error)
	memcacheGetMultiFail := func(c context.Context,
		keys []string) (map[string]*memcache.Item, error) {
		return nil, expectedErr
	}

	type memcacheAddMultiFunc func(c context.Context,
		items []*memcache.Item) error
	memcacheAddMultiFail := func(c context.Context,
		items []*memcache.Item) error {
		return expectedErr
	}

	type memcacheCompareAndSwapMultiFunc func(c context.Context,
		items []*memcache.Item) error
	memcacheCompareAndSwapMultiFail := func(c context.Context,
		items []*memcache.Item) error {
		return expectedErr
	}

	type datastoreGetMultiFunc func(c context.Context,
		keys []*datastore.Key, vals interface{}) error
	datastoreGetMultiFail := func(c context.Context,
		keys []*datastore.Key, vals interface{}) error {
		return expectedErr
	}

	type marshalFunc func(pl datastore.PropertyList) ([]byte, error)
	marshalFail := func(pl datastore.PropertyList) ([]byte, error) {
		return nil, expectedErr
	}

	type unmarshalFunc func(data []byte, pl *datastore.PropertyList) error
	/*
	   unmarshalFail := func(data []byte, pl *datastore.PropertyList) error {
	       return expectedErr
	   }
	*/

	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int64
	}

	keysVals := func(c context.Context, count int64) (
		[]*datastore.Key, []testEntity) {

		keys, vals := make([]*datastore.Key, count), make([]testEntity, count)
		for i := int64(0); i < count; i++ {
			keys[i] = datastore.NewKey(c, "Entity", "", i+1, nil)
			vals[i] = testEntity{i + 1}
		}
		return keys, vals
	}

	tests := []struct {
		description string

		// Number of keys used to as GetMulti params.
		keyCount int64

		// Number of times GetMulti is called.
		callCount int

		// There are 2 memcacheGetMulti calls for every GetMulti call.
		memcacheGetMultis           []memcacheGetMultiFunc
		memcacheAddMulti            memcacheAddMultiFunc
		memcacheCompareAndSwapMulti memcacheCompareAndSwapMultiFunc

		datastoreGetMulti datastoreGetMultiFunc

		marshal marshalFunc
		// There are 2 unmarshal calls for every GetMultiCall.
		unmarshals []unmarshalFunc

		expectedErrs []error
	}{
		{
			"no errors",
			20,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				memcache.GetMulti,
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil},
		},
		{
			"datastore unknown error",
			2,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				memcache.GetMulti,
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastoreGetMultiFail,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{expectedErr},
		},
		{
			"datastore unknown multierror",
			2,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				memcache.GetMulti,
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			func(c context.Context,
				keys []*datastore.Key, vals interface{}) error {

				me := make(appengine.MultiError, len(keys))
				for i := range me {
					me[i] = expectedErr
				}
				return me
			},
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{
				appengine.MultiError{expectedErr, expectedErr},
			},
		},
		{
			"marshal error",
			5,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				memcache.GetMulti,
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			marshalFail,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil},
		},
		{
			"total memcache fail",
			20,
			1,
			[]memcacheGetMultiFunc{
				memcacheGetMultiFail,
				memcacheGetMultiFail,
			},
			memcacheAddMultiFail,
			memcacheCompareAndSwapMultiFail,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil},
		},
		{
			"lock memcache fail",
			20,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				memcacheGetMultiFail,
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil},
		},
		{
			"memcache corrupt",
			2,
			2,
			[]memcacheGetMultiFunc{
				// Charge memcache.
				memcache.GetMulti,
				memcache.GetMulti,
				// Corrupt memcache.
				func(c context.Context, keys []string) (
					map[string]*memcache.Item, error) {
					items, err := memcache.GetMulti(c, keys)
					// Corrupt items.
					for _, item := range items {
						item.Value = []byte("corrupt string")
					}
					return items, err
				},
				memcache.GetMulti,
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil, nil},
		},
		{
			"memcache flag corrupt",
			2,
			2,
			[]memcacheGetMultiFunc{
				// Charge memcache.
				memcache.GetMulti,
				memcache.GetMulti,
				// Corrupt memcache flags.
				func(c context.Context, keys []string) (
					map[string]*memcache.Item, error) {
					items, err := memcache.GetMulti(c, keys)
					// Corrupt flags with unknown number.
					for _, item := range items {
						item.Flags = 56
					}
					return items, err
				},
				memcache.GetMulti,
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil, nil},
		},
		{
			"lock memcache value fail",
			20,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				func(c context.Context, keys []string) (
					map[string]*memcache.Item, error) {
					items, err := memcache.GetMulti(c, keys)
					// Corrupt flags with unknown number.
					for _, item := range items {
						item.Value = []byte("corrupt value")
					}
					return items, err
				},
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil},
		},
		{
			"lock memcache value none item",
			2,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				func(c context.Context, keys []string) (
					map[string]*memcache.Item, error) {
					items, err := memcache.GetMulti(c, keys)
					// Corrupt flags with unknown number.
					for _, item := range items {
						item.Flags = nds.NoneItem
					}
					return items, err
				},
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{
				appengine.MultiError{
					datastore.ErrNoSuchEntity,
					datastore.ErrNoSuchEntity,
				},
			},
		},
		{
			"memcache get no entity unmarshal fail",
			2,
			1,
			[]memcacheGetMultiFunc{
				memcache.GetMulti,
				func(c context.Context, keys []string) (
					map[string]*memcache.Item, error) {
					items, err := memcache.GetMulti(c, keys)
					// Corrupt flags with unknown number.
					for _, item := range items {
						item.Flags = nds.EntityItem
					}
					return items, err
				},
			},
			memcache.AddMulti,
			memcache.CompareAndSwapMulti,
			datastore.GetMulti,
			nds.MarshalPropertyList,
			[]unmarshalFunc{
				nds.UnmarshalPropertyList,
				nds.UnmarshalPropertyList,
			},
			[]error{nil},
		},
	}

	for _, test := range tests {
		t.Log("Start", test.description)

		keys, putVals := keysVals(c, test.keyCount)
		if _, err := nds.PutMulti(c, keys, putVals); err != nil {
			t.Fatal(err)
		}

		memcacheGetChan := make(chan memcacheGetMultiFunc,
			len(test.memcacheGetMultis))

		for _, fn := range test.memcacheGetMultis {
			memcacheGetChan <- fn
		}

		nds.SetMemcacheGetMulti(func(c context.Context, keys []string) (
			map[string]*memcache.Item, error) {
			fn := <-memcacheGetChan
			return fn(c, keys)
		})

		nds.SetMemcacheAddMulti(test.memcacheAddMulti)
		nds.SetMemcacheCompareAndSwapMulti(test.memcacheCompareAndSwapMulti)

		nds.SetDatastoreGetMulti(test.datastoreGetMulti)

		nds.SetMarshal(test.marshal)

		unmarshalChan := make(chan unmarshalFunc,
			len(test.unmarshals))

		for _, fn := range test.unmarshals {
			unmarshalChan <- fn
		}

		nds.SetUnmarshal(func(data []byte, pl *datastore.PropertyList) error {
			fn := <-unmarshalChan
			return fn(data, pl)
		})

		for i := 0; i < test.callCount; i++ {
			getVals := make([]testEntity, test.keyCount)
			err := nds.GetMulti(c, keys, getVals)

			expectedErr := test.expectedErrs[i]

			if expectedErr == nil {
				if err != nil {
					t.Fatal(err)
				}

				for i := range getVals {
					if getVals[i].IntVal != putVals[i].IntVal {
						t.Fatal("incorrect IntVal")
					}
				}
				continue
			}

			if err == nil {
				t.Fatal("expected error")
			}
			expectedMultiErr, isMultiErr := expectedErr.(appengine.MultiError)

			if isMultiErr {
				me, ok := err.(appengine.MultiError)
				if !ok {
					t.Fatal("expected appengine.MultiError but got", err)
				}

				if len(me) != len(expectedMultiErr) {
					t.Fatal("appengine.MultiError length incorrect")
				}

				for i, e := range me {
					if e != expectedMultiErr[i] {
						t.Fatal("non matching errors", e, expectedMultiErr[i])
					}

					if e == nil {
						if getVals[i].IntVal != putVals[i].IntVal {
							t.Fatal("incorrect IntVal")
						}
					}
				}
			}
		}

		// Reset App Engine API calls.
		nds.SetMemcacheGetMulti(memcache.GetMulti)
		nds.SetMemcacheAddMulti(memcache.AddMulti)
		nds.SetMemcacheCompareAndSwapMulti(memcache.CompareAndSwapMulti)
		nds.SetDatastoreGetMulti(datastore.GetMulti)
		nds.SetMarshal(nds.MarshalPropertyList)
		nds.SetUnmarshal(nds.UnmarshalPropertyList)

		if err := nds.DeleteMulti(c, keys); err != nil {
			t.Fatal(err)
		}
		t.Log("End", test.description)
	}
}
Пример #21
0
func TestGetMultiPropertyLoadSaver(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		IntVal int
	}

	keys := []*datastore.Key{}
	entities := []datastore.PropertyList{}

	for i := 1; i < 3; i++ {
		keys = append(keys, datastore.NewKey(c, "Entity", "", int64(i), nil))

		pl, err := datastore.SaveStruct(&testEntity{i})
		if err != nil {
			t.Fatal(err)
		}
		entities = append(entities, pl)
	}

	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	// Prime the cache.
	uncachedEntities := make([]datastore.PropertyList, len(keys))
	if err := nds.GetMulti(c, keys, uncachedEntities); err != nil {
		t.Fatal(err)
	}

	for i, e := range entities {
		if !reflect.DeepEqual(e, uncachedEntities[i]) {
			t.Fatal("uncachedEntities not equal", e, uncachedEntities[i])
		}
	}

	// Use cache.
	cachedEntities := make([]datastore.PropertyList, len(keys))
	if err := nds.GetMulti(c, keys, cachedEntities); err != nil {
		t.Fatal(err)
	}

	for i, e := range entities {
		if !reflect.DeepEqual(e, cachedEntities[i]) {
			t.Fatal("cachedEntities not equal", e, cachedEntities[i])
		}
	}

	// We know the datastore supports property load saver but we need to make
	// sure that memcache does by ensuring memcache does not error when we
	// change to fetching with structs.
	// Do this by making sure the datastore is not called on this following
	// GetMulti as memcache should have worked.
	nds.SetDatastoreGetMulti(func(c context.Context,
		keys []*datastore.Key, vals interface{}) error {
		if len(keys) != 0 {
			return errors.New("should not be called")
		}
		return nil
	})
	defer func() {
		nds.SetDatastoreGetMulti(datastore.GetMulti)
	}()
	tes := make([]testEntity, len(entities))
	if err := nds.GetMulti(c, keys, tes); err != nil {
		t.Fatal(err)
	}
}
Пример #22
0
func Process(ctx context.Context, r io.Reader) error {

	decoder := json.NewDecoder(r)

	// read off the opening object brace
	token, err := decoder.Token()
	if err != nil {
		return err
	}

	switch t := token.(type) {
	case json.Delim:
		if t.String() != "{" {
			return ErrInvalidStream
		}
	default:
		return ErrInvalidStream
	}

	keys := make([]*datastore.Key, 0, 1000)
	values := make([]datastore.PropertyList, 0, 1000)

	for decoder.More() {

		encodedEntityKey := ""
		nextEntity := entity{}
		nextPropertyList := []datastore.Property{}

		token, err = decoder.Token()
		if err != nil {
			return err
		}

		switch t := token.(type) {
		case string:
			encodedEntityKey = t
		default:
			return ErrInvalidStream
		}

		if entityKey, err := decodeDatastoreKey(ctx, encodedEntityKey); err != nil {
			return err
		} else {
			keys = append(keys, entityKey)
		}

		if err := decoder.Decode(&nextEntity); err != nil {
			return err
		} else if err := nextEntity.FetchProperties(ctx, &nextPropertyList); err != nil {
			return err
		} else {
			values = append(values, nextPropertyList)
		}

		// flush if necessary
		if len(keys) == cap(keys) {

			if _, err := nds.PutMulti(ctx, keys, values); err != nil {
				return err
			}

			keys = keys[:0]
			values = values[:0]

		}

	}

	// flush the buffer if there's anything left in it
	if len(keys) > 0 {
		_, err := nds.PutMulti(ctx, keys, []datastore.PropertyList(values))
		return err
	} else {
		return nil
	}

}
Пример #23
0
func TestInterfaces(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		Val int
	}

	incompleteKey := datastore.NewIncompleteKey(c, "Entity", nil)
	incompleteKeys := []*datastore.Key{incompleteKey}
	entities := []interface{}{&testEntity{43}}
	keys, err := nds.PutMulti(c, incompleteKeys, entities)
	if err != nil {
		t.Fatal(err)
	}
	if len(keys) != 1 {
		t.Fatal("len(keys) != 1")
	}

	if keys[0].Incomplete() {
		t.Fatal("Key is incomplete")
	}

	entities = []interface{}{&testEntity{}}
	if err := nds.GetMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	if entities[0].(*testEntity).Val != 43 {
		t.Fatal("te.Val != 43")
	}

	// Get from cache.
	entities = []interface{}{&testEntity{}}
	if err := nds.GetMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	if entities[0].(*testEntity).Val != 43 {
		t.Fatal("te.Val != 43")
	}

	// Change value.
	entities = []interface{}{&testEntity{64}}
	if _, err := nds.PutMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	// Get from nds with struct.
	entities = []interface{}{&testEntity{}}
	if err := nds.GetMulti(c, keys, entities); err != nil {
		t.Fatal(err)
	}

	if entities[0].(*testEntity).Val != 64 {
		t.Fatal("te.Val != 64")
	}

	if err := nds.DeleteMulti(c, keys); err != nil {
		t.Fatal(err)
	}

	entities = []interface{}{testEntity{}}
	err = nds.GetMulti(c, keys, entities)
	if me, ok := err.(appengine.MultiError); ok {

		if len(me) != 1 {
			t.Fatal("expected 1 appengine.MultiError")
		}
		if me[0] != datastore.ErrNoSuchEntity {
			t.Fatal("expected datastore.ErrNoSuchEntity")
		}
	} else {
		t.Fatal("expected datastore.ErrNoSuchEntity", err)
	}
}
Пример #24
0
func TestMultiCache(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		Val int
	}
	const entityCount = 88

	// Create entities.
	keys := []*datastore.Key{}
	entities := []testEntity{}
	for i := 0; i < entityCount; i++ {
		key := datastore.NewKey(c, "Test", strconv.Itoa(i), 0, nil)
		keys = append(keys, key)
		entities = append(entities, testEntity{i})
	}

	// Save every other entity.
	putKeys := []*datastore.Key{}
	putEntities := []testEntity{}
	for i, key := range keys {
		if i%2 == 0 {
			putKeys = append(putKeys, key)
			putEntities = append(putEntities, entities[i])
		}
	}
	if keys, err := nds.PutMulti(c, putKeys, putEntities); err != nil {
		t.Fatal(err)
	} else if len(keys) != len(putKeys) {
		t.Fatal("incorrect key len")
	}

	// Get from nds.
	respEntities := make([]testEntity, len(keys))
	err := nds.GetMulti(c, keys, respEntities)
	if err == nil {
		t.Fatal("should be errors")
	}

	me, ok := err.(appengine.MultiError)
	if !ok {
		t.Fatalf("not an appengine.MultiError: %+T, %s", err, err)
	}

	// Check respEntities are in order.
	for i, re := range respEntities {
		if i%2 == 0 {
			if re.Val != entities[i].Val {
				t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
					entities[i].Val)
			}
			if me[i] != nil {
				t.Fatalf("should be nil error: %s", me[i])
			}
		} else {
			if re.Val != 0 {
				t.Fatal("entity not zeroed")
			}
			if me[i] != datastore.ErrNoSuchEntity {
				t.Fatalf("incorrect error %+v, index %d, of %d",
					me, i, entityCount)
			}
		}
	}

	// Get from local cache.
	respEntities = make([]testEntity, len(keys))
	err = nds.GetMulti(c, keys, respEntities)
	if err == nil {
		t.Fatal("should be errors")
	}

	me, ok = err.(appengine.MultiError)
	if !ok {
		t.Fatalf("not an appengine.MultiError: %s", err)
	}

	// Check respEntities are in order.
	for i, re := range respEntities {
		if i%2 == 0 {
			if re.Val != entities[i].Val {
				t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
					entities[i].Val)
			}
			if me[i] != nil {
				t.Fatal("should be nil error")
			}
		} else {
			if re.Val != 0 {
				t.Fatal("entity not zeroed")
			}
			if me[i] != datastore.ErrNoSuchEntity {
				t.Fatalf("incorrect error %+v, index %d, of %d",
					me, i, entityCount)
			}
		}
	}

	// Get from memcache.
	respEntities = make([]testEntity, len(keys))
	err = nds.GetMulti(c, keys, respEntities)
	if err == nil {
		t.Fatal("should be errors")
	}

	me, ok = err.(appengine.MultiError)
	if !ok {
		t.Fatalf("not an appengine.MultiError: %+T", me)
	}

	// Check respEntities are in order.
	for i, re := range respEntities {
		if i%2 == 0 {
			if re.Val != entities[i].Val {
				t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
					entities[i].Val)
			}
			if me[i] != nil {
				t.Fatal("should be nil error")
			}
		} else {
			if re.Val != 0 {
				t.Fatal("entity not zeroed")
			}
			if me[i] != datastore.ErrNoSuchEntity {
				t.Fatalf("incorrect error %+v, index %d, of %d",
					me, i, entityCount)
			}
		}
	}
}
Пример #25
0
func TestGetMultiErrorMix(t *testing.T) {
	c, closeFunc := NewContext(t)
	defer closeFunc()

	type testEntity struct {
		Val int
	}

	for _, count := range []int{999, 1000, 1001} {

		// Create entities.
		keys := []*datastore.Key{}
		entities := []testEntity{}
		for i := 0; i < count; i++ {
			key := datastore.NewKey(c, "Test", strconv.Itoa(i), 0, nil)
			keys = append(keys, key)
			entities = append(entities, testEntity{i})
		}

		// Save every other entity.
		putKeys := []*datastore.Key{}
		putEntities := []testEntity{}
		for i, key := range keys {
			if i%2 == 0 {
				putKeys = append(putKeys, key)
				putEntities = append(putEntities, entities[i])
			}
		}

		if _, err := nds.PutMulti(c, putKeys, putEntities); err != nil {
			t.Fatal(err)
		}

		respEntities := make([]testEntity, len(keys))
		err := nds.GetMulti(c, keys, respEntities)
		if err == nil {
			t.Fatal("should be errors")
		}

		if me, ok := err.(appengine.MultiError); !ok {
			t.Fatal("not appengine.MultiError")
		} else if len(me) != len(keys) {
			t.Fatal("incorrect length appengine.MultiError")
		}

		// Check respEntities are in order.
		for i, re := range respEntities {
			if i%2 == 0 {
				if re.Val != entities[i].Val {
					t.Fatalf("respEntities in wrong order, %d vs %d", re.Val,
						entities[i].Val)
				}
			} else if me, ok := err.(appengine.MultiError); ok {
				if me[i] != datastore.ErrNoSuchEntity {
					t.Fatalf("incorrect error %+v, index %d, of %d",
						me, i, count)
				}
			} else {
				t.Fatalf("incorrect error, index %d", i)
			}
		}
	}
}