func TestDeleteInTransaction(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "TestEntity", "", 1, nil)
if _, err := nds.Put(c, key, &testEntity{2}); err != nil {
t.Fatal(err)
}
// Prime cache.
if err := nds.Get(c, key, &testEntity{}); err != nil {
t.Fatal(err)
}
if err := nds.RunInTransaction(c, func(tc context.Context) error {
return nds.DeleteMulti(tc, []*datastore.Key{key})
}, nil); err != nil {
t.Fatal(err)
}
if err := nds.Get(c, key, &testEntity{}); err == nil {
t.Fatal("expected no entity")
} else if err != datastore.ErrNoSuchEntity {
t.Fatal(err)
}
}
func TestProcess(t *testing.T) {
ctx, done, _ := aetest.NewContext()
defer done()
smallTest := strings.NewReader(`{
"User/jsmith": {
"Name": "John Smith",
"LotteryNumbers": [1,2,3,4,5],
"CreatedAt": {
"Type": "time",
"Value": "1993-05-01T12:31:00.000Z"
}
},
"User/jdoe": {
"Name": "Jane Doe",
"LotteryNumbers": [2,4,6,8,10],
"CreatedAt": {
"Type": "time",
"Value": "1992-01-30T08:01:00.000Z"
}
}
}`)
if err := Process(ctx, smallTest); err != nil {
t.Errorf("Unexpected error %s", err)
}
key1 := datastore.NewKey(ctx, "User", "jdoe", 0, nil)
key2 := datastore.NewKey(ctx, "User", "jsmith", 0, nil)
value1 := fakeUser{}
value2 := fakeUser{}
if err := nds.Get(ctx, key1, &value1); err != nil {
t.Errorf("Unexpected error %s retrieving value1", err)
} else if err := nds.Get(ctx, key2, &value2); err != nil {
t.Errorf("Unexpected error %s retrieving value2", err)
}
value1.CreatedAt = value1.CreatedAt.UTC()
value2.CreatedAt = value2.CreatedAt.UTC()
if !reflect.DeepEqual(value1, fakeUser{
Name: "Jane Doe",
LotteryNumbers: []int64{2, 4, 6, 8, 10},
CreatedAt: time.Date(1992, 1, 30, 8, 1, 0, 0, time.UTC),
}) {
t.Errorf("Unexpected value in value1: %+v", value1)
}
if !reflect.DeepEqual(value2, fakeUser{
Name: "John Smith",
LotteryNumbers: []int64{1, 2, 3, 4, 5},
CreatedAt: time.Date(1993, 5, 1, 12, 31, 0, 0, time.UTC),
}) {
t.Errorf("Unexpected value in value2: %+v", value1)
}
}
func TestGetArgs(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int64
}
if err := nds.Get(c, nil, &testEntity{}); err == nil {
t.Fatal("expected error for nil key")
}
key := datastore.NewKey(c, "Entity", "", 1, nil)
if err := nds.Get(c, key, nil); err != datastore.ErrInvalidEntityType {
t.Fatal("expected ErrInvalidEntityType for nil value")
}
if err := nds.Get(c, key, datastore.PropertyList{}); err == nil {
t.Fatal("expected error for datastore.PropertyList")
}
if err := nds.Get(c, key, testEntity{}); err == nil {
t.Fatal("expected error for struct")
}
rte := newReaderTestEntity()
if err := nds.Get(c, key, rte); err == nil {
t.Fatal("expected error for interface")
}
}
// TestGetNamespacedKey ensures issue https://goo.gl/rXU8nK is fixed so that
// memcache uses the namespace from the key instead of the context.
func TestGetNamespacedKey(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
const intVal = int64(12)
type testEntity struct {
IntVal int64
}
namespacedCtx, err := appengine.Namespace(c, "keyNamespace")
if err != nil {
t.Fatal(err)
}
key := datastore.NewKey(c, "Entity", "", 1, nil)
namespacedKey := datastore.NewKey(namespacedCtx,
"Entity", "", key.IntID(), nil)
entity := &testEntity{intVal}
if namespacedKey, err = nds.Put(c, namespacedKey, entity); err != nil {
t.Fatal(err)
}
// Prime cache.
if err := nds.Get(namespacedCtx, namespacedKey, &testEntity{}); err != nil {
t.Fatal(err)
}
// Ensure that we get a value back from the cache by checking if the
// datastore is called at all.
entityFromCache := true
nds.SetDatastoreGetMulti(func(c context.Context,
keys []*datastore.Key, vals interface{}) error {
if len(keys) != 0 {
entityFromCache = false
}
return nil
})
if err := nds.Get(c, namespacedKey, &testEntity{}); err != nil {
t.Fatal(err)
}
nds.SetDatastoreGetMulti(datastore.GetMulti)
if !entityFromCache {
t.Fatal("entity not obtained from cache")
}
if err := nds.Delete(namespacedCtx, namespacedKey); err != nil {
t.Fatal(err)
}
entity = &testEntity{}
if err := nds.Get(c, namespacedKey, entity); err == nil {
t.Fatalf("expected no such entity error but got %+v", entity)
} else if err != datastore.ErrNoSuchEntity {
t.Fatal(err)
}
}
// New creates and returns a new blank account. It returns an error if an account
// with the specified email address already exists.
func New(ctx context.Context, email, password string) (*Account, error) {
account := new(Account)
account.Email = email
account.CreatedAt = time.Now()
if err := account.SetPassword(password); err != nil {
return nil, err
}
err := nds.RunInTransaction(ctx, func(txCtx context.Context) error {
dsKey := account.Key(txCtx)
if err := nds.Get(txCtx, dsKey, account); err == nil {
return ErrAccountExists
} else if err != datastore.ErrNoSuchEntity {
return err
}
_, err := nds.Put(txCtx, dsKey, account)
return err
}, nil)
if err != nil {
return nil, err
}
account.flag = camethroughus
account.originalEmail = email
return account, nil
}
func TestMemcacheNamespace(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int
}
// Illegal namespace chars.
nds.SetMemcacheNamespace("£££")
key := datastore.NewKey(c, "Entity", "", 1, nil)
if err := nds.Get(c, key, &testEntity{}); err == nil {
t.Fatal("expected namespace error")
}
if _, err := nds.Put(c, key, &testEntity{}); err == nil {
t.Fatal("expected namespace error")
}
if err := nds.Delete(c, key); err == nil {
t.Fatal("expected namespace error")
}
if err := nds.RunInTransaction(c, func(tc context.Context) error {
return nil
}, nil); err == nil {
t.Fatal("expected namespace error")
}
nds.SetMemcacheNamespace("")
}
// Next processes the next item
func (x *example6) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
// we need to load the entity ourselves
photo := new(Photo)
if err := nds.Get(c, key, photo); err != nil {
return err
}
photo.ID = key.IntID()
suffix := photo.Taken.Format("20060102")
_, err := x.bq.Tabledata.InsertAll(x.appID, "datasetName", "tableName", &bigquery.TableDataInsertAllRequest{
TemplateSuffix: suffix,
Rows: []*bigquery.TableDataInsertAllRequestRows{
{
Json: map[string]bigquery.JsonValue{
"id": photo.ID,
"taken": photo.Taken,
"photographer": map[string]bigquery.JsonValue{
"id": photo.Photographer.ID,
"name": photo.Photographer.Name,
},
},
},
},
}).Context(c).Do()
return err
}
// TestClearNamespacedLocks tests to make sure that locks are cleared when
// RunInTransaction is using a namespace.
func TestClearNamespacedLocks(t *testing.T) {
c, closeFunc := NewContext(t, nil)
defer closeFunc()
c, err := appengine.Namespace(c, "testnamespace")
if err != nil {
t.Fatal(err)
}
type testEntity struct {
Val int
}
key := datastore.NewKey(c, "TestEntity", "", 1, nil)
// Prime cache.
if err := nds.Get(c, key, &testEntity{}); err == nil {
t.Fatal("expected no such entity")
} else if err != datastore.ErrNoSuchEntity {
t.Fatal(err)
}
if err := nds.RunInTransaction(c, func(tc context.Context) error {
if err := nds.Get(tc, key, &testEntity{}); err == nil {
return errors.New("expected no such entity")
} else if err != datastore.ErrNoSuchEntity {
return err
}
if _, err := nds.Put(tc, key, &testEntity{3}); err != nil {
return err
}
return nil
}, nil); err != nil {
t.Fatal(err)
}
entity := &testEntity{}
if err := nds.Get(c, key, entity); err != nil {
t.Fatal(err)
}
if entity.Val != 3 {
t.Fatal("incorrect val")
}
}
// retrieve obtains the application configuration as a datastore.PropertyList.
func retrieve(ctx context.Context) (datastore.PropertyList, error) {
p := datastore.PropertyList(make([]datastore.Property, 0, 8))
key := datastore.NewKey(ctx, Entity, Entity, 0, nil)
err := nds.Get(ctx, key, &p)
return p, err
}
// Next processes the next item
func (x *example1) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
// we need to load the entity ourselves
photo := new(Photo)
if err := nds.Get(c, key, photo); err != nil {
return err
}
counters.Increment(photo.Photographer.Name, 1)
return nil
}
// HasChanged checks the current state of an account in the datastore. It returns
// true if the saved version of the account has diverged from the state of the account
// as described in account.
func HasChanged(ctx context.Context, account *Account) (bool, error) {
var currentState Account
key := account.Key(ctx)
if err := nds.Get(ctx, key, ¤tState); err != nil {
return false, err
} else {
return reflect.DeepEqual(*account, currentState), nil
}
}
// Get retrieves the account identified by email and stores it in
// the value pointed to by account.
func Get(ctx context.Context, email string, account *Account) error {
if err := nds.Get(ctx, datastore.NewKey(ctx, Entity, email, 0, nil), account); err != nil {
return err
} else {
account.flag = camethroughus
account.originalEmail = account.Email
return nil
}
}
// Next processes the next item
func (x *example3) Next(c context.Context, counters mapper.Counters, key *datastore.Key) error {
photo := new(Photo)
if err := nds.Get(c, key, photo); err != nil {
log.Errorf(c, err.Error())
return err
}
photo.ID = key.IntID()
counters.Increment(photo.Photographer.Name, 1)
return nil
}
// Save changes the application configuration to
// the values in conf. All HTTP requests subsequent to this one
// are guaranteed to use the new values in their configuration.
//
// Note that subsequent calls to Get with the same request context
// will continue to retrieve the old version of the configuration.
//
// As a special case, calling Save with a *config.Config will replace
// the entire contents of the configuration with the contents of Config.
func Save(ctx context.Context, conf interface{}) error {
if typedConfig, ok := conf.(*Config); ok {
pl := datastore.PropertyList(*typedConfig)
replaceKey := datastore.NewKey(ctx, Entity, Entity, 0, nil)
_, replaceErr := nds.Put(ctx, replaceKey, &pl)
return replaceErr
}
return datastore.RunInTransaction(ctx, func(txCtx context.Context) error {
props := datastore.PropertyList{}
key := datastore.NewKey(txCtx, Entity, Entity, 0, nil)
if err := nds.Get(txCtx, key, &props); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
// merge existing config with the new values
if newProps, err := datastore.SaveStruct(conf); err != nil {
return err
} else {
for _, newProp := range newProps {
newProp.NoIndex = true
replacing := false
for _, prop := range props {
// make sure NoIndex is set
prop.NoIndex = true
if prop.Name == newProp.Name {
replacing = true
prop.Value = newProp.Value
break
}
}
if !replacing {
// append
props = append(props, newProp)
}
}
}
_, err := nds.Put(txCtx, key, &props)
return err
}, nil)
}
func TestGetSliceProperty(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVals []int64
}
key := datastore.NewKey(c, "Entity", "", 1, nil)
intVals := []int64{0, 1, 2, 3}
val := &testEntity{intVals}
if _, err := nds.Put(c, key, val); err != nil {
t.Fatal(err)
}
// Get from datastore.
newVal := &testEntity{}
if err := nds.Get(c, key, newVal); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(val.IntVals, intVals) {
t.Fatal("slice properties not equal", val.IntVals)
}
// Get from memcache.
newVal = &testEntity{}
if err := nds.Get(c, key, newVal); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(val.IntVals, intVals) {
t.Fatal("slice properties not equal", val.IntVals)
}
}
// IncrementX is the version of Increment you should call if you wish to increment a
// counter inside a transaction.
func (c *Counter) IncrementX(txCtx context.Context, delta int64) error {
val := count{}
// pick a key at random and alter its value by delta
key := datastore.NewKey(txCtx, c.entity, c.shardKeys[rand.Int63()%int64(c.shardCount)], 0, nil)
if err := nds.Get(txCtx, key, &val); err != nil && err != datastore.ErrNoSuchEntity {
return err
}
val.C += delta
if _, err := nds.Put(txCtx, key, &val); err != nil {
return err
}
return nil
}
func warmupHandler(c *echo.Context) error {
if appengine.IsDevAppServer() {
k := datastore.NewKey(c, "photo", "", 1, nil)
p := new(Photo)
err := nds.Get(c, k, p)
if err != datastore.ErrNoSuchEntity {
return c.NoContent(http.StatusOK)
}
photographers := []Photographer{
{1, "Mr Canon"},
{2, "Miss Nikon"},
{3, "Mrs Pentax"},
{4, "Ms Sony"},
}
// create some dummy data
var id int64
for m := 1; m <= 12; m++ {
for d := 1; d < 28; d++ {
taken := time.Date(2015, time.Month(m), d, 12, 0, 0, 0, time.UTC)
photographer := photographers[rand.Int31n(4)]
p = &Photo{
Photographer: photographer,
Uploaded: time.Now().UTC(),
Width: 8000,
Height: 6000,
Taken: taken,
}
id++
k = datastore.NewKey(c, "photo", "", id, nil)
nds.Put(c, k, p)
}
}
}
return c.NoContent(http.StatusOK)
}
// LoadConfig retrieves the most recent state of the configuration
// from the datastore into the value pointed to by conf.
func LoadConfig(ctx context.Context, conf interface{}) error {
return nds.Get(ctx, datastore.NewKey(ctx, config.Entity, config.Entity, 0, nil), conf)
}
func TestPutGetDelete(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int
}
// Check we set memcahce, put datastore and delete memcache.
seq := make(chan string, 3)
nds.SetMemcacheSetMulti(func(c context.Context,
items []*memcache.Item) error {
seq <- "memcache.SetMulti"
return memcache.SetMulti(c, items)
})
nds.SetDatastorePutMulti(func(c context.Context,
keys []*datastore.Key, vals interface{}) ([]*datastore.Key, error) {
seq <- "datastore.PutMulti"
return datastore.PutMulti(c, keys, vals)
})
nds.SetMemcacheDeleteMulti(func(c context.Context,
keys []string) error {
seq <- "memcache.DeleteMulti"
close(seq)
return memcache.DeleteMulti(c, keys)
})
incompleteKey := datastore.NewIncompleteKey(c, "Entity", nil)
key, err := nds.Put(c, incompleteKey, &testEntity{43})
if err != nil {
t.Fatal(err)
}
nds.SetMemcacheSetMulti(memcache.SetMulti)
nds.SetDatastorePutMulti(datastore.PutMulti)
nds.SetMemcacheDeleteMulti(memcache.DeleteMulti)
if s := <-seq; s != "memcache.SetMulti" {
t.Fatal("memcache.SetMulti not", s)
}
if s := <-seq; s != "datastore.PutMulti" {
t.Fatal("datastore.PutMulti not", s)
}
if s := <-seq; s != "memcache.DeleteMulti" {
t.Fatal("memcache.DeleteMulti not", s)
}
// Check chan is closed.
<-seq
if key.Incomplete() {
t.Fatal("Key is incomplete")
}
te := &testEntity{}
if err := nds.Get(c, key, te); err != nil {
t.Fatal(err)
}
if te.IntVal != 43 {
t.Fatal("te.Val != 43", te.IntVal)
}
// Get from cache.
te = &testEntity{}
if err := nds.Get(c, key, te); err != nil {
t.Fatal(err)
}
if te.IntVal != 43 {
t.Fatal("te.Val != 43", te.IntVal)
}
// Change value.
if _, err := nds.Put(c, key, &testEntity{64}); err != nil {
t.Fatal(err)
}
// Get from cache.
te = &testEntity{}
if err := nds.Get(c, key, te); err != nil {
t.Fatal(err)
}
if te.IntVal != 64 {
t.Fatal("te.Val != 64", te.IntVal)
}
if err := nds.Delete(c, key); err != nil {
t.Fatal(err)
}
if err := nds.Get(c, key, &testEntity{}); err != datastore.ErrNoSuchEntity {
t.Fatal("expected datastore.ErrNoSuchEntity")
}
}
func processPhotos(c context.Context, processor PhotoProcessor) error {
// use the full 10 minutes allowed (assuming front-end instance type)
c, _ = context.WithTimeout(c, time.Duration(10)*time.Minute)
r := processor.Start(c)
log.Debugf(c, "processPhotos from %s to %s cursor %s", r.From.Format(dateFormat), r.To.Format(dateFormat), r.Start)
// TODO: describe pros & cons of different querying + continuation strategies
q := datastore.NewQuery("photo")
q = q.Filter("taken >=", r.From)
q = q.Filter("taken <", r.To)
q = q.Order("taken")
// I use keys only because it saves on cost - entities come from memcache if possible
q = q.KeysOnly()
var cursor *datastore.Cursor
if r.Start != "" {
newCursor, err := datastore.DecodeCursor(r.Start)
if err != nil {
log.Errorf(c, "get start cursor error %s", err.Error())
return err
}
cursor = &newCursor
}
// only one entity is loaded at a time
p := new(Photo)
timeout := make(chan bool, 1)
timer := time.AfterFunc(r.Timeout, func() {
timeout <- true
})
defer timer.Stop()
Loop:
for {
// check if we've timed out or whether to keep going
select {
case <-timeout:
break Loop
default:
}
processed := 0
q = q.Limit(r.Size)
if cursor != nil {
q = q.Start(*cursor)
}
it := q.Run(c)
for {
// if not using keys only then we would load the actual entity here using
// key, err := it.Next(p)
key, err := it.Next(nil)
if err == datastore.Done {
break
}
if err != nil {
log.Errorf(c, "get key error %s", err.Error())
return err
}
// loads the actual entity from memcache / datastore
err = nds.Get(c, key, p)
if err != nil {
log.Errorf(c, "get photo error %s", err.Error())
return err
}
// call the processor with the entity
p.ID = key.IntID()
processor.Process(c, p)
processed++
}
// did we process a full batch? if so, there may be more
if processed == r.Size {
newCursor, err := it.Cursor()
if err != nil {
log.Errorf(c, "get next cursor error %s", err.Error())
return err
}
cursor = &newCursor
} else {
// otherwise we're finished
cursor = nil
break
}
}
// let the processor write any aggregation entries / tasks etc...
processor.Complete(c)
// if we didn't complete everything then continue from the cursor
if cursor != nil {
r.Start = cursor.String()
processPhotosFunc.Call(c, processor)
}
return nil
}
// ChangeEmail changes the email address of an account from oldEmail to newEmail.
// It performs this operation atomically.
func ChangeEmail(ctx context.Context, oldEmail, newEmail string) error {
return nds.RunInTransaction(ctx, func(txCtx context.Context) error {
// read out both the account at the old and the new email addresses
var fromAccount, toAccount Account
var errFrom, errTo error
fromAccountKey := datastore.NewKey(txCtx, Entity, oldEmail, 0, nil)
toAccountKey := datastore.NewKey(txCtx, Entity, newEmail, 0, nil)
var s sync.WaitGroup
s.Add(2)
go func() {
errFrom = nds.Get(txCtx, fromAccountKey, &fromAccount)
s.Done()
}()
go func() {
errTo = nds.Get(txCtx, toAccountKey, &toAccount)
s.Done()
}()
s.Wait()
if errFrom != nil {
return errFrom
} else if errTo != datastore.ErrNoSuchEntity {
return ErrAccountExists
}
// at this point, we set FromAccount's email address to the new one
fromAccount.Email = newEmail
fromAccount.LastUpdatedAt = time.Now()
s.Add(2)
go func() {
// delete the account at the old key
errFrom = nds.Delete(txCtx, fromAccountKey)
s.Done()
}()
go func() {
// save the account at the new key
_, errTo = nds.Put(txCtx, toAccountKey, &fromAccount)
s.Done()
}()
s.Wait()
if errFrom != nil {
return errFrom
} else if errTo != nil {
return errTo
}
return nil
}, xgTransaction)
}
