func (a *MemcacheDatastoreAccessor) uncacheIdiom(c context.Context, idiom *Idiom) error {
cacheKeys := make([]string, 1+len(idiom.Implementations))
cacheKeys[0] = fmt.Sprintf("getIdiom(%v)", idiom.Id)
for i, impl := range idiom.Implementations {
cacheKeys[1+i] = fmt.Sprintf("getIdiomByImplID(%v)", impl.Id)
}
err := memcache.DeleteMulti(c, cacheKeys)
if err != nil {
log.Errorf(c, err.Error())
}
return err
}
// RunInTransaction runs f in a transaction. It calls f with a transaction
// context tg that f should use for all App Engine operations. Neither cache nor
// memcache are used or set during a transaction.
//
// Otherwise similar to appengine/datastore.RunInTransaction:
// https://developers.google.com/appengine/docs/go/datastore/reference#RunInTransaction
func (g *Goon) RunInTransaction(f func(tg *Goon) error, opts *datastore.TransactionOptions) error {
var ng *Goon
err := datastore.RunInTransaction(g.Context, func(tc context.Context) error {
ng = &Goon{
Context: tc,
inTransaction: true,
toSet: make(map[string]interface{}),
toDelete: make(map[string]bool),
toDeleteMC: make(map[string]bool),
KindNameResolver: g.KindNameResolver,
}
return f(ng)
}, opts)
if err == nil {
if len(ng.toDeleteMC) > 0 {
var memkeys []string
for k := range ng.toDeleteMC {
memkeys = append(memkeys, k)
}
memcache.DeleteMulti(g.Context, memkeys)
}
g.cacheLock.Lock()
defer g.cacheLock.Unlock()
for k, v := range ng.toSet {
g.cache[k] = v
}
for k := range ng.toDelete {
delete(g.cache, k)
}
} else {
g.error(err)
}
return err
}
func (mc *gaeMemcache) deleteMulti(c context.Context, keys []string) error {
return memcache.DeleteMulti(c, keys)
}
// DeleteMulti deletes the multiple keys
func (d *Driver) DeleteMulti(keys []string) error {
return memcache.DeleteMulti(d.ctx, keys)
}
func TestPutGetDelete(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
type testEntity struct {
IntVal int
}
// Check we set memcahce, put datastore and delete memcache.
seq := make(chan string, 3)
nds.SetMemcacheSetMulti(func(c context.Context,
items []*memcache.Item) error {
seq <- "memcache.SetMulti"
return memcache.SetMulti(c, items)
})
nds.SetDatastorePutMulti(func(c context.Context,
keys []*datastore.Key, vals interface{}) ([]*datastore.Key, error) {
seq <- "datastore.PutMulti"
return datastore.PutMulti(c, keys, vals)
})
nds.SetMemcacheDeleteMulti(func(c context.Context,
keys []string) error {
seq <- "memcache.DeleteMulti"
close(seq)
return memcache.DeleteMulti(c, keys)
})
incompleteKey := datastore.NewIncompleteKey(c, "Entity", nil)
key, err := nds.Put(c, incompleteKey, &testEntity{43})
if err != nil {
t.Fatal(err)
}
nds.SetMemcacheSetMulti(memcache.SetMulti)
nds.SetDatastorePutMulti(datastore.PutMulti)
nds.SetMemcacheDeleteMulti(memcache.DeleteMulti)
if s := <-seq; s != "memcache.SetMulti" {
t.Fatal("memcache.SetMulti not", s)
}
if s := <-seq; s != "datastore.PutMulti" {
t.Fatal("datastore.PutMulti not", s)
}
if s := <-seq; s != "memcache.DeleteMulti" {
t.Fatal("memcache.DeleteMulti not", s)
}
// Check chan is closed.
<-seq
if key.Incomplete() {
t.Fatal("Key is incomplete")
}
te := &testEntity{}
if err := nds.Get(c, key, te); err != nil {
t.Fatal(err)
}
if te.IntVal != 43 {
t.Fatal("te.Val != 43", te.IntVal)
}
// Get from cache.
te = &testEntity{}
if err := nds.Get(c, key, te); err != nil {
t.Fatal(err)
}
if te.IntVal != 43 {
t.Fatal("te.Val != 43", te.IntVal)
}
// Change value.
if _, err := nds.Put(c, key, &testEntity{64}); err != nil {
t.Fatal(err)
}
// Get from cache.
te = &testEntity{}
if err := nds.Get(c, key, te); err != nil {
t.Fatal(err)
}
if te.IntVal != 64 {
t.Fatal("te.Val != 64", te.IntVal)
}
if err := nds.Delete(c, key); err != nil {
t.Fatal(err)
}
if err := nds.Get(c, key, &testEntity{}); err != datastore.ErrNoSuchEntity {
t.Fatal("expected datastore.ErrNoSuchEntity")
}
}
// DeleteMulti is a batch version of Delete.
func (g *Goon) DeleteMulti(keys []*datastore.Key) error {
if len(keys) == 0 {
return nil
// not an error, and it was "successful", so return nil
}
memkeys := make([]string, len(keys))
g.cacheLock.Lock()
for i, k := range keys {
mk := memkey(k)
memkeys[i] = mk
if g.inTransaction {
delete(g.toSet, mk)
g.toDelete[mk] = true
} else {
delete(g.cache, mk)
}
}
g.cacheLock.Unlock()
// Memcache needs to be updated after the datastore to prevent a common race condition,
// where a concurrent request will fetch the not-yet-updated data from the datastore
// and populate memcache with it.
if g.inTransaction {
for _, mk := range memkeys {
g.toDeleteMC[mk] = true
}
} else {
defer memcache.DeleteMulti(g.Context, memkeys)
}
multiErr, any := make(appengine.MultiError, len(keys)), false
goroutines := (len(keys)-1)/deleteMultiLimit + 1
var wg sync.WaitGroup
wg.Add(goroutines)
for i := 0; i < goroutines; i++ {
go func(i int) {
defer wg.Done()
lo := i * deleteMultiLimit
hi := (i + 1) * deleteMultiLimit
if hi > len(keys) {
hi = len(keys)
}
dmerr := datastore.DeleteMulti(g.Context, keys[lo:hi])
if dmerr != nil {
any = true // this flag tells DeleteMulti to return multiErr later
merr, ok := dmerr.(appengine.MultiError)
if !ok {
g.error(dmerr)
for j := lo; j < hi; j++ {
multiErr[j] = dmerr
}
return
}
copy(multiErr[lo:hi], merr)
}
}(i)
}
wg.Wait()
if any {
return realError(multiErr)
}
return nil
}
// PutMulti is a batch version of Put.
//
// src must be a *[]S, *[]*S, *[]I, []S, []*S, or []I, for some struct type S,
// or some interface type I. If *[]I or []I, each element must be a struct pointer.
func (g *Goon) PutMulti(src interface{}) ([]*datastore.Key, error) {
keys, err := g.extractKeys(src, true) // allow incomplete keys on a Put request
if err != nil {
return nil, err
}
var memkeys []string
for _, key := range keys {
if !key.Incomplete() {
memkeys = append(memkeys, memkey(key))
}
}
// Memcache needs to be updated after the datastore to prevent a common race condition,
// where a concurrent request will fetch the not-yet-updated data from the datastore
// and populate memcache with it.
if g.inTransaction {
for _, mk := range memkeys {
g.toDeleteMC[mk] = true
}
} else {
defer memcache.DeleteMulti(g.Context, memkeys)
}
v := reflect.Indirect(reflect.ValueOf(src))
multiErr, any := make(appengine.MultiError, len(keys)), false
goroutines := (len(keys)-1)/putMultiLimit + 1
var wg sync.WaitGroup
wg.Add(goroutines)
for i := 0; i < goroutines; i++ {
go func(i int) {
defer wg.Done()
lo := i * putMultiLimit
hi := (i + 1) * putMultiLimit
if hi > len(keys) {
hi = len(keys)
}
rkeys, pmerr := datastore.PutMulti(g.Context, keys[lo:hi], v.Slice(lo, hi).Interface())
if pmerr != nil {
any = true // this flag tells PutMulti to return multiErr later
merr, ok := pmerr.(appengine.MultiError)
if !ok {
g.error(pmerr)
for j := lo; j < hi; j++ {
multiErr[j] = pmerr
}
return
}
copy(multiErr[lo:hi], merr)
}
for i, key := range keys[lo:hi] {
if multiErr[lo+i] != nil {
continue // there was an error writing this value, go to next
}
vi := v.Index(lo + i).Interface()
if key.Incomplete() {
g.setStructKey(vi, rkeys[i])
keys[i] = rkeys[i]
}
if g.inTransaction {
mk := memkey(rkeys[i])
delete(g.toDelete, mk)
g.toSet[mk] = vi
} else {
g.putMemory(vi)
}
}
}(i)
}
wg.Wait()
if any {
return keys, realError(multiErr)
}
return keys, nil
}
// clearCache removes all data for the given namespace from the cache
func clearCache(con *Context, namespace string) error {
namespace = strings.ToLower(namespace)
return memcache.DeleteMulti(con.C, []string{"RSS" + namespace, "JSON" + namespace})
}
func (m mcImpl) DeleteMulti(keys []string, cb mc.RawCB) error {
return doCB(memcache.DeleteMulti(m.aeCtx, keys), cb)
}
