func (s *supportContext) mkRandKeys(keys []*ds.Key, metas ds.MultiMetaGetter) []string {
ret := []string(nil)
for i, key := range keys {
mg := metas.GetSingle(i)
if !ds.GetMetaDefault(mg, CacheEnableMeta, true).(bool) {
continue
}
shards := s.numShards(key)
if shards == 0 {
continue
}
if ret == nil {
ret = make([]string, len(keys))
}
ret[i] = MakeMemcacheKey(s.mr.Intn(shards), key)
}
return ret
}
func (t *txnBufState) getMulti(keys []*datastore.Key, metas datastore.MultiMetaGetter, cb datastore.GetMultiCB, haveLock bool) error {
encKeys, roots := toEncoded(keys)
data := make([]item, len(keys))
idxMap := []int(nil)
toGetKeys := []*datastore.Key(nil)
lme := errors.NewLazyMultiError(len(keys))
err := func() error {
if !haveLock {
t.Lock()
defer t.Unlock()
}
if err := t.updateRootsLocked(roots); err != nil {
return err
}
for i, key := range keys {
data[i].key = key
data[i].encKey = encKeys[i]
if size, ok := t.entState.get(data[i].getEncKey()); ok {
data[i].buffered = true
if size > 0 {
idxMap = append(idxMap, i)
toGetKeys = append(toGetKeys, key)
}
}
}
if len(toGetKeys) > 0 {
j := 0
t.bufDS.GetMulti(toGetKeys, nil, func(pm datastore.PropertyMap, err error) error {
impossible(err)
data[idxMap[j]].data = pm
j++
return nil
})
}
idxMap = nil
getKeys := []*datastore.Key(nil)
getMetas := datastore.MultiMetaGetter(nil)
for i, itm := range data {
if !itm.buffered {
idxMap = append(idxMap, i)
getKeys = append(getKeys, itm.key)
getMetas = append(getMetas, metas.GetSingle(i))
}
}
if len(idxMap) > 0 {
j := 0
err := t.parentDS.GetMulti(getKeys, getMetas, func(pm datastore.PropertyMap, err error) error {
if err != datastore.ErrNoSuchEntity {
i := idxMap[j]
if !lme.Assign(i, err) {
data[i].data = pm
}
}
j++
return nil
})
if err != nil {
return err
}
}
return nil
}()
if err != nil {
return err
}
for i, itm := range data {
err := lme.GetOne(i)
if err != nil {
cb(nil, err)
} else if itm.data == nil {
cb(nil, datastore.ErrNoSuchEntity)
} else {
cb(itm.data, nil)
}
}
return nil
}
func (d *dsCache) GetMulti(keys []*ds.Key, metas ds.MultiMetaGetter, cb ds.GetMultiCB) error {
lockItems, nonce := d.mkRandLockItems(keys, metas)
if len(lockItems) == 0 {
return d.RawInterface.GetMulti(keys, metas, cb)
}
if err := d.mc.AddMulti(lockItems); err != nil {
// Ignore this error. Either we couldn't add them because they exist
// (so, not an issue), or because memcache is having sad times (in which
// case we'll see so in the GetMulti which immediately follows this).
}
if err := d.mc.GetMulti(lockItems); err != nil {
(log.Fields{log.ErrorKey: err}).Warningf(
d.c, "dscache: GetMulti: memcache.GetMulti")
}
p := makeFetchPlan(d.c, d.aid, d.ns, &facts{keys, metas, lockItems, nonce})
if !p.empty() {
// looks like we have something to pull from datastore, and maybe some work
// to save stuff back to memcache.
toCas := []memcache.Item{}
j := 0
err := d.RawInterface.GetMulti(p.toGet, p.toGetMeta, func(pm ds.PropertyMap, err error) error {
i := p.idxMap[j]
toSave := p.toSave[j]
j++
data := []byte(nil)
// true: save entity to memcache
// false: lock entity in memcache forever
shouldSave := true
if err == nil {
p.decoded[i] = pm
if toSave != nil {
data = encodeItemValue(pm)
if len(data) > internalValueSizeLimit {
shouldSave = false
log.Warningf(
d.c, "dscache: encoded entity too big (%d/%d)!",
len(data), internalValueSizeLimit)
}
}
} else {
p.lme.Assign(i, err)
if err != ds.ErrNoSuchEntity {
return nil // aka continue to the next entry
}
}
if toSave != nil {
if shouldSave { // save
mg := metas.GetSingle(i)
expSecs := ds.GetMetaDefault(mg, CacheExpirationMeta, CacheTimeSeconds).(int64)
toSave.SetFlags(uint32(ItemHasData))
toSave.SetExpiration(time.Duration(expSecs) * time.Second)
toSave.SetValue(data)
} else {
// Set a lock with an infinite timeout. No one else should try to
// serialize this item to memcache until something Put/Delete's it.
toSave.SetFlags(uint32(ItemHasLock))
toSave.SetExpiration(0)
toSave.SetValue(nil)
}
toCas = append(toCas, toSave)
}
return nil
})
if err != nil {
return err
}
if len(toCas) > 0 {
// we have entries to save back to memcache.
if err := d.mc.CompareAndSwapMulti(toCas); err != nil {
(log.Fields{log.ErrorKey: err}).Warningf(
d.c, "dscache: GetMulti: memcache.CompareAndSwapMulti")
}
}
}
// finally, run the callback for all of the decoded items and the errors,
// if any.
for i, dec := range p.decoded {
cb(dec, p.lme.GetOne(i))
}
return nil
}