func (m *memcacheImpl) GetMulti(items []Item) error {
lme := errors.NewLazyMultiError(len(items))
realItems, idxMap := filterItems(lme, items, ErrCacheMiss)
if len(realItems) == 0 {
return lme.Get()
}
keys := make([]string, len(realItems))
for i, itm := range realItems {
keys[i] = itm.Key()
}
j := 0
err := m.RawInterface.GetMulti(keys, func(item Item, err error) {
i := idxMap[j]
if !lme.Assign(i, err) {
items[i].SetAll(item)
}
j++
})
if err == nil {
err = lme.Get()
}
return err
}
func (d *datastoreImpl) PutMulti(src interface{}) error {
slice := reflect.ValueOf(src)
mat := parseMultiArg(slice.Type())
if !mat.valid {
return fmt.Errorf("invalid PutMulti input type: %T", src)
}
keys, vals, err := mat.GetKeysPMs(d.NewKey, slice)
if err != nil {
return err
}
lme := errors.NewLazyMultiError(len(keys))
i := 0
err = d.RawInterface.PutMulti(keys, vals, func(key Key, err error) {
if key != keys[i] {
mat.setKey(slice.Index(i), key)
}
lme.Assign(i, err)
i++
})
if err == nil {
err = lme.Get()
}
return err
}
func (tcf *checkFilter) PutMulti(keys []Key, vals []PropertyMap, cb PutMultiCB) error {
if len(keys) != len(vals) {
return fmt.Errorf("datastore: PutMulti with mismatched keys/vals lengths (%d/%d)", len(keys), len(vals))
}
if len(keys) == 0 {
return nil
}
if cb == nil {
return fmt.Errorf("datastore: PutMulti callback is nil")
}
lme := errors.NewLazyMultiError(len(keys))
for i, k := range keys {
if !k.PartialValid(tcf.aid, tcf.ns) {
lme.Assign(i, ErrInvalidKey)
continue
}
v := vals[i]
if v == nil {
lme.Assign(i, errors.New("datastore: PutMulti got nil vals entry"))
}
}
if me := lme.Get(); me != nil {
for _, err := range me.(errors.MultiError) {
cb(nil, err)
}
return nil
}
return tcf.RawInterface.PutMulti(keys, vals, cb)
}
func (d *datastoreImpl) GetMulti(dst interface{}) error {
slice := reflect.ValueOf(dst)
mat := parseMultiArg(slice.Type())
keys, pms, err := mat.GetKeysPMs(d.aid, d.ns, slice, true)
if err != nil {
return err
}
lme := errors.NewLazyMultiError(len(keys))
i := 0
meta := NewMultiMetaGetter(pms)
err = d.RawInterface.GetMulti(keys, meta, func(pm PropertyMap, err error) error {
if !lme.Assign(i, err) {
lme.Assign(i, mat.setPM(slice.Index(i), pm))
}
i++
return nil
})
if err == nil {
err = lme.Get()
}
return err
}
func (t *txnBufState) fixKeys(keys []*datastore.Key) ([]*datastore.Key, error) {
lme := errors.NewLazyMultiError(len(keys))
realKeys := []*datastore.Key(nil)
for i, key := range keys {
if key.Incomplete() {
// intentionally call AllocateIDs without lock.
start, err := t.parentDS.AllocateIDs(key, 1)
if !lme.Assign(i, err) {
if realKeys == nil {
realKeys = make([]*datastore.Key, len(keys))
copy(realKeys, keys)
}
aid, ns, toks := key.Split()
toks[len(toks)-1].IntID = start
realKeys[i] = datastore.NewKeyToks(aid, ns, toks)
}
}
}
err := lme.Get()
if realKeys != nil {
return realKeys, err
}
return keys, err
}
func (d *datastoreImpl) GetMulti(dst interface{}) error {
slice := reflect.ValueOf(dst)
mat := parseMultiArg(slice.Type())
if !mat.valid {
return fmt.Errorf("invalid GetMulti input type: %T", dst)
}
keys, pms, err := mat.GetKeysPMs(d.NewKey, slice)
if err != nil {
return err
}
lme := errors.NewLazyMultiError(len(keys))
i := 0
meta := NewMultiMetaGetter(pms)
err = d.RawInterface.GetMulti(keys, meta, func(pm PropertyMap, err error) {
if !lme.Assign(i, err) {
lme.Assign(i, mat.setPM(slice.Index(i), pm))
}
i++
})
if err == nil {
err = lme.Get()
}
return err
}
// dsMF2R (DS multi-fake-to-fake) converts a slice of wrapped keys to SDK keys.
func dsMF2R(aeCtx context.Context, ks []*ds.Key) ([]*datastore.Key, error) {
lme := errors.NewLazyMultiError(len(ks))
ret := make([]*datastore.Key, len(ks))
err := error(nil)
for i, k := range ks {
ret[i], err = dsF2R(aeCtx, k)
lme.Assign(i, err)
}
return ret, lme.Get()
}
func (m *memcacheImpl) DeleteMulti(keys []string) error {
lme := errors.NewLazyMultiError(len(keys))
i := 0
err := m.RawInterface.DeleteMulti(keys, func(err error) {
lme.Assign(i, err)
i++
})
if err == nil {
err = lme.Get()
}
return err
}
func (t *taskqueueImpl) DeleteMulti(tasks []*Task, queueName string) error {
lme := errors.NewLazyMultiError(len(tasks))
i := 0
err := t.RawInterface.DeleteMulti(tasks, queueName, func(err error) {
lme.Assign(i, err)
i++
})
if err == nil {
err = lme.Get()
}
return err
}
func (d *datastoreImpl) DeleteMulti(keys []Key) (err error) {
lme := errors.NewLazyMultiError(len(keys))
i := 0
extErr := d.RawInterface.DeleteMulti(keys, func(internalErr error) {
lme.Assign(i, internalErr)
i++
})
err = lme.Get()
if err == nil {
err = extErr
}
return
}
func multiCall(items []Item, nilErr error, inner func(items []Item, cb RawCB) error) error {
lme := errors.NewLazyMultiError(len(items))
realItems, idxMap := filterItems(lme, items, nilErr)
j := 0
err := inner(realItems, func(err error) {
lme.Assign(idxMap[j], err)
j++
})
if err == nil {
err = lme.Get()
}
return err
}
func (t *taskqueueImpl) Stats(queueNames ...string) ([]Statistics, error) {
ret := make([]Statistics, len(queueNames))
lme := errors.NewLazyMultiError(len(queueNames))
i := 0
err := t.RawInterface.Stats(queueNames, func(s *Statistics, err error) {
if !lme.Assign(i, err) {
ret[i] = *s
}
i++
})
if err == nil {
err = lme.Get()
}
return ret, err
}
func commitToReal(s *txnBufState) error {
toPut, toPutKeys, toDel := s.effect()
return parallel.FanOutIn(func(ch chan<- func() error) {
if len(toPut) > 0 {
ch <- func() error {
mErr := errors.NewLazyMultiError(len(toPut))
i := 0
err := s.parentDS.PutMulti(toPutKeys, toPut, func(_ *datastore.Key, err error) error {
mErr.Assign(i, err)
i++
return nil
})
if err == nil {
err = mErr.Get()
}
return err
}
}
if len(toDel) > 0 {
ch <- func() error {
mErr := errors.NewLazyMultiError(len(toDel))
i := 0
err := s.parentDS.DeleteMulti(toDel, func(err error) error {
mErr.Assign(i, err)
i++
return nil
})
if err == nil {
err = mErr.Get()
}
return err
}
}
})
}
func (mat *multiArgType) GetKeysPMs(nk newKeyFunc, slice reflect.Value) ([]Key, []PropertyMap, error) {
retKey := make([]Key, slice.Len())
retPM := make([]PropertyMap, slice.Len())
lme := errors.NewLazyMultiError(len(retKey))
for i := range retKey {
key, err := mat.getKey(nk, slice.Index(i))
if !lme.Assign(i, err) {
retKey[i] = key
pm, err := mat.getPM(slice.Index(i))
if !lme.Assign(i, err) {
retPM[i] = pm
}
}
}
return retKey, retPM, lme.Get()
}
// makeFetchPlan takes the input facts and makes a plan about what to do with them.
//
// Possible scenarios:
// * all entries we got from memcache are valid data, and so we don't need
// to call through to the underlying datastore at all.
// * some entries are 'lock' entries, owned by us, and so we should get them
// from datastore and then attempt to save them back to memcache.
// * some entries are 'lock' entries, owned by something else, so we should
// get them from datastore and then NOT save them to memcache.
//
// Or some combination thereof. This also handles memcache enries with invalid
// data in them, cases where items have caching disabled entirely, etc.
func makeFetchPlan(c context.Context, aid, ns string, f *facts) *plan {
p := plan{
keepMeta: f.getMeta != nil,
decoded: make([]ds.PropertyMap, len(f.lockItems)),
lme: errors.NewLazyMultiError(len(f.lockItems)),
}
for i, lockItm := range f.lockItems {
m := f.getMeta.GetSingle(i)
getKey := f.getKeys[i]
if lockItm == nil {
// this item wasn't cacheable (e.g. the model had caching disabled,
// shardsForKey returned 0, etc.)
p.add(i, getKey, m, nil)
continue
}
switch FlagValue(lockItm.Flags()) {
case ItemHasLock:
if bytes.Equal(f.nonce, lockItm.Value()) {
// we have the lock
p.add(i, getKey, m, lockItm)
} else {
// someone else has the lock, don't save
p.add(i, getKey, m, nil)
}
case ItemHasData:
pmap, err := decodeItemValue(lockItm.Value(), aid, ns)
switch err {
case nil:
p.decoded[i] = pmap
case ds.ErrNoSuchEntity:
p.lme.Assign(i, ds.ErrNoSuchEntity)
default:
(logging.Fields{"error": err}).Warningf(c,
"dscache: error decoding %s, %s", lockItm.Key(), getKey)
p.add(i, getKey, m, nil)
}
default:
// have some other sort of object, or our AddMulti failed to add this item.
p.add(i, getKey, m, nil)
}
}
return &p
}
func (d *datastoreImpl) ExistsMulti(keys []*Key) ([]bool, error) {
lme := errors.NewLazyMultiError(len(keys))
ret := make([]bool, len(keys))
i := 0
err := d.RawInterface.GetMulti(keys, nil, func(_ PropertyMap, err error) error {
if err == nil {
ret[i] = true
} else if err != ErrNoSuchEntity {
lme.Assign(i, err)
}
i++
return nil
})
if err != nil {
return ret, err
}
return ret, lme.Get()
}
func (tcf *checkFilter) DeleteMulti(keys []Key, cb DeleteMultiCB) error {
if len(keys) == 0 {
return nil
}
if cb == nil {
return fmt.Errorf("datastore: DeleteMulti callback is nil")
}
lme := errors.NewLazyMultiError(len(keys))
for i, k := range keys {
if k.Incomplete() || !k.Valid(false, tcf.aid, tcf.ns) {
lme.Assign(i, ErrInvalidKey)
}
}
if me := lme.Get(); me != nil {
for _, err := range me.(errors.MultiError) {
cb(err)
}
return nil
}
return tcf.RawInterface.DeleteMulti(keys, cb)
}
func (mat *multiArgType) GetKeysPMs(aid, ns string, slice reflect.Value, meta bool) ([]*Key, []PropertyMap, error) {
retKey := make([]*Key, slice.Len())
retPM := make([]PropertyMap, slice.Len())
getter := mat.getPM
if meta {
getter = func(slot reflect.Value) (PropertyMap, error) {
return mat.getMetaPM(slot), nil
}
}
lme := errors.NewLazyMultiError(len(retKey))
for i := range retKey {
key, err := mat.getKey(aid, ns, slice.Index(i))
if !lme.Assign(i, err) {
retKey[i] = key
pm, err := getter(slice.Index(i))
if !lme.Assign(i, err) {
retPM[i] = pm
}
}
}
return retKey, retPM, lme.Get()
}
func (d *datastoreImpl) GetAll(q Query, dst interface{}) error {
v := reflect.ValueOf(dst)
if v.Kind() != reflect.Ptr {
return fmt.Errorf("invalid GetAll dst: must have a ptr-to-slice: %T", dst)
}
if !v.IsValid() || v.IsNil() {
return errors.New("invalid GetAll dst: <nil>")
}
if keys, ok := dst.(*[]Key); ok {
return d.RawInterface.Run(q.KeysOnly(), func(k Key, _ PropertyMap, _ CursorCB) bool {
*keys = append(*keys, k)
return true
})
}
slice := v.Elem()
mat := parseMultiArg(slice.Type())
if !mat.valid || mat.newElem == nil {
return fmt.Errorf("invalid GetAll input type: %T", dst)
}
lme := errors.NewLazyMultiError(slice.Len())
i := 0
err := d.RawInterface.Run(q, func(k Key, pm PropertyMap, _ CursorCB) bool {
slice.Set(reflect.Append(slice, mat.newElem()))
itm := slice.Index(i)
mat.setKey(itm, k)
lme.Assign(i, mat.setPM(itm, pm))
i++
return true
})
if err == nil {
err = lme.Get()
}
return err
}
func (d *datastoreImpl) PutMulti(src interface{}) error {
slice := reflect.ValueOf(src)
mat := parseMultiArg(slice.Type())
keys, vals, err := mat.GetKeysPMs(d.aid, d.ns, slice, false)
if err != nil {
return err
}
lme := errors.NewLazyMultiError(len(keys))
i := 0
err = d.RawInterface.PutMulti(keys, vals, func(key *Key, err error) error {
if !lme.Assign(i, err) && key != keys[i] {
mat.setKey(slice.Index(i), key)
}
i++
return nil
})
if err == nil {
err = lme.Get()
}
return err
}
func (t *txnBufState) getMulti(keys []*datastore.Key, metas datastore.MultiMetaGetter, cb datastore.GetMultiCB, haveLock bool) error {
encKeys, roots := toEncoded(keys)
data := make([]item, len(keys))
idxMap := []int(nil)
toGetKeys := []*datastore.Key(nil)
lme := errors.NewLazyMultiError(len(keys))
err := func() error {
if !haveLock {
t.Lock()
defer t.Unlock()
}
if err := t.updateRootsLocked(roots); err != nil {
return err
}
for i, key := range keys {
data[i].key = key
data[i].encKey = encKeys[i]
if size, ok := t.entState.get(data[i].getEncKey()); ok {
data[i].buffered = true
if size > 0 {
idxMap = append(idxMap, i)
toGetKeys = append(toGetKeys, key)
}
}
}
if len(toGetKeys) > 0 {
j := 0
t.bufDS.GetMulti(toGetKeys, nil, func(pm datastore.PropertyMap, err error) error {
impossible(err)
data[idxMap[j]].data = pm
j++
return nil
})
}
idxMap = nil
getKeys := []*datastore.Key(nil)
getMetas := datastore.MultiMetaGetter(nil)
for i, itm := range data {
if !itm.buffered {
idxMap = append(idxMap, i)
getKeys = append(getKeys, itm.key)
getMetas = append(getMetas, metas.GetSingle(i))
}
}
if len(idxMap) > 0 {
j := 0
err := t.parentDS.GetMulti(getKeys, getMetas, func(pm datastore.PropertyMap, err error) error {
if err != datastore.ErrNoSuchEntity {
i := idxMap[j]
if !lme.Assign(i, err) {
data[i].data = pm
}
}
j++
return nil
})
if err != nil {
return err
}
}
return nil
}()
if err != nil {
return err
}
for i, itm := range data {
err := lme.GetOne(i)
if err != nil {
cb(nil, err)
} else if itm.data == nil {
cb(nil, datastore.ErrNoSuchEntity)
} else {
cb(itm.data, nil)
}
}
return nil
}