Example #1
0
func (s *supportContext) mkRandKeys(keys []ds.Key, metas ds.MultiMetaGetter) []string {
	ret := []string(nil)
	for i, key := range keys {
		if !metas.GetMetaDefault(i, CacheEnableMeta, true).(bool) {
			continue
		}
		shards := s.numShards(key)
		if shards == 0 {
			continue
		}
		if ret == nil {
			ret = make([]string, len(keys))
		}
		ret[i] = MakeMemcacheKey(s.mr.Intn(shards), key)
	}
	return ret
}
Example #2
0
func (t *txnBufState) getMulti(keys []*datastore.Key, metas datastore.MultiMetaGetter, cb datastore.GetMultiCB, haveLock bool) error {
	encKeys, roots := toEncoded(keys)
	data := make([]item, len(keys))

	idxMap := []int(nil)
	toGetKeys := []*datastore.Key(nil)

	lme := errors.NewLazyMultiError(len(keys))
	err := func() error {
		if !haveLock {
			t.Lock()
			defer t.Unlock()
		}

		if err := t.updateRootsLocked(roots); err != nil {
			return err
		}

		for i, key := range keys {
			data[i].key = key
			data[i].encKey = encKeys[i]
			if size, ok := t.entState.get(data[i].getEncKey()); ok {
				data[i].buffered = true
				if size > 0 {
					idxMap = append(idxMap, i)
					toGetKeys = append(toGetKeys, key)
				}
			}
		}

		if len(toGetKeys) > 0 {
			j := 0
			t.bufDS.GetMulti(toGetKeys, nil, func(pm datastore.PropertyMap, err error) error {
				impossible(err)
				data[idxMap[j]].data = pm
				j++
				return nil
			})
		}

		idxMap = nil
		getKeys := []*datastore.Key(nil)
		getMetas := datastore.MultiMetaGetter(nil)

		for i, itm := range data {
			if !itm.buffered {
				idxMap = append(idxMap, i)
				getKeys = append(getKeys, itm.key)
				getMetas = append(getMetas, metas.GetSingle(i))
			}
		}

		if len(idxMap) > 0 {
			j := 0
			err := t.parentDS.GetMulti(getKeys, getMetas, func(pm datastore.PropertyMap, err error) error {
				if err != datastore.ErrNoSuchEntity {
					i := idxMap[j]
					if !lme.Assign(i, err) {
						data[i].data = pm
					}
				}
				j++
				return nil
			})
			if err != nil {
				return err
			}
		}
		return nil
	}()
	if err != nil {
		return err
	}

	for i, itm := range data {
		err := lme.GetOne(i)
		if err != nil {
			cb(nil, err)
		} else if itm.data == nil {
			cb(nil, datastore.ErrNoSuchEntity)
		} else {
			cb(itm.data, nil)
		}
	}
	return nil
}
Example #3
0
File: ds.go Project: martiniss/gae
func (d *dsCache) GetMulti(keys []ds.Key, metas ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	lockItems, nonce := d.mkRandLockItems(keys, metas)
	if len(lockItems) == 0 {
		return d.RawInterface.GetMulti(keys, metas, cb)
	}

	if err := d.mc.AddMulti(lockItems); err != nil {
		(log.Fields{log.ErrorKey: err}).Warningf(
			d.c, "dscache: GetMulti: memcache.AddMulti")

	}
	if err := d.mc.GetMulti(lockItems); err != nil {
		(log.Fields{log.ErrorKey: err}).Warningf(
			d.c, "dscache: GetMulti: memcache.GetMulti")
	}

	p := makeFetchPlan(d.c, d.aid, d.ns, &facts{keys, metas, lockItems, nonce})

	if !p.empty() {
		// looks like we have something to pull from datastore, and maybe some work
		// to save stuff back to memcache.

		toCas := []memcache.Item{}
		j := 0
		err := d.RawInterface.GetMulti(p.toGet, p.toGetMeta, func(pm ds.PropertyMap, err error) {
			i := p.idxMap[j]
			toSave := p.toSave[j]
			j++

			data := []byte(nil)

			// true: save entity to memcache
			// false: lock entity in memcache forever
			shouldSave := true
			if err == nil {
				p.decoded[i] = pm
				if toSave != nil {
					data = encodeItemValue(pm)
					if len(data) > internalValueSizeLimit {
						shouldSave = false
						log.Warningf(
							d.c, "dscache: encoded entity too big (%d/%d)!",
							len(data), internalValueSizeLimit)
					}
				}
			} else {
				p.lme.Assign(i, err)
				if err != ds.ErrNoSuchEntity {
					return // aka continue to the next entry
				}
			}

			if toSave != nil {
				if shouldSave { // save
					expSecs := metas.GetMetaDefault(i, CacheExpirationMeta, CacheTimeSeconds).(int64)
					toSave.SetFlags(uint32(ItemHasData))
					toSave.SetExpiration(time.Duration(expSecs) * time.Second)
					toSave.SetValue(data)
				} else {
					// Set a lock with an infinite timeout. No one else should try to
					// serialize this item to memcache until something Put/Delete's it.
					toSave.SetFlags(uint32(ItemHasLock))
					toSave.SetExpiration(0)
					toSave.SetValue(nil)
				}
				toCas = append(toCas, toSave)
			}
		})
		if err != nil {
			return err
		}
		if len(toCas) > 0 {
			// we have entries to save back to memcache.
			if err := d.mc.CompareAndSwapMulti(toCas); err != nil {
				(log.Fields{log.ErrorKey: err}).Warningf(
					d.c, "dscache: GetMulti: memcache.CompareAndSwapMulti")
			}
		}
	}

	// finally, run the callback for all of the decoded items and the errors,
	// if any.
	for i, dec := range p.decoded {
		cb(dec, p.lme.GetOne(i))
	}

	return nil
}