func (builder *stepsBuilder) LoadBatchFromCache(context appengine.Context) stream.OnDataFn {
return func(data stream.T, emitter stream.Emitter) {
batch := data.(*MemcacheLoadBatch)
items, err := memcache.GetMulti(context, batch.Keys)
if err != nil {
panic(err)
}
for id, item := range items {
if err := json.Unmarshal(item.Value, batch.Items[id]); err != nil {
panic(err)
}
// Set entity key back
batch.Items[id].Entity.SetKey(batch.Items[id].Key)
delete(batch.Items, id)
}
// In case of cache misses, send entities
// downstream to be handled by the next transformer
if !batch.Empty() {
for _, item := range batch.Items {
emitter.Emit(item.Entity)
}
}
}
}
// GetMulti is a batch version of Get. Cached values are returned from memcache, uncached values are returned from
// datastore and memcached for next time.
//
// dst must be a []S, []*S, []I or []P, for some struct type S, some interface type I, or some non-interface
// non-pointer type P such that P or *P implements PropertyLoadSaver. If an []I, each element must be a valid
// dst for Get: it must be a struct pointer or implement PropertyLoadSaver.
//
// As a special case, PropertyList is an invalid type for dst, even though a PropertyList is a slice of structs.
// It is treated as invalid to avoid being mistakenly passed when []PropertyList was intended.
func GetMulti(c appengine.Context, key []*datastore.Key, dst interface{}) error {
if len(key) == 0 {
return nil
}
// check cache
encodedKeys := encodeKeys(key)
itemMap, errm := memcache.GetMulti(c, encodedKeys)
if len(itemMap) != len(key) {
// TODO benchmark loading all vs loading missing
// load from datastore
errd := datastore.GetMulti(c, key, dst)
if Debug {
c.Debugf("reading from datastore: %#v", dst)
}
if errd != nil {
return errd
}
// cache for next time
errm = cache(key, dst, c)
} else {
errm = decodeItems(key, itemMap, dst)
if Debug {
c.Debugf("reading from memcache: %#v", dst)
}
}
return errm
}
func (cd Codec) GetMulti(c appengine.Context, keys []string,
dst interface{}) ([]*memcache.Item, error) {
itemsMap, err := memcache.GetMulti(c, keys)
if err != nil {
return nil, err
}
v := reflect.ValueOf(dst)
items := make([]*memcache.Item, len(keys))
multiErr := make(appengine.MultiError, len(keys))
for i, key := range keys {
if item, ok := itemsMap[key]; ok {
err := cd.Unmarshal(item.Value, v.Index(i).Interface())
if err != nil {
multiErr[i] = err
} else {
item.Object = v.Index(i).Interface()
items[i] = item
}
} else {
multiErr[i] = memcache.ErrCacheMiss
}
}
return items, multiErr
}
func memGetMulti(c appengine.Context, keys []string, dests []interface{}) (items []*memcache.Item, errors []error) {
items = make([]*memcache.Item, len(keys))
errors = make([]error, len(keys))
itemHash, err := memcache.GetMulti(c, keys)
if err != nil {
for index, _ := range errors {
errors[index] = err
}
return
}
var item *memcache.Item
var ok bool
for index, keyHash := range keys {
if item, ok = itemHash[keyHash]; ok {
items[index] = item
if err := MemCodec.Unmarshal(item.Value, dests[index]); err != nil {
errors[index] = err
}
} else {
errors[index] = memcache.ErrCacheMiss
}
}
return
}
/*
memGetMulti will look for all provided keys, and load them into the destinatinoPointers.
It will return the memcache.Items it found, and any errors the lookups caused.
If c is within a transaction no lookup will take place and errors will be slice of memcache.ErrCacheMiss.
*/
func memGetMulti(c TransactionContext, keys []string, destinationPointers []interface{}) (items []*memcache.Item, errors appengine.MultiError) {
items = make([]*memcache.Item, len(keys))
errors = make(appengine.MultiError, len(keys))
if !MemcacheEnabled || c.InTransaction() {
for index, _ := range errors {
errors[index] = memcache.ErrCacheMiss
}
return
}
itemHash, err := memcache.GetMulti(c, keys)
if err != nil {
c.Errorf("Error doing GetMulti: %v", err)
for index, _ := range errors {
errors[index] = ErrCacheMiss
}
err = errors
}
var item *memcache.Item
var ok bool
for index, keyHash := range keys {
if item, ok = itemHash[keyHash]; ok {
items[index] = item
if err := Codec.Unmarshal(item.Value, destinationPointers[index]); err != nil {
errors[index] = err
}
} else {
errors[index] = memcache.ErrCacheMiss
}
}
return
}
//RetrieveMultiFromStore attempts to retrieve all elements from memcache
func RetrieveMultiFromStore(c *appengine.Context, ids []string) ([]SteamAccountDetails, []string, error) {
allIDSet := mapset.NewSet()
for _, id := range ids {
allIDSet.Add(id)
}
foundIDs := mapset.NewSet()
var foundAccounts []SteamAccountDetails
items, err := memcache.GetMulti(*c, ids)
if err != nil {
return nil, nil, err
}
for key, item := range items {
var acc SteamAccountDetails
if err := json.Unmarshal(item.Value, &acc); err != nil {
return nil, nil, err
}
foundIDs.Add(key)
foundAccounts = append(foundAccounts, acc)
}
var missingIDs []string
missingIDSet := allIDSet.Difference(foundIDs)
for v := range missingIDSet.Iter() {
missingIDs = append(missingIDs, v.(string))
}
return foundAccounts, missingIDs, nil
}
// dst must be []Kind or []*Kind
func (d *Driver) GetMulti(keys []string, dst interface{}) error {
var errors = make([]error, 0)
if hits, err := memcache.GetMulti(d.ctx, keys); err == nil {
dstValue := reflect.ValueOf(dst)
for i, key := range keys {
if item, ok := hits[key]; ok {
objValue := dstValue.Index(i)
if objValue.Kind() == reflect.Ptr {
if string(item.Value) == _JSON_VALUE_NULL {
continue
}
obj := reflect.New(objValue.Type().Elem()).Interface()
if err = json.Unmarshal(item.Value, obj); err != nil {
errors = append(errors, fmt.Errorf("Could not unmarshal JSON: %v", err))
continue
}
objValue.Set(reflect.ValueOf(obj))
} else {
obj := objValue.Interface()
if err = json.Unmarshal(item.Value, &obj); err != nil {
errors = append(errors, fmt.Errorf("Could not unmarshal JSON: %v", err))
continue
}
}
}
}
if len(errors) > 0 {
return fmt.Errorf("MultiError: %v", errors)
}
return nil
} else {
return err
}
}
func (c *memcacheDriver) GetMulti(keys []string) (map[string][]byte, error) {
results, err := memcache.GetMulti(c.c, keys)
if err != nil && err != memcache.ErrCacheMiss {
return nil, err
}
value := make(map[string][]byte, len(results))
for k, v := range results {
value[k] = v.Value
}
return value, nil
}
func AdminDateFormats(c mpg.Context, w http.ResponseWriter, r *http.Request) {
type df struct {
URL, Format string
}
keys := make([]string, dateFormatCount)
for i := range keys {
keys[i] = fmt.Sprintf("_dateformat-%v", i)
}
items, _ := memcache.GetMulti(c, keys)
dfs := make(map[string]df)
for k, v := range items {
sp := strings.Split(string(v.Value), "|")
dfs[k] = df{sp[1], sp[0]}
}
if err := templates.ExecuteTemplate(w, "admin-date-formats.html", dfs); err != nil {
serveError(w, err)
}
}
func GetMulti(c appengine.Context, keys []string) ([]*memcache.Item, error) {
itemsMap, err := memcache.GetMulti(c, keys)
if err != nil {
return nil, err
}
items := make([]*memcache.Item, len(keys))
errs := make(appengine.MultiError, len(keys))
for i, key := range keys {
if item, ok := itemsMap[key]; ok {
items[i] = item
} else {
errs[i] = memcache.ErrCacheMiss
}
}
return items, errs
}
func (db FlightDB) flightsFromShardedMemcache(key string) ([]f.Flight, bool) {
/*
if _,err := memcache.Gob.Get(db.C, memKey, &flights); err == nil {
db.C.Infof(" ##== fdb cache hit for '%s'", memKey)
return flights, true
}*/
keys := []string{}
for i := 0; i < 32; i++ {
keys = append(keys, fmt.Sprintf("=%d=%s", i*chunksize, key))
}
if items, err := memcache.GetMulti(db.C, keys); err != nil {
db.C.Errorf("fdb memcache multiget: %v", err)
return nil, false
} else {
b := []byte{}
for i := 0; i < 32; i++ {
if item, exists := items[keys[i]]; exists == false {
break
} else {
db.C.Infof(" #--- Found '%s' !", item.Key)
b = append(b, item.Value...)
}
}
db.C.Infof(" #--- Final read len: %d", len(b))
buf := bytes.NewBuffer(b)
flights := []f.Flight{}
if err := gob.NewDecoder(buf).Decode(&flights); err != nil {
db.C.Errorf("fdb memcache multiget decode: %v", err)
return nil, false
}
db.C.Infof(" #--- Found all items ?")
return flights, true
}
}
// bool means 'found'
func BytesFromShardedMemcache(c appengine.Context, key string) ([]byte, bool) {
keys := []string{}
for i := 0; i < 32; i++ {
keys = append(keys, fmt.Sprintf("=%d=%s", i*chunksize, key))
}
if items, err := memcache.GetMulti(c, keys); err != nil {
c.Errorf("fdb memcache multiget: %v", err)
return nil, false
} else {
b := []byte{}
for i := 0; i < 32; i++ {
if item, exists := items[keys[i]]; exists == false {
break
} else {
c.Infof(" #=== Found '%s' !", item.Key)
b = append(b, item.Value...)
}
}
c.Infof(" #=== Final read len: %d", len(b))
/*
buf := bytes.NewBuffer(b)
flights := []Flight{}
if err := gob.NewDecoder(buf).Decode(&flights); err != nil {
db.C.Errorf("fdb memcache multiget decode: %v", err)
return nil,false
}
*/
if len(b) > 0 {
return b, true
} else {
return nil, false
}
}
}
func (u GAE) MemoryGet(items *[]MemoryItem) error {
if items == nil || len(*items) < 1 {
return nil
}
keys := make([]string, len(*items))
for index, item := range *items {
keys[index] = item.Key
}
gaeItems, err := memcache.GetMulti(u.context, keys)
if err != nil {
return err
}
for index, item := range *items {
if gaeItem, ok := gaeItems[item.Key]; ok {
(*items)[index].Value = string(gaeItem.Value)
}
}
return nil
}
// populateBuildingURLs populates each commit in Commits' buildingURLs map with the
// URLs of builds which are currently in progress.
func (td *uiTemplateData) populateBuildingURLs(ctx appengine.Context) {
// need are memcache keys: "building|<hash>|<gohash>|<builder>"
// The hash is of the main "go" repo, or the subrepo commit hash.
// The gohash is empty for the main repo, else it's the Go hash.
var need []string
commit := map[string]*Commit{} // commit hash -> Commit
// Gather pending commits for main repo.
for _, b := range td.Builders {
for _, c := range td.Commits {
if c.Result(b, "") == nil {
commit[c.Hash] = c
need = append(need, buildingKey(c.Hash, "", b))
}
}
}
// Gather pending commits for sub-repos.
for _, ts := range td.TagState {
goHash := ts.Tag.Hash
for _, b := range td.Builders {
for _, pkg := range ts.Packages {
c := pkg.Commit
commit[c.Hash] = c
if c.Result(b, goHash) == nil {
need = append(need, buildingKey(c.Hash, goHash, b))
}
}
}
}
if len(need) == 0 {
return
}
m, err := memcache.GetMulti(ctx, need)
if err != nil {
// oh well. this is a cute non-critical feature anyway.
ctx.Debugf("GetMulti of building keys: %v", err)
return
}
for k, it := range m {
f := strings.SplitN(k, "|", 4)
if len(f) != 4 {
continue
}
hash, goHash, builder := f[1], f[2], f[3]
c, ok := commit[hash]
if !ok {
continue
}
m := c.buildingURLs
if m == nil {
m = make(map[builderAndGoHash]string)
c.buildingURLs = m
}
m[builderAndGoHash{builder, goHash}] = string(it.Value)
}
}
func Index(w http.ResponseWriter, r *http.Request) {
keys := make([]string, modulus)
for i := range keys {
keys[i] = fmt.Sprintf(keyPart, i*distance)
}
c := context(r)
items, err := memcache.GetMulti(c, keys)
if err != nil {
return
}
ars := AllRequestStats{}
for _, v := range items {
t := stats_part{}
err := gob.NewDecoder(bytes.NewBuffer(v.Value)).Decode(&t)
if err != nil {
continue
}
r := RequestStats(t)
ars = append(ars, &r)
}
sort.Sort(reverse{ars})
requestById := make(map[int]*RequestStats, len(ars))
idByRequest := make(map[*RequestStats]int, len(ars))
requests := make(map[int]*StatByName)
byRequest := make(map[int]map[string]cVal)
for i, v := range ars {
idx := i + 1
requestById[idx] = v
idByRequest[v] = idx
requests[idx] = &StatByName{
RequestStats: v,
}
byRequest[idx] = make(map[string]cVal)
}
requestByPath := make(map[string][]int)
byCount := make(map[string]cVal)
byRPC := make(map[SKey]cVal)
for _, t := range ars {
id := idByRequest[t]
requestByPath[t.Path] = append(requestByPath[t.Path], id)
for _, r := range t.RPCStats {
rpc := r.Name()
v := byRequest[id][rpc]
v.count++
v.cost += r.Cost
byRequest[id][rpc] = v
v = byCount[rpc]
v.count++
v.cost += r.Cost
byCount[rpc] = v
v = byRPC[SKey{rpc, t.Path}]
v.count++
v.cost += r.Cost
byRPC[SKey{rpc, t.Path}] = v
}
}
for k, v := range byRequest {
stats := StatsByName{}
for rpc, s := range v {
stats = append(stats, &StatByName{
Name: rpc,
Count: s.count,
Cost: s.cost,
})
}
sort.Sort(reverse{stats})
requests[k].SubStats = stats
}
statsByRPC := make(map[string]StatsByName)
pathStats := make(map[string]StatsByName)
for k, v := range byRPC {
statsByRPC[k.a] = append(statsByRPC[k.a], &StatByName{
Name: k.b,
Count: v.count,
Cost: v.cost,
})
pathStats[k.b] = append(pathStats[k.b], &StatByName{
Name: k.a,
Count: v.count,
Cost: v.cost,
})
}
for k, v := range statsByRPC {
sort.Sort(reverse{v})
statsByRPC[k] = v
}
pathStatsByCount := StatsByName{}
for k, v := range pathStats {
total := 0
var cost int64
for _, stat := range v {
total += stat.Count
cost += stat.Cost
}
sort.Sort(reverse{v})
pathStatsByCount = append(pathStatsByCount, &StatByName{
Name: k,
Count: total,
Cost: cost,
SubStats: v,
Requests: len(requestByPath[k]),
RecentReqs: requestByPath[k],
})
}
sort.Sort(reverse{pathStatsByCount})
allStatsByCount := StatsByName{}
for k, v := range byCount {
allStatsByCount = append(allStatsByCount, &StatByName{
Name: k,
Count: v.count,
Cost: v.cost,
SubStats: statsByRPC[k],
})
}
sort.Sort(reverse{allStatsByCount})
v := struct {
Env map[string]string
Requests map[int]*StatByName
RequestStatsByCount map[int]*StatByName
AllStatsByCount StatsByName
PathStatsByCount StatsByName
}{
Env: map[string]string{
"APPLICATION_ID": appengine.AppID(c),
},
Requests: requests,
AllStatsByCount: allStatsByCount,
PathStatsByCount: pathStatsByCount,
}
_ = templates.ExecuteTemplate(w, "main", v)
}
// GetMulti is a batch version of Get.
//
// dst must be a *[]S, *[]*S, *[]I, []S, []*S, or []I, for some struct type S,
// or some interface type I. If *[]I or []I, each element must be a struct pointer.
func (g *Goon) GetMulti(dst interface{}) error {
keys, err := g.extractKeys(dst, false) // don't allow incomplete keys on a Get request
if err != nil {
return err
}
v := reflect.Indirect(reflect.ValueOf(dst))
if g.inTransaction {
// todo: support getMultiLimit in transactions
return datastore.GetMulti(g.Context, keys, v.Interface())
}
var dskeys []*datastore.Key
var dsdst []interface{}
var dixs []int
var memkeys []string
var mixs []int
g.cacheLock.RLock()
for i, key := range keys {
m := memkey(key)
vi := v.Index(i)
if vi.Kind() == reflect.Struct {
vi = vi.Addr()
}
if s, present := g.cache[m]; present {
if vi.Kind() == reflect.Interface {
vi = vi.Elem()
}
reflect.Indirect(vi).Set(reflect.Indirect(reflect.ValueOf(s)))
} else {
memkeys = append(memkeys, m)
mixs = append(mixs, i)
dskeys = append(dskeys, key)
dsdst = append(dsdst, vi.Interface())
dixs = append(dixs, i)
}
}
g.cacheLock.RUnlock()
if len(memkeys) == 0 {
return nil
}
multiErr, any := make(appengine.MultiError, len(keys)), false
memvalues, err := memcache.GetMulti(appengine.Timeout(g.Context, MemcacheGetTimeout), memkeys)
if appengine.IsTimeoutError(err) {
g.timeoutError(err)
err = nil
} else if err != nil {
g.error(err) // timing out or another error from memcache isn't something to fail over, but do log it
// No memvalues found, prepare the datastore fetch list already prepared above
} else if len(memvalues) > 0 {
// since memcache fetch was successful, reset the datastore fetch list and repopulate it
dskeys = dskeys[:0]
dsdst = dsdst[:0]
dixs = dixs[:0]
// we only want to check the returned map if there weren't any errors
// unlike the datastore, memcache will return a smaller map with no error if some of the keys were missed
for i, m := range memkeys {
d := v.Index(mixs[i]).Interface()
if v.Index(mixs[i]).Kind() == reflect.Struct {
d = v.Index(mixs[i]).Addr().Interface()
}
if s, present := memvalues[m]; present {
err := deserializeStruct(d, s.Value)
if err == datastore.ErrNoSuchEntity {
any = true // this flag tells GetMulti to return multiErr later
multiErr[mixs[i]] = err
} else if err != nil {
g.error(err)
return err
} else {
g.putMemory(d)
}
} else {
dskeys = append(dskeys, keys[mixs[i]])
dsdst = append(dsdst, d)
dixs = append(dixs, mixs[i])
}
}
if len(dskeys) == 0 {
if any {
return realError(multiErr)
}
return nil
}
}
goroutines := (len(dskeys)-1)/getMultiLimit + 1
var wg sync.WaitGroup
wg.Add(goroutines)
for i := 0; i < goroutines; i++ {
go func(i int) {
defer wg.Done()
var toCache []interface{}
var exists []byte
lo := i * getMultiLimit
hi := (i + 1) * getMultiLimit
if hi > len(dskeys) {
hi = len(dskeys)
}
gmerr := datastore.GetMulti(g.Context, dskeys[lo:hi], dsdst[lo:hi])
if gmerr != nil {
any = true // this flag tells GetMulti to return multiErr later
merr, ok := gmerr.(appengine.MultiError)
if !ok {
g.error(gmerr)
for j := lo; j < hi; j++ {
multiErr[j] = gmerr
}
return
}
for i, idx := range dixs[lo:hi] {
if merr[i] == nil {
toCache = append(toCache, dsdst[lo+i])
exists = append(exists, 1)
} else {
if merr[i] == datastore.ErrNoSuchEntity {
toCache = append(toCache, dsdst[lo+i])
exists = append(exists, 0)
}
multiErr[idx] = merr[i]
}
}
} else {
toCache = append(toCache, dsdst[lo:hi]...)
exists = append(exists, bytes.Repeat([]byte{1}, hi-lo)...)
}
if len(toCache) > 0 {
if err := g.putMemcache(toCache, exists); err != nil {
g.error(err)
// since putMemcache() gives no guarantee it will actually store the data in memcache
// we log and swallow this error
}
}
}(i)
}
wg.Wait()
if any {
return realError(multiErr)
}
return nil
}