func cacheSafeSet(c appengine.Context, item *memcache.Item) error { var buf bytes.Buffer err := gob.NewEncoder(&buf).Encode(item.Object) if err != nil { return err } swap := item.Value != nil item.Value = buf.Bytes() if swap { err = memcache.CompareAndSwap(c, item) switch err { case memcache.ErrCASConflict: // OK, cache item set by another request return nil case memcache.ErrNotStored: // Item expired. Try adding below. default: return err } } err = memcache.Add(c, item) if err == memcache.ErrNotStored { // OK, cache item set by another request err = nil } return err }
func cacheSet(c appengine.Context, item *memcache.Item) error { var buf bytes.Buffer err := gob.NewEncoder(&buf).Encode(item.Object) if err != nil { return err } item.Value = buf.Bytes() return memcache.Set(c, item) }
/* CAS will replace expected with replacement in memcache if expected is the current value. */ func CAS(c TransactionContext, key string, expected, replacement interface{}) (success bool, err error) { keyHash, err := Keyify(key) if err != nil { return } var item *memcache.Item if item, err = memcache.Get(c, keyHash); err != nil { if err == memcache.ErrCacheMiss { err = nil } else { err = errors.Errorf("Error doing Get %#v: %v", keyHash, err) } return } var encoded []byte if encoded, err = Codec.Marshal(expected); err != nil { return } if bytes.Compare(encoded, item.Value) != 0 { success = false return } if encoded, err = Codec.Marshal(replacement); err != nil { return } item.Value = encoded if err = memcache.CompareAndSwap(c, item); err != nil { if err == memcache.ErrCASConflict { err = nil } else { marshalled, _ := Codec.Marshal(replacement) err = errors.Errorf("Error doing CompareAndSwap %#v to %v bytes: %v", item.Key, len(marshalled), err) } return } success = true return }
// Returns prices for trades between 'begin' and 'end', in resolution 'res', recursing from tile size 'tile' down to the // most appropriate tile size, employing caching on the go. // Assumes that 'begin' is aligned with the current tile size. func queryPrices(c appengine.Context, article bitwrk.ArticleId, tile, res resolution, begin, end time.Time) ([]timeslot, error) { finest := res.finestTileResolution() if tile.finerThan(finest) { panic("Tile size too small") } // Does the interval fit into a single tile? If no, fan out and don't cache. if begin.Add(tile.interval).Before(end) { result := make([]timeslot, 0) count := 0 for begin.Before(end) { tileEnd := begin.Add(tile.interval) if tileEnd.After(end) { tileEnd = end } if r, err := queryPrices(c, article, tile, res, begin, tileEnd); err != nil { return nil, err } else { c.Infof("Fan-out to tile #%v returned %v slots", count, len(r)) result = append(result, r...) } begin = tileEnd count++ } c.Infof("Fan-out to %v tiles returned %v slots", count, len(result)) return result, nil } // First try to answer from cache key := fmt.Sprintf("prices-tile-%v/%v-%v-%v", tile.name, res.name, begin.Format(time.RFC3339), article) if item, err := memcache.Get(c, key); err == nil { result := make([]timeslot, 0) if err := json.Unmarshal(item.Value, &result); err != nil { // Shouldn't happen c.Errorf("Couldn't unmarshal memcache entry for: %v : %v", key, err) } else { return result, nil } } // Cache miss. Need to fetch data. // If tile size is the smallest for the desired resolution, ask the datastore. // Otherwise, recurse with next smaller tile size. var result []timeslot if tile == res.finestTileResolution() { count := 0 currentSlot := timeslot{ Begin: begin, End: begin.Add(res.interval), } result = make([]timeslot, 0) // Handler function that is called for every transaction in the current tile handler := func(key string, tx bitwrk.Transaction) { // Flush current interval to result list for currentSlot.End.Before(tx.Matched) { if currentSlot.Count > 0 { result = append(result, currentSlot) count += currentSlot.Count } currentSlot.advance() } currentSlot.addPrice(tx.Price) } // Query database if err := db.QueryTransactions(c, 10000, article, begin, end, handler); err != nil { return nil, err } // Flush last interval if currentSlot.Count > 0 { result = append(result, currentSlot) count += currentSlot.Count } c.Infof("QueryTransactions from %v to %v: %v slots/%v tx", begin, end, len(result), count) } else if r, err := queryPrices(c, article, tile.nextFiner(), res, begin, end); err != nil { return nil, err } else { result = r } // Before returning, update the cache. item := memcache.Item{Key: key} if data, err := json.Marshal(result); err != nil { // Shouldn't happen c.Errorf("Error marshalling result: %v", err) } else { item.Value = data } // Tiles very close to now expire after 10 seconds if begin.Add(tile.interval).After(time.Now().Add(-2 * time.Minute)) { item.Expiration = 10 * time.Second } if err := memcache.Add(c, &item); err != nil { c.Errorf("Error caching item for %v: %v", key, err) } return result, nil }