func cacheSafeSet(c appengine.Context, item *memcache.Item) error {
var buf bytes.Buffer
err := gob.NewEncoder(&buf).Encode(item.Object)
if err != nil {
return err
}
swap := item.Value != nil
item.Value = buf.Bytes()
if swap {
err = memcache.CompareAndSwap(c, item)
switch err {
case memcache.ErrCASConflict:
// OK, cache item set by another request
return nil
case memcache.ErrNotStored:
// Item expired. Try adding below.
default:
return err
}
}
err = memcache.Add(c, item)
if err == memcache.ErrNotStored {
// OK, cache item set by another request
err = nil
}
return err
}
func cacheSet(c appengine.Context, item *memcache.Item) error {
var buf bytes.Buffer
err := gob.NewEncoder(&buf).Encode(item.Object)
if err != nil {
return err
}
item.Value = buf.Bytes()
return memcache.Set(c, item)
}
/*
CAS will replace expected with replacement in memcache if expected is the current value.
*/
func CAS(c TransactionContext, key string, expected, replacement interface{}) (success bool, err error) {
keyHash, err := Keyify(key)
if err != nil {
return
}
var item *memcache.Item
if item, err = memcache.Get(c, keyHash); err != nil {
if err == memcache.ErrCacheMiss {
err = nil
} else {
err = errors.Errorf("Error doing Get %#v: %v", keyHash, err)
}
return
}
var encoded []byte
if encoded, err = Codec.Marshal(expected); err != nil {
return
}
if bytes.Compare(encoded, item.Value) != 0 {
success = false
return
}
if encoded, err = Codec.Marshal(replacement); err != nil {
return
}
item.Value = encoded
if err = memcache.CompareAndSwap(c, item); err != nil {
if err == memcache.ErrCASConflict {
err = nil
} else {
marshalled, _ := Codec.Marshal(replacement)
err = errors.Errorf("Error doing CompareAndSwap %#v to %v bytes: %v", item.Key, len(marshalled), err)
}
return
}
success = true
return
}
// Returns prices for trades between 'begin' and 'end', in resolution 'res', recursing from tile size 'tile' down to the
// most appropriate tile size, employing caching on the go.
// Assumes that 'begin' is aligned with the current tile size.
func queryPrices(c appengine.Context, article bitwrk.ArticleId, tile, res resolution, begin, end time.Time) ([]timeslot, error) {
finest := res.finestTileResolution()
if tile.finerThan(finest) {
panic("Tile size too small")
}
// Does the interval fit into a single tile? If no, fan out and don't cache.
if begin.Add(tile.interval).Before(end) {
result := make([]timeslot, 0)
count := 0
for begin.Before(end) {
tileEnd := begin.Add(tile.interval)
if tileEnd.After(end) {
tileEnd = end
}
if r, err := queryPrices(c, article, tile, res, begin, tileEnd); err != nil {
return nil, err
} else {
c.Infof("Fan-out to tile #%v returned %v slots", count, len(r))
result = append(result, r...)
}
begin = tileEnd
count++
}
c.Infof("Fan-out to %v tiles returned %v slots", count, len(result))
return result, nil
}
// First try to answer from cache
key := fmt.Sprintf("prices-tile-%v/%v-%v-%v", tile.name, res.name, begin.Format(time.RFC3339), article)
if item, err := memcache.Get(c, key); err == nil {
result := make([]timeslot, 0)
if err := json.Unmarshal(item.Value, &result); err != nil {
// Shouldn't happen
c.Errorf("Couldn't unmarshal memcache entry for: %v : %v", key, err)
} else {
return result, nil
}
}
// Cache miss. Need to fetch data.
// If tile size is the smallest for the desired resolution, ask the datastore.
// Otherwise, recurse with next smaller tile size.
var result []timeslot
if tile == res.finestTileResolution() {
count := 0
currentSlot := timeslot{
Begin: begin,
End: begin.Add(res.interval),
}
result = make([]timeslot, 0)
// Handler function that is called for every transaction in the current tile
handler := func(key string, tx bitwrk.Transaction) {
// Flush current interval to result list
for currentSlot.End.Before(tx.Matched) {
if currentSlot.Count > 0 {
result = append(result, currentSlot)
count += currentSlot.Count
}
currentSlot.advance()
}
currentSlot.addPrice(tx.Price)
}
// Query database
if err := db.QueryTransactions(c, 10000, article, begin, end, handler); err != nil {
return nil, err
}
// Flush last interval
if currentSlot.Count > 0 {
result = append(result, currentSlot)
count += currentSlot.Count
}
c.Infof("QueryTransactions from %v to %v: %v slots/%v tx",
begin, end, len(result), count)
} else if r, err := queryPrices(c, article, tile.nextFiner(), res, begin, end); err != nil {
return nil, err
} else {
result = r
}
// Before returning, update the cache.
item := memcache.Item{Key: key}
if data, err := json.Marshal(result); err != nil {
// Shouldn't happen
c.Errorf("Error marshalling result: %v", err)
} else {
item.Value = data
}
// Tiles very close to now expire after 10 seconds
if begin.Add(tile.interval).After(time.Now().Add(-2 * time.Minute)) {
item.Expiration = 10 * time.Second
}
if err := memcache.Add(c, &item); err != nil {
c.Errorf("Error caching item for %v: %v", key, err)
}
return result, nil
}