Пример #1
0
func cacheSafeSet(c appengine.Context, item *memcache.Item) error {
	var buf bytes.Buffer
	err := gob.NewEncoder(&buf).Encode(item.Object)
	if err != nil {
		return err
	}

	swap := item.Value != nil
	item.Value = buf.Bytes()

	if swap {
		err = memcache.CompareAndSwap(c, item)
		switch err {
		case memcache.ErrCASConflict:
			// OK, cache item set by another request
			return nil
		case memcache.ErrNotStored:
			// Item expired. Try adding below.
		default:
			return err
		}
	}

	err = memcache.Add(c, item)
	if err == memcache.ErrNotStored {
		// OK, cache item set by another request
		err = nil
	}
	return err
}
Пример #2
0
func cacheSet(c appengine.Context, item *memcache.Item) error {
	var buf bytes.Buffer
	err := gob.NewEncoder(&buf).Encode(item.Object)
	if err != nil {
		return err
	}
	item.Value = buf.Bytes()
	return memcache.Set(c, item)
}
Пример #3
0
/*
CAS will replace expected with replacement in memcache if expected is the current value.
*/
func CAS(c TransactionContext, key string, expected, replacement interface{}) (success bool, err error) {
	keyHash, err := Keyify(key)
	if err != nil {
		return
	}
	var item *memcache.Item
	if item, err = memcache.Get(c, keyHash); err != nil {
		if err == memcache.ErrCacheMiss {
			err = nil
		} else {
			err = errors.Errorf("Error doing Get %#v: %v", keyHash, err)
		}
		return
	}
	var encoded []byte
	if encoded, err = Codec.Marshal(expected); err != nil {
		return
	}
	if bytes.Compare(encoded, item.Value) != 0 {
		success = false
		return
	}
	if encoded, err = Codec.Marshal(replacement); err != nil {
		return
	}
	item.Value = encoded
	if err = memcache.CompareAndSwap(c, item); err != nil {
		if err == memcache.ErrCASConflict {
			err = nil
		} else {
			marshalled, _ := Codec.Marshal(replacement)
			err = errors.Errorf("Error doing CompareAndSwap %#v to %v bytes: %v", item.Key, len(marshalled), err)
		}
		return
	}
	success = true
	return
}
Пример #4
0
// Returns prices for trades between 'begin' and 'end', in resolution 'res', recursing from tile size 'tile' down to the
// most appropriate tile size, employing caching on the go.
// Assumes that 'begin' is aligned with the current tile size.
func queryPrices(c appengine.Context, article bitwrk.ArticleId, tile, res resolution, begin, end time.Time) ([]timeslot, error) {
	finest := res.finestTileResolution()
	if tile.finerThan(finest) {
		panic("Tile size too small")
	}

	// Does the interval fit into a single tile? If no, fan out and don't cache.
	if begin.Add(tile.interval).Before(end) {
		result := make([]timeslot, 0)
		count := 0
		for begin.Before(end) {
			tileEnd := begin.Add(tile.interval)
			if tileEnd.After(end) {
				tileEnd = end
			}

			if r, err := queryPrices(c, article, tile, res, begin, tileEnd); err != nil {
				return nil, err
			} else {
				c.Infof("Fan-out to tile #%v returned %v slots", count, len(r))
				result = append(result, r...)
			}
			begin = tileEnd
			count++
		}
		c.Infof("Fan-out to %v tiles returned %v slots", count, len(result))
		return result, nil
	}

	// First try to answer from cache
	key := fmt.Sprintf("prices-tile-%v/%v-%v-%v", tile.name, res.name, begin.Format(time.RFC3339), article)
	if item, err := memcache.Get(c, key); err == nil {
		result := make([]timeslot, 0)
		if err := json.Unmarshal(item.Value, &result); err != nil {
			// Shouldn't happen
			c.Errorf("Couldn't unmarshal memcache entry for: %v : %v", key, err)
		} else {
			return result, nil
		}
	}

	// Cache miss. Need to fetch data.
	// If tile size is the smallest for the desired resolution, ask the datastore.
	// Otherwise, recurse with next smaller tile size.
	var result []timeslot
	if tile == res.finestTileResolution() {

		count := 0
		currentSlot := timeslot{
			Begin: begin,
			End:   begin.Add(res.interval),
		}
		result = make([]timeslot, 0)

		// Handler function that is called for every transaction in the current tile
		handler := func(key string, tx bitwrk.Transaction) {
			// Flush current interval to result list
			for currentSlot.End.Before(tx.Matched) {
				if currentSlot.Count > 0 {
					result = append(result, currentSlot)
					count += currentSlot.Count
				}
				currentSlot.advance()
			}
			currentSlot.addPrice(tx.Price)
		}

		// Query database
		if err := db.QueryTransactions(c, 10000, article, begin, end, handler); err != nil {
			return nil, err
		}

		// Flush last interval
		if currentSlot.Count > 0 {
			result = append(result, currentSlot)
			count += currentSlot.Count
		}

		c.Infof("QueryTransactions from %v to %v: %v slots/%v tx",
			begin, end, len(result), count)
	} else if r, err := queryPrices(c, article, tile.nextFiner(), res, begin, end); err != nil {
		return nil, err
	} else {
		result = r
	}

	// Before returning, update the cache.
	item := memcache.Item{Key: key}
	if data, err := json.Marshal(result); err != nil {
		// Shouldn't happen
		c.Errorf("Error marshalling result: %v", err)
	} else {
		item.Value = data
	}

	// Tiles very close to now expire after 10 seconds
	if begin.Add(tile.interval).After(time.Now().Add(-2 * time.Minute)) {
		item.Expiration = 10 * time.Second
	}

	if err := memcache.Add(c, &item); err != nil {
		c.Errorf("Error caching item for %v: %v", key, err)
	}

	return result, nil
}