// SetupDbs takes connection parameters for redis and mongo and returns active sessions.
// The caller is responsible for closing the returned connections.
func SetupDbs(mongoURL, redisURL string) (*mgo.Database, redis.Conn, error) {
mongoSession, err := mgo.Dial(mongoURL)
if err != nil {
return nil, nil, err
}
// use 'monotonic' consistency mode. Since we only do reads, this doesn't have an actual effect
// other than letting us read from secondaries if the connection string has the ?connect=direct param.
mongoSession.SetMode(mgo.Monotonic, false)
// empty db string uses the db from the connection url
mongoDB := mongoSession.DB("")
logger.Info("Connected to mongo", logger.M{"mongo_url": mongoURL})
redisURL, err = resolveRedis(redisURL)
if err != nil {
return nil, nil, err
}
redisConn, err := redis.DialTimeout("tcp", redisURL, 15*time.Second, 10*time.Second, 10*time.Second)
if err != nil {
return nil, nil, err
}
logger.Info("Connected to redis", logger.M{"redis_url": redisURL})
return mongoDB, redisConn, nil
}
func processCollections(cacheConfig Config, params Params, mongoDb *mgo.Database, redisConn redis.Conn) error {
redisWriter := NewRedisWriter(redisConn)
for _, collection := range cacheConfig.Collections {
query, err := ParseTemplatedJSON(collection.Query, params)
if err != nil {
logger.Error("Failed to parse query", err)
return err
}
var iter MongoIter
var projection map[string]interface{}
if collection.Projection != "" {
var err error
projection, err = ParseTemplatedJSON(collection.Projection, params)
if err != nil {
logger.Error("Error applying projection template", err)
}
iter = mongoDb.C(collection.Collection).Find(query).Select(projection).Iter()
} else {
iter = mongoDb.C(collection.Collection).Find(query).Iter()
}
if err := SetRedisHashKeys(redisConn, &collection); err != nil {
logger.Error("Error setting up redis map keys", err)
return err
}
if err := ParseTemplates(&collection); err != nil {
logger.Error("Error parsing templates", err)
return err
}
logger.Info("Processing query for collection", logger.M{
"query": query,
"collection": collection.Collection,
"projection": projection,
})
if err := ProcessQuery(redisWriter, iter, collection.Maps); err != nil {
logger.Error("Error processing query", err)
return err
}
if err := redisWriter.Flush(); err != nil {
logger.Error("Error flushing redis conn", err)
return err
}
for _, rmap := range collection.Maps {
if err := UpdateRedisMapReference(redisConn, params, rmap); err != nil {
logger.Error("Failed to update map reference", err)
return err
}
}
}
logger.Info("Completed populating cache", logger.M{"cache": cacheConfig.Name})
return nil
}
// UpdateRedisMapReference updates the map specified in redis to point to the newly populated hashes,
// then deletes the previously referenced hash. The hash reference is updated atomically.
func UpdateRedisMapReference(conn redis.Conn, params Params, mapConfig MapConfig) error {
mapName, err := ApplyTemplate(mapConfig.Name, params.Bson())
if err != nil {
return err
}
oldMap, err := redis.String(conn.Do("GETSET", mapName, mapConfig.HashKey))
logger.Info("Updating map reference", logger.M{"map": mapName, "oldref": oldMap, "newref": mapConfig.HashKey})
if err == redis.ErrNil {
// no old map, just return
return nil
}
if err != nil {
return err
}
logger.Info("Deleting old referenced map", logger.M{"map": oldMap})
if _, err := conn.Do("DEL", oldMap); err != nil {
return err
}
return nil
}
// BuildCache builds a redis cache according to the passed in config.
func BuildCache(cacheConfig Config, params Params, redisURL string, mongoURL string) error {
logger.Info("Populating cache.", logger.M{"cache": cacheConfig.Name})
// set up mongo/redis connections
mongoDb, redisConn, err := SetupDbs(mongoURL, redisURL)
if err != nil {
logger.Error("Failed to connect to dbs", err)
return err
}
defer mongoDb.Session.Close()
defer redisConn.Close()
if err := processCollections(cacheConfig, params, mongoDb, redisConn); err != nil {
return err
}
return nil
}
// ProcessQuery iterates through all of the documents contained within iter, and maps
// keys to values in a redis hash according to your mapping config.
func ProcessQuery(writer RedisWriter, iter MongoIter, maps []MapConfig) error {
processed := 0
var result bson.M
var b bytes.Buffer
for iter.Next(&result) {
for _, rmap := range maps {
if err := rmap.KeyTemplate.Execute(&b, result); err != nil {
logger.Error("Could not execute key template", err)
return err
}
key := b.String()
b.Reset()
if key == "" || key == "<no value>" {
continue
}
if err := rmap.ValueTemplate.Execute(&b, result); err != nil {
logger.Error("Could not execute value template", err)
return err
}
val := b.String()
b.Reset()
if err := writer.Send("HSET", rmap.HashKey, key, val); err != nil {
logger.Error("Could not send HSET", err)
return err
}
}
processed++
}
if err := iter.Err(); err != nil {
logger.Error("Iteration error", err)
return err
}
if err := iter.Close(); err != nil {
logger.Error("Iter.Close() error", err)
return err
}
if err := writer.Flush(); err != nil {
logger.Error("Error flushing", err)
return err
}
logger.Info("Processed all documents for query", logger.M{"processed": processed})
return nil
}