func processCollections(cacheConfig Config, params Params, mongoDb *mgo.Database, redisConn redis.Conn) error {
redisWriter := NewRedisWriter(redisConn)
for _, collection := range cacheConfig.Collections {
query, err := ParseTemplatedJSON(collection.Query, params)
if err != nil {
logger.Error("Failed to parse query", err)
return err
}
var iter MongoIter
var projection map[string]interface{}
if collection.Projection != "" {
var err error
projection, err = ParseTemplatedJSON(collection.Projection, params)
if err != nil {
logger.Error("Error applying projection template", err)
}
iter = mongoDb.C(collection.Collection).Find(query).Select(projection).Iter()
} else {
iter = mongoDb.C(collection.Collection).Find(query).Iter()
}
if err := SetRedisHashKeys(redisConn, &collection); err != nil {
logger.Error("Error setting up redis map keys", err)
return err
}
if err := ParseTemplates(&collection); err != nil {
logger.Error("Error parsing templates", err)
return err
}
logger.Info("Processing query for collection", logger.M{
"query": query,
"collection": collection.Collection,
"projection": projection,
})
if err := ProcessQuery(redisWriter, iter, collection.Maps); err != nil {
logger.Error("Error processing query", err)
return err
}
if err := redisWriter.Flush(); err != nil {
logger.Error("Error flushing redis conn", err)
return err
}
for _, rmap := range collection.Maps {
if err := UpdateRedisMapReference(redisConn, params, rmap); err != nil {
logger.Error("Failed to update map reference", err)
return err
}
}
}
logger.Info("Completed populating cache", logger.M{"cache": cacheConfig.Name})
return nil
}
// BuildCache builds a redis cache according to the passed in config.
func BuildCache(cacheConfig Config, params Params, redisURL string, mongoURL string) error {
logger.Info("Populating cache.", logger.M{"cache": cacheConfig.Name})
// set up mongo/redis connections
mongoDb, redisConn, err := SetupDbs(mongoURL, redisURL)
if err != nil {
logger.Error("Failed to connect to dbs", err)
return err
}
defer mongoDb.Session.Close()
defer redisConn.Close()
if err := processCollections(cacheConfig, params, mongoDb, redisConn); err != nil {
return err
}
return nil
}
func main() {
flag.Usage = PrintUsage
flag.Parse()
// grab connection from env or default if not in flags
mongoURL = FlagEnvOrDefault(mongoURL, "MONGO_URL", DefaultMongoURL)
redisURL = FlagEnvOrDefault(redisURL, "REDIS_URL", DefaultRedisURL)
conf, err := moredis.LoadConfig(configFilePath)
if err != nil {
logger.Error("Error loading config.", err)
os.Exit(1)
}
if err := moredis.BuildCache(conf, params, redisURL, mongoURL); err != nil {
fmt.Fprint(os.Stderr, err)
os.Exit(1)
}
}
// ProcessQuery iterates through all of the documents contained within iter, and maps
// keys to values in a redis hash according to your mapping config.
func ProcessQuery(writer RedisWriter, iter MongoIter, maps []MapConfig) error {
processed := 0
var result bson.M
var b bytes.Buffer
for iter.Next(&result) {
for _, rmap := range maps {
if err := rmap.KeyTemplate.Execute(&b, result); err != nil {
logger.Error("Could not execute key template", err)
return err
}
key := b.String()
b.Reset()
if key == "" || key == "<no value>" {
continue
}
if err := rmap.ValueTemplate.Execute(&b, result); err != nil {
logger.Error("Could not execute value template", err)
return err
}
val := b.String()
b.Reset()
if err := writer.Send("HSET", rmap.HashKey, key, val); err != nil {
logger.Error("Could not send HSET", err)
return err
}
}
processed++
}
if err := iter.Err(); err != nil {
logger.Error("Iteration error", err)
return err
}
if err := iter.Close(); err != nil {
logger.Error("Iter.Close() error", err)
return err
}
if err := writer.Flush(); err != nil {
logger.Error("Error flushing", err)
return err
}
logger.Info("Processed all documents for query", logger.M{"processed": processed})
return nil
}