Exemple #1
0
func processCollections(cacheConfig Config, params Params, mongoDb *mgo.Database, redisConn redis.Conn) error {
	redisWriter := NewRedisWriter(redisConn)
	for _, collection := range cacheConfig.Collections {
		query, err := ParseTemplatedJSON(collection.Query, params)
		if err != nil {
			logger.Error("Failed to parse query", err)
			return err
		}

		var iter MongoIter
		var projection map[string]interface{}
		if collection.Projection != "" {
			var err error
			projection, err = ParseTemplatedJSON(collection.Projection, params)
			if err != nil {
				logger.Error("Error applying projection template", err)
			}
			iter = mongoDb.C(collection.Collection).Find(query).Select(projection).Iter()
		} else {
			iter = mongoDb.C(collection.Collection).Find(query).Iter()
		}

		if err := SetRedisHashKeys(redisConn, &collection); err != nil {
			logger.Error("Error setting up redis map keys", err)
			return err
		}

		if err := ParseTemplates(&collection); err != nil {
			logger.Error("Error parsing templates", err)
			return err
		}

		logger.Info("Processing query for collection", logger.M{
			"query":      query,
			"collection": collection.Collection,
			"projection": projection,
		})
		if err := ProcessQuery(redisWriter, iter, collection.Maps); err != nil {
			logger.Error("Error processing query", err)
			return err
		}
		if err := redisWriter.Flush(); err != nil {
			logger.Error("Error flushing redis conn", err)
			return err
		}

		for _, rmap := range collection.Maps {
			if err := UpdateRedisMapReference(redisConn, params, rmap); err != nil {
				logger.Error("Failed to update map reference", err)
				return err
			}
		}
	}
	logger.Info("Completed populating cache", logger.M{"cache": cacheConfig.Name})
	return nil
}
Exemple #2
0
// BuildCache builds a redis cache according to the passed in config.
func BuildCache(cacheConfig Config, params Params, redisURL string, mongoURL string) error {
	logger.Info("Populating cache.", logger.M{"cache": cacheConfig.Name})

	// set up mongo/redis connections
	mongoDb, redisConn, err := SetupDbs(mongoURL, redisURL)
	if err != nil {
		logger.Error("Failed to connect to dbs", err)
		return err
	}
	defer mongoDb.Session.Close()
	defer redisConn.Close()

	if err := processCollections(cacheConfig, params, mongoDb, redisConn); err != nil {
		return err
	}
	return nil
}
Exemple #3
0
func main() {
	flag.Usage = PrintUsage
	flag.Parse()

	// grab connection from env or default if not in flags
	mongoURL = FlagEnvOrDefault(mongoURL, "MONGO_URL", DefaultMongoURL)
	redisURL = FlagEnvOrDefault(redisURL, "REDIS_URL", DefaultRedisURL)

	conf, err := moredis.LoadConfig(configFilePath)
	if err != nil {
		logger.Error("Error loading config.", err)
		os.Exit(1)
	}

	if err := moredis.BuildCache(conf, params, redisURL, mongoURL); err != nil {
		fmt.Fprint(os.Stderr, err)
		os.Exit(1)
	}
}
Exemple #4
0
// ProcessQuery iterates through all of the documents contained within iter, and maps
// keys to values in a redis hash according to your mapping config.
func ProcessQuery(writer RedisWriter, iter MongoIter, maps []MapConfig) error {
	processed := 0
	var result bson.M
	var b bytes.Buffer
	for iter.Next(&result) {
		for _, rmap := range maps {
			if err := rmap.KeyTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute key template", err)
				return err
			}
			key := b.String()
			b.Reset()

			if key == "" || key == "<no value>" {
				continue
			}

			if err := rmap.ValueTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute value template", err)
				return err
			}
			val := b.String()
			b.Reset()

			if err := writer.Send("HSET", rmap.HashKey, key, val); err != nil {
				logger.Error("Could not send HSET", err)
				return err
			}
		}
		processed++
	}
	if err := iter.Err(); err != nil {
		logger.Error("Iteration error", err)
		return err
	}
	if err := iter.Close(); err != nil {
		logger.Error("Iter.Close() error", err)
		return err
	}
	if err := writer.Flush(); err != nil {
		logger.Error("Error flushing", err)
		return err
	}
	logger.Info("Processed all documents for query", logger.M{"processed": processed})
	return nil
}