Example #1
0
// SetupDbs takes connection parameters for redis and mongo and returns active sessions.
// The caller is responsible for closing the returned connections.
func SetupDbs(mongoURL, redisURL string) (*mgo.Database, redis.Conn, error) {
	mongoSession, err := mgo.Dial(mongoURL)
	if err != nil {
		return nil, nil, err
	}
	// use 'monotonic' consistency mode.  Since we only do reads, this doesn't have an actual effect
	// other than letting us read from secondaries if the connection string has the ?connect=direct param.
	mongoSession.SetMode(mgo.Monotonic, false)

	// empty db string uses the db from the connection url
	mongoDB := mongoSession.DB("")
	logger.Info("Connected to mongo", logger.M{"mongo_url": mongoURL})

	redisURL, err = resolveRedis(redisURL)
	if err != nil {
		return nil, nil, err
	}

	redisConn, err := redis.DialTimeout("tcp", redisURL, 15*time.Second, 10*time.Second, 10*time.Second)
	if err != nil {
		return nil, nil, err
	}
	logger.Info("Connected to redis", logger.M{"redis_url": redisURL})
	return mongoDB, redisConn, nil
}
Example #2
0
func processCollections(cacheConfig Config, params Params, mongoDb *mgo.Database, redisConn redis.Conn) error {
	redisWriter := NewRedisWriter(redisConn)
	for _, collection := range cacheConfig.Collections {
		query, err := ParseTemplatedJSON(collection.Query, params)
		if err != nil {
			logger.Error("Failed to parse query", err)
			return err
		}

		var iter MongoIter
		var projection map[string]interface{}
		if collection.Projection != "" {
			var err error
			projection, err = ParseTemplatedJSON(collection.Projection, params)
			if err != nil {
				logger.Error("Error applying projection template", err)
			}
			iter = mongoDb.C(collection.Collection).Find(query).Select(projection).Iter()
		} else {
			iter = mongoDb.C(collection.Collection).Find(query).Iter()
		}

		if err := SetRedisHashKeys(redisConn, &collection); err != nil {
			logger.Error("Error setting up redis map keys", err)
			return err
		}

		if err := ParseTemplates(&collection); err != nil {
			logger.Error("Error parsing templates", err)
			return err
		}

		logger.Info("Processing query for collection", logger.M{
			"query":      query,
			"collection": collection.Collection,
			"projection": projection,
		})
		if err := ProcessQuery(redisWriter, iter, collection.Maps); err != nil {
			logger.Error("Error processing query", err)
			return err
		}
		if err := redisWriter.Flush(); err != nil {
			logger.Error("Error flushing redis conn", err)
			return err
		}

		for _, rmap := range collection.Maps {
			if err := UpdateRedisMapReference(redisConn, params, rmap); err != nil {
				logger.Error("Failed to update map reference", err)
				return err
			}
		}
	}
	logger.Info("Completed populating cache", logger.M{"cache": cacheConfig.Name})
	return nil
}
Example #3
0
// UpdateRedisMapReference updates the map specified in redis to point to the newly populated hashes,
// then deletes the previously referenced hash.  The hash reference is updated atomically.
func UpdateRedisMapReference(conn redis.Conn, params Params, mapConfig MapConfig) error {
	mapName, err := ApplyTemplate(mapConfig.Name, params.Bson())
	if err != nil {
		return err
	}
	oldMap, err := redis.String(conn.Do("GETSET", mapName, mapConfig.HashKey))
	logger.Info("Updating map reference", logger.M{"map": mapName, "oldref": oldMap, "newref": mapConfig.HashKey})
	if err == redis.ErrNil {
		// no old map, just return
		return nil
	}
	if err != nil {
		return err
	}

	logger.Info("Deleting old referenced map", logger.M{"map": oldMap})
	if _, err := conn.Do("DEL", oldMap); err != nil {
		return err
	}
	return nil
}
Example #4
0
// BuildCache builds a redis cache according to the passed in config.
func BuildCache(cacheConfig Config, params Params, redisURL string, mongoURL string) error {
	logger.Info("Populating cache.", logger.M{"cache": cacheConfig.Name})

	// set up mongo/redis connections
	mongoDb, redisConn, err := SetupDbs(mongoURL, redisURL)
	if err != nil {
		logger.Error("Failed to connect to dbs", err)
		return err
	}
	defer mongoDb.Session.Close()
	defer redisConn.Close()

	if err := processCollections(cacheConfig, params, mongoDb, redisConn); err != nil {
		return err
	}
	return nil
}
Example #5
0
// ProcessQuery iterates through all of the documents contained within iter, and maps
// keys to values in a redis hash according to your mapping config.
func ProcessQuery(writer RedisWriter, iter MongoIter, maps []MapConfig) error {
	processed := 0
	var result bson.M
	var b bytes.Buffer
	for iter.Next(&result) {
		for _, rmap := range maps {
			if err := rmap.KeyTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute key template", err)
				return err
			}
			key := b.String()
			b.Reset()

			if key == "" || key == "<no value>" {
				continue
			}

			if err := rmap.ValueTemplate.Execute(&b, result); err != nil {
				logger.Error("Could not execute value template", err)
				return err
			}
			val := b.String()
			b.Reset()

			if err := writer.Send("HSET", rmap.HashKey, key, val); err != nil {
				logger.Error("Could not send HSET", err)
				return err
			}
		}
		processed++
	}
	if err := iter.Err(); err != nil {
		logger.Error("Iteration error", err)
		return err
	}
	if err := iter.Close(); err != nil {
		logger.Error("Iter.Close() error", err)
		return err
	}
	if err := writer.Flush(); err != nil {
		logger.Error("Error flushing", err)
		return err
	}
	logger.Info("Processed all documents for query", logger.M{"processed": processed})
	return nil
}