func topDestinations(rw http.ResponseWriter, req *http.Request) {
client, err := redis.Dial("tcp", "localhost:6379")
if err != nil {
fmt.Println("Problem communicating to Redis...")
log.Fatal(err)
} else {
defer client.Close()
}
response := client.Cmd("ZREVRANGE", "popularity", "0", "-1", "WITHSCORES")
// here we create a byte buffer to construct the html response
buffer := bytes.NewBufferString("<TABLE>")
l, _ := response.List()
for _, elemStr := range l {
buffer.WriteString("<TR><TD>")
buffer.WriteString(elemStr)
buffer.WriteString("<TR><TD>")
}
buffer.WriteString("</TABLE>")
p := loadTopPage(buffer.Bytes())
renderTemplate(rw, "top", p)
}
func newRedisDriver(c *config) (*redisDriver, error) {
r, err := redis.Dial("tcp", c.RedisAddress)
if err != nil {
return nil, err
}
return &redisDriver{c: r}, nil
}
// Test a basic manual failover
func TestFailover(t *T) {
s := getSentinel(t)
sc, err := redis.Dial("tcp", "127.0.0.1:28000")
require.Nil(t, err)
k := randStr()
c, err := s.GetMaster("test")
require.Nil(t, err)
require.Nil(t, c.Cmd("SET", k, "foo").Err)
s.PutMaster("test", c)
require.Nil(t, sc.Cmd("SENTINEL", "FAILOVER", "test").Err)
c, err = s.GetMaster("test")
require.Nil(t, err)
foo, err := c.Cmd("GET", k).Str()
require.Nil(t, err)
assert.Equal(t, "foo", foo)
require.Nil(t, c.Cmd("SET", k, "bar").Err)
s.PutMaster("test", c)
time.Sleep(10 * time.Second)
require.Nil(t, sc.Cmd("SENTINEL", "FAILOVER", "test").Err)
c, err = s.GetMaster("test")
require.Nil(t, err)
bar, err := c.Cmd("GET", k).Str()
require.Nil(t, err)
assert.Equal(t, "bar", bar)
s.PutMaster("test", c)
}
func TestPublishEvent(t *testing.T) {
db := redisdb.New(dburl)
if err := db.Connect(); err != nil {
t.Fatalf("Failed to connect to redis: %s", err.Error())
}
c, err := redis.Dial("tcp", dburl)
if err != nil {
t.Fatalf("Failed to connect to redis2: %s", err.Error())
}
ps := pubsub.NewSubClient(c)
ps.Subscribe("thechan")
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
resp := ps.Receive()
if resp.Message != "moar" {
t.Errorf("Expected moar but got %s", resp.Message)
}
wg.Done()
}()
go func() {
buf := bytes.NewBufferString("moar")
if err := db.PublishEvent("thechan", buf); err != nil {
t.Fatalf("Should have have errored publishing: %s", err.Error())
}
}()
wg.Wait()
}
func TestLuaEval(t *T) {
c1, err := redis.Dial("tcp", "127.0.0.1:6379")
require.Nil(t, err)
c2, err := cluster.New("127.0.0.1:7000")
require.Nil(t, err)
cs := []Cmder{c1, c2}
for _, c := range cs {
script, key, val := randTestScript()
s, err := LuaEval(c, script, 1, key, val).Str()
require.Nil(t, err)
assert.Equal(t, "OK", s)
// The second time the command will be hashed
script, key, val = randTestScript()
s, err = LuaEval(c, script, 1, key, val).Str()
require.Nil(t, err)
assert.Equal(t, "OK", s)
s, err = c.Cmd("GET", key).Str()
require.Nil(t, err)
assert.Equal(t, val, s)
}
}
func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns int) (*RedisStorage, error) {
df := func(network, addr string) (*redis.Client, error) {
client, err := redis.Dial(network, addr)
if err != nil {
return nil, err
}
if len(pass) != 0 {
if err = client.Cmd("AUTH", pass).Err; err != nil {
client.Close()
return nil, err
}
}
if db != 0 {
if err = client.Cmd("SELECT", db).Err; err != nil {
client.Close()
return nil, err
}
}
return client, nil
}
p, err := pool.NewCustom("tcp", address, maxConns, df)
if err != nil {
return nil, err
}
var mrshler Marshaler
if mrshlerStr == utils.MSGPACK {
mrshler = NewCodecMsgpackMarshaler()
} else if mrshlerStr == utils.JSON {
mrshler = new(JSONMarshaler)
} else {
return nil, fmt.Errorf("Unsupported marshaler: %v", mrshlerStr)
}
return &RedisStorage{db: p, ms: mrshler}, nil
}
func NewRedisProvider(network, addr, auth string, idleConns int) (*RedisProvider, error) {
df := func(n, a string) (*redis.Client, error) {
c, err := redis.Dial(n, a)
if err != nil {
return nil, err
}
if err = c.Cmd("AUTH", auth).Err; err != nil {
c.Close()
return nil, err
}
return c, nil
}
p, err := rPool.NewCustom(network, addr, idleConns, df)
if err != nil {
return nil, err
}
rp := &RedisProvider{
Pool: p,
}
return rp, nil
}
func setupRedis() {
c, err := redis.Dial("tcp", "localhost:6379")
client = c
if err != nil {
//handle error
}
}
// NewConnection establishes a new connection to a Redis instance
func NewConnection(host, port string) *redis.Client {
client, err := redis.Dial("tcp", host+":"+port)
if err != nil {
log.Println("Error while connecting:" + err.Error())
return nil
}
return client
}
// Connect establishes a redis client connection.
func (db *DB) Connect() error {
c, err := redis.Dial("tcp", db.addr)
if err != nil {
return err
}
db.client = c
return nil
}
func initConnection(connDetails connectionDetails) *redis.Client {
// Connection Establishment
client, err := redis.Dial("tcp", connDetails.host+":"+connDetails.port)
if err != nil {
fmt.Println("Error while connecting:" + err.Error())
return nil
}
return client
}
// Redis is needed to run this test
func pingRedis() *redis.Client {
redisClient, err := redis.Dial("tcp", getRedisHost())
if err != nil {
log.Warnln("Cannot find Redis (standalone) at 'localhost:6379'.")
log.Infoln("Trying to connect to Redis sentinel ...")
redisClient2, err2 := redis.Dial("tcp", getRedisSentinelHost())
if err2 != nil {
log.Infoln("Please start a local Redis or Redis sentinel.")
log.Infoln("Please specify TEST_REDIS_HOST or TEST_REDIS_SENTINEL_HOST (and optionally TEST_REDIS_MASTER)")
panic("Cannot find Redis server.")
}
log.Infof("Successfully connected to Redis Sentinel '%s'", redisClient2.Addr)
return redisClient2
}
log.Infof("Successfully connected to Redis '%s'", redisClient.Addr)
return redisClient
}
func init() {
flag.IntVar(&redisPort, "port", 0, "Port of the redis server.")
flag.StringVar(&zset, "zset", "", "Name of the ZSET to store into redis.")
flag.Parse()
client, err = redis.Dial("tcp", fmt.Sprintf("localhost:%d", redisPort))
if err != nil {
log.Fatal("Couldn't open a connection to the Redis Server, please check that redis-server is running.")
}
}
func (r *rutil) Client() *redis.Client {
if r.cli == nil {
cli, err := redis.Dial("tcp", fmt.Sprintf("%s:%d", r.Host, r.Port))
checkErr(err)
if r.Auth != "" {
res := cli.Cmd("AUTH", r.Auth)
checkErr(res.Err)
}
r.cli = cli
}
return r.cli
}
func main() {
client, err := redis.Dial("tcp", "localhost:6379")
if err != nil {
log.Fatalf("Couldn't connect to Redis server")
}
defer client.Close()
val, err := client.Cmd("ZCARD", "myxyht").Int64()
if err != nil {
log.Fatalf("Error retrieving value: %s", err)
}
fmt.Printf("value = %d", val)
}
func main() {
args := os.Args[1:]
if len(args) == 0 {
log.Fatal("Towns db file path is not specified")
}
if len(args) == 1 {
log.Fatal("Cashpoints db file path is not specified")
}
if len(args) == 2 {
log.Fatal("Banks db file path is not specified")
}
if len(args) == 3 {
log.Fatal("Redis database url is not specified")
}
townsDbPath := args[0]
cashpointsDbPath := args[1]
banksDbPath := args[2]
redisUrl := args[3]
townsDb, err := sql.Open("sqlite3", townsDbPath)
if err != nil {
log.Fatal(err)
}
defer townsDb.Close()
cashpointsDb, err := sql.Open("sqlite3", cashpointsDbPath)
if err != nil {
log.Fatal(err)
}
defer cashpointsDb.Close()
banksDb, err := sql.Open("sqlite3", banksDbPath)
if err != nil {
log.Fatal(err)
}
defer banksDb.Close()
redisCli, err := redis.Dial("tcp", redisUrl)
if err != nil {
log.Fatal(err)
}
defer redisCli.Close()
migrate(townsDb, cashpointsDb, banksDb, redisCli)
}
// NewClientCustom is the same as NewClient, except it takes in a DialFunc which
// will be used to create all new connections to the master instances. This can
// be used to implement authentication, custom timeouts, etc...
func NewClientCustom(
network, address string, poolSize int, df DialFunc, names ...string,
) (
*Client, error,
) {
// We use this to fetch initial details about masters before we upgrade it
// to a pubsub client
client, err := redis.Dial(network, address)
if err != nil {
return nil, &ClientError{err: err}
}
masterPools := map[string]*pool.Pool{}
for _, name := range names {
r := client.Cmd("SENTINEL", "MASTER", name)
l, err := r.List()
if err != nil {
return nil, &ClientError{err: err, SentinelErr: true}
}
addr := l[3] + ":" + l[5]
pool, err := pool.NewCustom("tcp", addr, poolSize, (pool.DialFunc)(df))
if err != nil {
return nil, &ClientError{err: err}
}
masterPools[name] = pool
}
subClient := pubsub.NewSubClient(client)
r := subClient.Subscribe("+switch-master")
if r.Err != nil {
return nil, &ClientError{err: r.Err, SentinelErr: true}
}
c := &Client{
poolSize: poolSize,
masterPools: masterPools,
subClient: subClient,
dialFunc: (pool.DialFunc)(df),
getCh: make(chan *getReq),
putCh: make(chan *putReq),
closeCh: make(chan struct{}),
alwaysErrCh: make(chan *ClientError),
switchMasterCh: make(chan *switchMaster),
}
go c.subSpin()
go c.spin()
return c, nil
}
func main() {
flag.Parse()
var err error
redis_client, err = redis.Dial(*redis_prot, *redis_addr)
if err != nil {
fatal("Unable to connect to redis", err)
}
go cleaner(redis_client)
startListening(*address, *port)
}
func main() {
conn, err := redis.Dial("tcp", "localhost:6379")
if err != nil {
log.Fatal(err)
}
defer conn.Close()
keys, err := conn.Cmd("KEYS", "image:cache:*").List()
if err != nil {
log.Fatal(err)
}
for _, key := range keys {
conn.Cmd("DEL", key)
}
}
// returns the top x addresses, responds to GET <statserver>/topip?limit=x
func jsonTop(w http.ResponseWriter, req *http.Request) {
number := req.FormValue("limit") // read the limit from the request
client, err := redis.Dial("tcp", "localhost:6379")
if err != nil {
log.Fatal(err)
} else {
defer client.Close()
}
response := client.Cmd("ZREVRANGEBYSCORE", "popularity", "+inf", "-inf", "WITHSCORES", "LIMIT", "0", number)
list, _ := response.List()
encoder := json.NewEncoder(w)
encoder.Encode(list)
}
func getPage(host string, port int) (string, error) {
url := fmt.Sprintf("%s:%d", host, port)
client, err := redis.Dial("tcp", url)
if err != nil {
log.Error("Unable to connect to Redis: %v", err)
return "", err
}
defer client.Close()
resp, err := client.Cmd("INFO").Str()
if err != nil {
log.Error("Unable to retrieve INFO: %v", err)
return "", err
}
return resp, nil
}
// JSON end point for total number of IPs being tracked, responds to GET <statserver>/total
func jsonTotal(w http.ResponseWriter, req *http.Request) {
client, err := redis.Dial("tcp", "localhost:6379")
if err != nil {
log.Fatal(err)
} else {
defer client.Close()
}
response := client.Cmd("ZCARD", "popularity")
count, _ := response.Int() // get the count value
// get a json encoder
encoder := json.NewEncoder(w)
// create a map with key and value that json encoder will respond with
n := map[string]int{"count": count}
encoder.Encode(n)
}
func (config *PoolConnectionConfig) DialFunction(network string, addr string) (*redis.Client, error) {
client, err := redis.Dial(network, addr)
if err != nil {
return nil, err
}
if len(config.password) != 0 {
if err = client.Cmd("AUTH", config.password).Err; err != nil {
client.Close()
return nil, err
}
}
if config.db != 0 {
if err = client.Cmd("SELECT", config.db).Err; err != nil {
client.Close()
return nil, err
}
}
return client, nil
}
//------------- Connection Pool for redis
func initialize_redis_connection() {
//connect to redis server
fmt.Println("|---------------------------------------------------------------|")
fmt.Println("| [scanner]\tInitializing Redis Client Configuration\t\t|")
client, err = redis.Dial(redisProtocol, redisServerURL)
if err != nil {
fmt.Println("| WARNING: Problem connecting to Redis Server: " + redisServerURL + "\t\t|")
fmt.Println("| " + err.Error() + "\t|")
fmt.Println("| \tPACKET_LOGGING_MODE: PRINT_TO_SCREEN\t\t\t|")
fmt.Println("|---------------------------------------------------------------|")
} else {
fmt.Println("|\t\tConnected to Redis Server: " + redisServerURL + "|" + redisProtocol + "\t|")
fmt.Println("|\t\tPACKET_LOGGING_MODE: SEND_TO_REDIS \t\t|")
fmt.Println("|---------------------------------------------------------------|")
redisConnectionInitialized = true
}
}
func loader() {
rdb, err := redis.Dial("tcp", "localhost:6379")
if err != nil {
fmt.Println("Can not connect to redis db")
os.Exit(2)
}
for {
res := rdb.Cmd("BRPOP", "buff", 0)
if res.Err != nil {
fmt.Println("Can't BRPOP buff", res.Err)
continue
}
list, err := res.List()
if err != nil {
fmt.Println("Can't read response", err)
continue
}
entry := list[1]
key_val := strings.Split(entry, "\t")
// key_bin := binariseKey(key_val[0])
values := strings.Split(key_val[1], " ")
value := values[len(values)-1]
value_bin := encode(value)
// res = rdb.Cmd("HMSET", "cache", key_bin, value_bin)
// if res.Err != nil {
// fmt.Println(res.Err)
// }
fmt.Println(key_val[0], value_bin)
cache_mutex.Lock()
cache[key_val[0]] = value_bin
cache_mutex.Unlock()
}
}
func loader() {
rdb, err := redis.Dial("tcp", "localhost:6379")
if err != nil {
fmt.Println("Can not connect to redis db")
os.Exit(2)
}
for {
res := rdb.Cmd("BRPOP", "buff", 0)
if res.Err != nil {
fmt.Println("Can't BRPOP buff", res.Err)
continue
}
list, err := res.List()
if err != nil {
fmt.Println("Can't read response", err)
continue
}
entry := list[1]
key_val := strings.Split(entry, "\t")
key_bin, err := base64.RawURLEncoding.DecodeString(key_val[0])
if err != nil {
fmt.Println("Can't decode key", err, key_val)
continue
}
values := strings.Split(key_val[1], " ")
value := values[len(values)-1]
value_bin := encode(value)
res = rdb.Cmd("HMSET", "cache", key_bin, value_bin)
if res.Err != nil {
fmt.Println(res.Err)
}
}
}
func NewRedisStorage(address string, db int, pass, mrshlerStr string, maxConns int, cacheDumpDir string, loadHistorySize int) (*RedisStorage, error) {
df := func(network, addr string) (*redis.Client, error) {
client, err := redis.Dial(network, addr)
if err != nil {
return nil, err
}
if len(pass) != 0 {
if err = client.Cmd("AUTH", pass).Err; err != nil {
client.Close()
return nil, err
}
}
if db != 0 {
if err = client.Cmd("SELECT", db).Err; err != nil {
client.Close()
return nil, err
}
}
return client, nil
}
p, err := pool.NewCustom("tcp", address, maxConns, df)
if err != nil {
return nil, err
}
var mrshler Marshaler
if mrshlerStr == utils.MSGPACK {
mrshler = NewCodecMsgpackMarshaler()
} else if mrshlerStr == utils.JSON {
mrshler = new(JSONMarshaler)
} else {
return nil, fmt.Errorf("Unsupported marshaler: %v", mrshlerStr)
}
if cacheDumpDir != "" {
if err := CacheSetDumperPath(cacheDumpDir); err != nil {
utils.Logger.Info("<cache dumper> init error: " + err.Error())
}
}
return &RedisStorage{db: p, ms: mrshler, cacheDumpDir: cacheDumpDir, loadHistorySize: loadHistorySize}, nil
}
func TestScan(t *T) {
client, err := redis.Dial("tcp", "127.0.0.1:6379")
require.Nil(t, err)
prefix := "scanTestPrefix"
fullMap := map[string]bool{}
for i := 0; i < 100; i++ {
key := prefix + ":" + strconv.Itoa(i)
fullMap[key] = true
require.Nil(t, client.Cmd("SET", key, "1").Err)
}
// make sure we get all results when scanning with an existing prefix
ch := make(chan string)
go func() {
err = Scan(client, ch, "SCAN", "", prefix+":*")
}()
testMap := map[string]bool{}
for key := range ch {
testMap[key] = true
}
require.Nil(t, err)
assert.Equal(t, fullMap, testMap)
// make sure we don't get any results when scanning with a non-existing
// prefix
ch = make(chan string)
go func() {
err = Scan(client, ch, "SCAN", "", prefix+"DNE:*")
}()
testMap = map[string]bool{}
for key := range ch {
testMap[key] = true
}
require.Nil(t, err)
assert.Equal(t, map[string]bool{}, testMap)
}
// Similar to TestScan, but scans over a set instead of the whole key space
func TestSScan(t *T) {
client, err := redis.Dial("tcp", "127.0.0.1:6379")
require.Nil(t, err)
key := "scanTestSet"
fullMap := map[string]bool{}
for i := 0; i < 100; i++ {
elem := strconv.Itoa(i)
fullMap[elem] = true
require.Nil(t, client.Cmd("SADD", key, elem).Err)
}
// make sure we get all results when scanning with an existing prefix
ch := make(chan string)
go func() {
err = Scan(client, ch, "SSCAN", key, "*")
}()
testMap := map[string]bool{}
for elem := range ch {
testMap[elem] = true
}
require.Nil(t, err)
assert.Equal(t, fullMap, testMap)
// make sure we don't get any results when scanning with a non-existing
// prefix
ch = make(chan string)
go func() {
err = Scan(client, ch, "SSCAN", key+"DNE", "*")
}()
testMap = map[string]bool{}
for elem := range ch {
testMap[elem] = true
}
require.Nil(t, err)
assert.Equal(t, map[string]bool{}, testMap)
}
// Similar to TestScanner, but scans over a set instead of the whole key space
func TestScannerSet(t *T) {
client, err := redis.Dial("tcp", "127.0.0.1:6379")
require.Nil(t, err)
key, fullMap := randSet(t, client, 100)
// make sure we get all results when scanning an existing set
testMap := map[string]bool{}
sc := NewScanner(client, ScanOpts{Command: "SSCAN", Key: key})
for sc.HasNext() {
testMap[sc.Next()] = true
}
require.Nil(t, sc.Err())
assert.Equal(t, fullMap, testMap)
// make sure we don't get any results when scanning a non-existent set
testMap = map[string]bool{}
sc = NewScanner(client, ScanOpts{Command: "SSCAN", Key: key + "DNE"})
for sc.HasNext() {
testMap[sc.Next()] = true
}
require.Nil(t, sc.Err())
assert.Empty(t, testMap)
}