func TestReadDeadline(t *testing.T) {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("net.Listen returned %v", err)
}
defer l.Close()
go func() {
for {
c, err := l.Accept()
if err != nil {
return
}
go func() {
time.Sleep(time.Second)
c.Write([]byte("+OK\r\n"))
c.Close()
}()
}
}()
c1, err := redis.DialTimeout(l.Addr().Network(), l.Addr().String(), 0, time.Millisecond, 0)
if err != nil {
t.Fatalf("redis.Dial returned %v", err)
}
defer c1.Close()
_, err = c1.Do("PING")
if err == nil {
t.Fatalf("c1.Do() returned nil, expect error")
}
if c1.Err() == nil {
t.Fatalf("c1.Err() = nil, expect error")
}
c2, err := redis.DialTimeout(l.Addr().Network(), l.Addr().String(), 0, time.Millisecond, 0)
if err != nil {
t.Fatalf("redis.Dial returned %v", err)
}
defer c2.Close()
c2.Send("PING")
c2.Flush()
_, err = c2.Receive()
if err == nil {
t.Fatalf("c2.Receive() returned nil, expect error")
}
if c2.Err() == nil {
t.Fatalf("c2.Err() = nil, expect error")
}
}
func InitRedisConnPool() {
if !Config().Alarm.Enabled {
return
}
dsn := Config().Alarm.Redis.Dsn
maxIdle := Config().Alarm.Redis.MaxIdle
idleTimeout := 240 * time.Second
connTimeout := time.Duration(Config().Alarm.Redis.ConnTimeout) * time.Millisecond
readTimeout := time.Duration(Config().Alarm.Redis.ReadTimeout) * time.Millisecond
writeTimeout := time.Duration(Config().Alarm.Redis.WriteTimeout) * time.Millisecond
RedisConnPool = &redis.Pool{
MaxIdle: maxIdle,
IdleTimeout: idleTimeout,
Dial: func() (redis.Conn, error) {
c, err := redis.DialTimeout("tcp", dsn, connTimeout, readTimeout, writeTimeout)
if err != nil {
return nil, err
}
return c, err
},
TestOnBorrow: PingRedis,
}
}
//创建一个新的redis连接
func NewRedisCache(ip string, port string, password string) (*RedisCache, error) {
var (
c redis.Conn
err error
)
c, err = redis.DialTimeout("tcp", ip+":"+port, 0, 1*time.Second, 1*time.Second)
if err != nil {
glog.Error("Error:", err)
return nil, err
}
if password != "" {
_, err = c.Do("AUTH", password)
if err != nil {
c.Close()
glog.Error("Error:", err)
return nil, err
}
}
return &RedisCache{
session: c,
}, err
}
// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCache(host string, password string, defaultExpiration time.Duration) RedisCache {
var pool = &redis.Pool{
MaxIdle: 5,
IdleTimeout: 120 * time.Second,
Dial: func() (redis.Conn, error) {
protocol := "tcp"
c, err := redis.DialTimeout(protocol, host, 1*time.Second, 1*time.Second, 1*time.Second)
if err != nil {
return nil, err
}
// check with PING
if _, err := c.Do("PING"); err != nil {
c.Close()
return nil, err
}
return c, err
},
// custom connection test method
TestOnBorrow: func(c redis.Conn, t time.Time) error {
if _, err := c.Do("PING"); err != nil {
return err
}
return nil
},
}
return RedisCache{pool, defaultExpiration}
}
func main() {
Pool = &redis.Pool{
MaxActive: 1000,
MaxIdle: 3,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
d, _ := time.ParseDuration("250ms")
c, err := redis.DialTimeout("tcp", *redisServer, d, d, d)
if err != nil {
log.Println("Redis: connection error")
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
http.Handle("/", &RedirHttpHandler{})
http.ListenAndServe(*httpPort, nil)
log.Println("end main")
}
func initRedisQueue(option *config.Option) map[string][]*poolwrapper {
redispool := make(map[string][]*poolwrapper, 0)
//创建redis的消费连接
for _, v := range option.QueueHostPorts {
hp := v
pool := redis.NewPool(func() (conn redis.Conn, err error) {
conn, err = redis.DialTimeout("tcp", hp.Host+":"+strconv.Itoa(hp.Port),
time.Duration(hp.Timeout)*time.Second,
time.Duration(hp.Timeout)*time.Second,
time.Duration(hp.Timeout)*time.Second)
return
}, hp.Maxconn/2)
pools, ok := redispool[v.QueueName]
if !ok {
pools = make([]*poolwrapper, 0)
redispool[v.QueueName] = pools
}
poolw := &poolwrapper{}
poolw.hostport = v.HostPort
poolw.rpool = pool
redispool[v.QueueName] = append(pools, poolw)
}
return redispool
}
func open(addr string) (redis.Conn, error) {
return redis.DialTimeout("tcp",
addr,
1*time.Second,
1*time.Second,
1*time.Second)
}
func makeConn(addr, pwd string) (conn *redisConn, err error) {
log.Info("makeConn|%v|%v", addr, pwd)
conn = nil
const dataTimeout = 5 * time.Second
const connTimeout = 2 * time.Second
var c redis.Conn
if c, err = redis.DialTimeout("tcp", addr, connTimeout, dataTimeout, dataTimeout); err != nil {
log.Error("makeConn|DialTimeout|%v", err)
return
}
if pwd != "" {
if _, err = c.Do("AUTH", pwd); err != nil {
log.Error("makeConn|auth|%v", err)
c.Close()
return
}
}
if _, err = c.Do("get", "__test"); err != nil {
log.Error("makeConn|get|%v", err)
c.Close()
return
}
log.Info("makeConn|ok|%v", addr)
var now = time.Now().Unix()
conn = &redisConn{c, addr, seed, now + pingInterval, now}
seed++
return
}
func SlotsInfo(addr string, fromSlot, toSlot int) (map[int]int, error) {
c, err := redis.DialTimeout("tcp", addr, defaultTimeout, defaultTimeout, defaultTimeout)
if err != nil {
return nil, errors.Trace(err)
}
defer c.Close()
infos, err := redis.Values(c.Do("SLOTSINFO", fromSlot, toSlot-fromSlot+1))
if err != nil {
return nil, errors.Trace(err)
}
slots := make(map[int]int)
if infos != nil {
for i := 0; i < len(infos); i++ {
info, err := redis.Values(infos[i], nil)
if err != nil {
return nil, errors.Trace(err)
}
var slotid, slotsize int
if _, err := redis.Scan(info, &slotid, &slotsize); err != nil {
return nil, errors.Trace(err)
} else {
slots[slotid] = slotsize
}
}
}
return slots, nil
}
func msetTestSingle(ch chan bool, cn, n int) error {
var err error
addr := utils.Addrcat(host, port)
conn, err := redis.DialTimeout("tcp", addr, 0, 1*time.Second, 1*time.Second)
if err != nil {
log.Printf("redis conn error: %s", err)
return err
}
defer conn.Close()
v := make([]byte, dataSize)
b := make([]interface{}, bucket+1)
b[0] = topicName
for i := 1; i < bucket+1; i++ {
b[i] = v
}
count := n / bucket
for i := 0; i < count; i++ {
start := time.Now()
_, err = conn.Do("MSET", b...)
if err != nil {
log.Printf("set error: c%d %v", cn, err)
} else {
end := time.Now()
duration := end.Sub(start).Seconds()
log.Printf("set succ: %s spend: %.3fms", topicName, duration*1000)
}
}
ch <- true
return nil
}
func NewRedisRegistryWithConfig(conf *RedisConfig) (registry Registry) {
pool := &redis.Pool{
MaxIdle: conf.MaxIdle,
IdleTimeout: time.Duration(conf.IdleTimeoutS) * time.Second,
MaxActive: conf.MaxActive,
Dial: func() (redis.Conn, error) {
var c redis.Conn
var err error
c, err = redis.DialTimeout("tcp", conf.Address,
time.Duration(conf.ConnectTimeoutMs)*time.Millisecond,
time.Duration(conf.ReadTimeoutMs)*time.Millisecond,
time.Duration(conf.WriteTimeoutMs)*time.Millisecond)
if err != nil {
return nil, err
}
//password authentication
if len(conf.Password) > 0 {
if _, err_pass := c.Do("AUTH", conf.Password); err_pass != nil {
c.Close()
}
}
return c, err
},
}
return &redisRegistry{pool}
}
func getTestSingle(ch chan bool, cn, n int) error {
addr := utils.Addrcat(host, port)
conn, err := redis.DialTimeout("tcp", addr, 0, 1*time.Second, 1*time.Second)
if err != nil {
log.Printf("redis conn error: %s", err)
return err
}
defer conn.Close()
key := topicName + "/" + lineName
for i := 0; i < n; i++ {
start := time.Now()
reply, err := conn.Do("GET", key)
if err != nil {
log.Printf("get error: c%d %v", cn, err)
} else {
rpl, err := redis.Strings(reply, err)
if err != nil {
fmt.Printf("redis.values error: c%d %v\n", cn, err)
return err
}
// fmt.Printf("redis.strings %v\n", v)
end := time.Now()
duration := end.Sub(start).Seconds()
id := rpl[1]
log.Printf("get succ: %s spend: %.3fms", id, duration*1000)
}
}
ch <- true
return nil
}
func (p *connectionPool) get() (redis.Conn, error) {
p.mu.Lock()
for {
available := len(p.available)
switch {
case available <= 0 && p.outstanding >= p.max:
// Worst case. No connection available, and we can't dial a new one.
p.co.Wait() // TODO starvation is possible here
case available <= 0 && p.outstanding < p.max:
// No connection available, but we can dial a new one.
//
// We shouldn't wait for a connection to be successfully established
// before incrementing our outstanding counter, because additional
// goroutines may sneak in with a get() request while we're dialing,
// and bump outstanding above p.max.
//
// So, clients of get() should always put() the resulting conn, even
// if it is nil. put() must handle that circumstance.
p.outstanding++
p.mu.Unlock()
return redis.DialTimeout("tcp", p.address, p.connect, p.read, p.write)
case available > 0:
// Best case. We can directly use an available connection.
var conn redis.Conn
conn, p.available = p.available[0], p.available[1:]
if p.outstanding < p.max {
p.outstanding++
}
p.mu.Unlock()
return conn, nil
}
}
}
// Redis DB Pool
func NewRDBpool(address string) *RDBpool {
pool := redis.Pool{
MaxActive: 0,
MaxIdle: 3,
Dial: func() (redis.Conn, error) {
c, err := redis.DialTimeout(
"tcp",
address,
time.Duration(1)*time.Second,
time.Duration(1)*time.Second,
time.Duration(1)*time.Second,
)
if err != nil {
return nil, err
}
return c, err
},
}
conn := pool.Get()
defer conn.Close()
if conn.Err() != nil {
panic(fmt.Sprintf("Can not connect to redis %s", address))
}
return &RDBpool{pool: pool}
}
func (n *Node) doCommand(cmd string, args ...interface{}) (interface{}, error) {
var err error
var v interface{}
for i := 0; i < 3; i++ {
if n.conn == nil {
n.conn, err = redis.DialTimeout("tcp", n.Addr, 5*time.Second, 0, 0)
if err != nil {
log.Errorf("dial %s error: %v, try again", n.Addr, err)
continue
}
}
v, err = n.conn.Do(cmd, args...)
if err != nil {
log.Errorf("do %s command for %s error: %v, try again", cmd, n.Addr, err)
n.conn.Close()
n.conn = nil
continue
} else {
return v, nil
}
}
// go here means do command error, maybe redis is down.
return nil, err
}
func GetRedisStat(addr string) (map[string]string, error) {
c, err := redis.DialTimeout("tcp", addr, defaultTimeout, defaultTimeout, defaultTimeout)
if err != nil {
return nil, errors.Trace(err)
}
defer c.Close()
ret, err := redis.String(c.Do("INFO"))
if err != nil {
return nil, errors.Trace(err)
}
m := make(map[string]string)
lines := strings.Split(ret, "\n")
for _, line := range lines {
kv := strings.SplitN(line, ":", 2)
if len(kv) == 2 {
k, v := strings.TrimSpace(kv[0]), strings.TrimSpace(kv[1])
m[k] = v
}
}
reply, err := redis.Strings(c.Do("config", "get", "maxmemory"))
if err != nil {
return nil, errors.Trace(err)
}
// we got result
if len(reply) == 2 {
if reply[1] != "0" {
m["maxmemory"] = reply[1]
} else {
m["maxmemory"] = "∞"
}
}
return m, nil
}
// SetupDbs takes connection parameters for redis and mongo and returns active sessions.
// The caller is responsible for closing the returned connections.
func SetupDbs(mongoURL, redisURL string) (*mgo.Database, redis.Conn, error) {
mongoSession, err := mgo.Dial(mongoURL)
if err != nil {
return nil, nil, err
}
// use 'monotonic' consistency mode. Since we only do reads, this doesn't have an actual effect
// other than letting us read from secondaries if the connection string has the ?connect=direct param.
mongoSession.SetMode(mgo.Monotonic, false)
// empty db string uses the db from the connection url
mongoDB := mongoSession.DB("")
logger.Info("Connected to mongo", logger.M{"mongo_url": mongoURL})
redisURL, err = resolveRedis(redisURL)
if err != nil {
return nil, nil, err
}
redisConn, err := redis.DialTimeout("tcp", redisURL, 15*time.Second, 10*time.Second, 10*time.Second)
if err != nil {
return nil, nil, err
}
logger.Info("Connected to redis", logger.M{"redis_url": redisURL})
return mongoDB, redisConn, nil
}
// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCache(host string, idle, max int, toc, tor, tow, defaultExpiration time.Duration) RedisCache {
var pool = &redis.Pool{
MaxIdle: idle,
MaxActive: max,
IdleTimeout: 240 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.DialTimeout("tcp", host, toc, tor, tow)
if err != nil {
return nil, err
}
// check with PING
if _, err := c.Do("PING"); err != nil {
c.Close()
return nil, err
}
return c, err
},
// custom connection test method
TestOnBorrow: func(c redis.Conn, t time.Time) error {
if _, err := c.Do("PING"); err != nil {
return err
}
return nil
},
}
return RedisCache{pool, defaultExpiration}
}
// create new connection pool to redis.
func (rc *RedisCli) initConnPool(conf *RedisConf) {
if rc.dialFunc == nil {
rc.dialFunc = func() (c redis.Conn, err error) {
c, err = redis.DialTimeout("tcp", conf.Addr, conf.Timeout, conf.Timeout, conf.Timeout)
if err != nil {
return nil, err
}
if conf.Password != "" {
if _, err := c.Do("AUTH", conf.Password); err != nil {
c.Close()
return nil, err
}
}
_, selecterr := c.Do("SELECT", conf.DbNo)
if selecterr != nil {
c.Close()
return nil, selecterr
}
return
}
}
// initialize a new pool
rc.p = &redis.Pool{
MaxIdle: MaxRedisIdleConn,
IdleTimeout: 180 * time.Second,
Dial: rc.dialFunc,
}
}
// Dial dials the local Redis server and selects database 9. To prevent
// stomping on real data, DialTestDB fails if database 9 contains data. The
// returned connection flushes database 9 on close.
func Dial() (redis.Conn, error) {
c, err := redis.DialTimeout("tcp", ":6379", 0, 1*time.Second, 1*time.Second)
if err != nil {
return nil, err
}
_, err = c.Do("SELECT", "9")
if err != nil {
c.Close()
return nil, err
}
n, err := redis.Int(c.Do("DBSIZE"))
if err != nil {
c.Close()
return nil, err
}
if n != 0 {
c.Close()
return nil, errors.New("database #9 is not empty, test can not continue")
}
return testConn{c}, nil
}
func sender(conn net.Conn) {
redis_conn, _ := redis.DialTimeout("tcp", ":6379", 0, 1*time.Second, 1*time.Second)
for {
lang, _ := redis.String(redis_conn.Do("LPOP", "lang"))
if lang != "" {
words := lang
if words[0] == 48 {
os.Exit(0)
} else {
conn.Write(protocol.Packet([]byte(words)))
fmt.Println(conn)
}
}
// reader := bufio.NewReader(os.Stdin)
// words, _, _ := reader.ReadLine()
// words := "{\"Id\":1,\"Name\":\"golang\",\"Message\":\"message\"}"
}
fmt.Println("send over")
}
func init() {
u, err := url.Parse(os.Getenv("REDIS_URL"))
if err != nil {
fmt.Printf("error=%q\n", "Missing REDIS_URL.")
os.Exit(1)
}
server := u.Host
password, set := u.User.Password()
if !set {
fmt.Printf("at=error error=%q\n", "password not set")
os.Exit(1)
}
redisPool = &redis.Pool{
MaxIdle: 10,
IdleTimeout: 10 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.DialTimeout("tcp", server, time.Second, time.Second, time.Second)
if err != nil {
return nil, err
}
c.Do("AUTH", password)
return c, err
},
}
}
// checkRole check the current redis role.
func checkRole(addr string, role string) bool {
if addr == "" {
return false
}
// Step 3: call the ROLE command in the target instance
conn, err := redis.DialTimeout("tcp", addr, sentinelTimeout, sentinelTimeout, sentinelTimeout)
if err != nil {
log.Error("redis.DialTimeout(\"tcp\", \"%s\", 500ms...) error(%v)", addr, err)
return false
}
defer conn.Close()
replies, err := redis.Values(conn.Do("ROLE"))
if err != nil {
log.Error("conn.Do(\"ROLE\") error(%v)", err)
return false
}
if len(replies) < 1 {
return false
}
curRole, err := redis.String(replies[0], nil)
if err != nil {
log.Error("redis.String(replies[0], nil) error(%v)", err)
return false
}
log.Info("redis: \"%s\" role: \"%s\"", addr, curRole)
if curRole == role {
return true
}
return false
}
func InitRedisConnPool() {
dsn := Config().Redis.Dsn
maxIdle := Config().Redis.MaxIdle
idleTimeout := 240 * time.Second
connTimeout := time.Duration(Config().Redis.ConnTimeout) * time.Millisecond
readTimeout := time.Duration(Config().Redis.ReadTimeout) * time.Millisecond
writeTimeout := time.Duration(Config().Redis.WriteTimeout) * time.Millisecond
RedisConnPool = &redis.Pool{
MaxIdle: maxIdle,
IdleTimeout: idleTimeout,
Dial: func() (redis.Conn, error) {
c, err := redis.DialTimeout("tcp", dsn, connTimeout, readTimeout, writeTimeout)
if err != nil {
return nil, err
}
if _, err := c.Do("AUTH", Config().Redis.Passwd); err != nil {
c.Close()
return nil, err
}
return c, err
},
TestOnBorrow: PingRedis,
}
}
func NewRedisPool(config *RedisConfig) *redis.Pool {
setDefaultRedisConfig(config)
return &redis.Pool{
MaxIdle: config.MaxIdle,
MaxActive: config.MaxActive,
IdleTimeout: config.IdleTimeout,
Dial: func() (redis.Conn, error) {
c, err := redis.DialTimeout(config.Protocol, config.Host, config.ConnectTimeout, config.ReadTimeout, config.WriteTimeout)
if err != nil {
return nil, err
}
if len(config.Password) > 0 {
if _, err := c.Do("AUTH", config.Password); err != nil {
c.Close()
return nil, err
}
} else {
// check with PING
if _, err := c.Do("PING"); err != nil {
c.Close()
return nil, err
}
}
return c, err
},
// custom connection test method
TestOnBorrow: func(c redis.Conn, t time.Time) error {
if _, err := c.Do("PING"); err != nil {
return err
}
return nil
},
}
}
func newRedisStorage(server string, pass string, maxActive, maxIdle, idleTimeout, retry, cto, rto, wto int) *RedisStorage {
return &RedisStorage{
pool: &redis.Pool{
MaxActive: maxActive,
MaxIdle: maxIdle,
IdleTimeout: time.Duration(idleTimeout) * time.Second,
Dial: func() (redis.Conn, error) {
//c, err := redis.Dial("tcp", server)
c, err := redis.DialTimeout(
"tcp", server,
time.Duration(cto)*time.Second,
time.Duration(rto)*time.Second,
time.Duration(wto)*time.Second)
if err != nil {
log.Warnf("failed to connect Redis, (%s)", err)
return nil, err
}
if pass != "" {
if _, err := c.Do("AUTH", pass); err != nil {
log.Warnf("failed to auth Redis, (%s)", err)
return nil, err
}
}
//log.Debugf("connected with Redis (%s)", server)
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
},
retry: retry,
}
}
// get redis's slot size
func SlotsInfo(addr string, fromSlot, toSlot int) (map[int]int, error) {
c, err := redis.DialTimeout("tcp", addr, defaultTimeout, defaultTimeout, defaultTimeout)
if err != nil {
return nil, err
}
defer c.Close()
var reply []interface{}
var val []interface{}
reply, err = redis.Values(c.Do("SLOTSINFO", fromSlot, toSlot-fromSlot+1))
if err != nil {
return nil, err
}
ret := make(map[int]int)
for {
if reply == nil || len(reply) == 0 {
break
}
if reply, err = redis.Scan(reply, &val); err != nil {
return nil, err
}
var slot, keyCount int
_, err := redis.Scan(val, &slot, &keyCount)
if err != nil {
return nil, err
}
ret[slot] = keyCount
}
return ret, nil
}
func (out *RedisOutput) RedisConnect(db int) (redis.Conn, error) {
conn, err := redis.DialTimeout(
"tcp",
out.Hostname,
out.Timeout, out.Timeout, out.Timeout)
if err != nil {
return nil, err
}
if len(out.Password) > 0 {
_, err = conn.Do("AUTH", out.Password)
if err != nil {
return nil, err
}
}
_, err = conn.Do("PING")
if err != nil {
return nil, err
}
_, err = conn.Do("SELECT", db)
if err != nil {
return nil, err
}
return conn, nil
}
func (r *RedisCache) Dial(addr string) error {
var err error
r.C, err = redis.DialTimeout("tcp", addr, CONNECTION_TIMEOUT, CONNECTION_TIMEOUT, CONNECTION_TIMEOUT)
if err != nil {
return errors.New("[cache/redis] " + err.Error())
}
return nil
}
func NewRedisConn(redisConf RedisConf) *RedisConn {
c, err := redis.DialTimeout(redisConf.Network, redisConf.Address, time.Duration(redisConf.ConnectTimeout), time.Duration(redisConf.ReadTimeout), time.Duration(redisConf.WriteTimeout))
if err != nil {
log.Fatal("(NewRedisConn) ", err)
}
return &RedisConn{
conn: c,
}
}
