// verifyMaster verifies that the decided master node has fully transitioned
func verifyMaster(addr, pass string) error {
// connect to redis in order to verify its state
r, err := redis.DialURL("redis://"+addr, redis.DialConnectTimeout(config.TimeoutNotReady), redis.DialPassword(pass))
if err != nil {
return fmt.Errorf("Failed to reach redis at: '%v'", addr)
}
// give redis some time to transition
timeout := time.After(config.TimeoutMasterWait)
for {
select {
case <-timeout:
return fmt.Errorf("Timed out waiting for redis to transition to master")
default:
// retrieve the redis node's role
info, err := redis.Bytes(r.Do("INFO", "replication"))
if err != nil {
return fmt.Errorf("Failed to get INFO - %v", err)
}
// check if node is master
if strings.Contains(string(info), "role:master") {
return nil
}
}
}
// cleanup after ourselves
r.Close()
return nil
}
// updateMaster gets the address of the master node from sentinel
func updateMaster() error {
config.Log.Debug("Contacting sentinel for address of master...")
// connect to sentinel in order to query for the master address
r, err := redis.DialURL("redis://"+config.SentinelAddress, redis.DialConnectTimeout(config.TimeoutNotReady), redis.DialReadTimeout(config.TimeoutSentinelPoll), redis.DialPassword(config.SentinelPassword))
if err != nil {
return fmt.Errorf("Failed to reach sentinel - %v", err)
}
// retrieve the master redis address
addr, err := redis.Strings(r.Do("SENTINEL", "get-master-addr-by-name", config.MonitorName))
if err != nil {
return fmt.Errorf("Failed to get-master-addr-by-name - %v", err)
}
// cleanup after ourselves
r.Close()
// construct a useable address from sentinel's response
masterAddr = fmt.Sprintf("%v:%v", addr[0], addr[1])
config.Log.Debug("Master address: '%v'", masterAddr)
// wait for redis to transition to master
if err = verifyMaster(masterAddr, config.SentinelPassword); err != nil {
return fmt.Errorf("Could not verify master - %v", err)
}
return nil
}
// Iterate through `machines`, trying to connect to each in turn.
// Returns the first successful connection or the last error encountered.
// Assumes that `machines` is non-empty.
func tryConnect(machines []string, password string) (redis.Conn, error) {
var err error
for _, address := range machines {
var conn redis.Conn
network := "tcp"
if _, err = os.Stat(address); err == nil {
network = "unix"
}
log.Debug(fmt.Sprintf("Trying to connect to redis node %s", address))
dialops := []redis.DialOption{
redis.DialConnectTimeout(time.Second),
redis.DialReadTimeout(time.Second),
redis.DialWriteTimeout(time.Second),
}
if password != "" {
dialops = append(dialops, redis.DialPassword(password))
}
conn, err = redis.Dial(network, address, dialops...)
if err != nil {
continue
}
return conn, nil
}
return nil, err
}
// CreatePool creates a redis connection pool
func CreatePool(
host, password, network string,
maxConn int,
idleTimeout, connTimeout time.Duration,
) *rd.Pool {
return &rd.Pool{
MaxIdle: maxConn,
IdleTimeout: idleTimeout,
Dial: func() (rd.Conn, error) {
c, err := rd.Dial(network, host,
rd.DialConnectTimeout(connTimeout),
rd.DialReadTimeout(connTimeout),
rd.DialWriteTimeout(connTimeout))
if err != nil {
return nil, err
}
if password != "" {
if _, err := c.Do("AUTH", password); err != nil {
c.Close()
return nil, err
}
}
return c, err
},
}
}
func TestMain(m *testing.M) {
// clean test dir
os.RemoveAll("/tmp/clusterTest")
// initialize backend if redis-server found
initialize()
conn, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(30*time.Second), redis.DialPassword(config.ClusterToken))
if err != nil {
return
}
hostname, _ := os.Hostname()
self := fmt.Sprintf("%v:%v", hostname, config.ApiPort)
defer conn.Do("SREM", "members", self)
defer conn.Close()
rtn := m.Run()
// clean test dir
os.RemoveAll("/tmp/clusterTest")
// just in case, ensure clean members
conn.Do("SREM", "members", self)
conn.Close()
os.Exit(rtn)
}
// ping executes a PING command against addr until timeout occurs.
func (p *Process) ping(addr string, timeout time.Duration) error {
// Default to local process if addr not specified.
if addr == "" {
addr = fmt.Sprintf("localhost:%s", p.Port)
}
logger := p.Logger.New("fn", "ping", "addr", addr, "timeout", timeout)
logger.Info("sending")
timer := time.NewTimer(timeout)
defer timer.Stop()
ticker := time.NewTicker(checkInterval)
defer ticker.Stop()
for {
// Attempt to ping the server.
if ok := func() bool {
logger.Info("sending PING")
conn, err := redis.Dial("tcp", addr,
redis.DialPassword(p.Password),
redis.DialConnectTimeout(timeout),
redis.DialReadTimeout(timeout),
redis.DialWriteTimeout(timeout),
)
if err != nil {
logger.Error("conn error", "err", err)
return false
}
defer conn.Close()
if _, err := conn.Do("PING"); err != nil {
logger.Error("error getting upstream status", "err", err)
return false
}
logger.Info("PONG received")
return true
}(); ok {
return nil
}
select {
case <-timer.C:
logger.Info("timeout")
return ErrTimeout
case <-ticker.C:
}
}
}
func (l *RedisInput) Run(output chan common.MapStr) error {
logp.Debug("redisinput", "Running Redis Input")
var keysScript = redis.NewScript(1, `return redis.call('KEYS', KEYS[1])`)
go func() {
redisURL := fmt.Sprintf("redis://%s:%d/%d", l.Host, l.Port, l.DB)
dialConnectTimeout := redis.DialConnectTimeout(3 * time.Second)
dialReadTimeout := redis.DialReadTimeout(10 * time.Second)
var backOffCount = 0
var backOffDuration time.Duration = 5 * time.Second
for {
logp.Debug("redisinput", "Connecting to: %s", redisURL)
server, err := redis.DialURL(redisURL, dialConnectTimeout, dialReadTimeout)
if err != nil {
logp.Err("couldn't start listening: " + err.Error())
return
}
logp.Debug("redisinput", "Connected to Redis Server")
reply, err := keysScript.Do(server, "*")
if err != nil {
logp.Err("An error occured while executing KEYS command: %s\n", err)
return
}
keys, err := redis.Strings(reply, err)
if err != nil {
logp.Err("An error occured while converting reply to String: %s\n", err)
return
}
for _, key := range keys {
logp.Debug("redisinput", "key is %s", key)
lineCount, err := l.handleConn(server, output, key)
if err == nil {
logp.Debug("redisinput", "Read %v events", lineCount)
backOffCount = 0
backOffDuration = time.Duration(backOffCount) * time.Second
time.Sleep(backOffDuration)
} else {
backOffCount++
backOffDuration = time.Duration(backOffCount) * time.Second
time.Sleep(backOffDuration)
}
}
defer server.Close()
}
}()
return nil
}
// RedisInfo executes an INFO command against a Redis server and returns the results.
func (p *Process) RedisInfo(addr string, timeout time.Duration) (*RedisInfo, error) {
// Default to local process if addr not specified.
if addr == "" {
addr = fmt.Sprintf("localhost:%s", p.Port)
}
logger := p.Logger.New("fn", "replInfo", "addr", addr)
logger.Info("sending INFO")
// Connect to the redis server.
conn, err := redis.Dial("tcp", addr,
redis.DialPassword(p.Password),
redis.DialConnectTimeout(timeout),
redis.DialReadTimeout(timeout),
redis.DialWriteTimeout(timeout),
)
if err != nil {
logger.Info("dial error", "err", err)
return nil, err
}
defer conn.Close()
// Execute INFO command.
reply, err := conn.Do("INFO")
if err != nil {
logger.Error("info error", "err", err)
return nil, err
}
buf, ok := reply.([]byte)
if !ok {
logger.Error("info reply type error", "type", fmt.Sprintf("%T", buf))
return nil, fmt.Errorf("unexpected INFO reply format: %T", buf)
}
// Parse the bulk string reply info a typed object.
info, err := ParseRedisInfo(string(buf))
if err != nil {
logger.Error("parse info error", "err", err)
return nil, fmt.Errorf("parse info: %s", err)
}
logger.Info("INFO received")
return info, nil
}
// creates a redis connection pool to use
func (r Redis) newPool(server, password string) *redis.Pool {
return &redis.Pool{
MaxIdle: 3,
IdleTimeout: 5 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.DialURL(server, redis.DialConnectTimeout(30*time.Second),
redis.DialWriteTimeout(10*time.Second), redis.DialPassword(password))
if err != nil {
return nil, fmt.Errorf("Failed to reach redis - %v", err)
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
func newPool() *redis.Pool {
return &redis.Pool{
MaxIdle: 20,
MaxActive: 1024, // max number of connections
IdleTimeout: 5 * time.Second,
Dial: func() (redis.Conn, error) {
c, err := redis.Dial("unix", "/tmp/redis.sock",
redis.DialConnectTimeout(15*time.Second))
if err != nil {
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
}
// Connect establishes a connection to Redis which provides the
// pub/sub implementation.
func (b *redisBroker) Connect() error {
if b.pool != nil {
return nil
}
var addr string
if len(b.opts.Addrs) == 0 || b.opts.Addrs[0] == "" {
addr = "redis://127.0.0.1:6379"
} else {
addr = b.opts.Addrs[0]
if !strings.HasPrefix("redis://", addr) {
addr = "redis://" + addr
}
}
b.addr = addr
b.pool = &redis.Pool{
MaxIdle: b.bopts.maxIdle,
MaxActive: b.bopts.maxActive,
IdleTimeout: b.bopts.idleTimeout,
Dial: func() (redis.Conn, error) {
return redis.DialURL(
b.addr,
redis.DialConnectTimeout(b.bopts.connectTimeout),
redis.DialReadTimeout(b.bopts.readTimeout),
redis.DialWriteTimeout(b.bopts.writeTimeout),
)
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
return nil
}
func newPool(conf *Configure, host string, dbnum int) *RedisPool {
pool := &redis.Pool{
MaxIdle: conf.Redis.MaxIdle,
MaxActive: conf.Redis.MaxActive,
IdleTimeout: time.Duration(conf.Redis.IdleTimeout) * time.Second,
Dial: func() (redis.Conn, error) {
opt_timeout := redis.DialConnectTimeout(time.Duration(conf.Redis.ConnTimeout) * time.Second)
opt_selectdb := redis.DialDatabase(dbnum)
c, err := redis.Dial("tcp", host, opt_timeout, opt_selectdb)
if err != nil {
return nil, err
}
return c, err
},
TestOnBorrow: func(c redis.Conn, t time.Time) error {
_, err := c.Do("PING")
return err
},
}
p := &RedisPool{p: pool, db: dbnum, host: host}
log.Printf("[redis_pool]on host:%s:%d, create redis pool success.\n", host, dbnum)
return p
}
func (m MRedis) Sub(ch string) {
c, err := redis.Dial("tcp", "192.168.176.3:6379", redis.DialConnectTimeout(10*time.Second))
if err != nil {
log.Fatal(err)
}
defer c.Close()
var wg sync.WaitGroup
wg.Add(1)
psc := redis.PubSubConn{Conn: c}
if err = psc.Subscribe(ch); err != nil {
log.Fatal(err)
}
go func() {
defer wg.Done()
for {
switch n := psc.Receive().(type) {
case redis.Message:
fmt.Printf("Message: %s %s\n", n.Channel, n.Data)
case redis.PMessage:
fmt.Printf("PMessage: %s %s %s\n", n.Pattern, n.Channel, n.Data)
case redis.Subscription:
fmt.Printf("Subscription: %s %s %d\n", n.Kind, n.Channel, n.Count)
if n.Count == 0 {
return
}
case error:
fmt.Printf("error: %v\n", n)
return
}
}
}()
wg.Wait()
}
func (this *Conf) Dail() (conn redis.Conn, err error) {
option := redis.DialConnectTimeout(time.Duration(this.Timeout) * time.Second)
conn, err = redis.Dial(this.NetWork, this.Address, option)
return
}
// GetCerts gets a list of certs from the database, or another cluster member.
func (r *Redis) GetCerts() ([]core.CertBundle, error) {
if database.CentralStore {
return database.GetCerts()
}
conn := pool.Get()
defer conn.Close()
// get known members(other than me) to 'poll' for certs
members, _ := redis.Strings(conn.Do("SMEMBERS", "members"))
if len(members) == 0 {
// should only happen on new cluster
// assume i'm ok to be master so don't reset imported certs
config.Log.Trace("[cluster] - Assuming OK to be master, using certs from my database...")
return common.GetCerts()
}
for i := range members {
if members[i] == self {
// if i'm in the list of members, new requests should have failed while `waitForMembers`ing
config.Log.Trace("[cluster] - Assuming I was in sync, using certs from my database...")
return common.GetCerts()
}
}
c, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(15*time.Second), redis.DialPassword(config.ClusterToken))
if err != nil {
return nil, fmt.Errorf("Failed to reach redis for certs subscriber - %v", err)
}
defer c.Close()
message := make(chan interface{})
subconn := redis.PubSubConn{c}
// subscribe to channel that certs will be published on
if err := subconn.Subscribe("certs"); err != nil {
return nil, fmt.Errorf("Failed to reach redis for certs subscriber - %v", err)
}
defer subconn.Close()
// listen always
go func() {
for {
message <- subconn.Receive()
}
}()
// todo: maybe use ttl?
// timeout is how long to wait for the listed members to come back online
timeout := time.After(time.Duration(20) * time.Second)
// loop attempts for timeout, allows last dead members to start back up
for {
select {
case <-timeout:
return nil, fmt.Errorf("Timed out waiting for certs from %v", strings.Join(members, ", "))
default:
// request certs from each member until successful
for _, member := range members {
// memberTimeout is how long to wait for a member to respond with list of certs
memberTimeout := time.After(3 * time.Second)
// ask a member for its certs
config.Log.Trace("[cluster] - Attempting to request certs from %v...", member)
_, err := conn.Do("PUBLISH", "portal", fmt.Sprintf("get-certs %s", member))
if err != nil {
return nil, err
}
// wait for member to respond
for {
select {
case <-memberTimeout:
config.Log.Debug("[cluster] - Timed out waiting for certs from %v", member)
goto nextCertMember
case msg := <-message:
switch v := msg.(type) {
case redis.Message:
config.Log.Trace("[cluster] - Received message on 'certs' channel")
var certs []core.CertBundle
err = parseBody(v.Data, &certs)
if err != nil {
return nil, fmt.Errorf("Failed to marshal certs - %v", err.Error())
}
config.Log.Trace("[cluster] - Certs from cluster: %#v\n", certs)
return certs, nil
case error:
return nil, fmt.Errorf("Subscriber failed to receive certs - %v", v.Error())
}
}
}
nextCertMember:
}
}
}
}
func (r *Redis) Init() error {
hostname, _ := os.Hostname()
self = fmt.Sprintf("%v:%v", hostname, config.ApiPort)
pool = r.newPool(config.ClusterConnection, config.ClusterToken)
// get services
services, err := r.GetServices()
if err != nil {
return fmt.Errorf("Failed to get services - %v", err)
}
// write services
if services != nil {
config.Log.Trace("[cluster] - Setting services...")
err = common.SetServices(services)
if err != nil {
return fmt.Errorf("Failed to set services - %v", err)
}
}
// get routes
routes, err := r.GetRoutes()
if err != nil {
return fmt.Errorf("Failed to get routes - %v", err)
}
// write routes
if routes != nil {
config.Log.Trace("[cluster] - Setting routes...")
err = common.SetRoutes(routes)
if err != nil {
return fmt.Errorf("Failed to set routes - %v", err)
}
}
// get certs
certs, err := r.GetCerts()
if err != nil {
return fmt.Errorf("Failed to get certs - %v", err)
}
// write certs
if certs != nil {
config.Log.Trace("[cluster] - Setting certs...")
err = common.SetCerts(certs)
if err != nil {
return fmt.Errorf("Failed to set certs - %v", err)
}
}
// get vips
vips, err := r.GetVips()
if err != nil {
return fmt.Errorf("Failed to get vips - %v", err)
}
// write vips
if vips != nil {
config.Log.Trace("[cluster] - Setting vips...")
err = common.SetVips(vips)
if err != nil {
return fmt.Errorf("Failed to set vips - %v", err)
}
}
// note: keep subconn connection initialization out here or sleep after `go r.subscribe()`
// don't set read timeout on subscriber - it dies if no 'updates' within that time
s, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(30*time.Second), redis.DialPassword(config.ClusterToken))
if err != nil {
return fmt.Errorf("Failed to reach redis for subconn - %v", err)
}
r.subconn = redis.PubSubConn{s}
r.subconn.Subscribe("portal")
p := pool.Get()
defer p.Close()
p.Do("SET", self, "alive", "EX", ttl)
_, err = p.Do("SADD", "members", self)
if err != nil {
return fmt.Errorf("Failed to add myself to list of members - %v", err)
}
go r.subscribe()
go r.heartbeat()
go r.cleanup()
return nil
}