Exemple #1
0
// verifyMaster verifies that the decided master node has fully transitioned
func verifyMaster(addr, pass string) error {
	// connect to redis in order to verify its state
	r, err := redis.DialURL("redis://"+addr, redis.DialConnectTimeout(config.TimeoutNotReady), redis.DialPassword(pass))
	if err != nil {
		return fmt.Errorf("Failed to reach redis at: '%v'", addr)
	}

	// give redis some time to transition
	timeout := time.After(config.TimeoutMasterWait)

	for {
		select {
		case <-timeout:
			return fmt.Errorf("Timed out waiting for redis to transition to master")
		default:
			// retrieve the redis node's role
			info, err := redis.Bytes(r.Do("INFO", "replication"))
			if err != nil {
				return fmt.Errorf("Failed to get INFO - %v", err)
			}

			// check if node is master
			if strings.Contains(string(info), "role:master") {
				return nil
			}
		}
	}

	// cleanup after ourselves
	r.Close()

	return nil
}
Exemple #2
0
// updateMaster gets the address of the master node from sentinel
func updateMaster() error {
	config.Log.Debug("Contacting sentinel for address of master...")

	// connect to sentinel in order to query for the master address
	r, err := redis.DialURL("redis://"+config.SentinelAddress, redis.DialConnectTimeout(config.TimeoutNotReady), redis.DialReadTimeout(config.TimeoutSentinelPoll), redis.DialPassword(config.SentinelPassword))
	if err != nil {
		return fmt.Errorf("Failed to reach sentinel - %v", err)
	}

	// retrieve the master redis address
	addr, err := redis.Strings(r.Do("SENTINEL", "get-master-addr-by-name", config.MonitorName))
	if err != nil {
		return fmt.Errorf("Failed to get-master-addr-by-name - %v", err)
	}

	// cleanup after ourselves
	r.Close()

	// construct a useable address from sentinel's response
	masterAddr = fmt.Sprintf("%v:%v", addr[0], addr[1])
	config.Log.Debug("Master address: '%v'", masterAddr)

	// wait for redis to transition to master
	if err = verifyMaster(masterAddr, config.SentinelPassword); err != nil {
		return fmt.Errorf("Could not verify master - %v", err)
	}

	return nil
}
Exemple #3
0
// Iterate through `machines`, trying to connect to each in turn.
// Returns the first successful connection or the last error encountered.
// Assumes that `machines` is non-empty.
func tryConnect(machines []string, password string) (redis.Conn, error) {
	var err error
	for _, address := range machines {
		var conn redis.Conn
		network := "tcp"
		if _, err = os.Stat(address); err == nil {
			network = "unix"
		}
		log.Debug(fmt.Sprintf("Trying to connect to redis node %s", address))

		dialops := []redis.DialOption{
			redis.DialConnectTimeout(time.Second),
			redis.DialReadTimeout(time.Second),
			redis.DialWriteTimeout(time.Second),
		}

		if password != "" {
			dialops = append(dialops, redis.DialPassword(password))
		}

		conn, err = redis.Dial(network, address, dialops...)

		if err != nil {
			continue
		}
		return conn, nil
	}
	return nil, err
}
Exemple #4
0
// CreatePool creates a redis connection pool
func CreatePool(
	host, password, network string,
	maxConn int,
	idleTimeout, connTimeout time.Duration,
) *rd.Pool {
	return &rd.Pool{
		MaxIdle:     maxConn,
		IdleTimeout: idleTimeout,
		Dial: func() (rd.Conn, error) {
			c, err := rd.Dial(network, host,
				rd.DialConnectTimeout(connTimeout),
				rd.DialReadTimeout(connTimeout),
				rd.DialWriteTimeout(connTimeout))
			if err != nil {
				return nil, err
			}
			if password != "" {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					return nil, err
				}
			}
			return c, err
		},
	}
}
Exemple #5
0
func TestMain(m *testing.M) {
	// clean test dir
	os.RemoveAll("/tmp/clusterTest")

	// initialize backend if redis-server found
	initialize()

	conn, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(30*time.Second), redis.DialPassword(config.ClusterToken))
	if err != nil {
		return
	}
	hostname, _ := os.Hostname()
	self := fmt.Sprintf("%v:%v", hostname, config.ApiPort)
	defer conn.Do("SREM", "members", self)
	defer conn.Close()

	rtn := m.Run()

	// clean test dir
	os.RemoveAll("/tmp/clusterTest")
	// just in case, ensure clean members
	conn.Do("SREM", "members", self)
	conn.Close()

	os.Exit(rtn)
}
Exemple #6
0
// ping executes a PING command against addr until timeout occurs.
func (p *Process) ping(addr string, timeout time.Duration) error {
	// Default to local process if addr not specified.
	if addr == "" {
		addr = fmt.Sprintf("localhost:%s", p.Port)
	}

	logger := p.Logger.New("fn", "ping", "addr", addr, "timeout", timeout)
	logger.Info("sending")

	timer := time.NewTimer(timeout)
	defer timer.Stop()

	ticker := time.NewTicker(checkInterval)
	defer ticker.Stop()

	for {
		// Attempt to ping the server.
		if ok := func() bool {
			logger.Info("sending PING")

			conn, err := redis.Dial("tcp", addr,
				redis.DialPassword(p.Password),
				redis.DialConnectTimeout(timeout),
				redis.DialReadTimeout(timeout),
				redis.DialWriteTimeout(timeout),
			)
			if err != nil {
				logger.Error("conn error", "err", err)
				return false
			}
			defer conn.Close()

			if _, err := conn.Do("PING"); err != nil {
				logger.Error("error getting upstream status", "err", err)
				return false
			}

			logger.Info("PONG received")
			return true
		}(); ok {
			return nil
		}

		select {
		case <-timer.C:
			logger.Info("timeout")
			return ErrTimeout
		case <-ticker.C:
		}
	}
}
Exemple #7
0
func (l *RedisInput) Run(output chan common.MapStr) error {
	logp.Debug("redisinput", "Running Redis Input")
	var keysScript = redis.NewScript(1, `return redis.call('KEYS', KEYS[1])`)

	go func() {
		redisURL := fmt.Sprintf("redis://%s:%d/%d", l.Host, l.Port, l.DB)
		dialConnectTimeout := redis.DialConnectTimeout(3 * time.Second)
		dialReadTimeout := redis.DialReadTimeout(10 * time.Second)
		var backOffCount = 0
		var backOffDuration time.Duration = 5 * time.Second
		for {
			logp.Debug("redisinput", "Connecting to: %s", redisURL)
			server, err := redis.DialURL(redisURL, dialConnectTimeout, dialReadTimeout)
			if err != nil {
				logp.Err("couldn't start listening: " + err.Error())
				return
			}
			logp.Debug("redisinput", "Connected to Redis Server")

			reply, err := keysScript.Do(server, "*")
			if err != nil {
				logp.Err("An error occured while executing KEYS command: %s\n", err)
				return
			}

			keys, err := redis.Strings(reply, err)
			if err != nil {
				logp.Err("An error occured while converting reply to String: %s\n", err)
				return
			}

			for _, key := range keys {
				logp.Debug("redisinput", "key is %s", key)
				lineCount, err := l.handleConn(server, output, key)
				if err == nil {
					logp.Debug("redisinput", "Read %v events", lineCount)
					backOffCount = 0
					backOffDuration = time.Duration(backOffCount) * time.Second
					time.Sleep(backOffDuration)
				} else {
					backOffCount++
					backOffDuration = time.Duration(backOffCount) * time.Second
					time.Sleep(backOffDuration)
				}
			}
			defer server.Close()
		}
	}()
	return nil
}
Exemple #8
0
// RedisInfo executes an INFO command against a Redis server and returns the results.
func (p *Process) RedisInfo(addr string, timeout time.Duration) (*RedisInfo, error) {
	// Default to local process if addr not specified.
	if addr == "" {
		addr = fmt.Sprintf("localhost:%s", p.Port)
	}

	logger := p.Logger.New("fn", "replInfo", "addr", addr)
	logger.Info("sending INFO")

	// Connect to the redis server.
	conn, err := redis.Dial("tcp", addr,
		redis.DialPassword(p.Password),
		redis.DialConnectTimeout(timeout),
		redis.DialReadTimeout(timeout),
		redis.DialWriteTimeout(timeout),
	)
	if err != nil {
		logger.Info("dial error", "err", err)
		return nil, err
	}
	defer conn.Close()

	// Execute INFO command.
	reply, err := conn.Do("INFO")
	if err != nil {
		logger.Error("info error", "err", err)
		return nil, err
	}

	buf, ok := reply.([]byte)
	if !ok {
		logger.Error("info reply type error", "type", fmt.Sprintf("%T", buf))
		return nil, fmt.Errorf("unexpected INFO reply format: %T", buf)
	}

	// Parse the bulk string reply info a typed object.
	info, err := ParseRedisInfo(string(buf))
	if err != nil {
		logger.Error("parse info error", "err", err)
		return nil, fmt.Errorf("parse info: %s", err)
	}

	logger.Info("INFO received")
	return info, nil
}
Exemple #9
0
// creates a redis connection pool to use
func (r Redis) newPool(server, password string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     3,
		IdleTimeout: 5 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.DialURL(server, redis.DialConnectTimeout(30*time.Second),
				redis.DialWriteTimeout(10*time.Second), redis.DialPassword(password))

			if err != nil {
				return nil, fmt.Errorf("Failed to reach redis - %v", err)
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
Exemple #10
0
func newPool() *redis.Pool {
	return &redis.Pool{
		MaxIdle:     20,
		MaxActive:   1024, // max number of connections
		IdleTimeout: 5 * time.Second,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("unix", "/tmp/redis.sock",
				redis.DialConnectTimeout(15*time.Second))
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
Exemple #11
0
// Connect establishes a connection to Redis which provides the
// pub/sub implementation.
func (b *redisBroker) Connect() error {
	if b.pool != nil {
		return nil
	}

	var addr string

	if len(b.opts.Addrs) == 0 || b.opts.Addrs[0] == "" {
		addr = "redis://127.0.0.1:6379"
	} else {
		addr = b.opts.Addrs[0]

		if !strings.HasPrefix("redis://", addr) {
			addr = "redis://" + addr
		}
	}

	b.addr = addr

	b.pool = &redis.Pool{
		MaxIdle:     b.bopts.maxIdle,
		MaxActive:   b.bopts.maxActive,
		IdleTimeout: b.bopts.idleTimeout,
		Dial: func() (redis.Conn, error) {
			return redis.DialURL(
				b.addr,
				redis.DialConnectTimeout(b.bopts.connectTimeout),
				redis.DialReadTimeout(b.bopts.readTimeout),
				redis.DialWriteTimeout(b.bopts.writeTimeout),
			)
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}

	return nil
}
Exemple #12
0
func newPool(conf *Configure, host string, dbnum int) *RedisPool {
	pool := &redis.Pool{
		MaxIdle:     conf.Redis.MaxIdle,
		MaxActive:   conf.Redis.MaxActive,
		IdleTimeout: time.Duration(conf.Redis.IdleTimeout) * time.Second,
		Dial: func() (redis.Conn, error) {
			opt_timeout := redis.DialConnectTimeout(time.Duration(conf.Redis.ConnTimeout) * time.Second)
			opt_selectdb := redis.DialDatabase(dbnum)
			c, err := redis.Dial("tcp", host, opt_timeout, opt_selectdb)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
	p := &RedisPool{p: pool, db: dbnum, host: host}
	log.Printf("[redis_pool]on host:%s:%d, create redis pool success.\n", host, dbnum)
	return p
}
Exemple #13
0
func (m MRedis) Sub(ch string) {

	c, err := redis.Dial("tcp", "192.168.176.3:6379", redis.DialConnectTimeout(10*time.Second))
	if err != nil {
		log.Fatal(err)
	}
	defer c.Close()

	var wg sync.WaitGroup
	wg.Add(1)
	psc := redis.PubSubConn{Conn: c}
	if err = psc.Subscribe(ch); err != nil {
		log.Fatal(err)
	}
	go func() {
		defer wg.Done()
		for {
			switch n := psc.Receive().(type) {
			case redis.Message:
				fmt.Printf("Message: %s %s\n", n.Channel, n.Data)
			case redis.PMessage:
				fmt.Printf("PMessage: %s %s %s\n", n.Pattern, n.Channel, n.Data)
			case redis.Subscription:
				fmt.Printf("Subscription: %s %s %d\n", n.Kind, n.Channel, n.Count)
				if n.Count == 0 {
					return
				}
			case error:
				fmt.Printf("error: %v\n", n)
				return
			}
		}
	}()
	wg.Wait()

}
Exemple #14
0
func (this *Conf) Dail() (conn redis.Conn, err error) {
	option := redis.DialConnectTimeout(time.Duration(this.Timeout) * time.Second)
	conn, err = redis.Dial(this.NetWork, this.Address, option)
	return
}
Exemple #15
0
// GetCerts gets a list of certs from the database, or another cluster member.
func (r *Redis) GetCerts() ([]core.CertBundle, error) {
	if database.CentralStore {
		return database.GetCerts()
	}

	conn := pool.Get()
	defer conn.Close()

	// get known members(other than me) to 'poll' for certs
	members, _ := redis.Strings(conn.Do("SMEMBERS", "members"))
	if len(members) == 0 {
		// should only happen on new cluster
		// assume i'm ok to be master so don't reset imported certs
		config.Log.Trace("[cluster] - Assuming OK to be master, using certs from my database...")
		return common.GetCerts()
	}
	for i := range members {
		if members[i] == self {
			// if i'm in the list of members, new requests should have failed while `waitForMembers`ing
			config.Log.Trace("[cluster] - Assuming I was in sync, using certs from my database...")
			return common.GetCerts()
		}
	}

	c, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(15*time.Second), redis.DialPassword(config.ClusterToken))
	if err != nil {
		return nil, fmt.Errorf("Failed to reach redis for certs subscriber - %v", err)
	}
	defer c.Close()

	message := make(chan interface{})
	subconn := redis.PubSubConn{c}

	// subscribe to channel that certs will be published on
	if err := subconn.Subscribe("certs"); err != nil {
		return nil, fmt.Errorf("Failed to reach redis for certs subscriber - %v", err)
	}
	defer subconn.Close()

	// listen always
	go func() {
		for {
			message <- subconn.Receive()
		}
	}()

	// todo: maybe use ttl?
	// timeout is how long to wait for the listed members to come back online
	timeout := time.After(time.Duration(20) * time.Second)

	// loop attempts for timeout, allows last dead members to start back up
	for {
		select {
		case <-timeout:
			return nil, fmt.Errorf("Timed out waiting for certs from %v", strings.Join(members, ", "))
		default:
			// request certs from each member until successful
			for _, member := range members {
				// memberTimeout is how long to wait for a member to respond with list of certs
				memberTimeout := time.After(3 * time.Second)

				// ask a member for its certs
				config.Log.Trace("[cluster] - Attempting to request certs from %v...", member)
				_, err := conn.Do("PUBLISH", "portal", fmt.Sprintf("get-certs %s", member))
				if err != nil {
					return nil, err
				}

				// wait for member to respond
				for {
					select {
					case <-memberTimeout:
						config.Log.Debug("[cluster] - Timed out waiting for certs from %v", member)
						goto nextCertMember
					case msg := <-message:
						switch v := msg.(type) {
						case redis.Message:
							config.Log.Trace("[cluster] - Received message on 'certs' channel")
							var certs []core.CertBundle
							err = parseBody(v.Data, &certs)
							if err != nil {
								return nil, fmt.Errorf("Failed to marshal certs - %v", err.Error())
							}
							config.Log.Trace("[cluster] - Certs from cluster: %#v\n", certs)
							return certs, nil
						case error:
							return nil, fmt.Errorf("Subscriber failed to receive certs - %v", v.Error())
						}
					}
				}
			nextCertMember:
			}
		}
	}
}
Exemple #16
0
func (r *Redis) Init() error {
	hostname, _ := os.Hostname()
	self = fmt.Sprintf("%v:%v", hostname, config.ApiPort)
	pool = r.newPool(config.ClusterConnection, config.ClusterToken)

	// get services
	services, err := r.GetServices()
	if err != nil {
		return fmt.Errorf("Failed to get services - %v", err)
	}
	// write services
	if services != nil {
		config.Log.Trace("[cluster] - Setting services...")
		err = common.SetServices(services)
		if err != nil {
			return fmt.Errorf("Failed to set services - %v", err)
		}
	}

	// get routes
	routes, err := r.GetRoutes()
	if err != nil {
		return fmt.Errorf("Failed to get routes - %v", err)
	}
	// write routes
	if routes != nil {
		config.Log.Trace("[cluster] - Setting routes...")
		err = common.SetRoutes(routes)
		if err != nil {
			return fmt.Errorf("Failed to set routes - %v", err)
		}
	}

	// get certs
	certs, err := r.GetCerts()
	if err != nil {
		return fmt.Errorf("Failed to get certs - %v", err)
	}
	// write certs
	if certs != nil {
		config.Log.Trace("[cluster] - Setting certs...")
		err = common.SetCerts(certs)
		if err != nil {
			return fmt.Errorf("Failed to set certs - %v", err)
		}
	}

	// get vips
	vips, err := r.GetVips()
	if err != nil {
		return fmt.Errorf("Failed to get vips - %v", err)
	}
	// write vips
	if vips != nil {
		config.Log.Trace("[cluster] - Setting vips...")
		err = common.SetVips(vips)
		if err != nil {
			return fmt.Errorf("Failed to set vips - %v", err)
		}
	}

	// note: keep subconn connection initialization out here or sleep after `go r.subscribe()`
	// don't set read timeout on subscriber - it dies if no 'updates' within that time
	s, err := redis.DialURL(config.ClusterConnection, redis.DialConnectTimeout(30*time.Second), redis.DialPassword(config.ClusterToken))
	if err != nil {
		return fmt.Errorf("Failed to reach redis for subconn - %v", err)
	}

	r.subconn = redis.PubSubConn{s}
	r.subconn.Subscribe("portal")

	p := pool.Get()
	defer p.Close()

	p.Do("SET", self, "alive", "EX", ttl)
	_, err = p.Do("SADD", "members", self)
	if err != nil {
		return fmt.Errorf("Failed to add myself to list of members - %v", err)
	}

	go r.subscribe()
	go r.heartbeat()
	go r.cleanup()

	return nil
}