// Returns a client back to the pool. If the pool is full the client is closed
// instead. If the client is already closed (due to connection failure or
// what-have-you) it should not be put back in the pool. The pool will create
// more connections as needed.
func (p *Pool) Put(conn *redis.Client) {
select {
case p.pool <- conn:
default:
conn.Close()
}
}
func (rp *redisPool) FreeClient(c *redis.Client) {
select {
case rp.client_chan <- c:
// redisClient on free list; nothing more to do.
default:
// Free list full, close connection and let it get GC'd
log.Info("Free list is full - closing redis connection")
go c.Close()
}
}
// Removes and calls Close() on all the connections currently in the pool.
// Assuming there are no other connections waiting to be Put back this method
// effectively closes and cleans up the pool.
func (p *Pool) Empty() {
var conn *redis.Client
for {
select {
case conn = <-p.pool:
conn.Close()
default:
return
}
}
}
// Puts the connection back in the pool for the given address. Takes in a
// pointer to an error which can be used to decide whether or not to put the
// connection back. It's a pointer because this method is deferable (like
// CarefullyPut)
func (c *Cluster) putConn(addr string, conn *redis.Client, maybeErr *error) {
c.callCh <- func(c *Cluster) {
pool := c.pools[addr]
if pool == nil {
conn.Close()
return
}
pool.CarefullyPut(conn, maybeErr)
}
}
// A useful helper method which acts as a wrapper around Put. It will only
// actually Put the conn back if potentialErr is not an error or is a
// redis.CmdError. It would be used like the following:
//
// func doSomeThings(p *Pool) error {
// conn, redisErr := p.Get()
// if redisErr != nil {
// return redisErr
// }
// defer p.CarefullyPut(conn, &redisErr)
//
// var i int
// i, redisErr = conn.Cmd("GET", "foo").Int()
// if redisErr != nil {
// return redisErr
// }
//
// redisErr = conn.Cmd("SET", "foo", i * 3).Err
// return redisErr
// }
//
// If we were just using the normal Put we wouldn't be able to defer it because
// we don't want to Put back a connection which is broken. This method takes
// care of doing that check so we can still use the convenient defer
func (p *Pool) CarefullyPut(conn *redis.Client, potentialErr *error) {
if potentialErr != nil && *potentialErr != nil {
// We don't care about command errors, they don't indicate anything
// about the connection integrity
if _, ok := (*potentialErr).(*redis.CmdError); !ok {
conn.Close()
return
}
}
p.Put(conn)
}
// A useful helper method, analagous to the pool package's CarefullyPut method.
// Since we don't want to Put a connection which is having connectivity
// issues, this can be defered inside a function to make sure we only put back a
// connection when we should. It should be used like the following:
//
// func doSomeThings(c *Client) error {
// conn, redisErr := c.GetMaster("bucket0")
// if redisErr != nil {
// return redisErr
// }
// defer c.CarefullyPutMaster("bucket0", conn, &redisErr)
//
// var i int
// i, redisErr = conn.Cmd("GET", "foo").Int()
// if redisErr != nil {
// return redisErr
// }
//
// redisErr = conn.Cmd("SET", "foo", i * 3).Err
// return redisErr
// }
func (c *Client) CarefullyPutMaster(
name string, client *redis.Client, potentialErr *error,
) {
if potentialErr != nil && *potentialErr != nil {
// If the client sent back that it's READONLY then we don't want to keep
// this connection around. Otherwise, we don't care about command errors
if cerr, ok := (*potentialErr).(*redis.CmdError); !ok || cerr.Readonly() {
client.Close()
return
}
}
c.PutMaster(name, client)
}
func (self *OutputConfig) closeRedisClient() (err error) {
var (
client *redis.Client
)
for _, client = range self.clients {
client.Close()
}
self.clients = self.clients[:0]
return
}
// Release pushes back client to pool (if number of available clients in the pool is < MaxClients),
// otherwise closes client
func Release(client *redis.Client) {
if client != nil {
if len(clients) < config.Cfg.DB.MaxClients {
log.Println("Releasing Db Client")
clients <- client
log.Println("Number of idle Db Clients: ", len(clients))
} else {
log.Println("Closing Db Client")
if err := client.Close(); err != nil {
log.Println("Close error: ", err)
}
}
}
}
func NewConnection() (*redis.Client, chan bool) {
var r *redis.Client
// we try to get a connection from the pool a number of times
for retries := 0; retries < 20; retries++ {
r = GetConnectionFromPool()
if r != nil {
break
}
redisLock.Lock()
breakOnCount := *connectionsAllocated < (connectionPoolCount / 4)
redisLock.Unlock()
if breakOnCount {
break
}
time.Sleep(time.Millisecond * 1)
}
if r == nil {
redisLock.Lock()
*connectionsAllocated++
r = redisinterface.SetupRedisConnection()
redisLock.Unlock()
fmt.Print("Connections initialized: ", *connectionsAllocated, "\n")
}
// buffered channel, as the connection
// may time out
returnChannel := make(chan bool, 1)
defer func(rChan chan bool) {
go func() {
select {
case _ = <-rChan:
AddConnectionToPool(r)
case <-time.After(time.Second * 60):
fmt.Print("Connection Timeout")
r.Close()
}
}()
}(returnChannel)
return r, returnChannel
}
func AddConnectionToPool(client *redis.Client) {
var i int
var c *redis.Client
poolSpotFound := false
connectionsLock.Lock()
for i, c = range connections {
if c == nil {
poolSpotFound = true
break
}
}
if poolSpotFound {
connections[i] = client
} else {
fmt.Print("Closing This connection, Pool full\n")
client.Close()
}
connectionsLock.Unlock()
}
func cleanup(conn *redis.Client) {
conn.Cmd("FLUSHDB")
conn.Close()
}
func main() {
var yml_config_file string
y_c, e := os.LookupEnv("REDSHIFT_CONFIG")
if e == false {
yml_config_file = "config.yml"
} else {
yml_config_file = y_c
}
settings, err := yaml.Open(yml_config_file)
s.CheckError(err, true)
debug := settings.Get("debug").(bool)
senders := make([]string, 0)
k := settings.Get("sends")
for m, _ := range k.(map[interface{}]interface{}) {
n := fmt.Sprintf("%sGo", strings.Title(m.(string)))
senders = append(senders, n)
}
// There's probably a better way to do this
redis_is_cluster := settings.Get("redis_is_cluster").(bool)
redis_list := settings.Get("redis_list").(string)
redis_watch_interval := settings.Get("redis_watch_interval").(int)
t := time.Now()
ts := t.Format("Mon Jan 2 15:04:05 -0700 MST 2006")
var rClient *redis.Client
var rCluster *cluster.Cluster
if redis_is_cluster {
if debug {
fmt.Printf("[%s] INFO Starting up in cluster mode\n", ts)
}
rCluster = PullCluster(settings)
defer rCluster.Close()
} else {
redis_db := settings.Get("redis_db").(int)
if debug {
fmt.Printf("[%s] INFO Starting up in single node mode\n", ts)
}
rClient = PullNode(settings)
defer rClient.Close()
rClient.Cmd("SELECT", redis_db)
}
var r *redis.Reply
if redis_is_cluster {
r = rCluster.Cmd("RPOP", redis_list)
} else {
r = rClient.Cmd("RPOP", redis_list)
}
for {
if redis_is_cluster {
r = rCluster.Cmd("RPOP", redis_list)
} else {
r = rClient.Cmd("RPOP", redis_list)
}
t = time.Now()
ts = t.Format("Mon Jan 2 15:04:05 -0700 MST 2006")
// If the response from redis is actual data, process and then pop another response off
// If not, who cares, go through the old switch case
for r.Type == redis.BulkReply {
t = time.Now()
ts = t.Format("Mon Jan 2 15:04:05 -0700 MST 2006")
data, err := r.Bytes()
if debug {
fmt.Printf("[%s] INFO BulkReply reply received. Data length %d\n", ts, len(data))
}
if err != nil {
fmt.Printf("[%s] ERROR Error received: %s\n", ts, err)
} else {
err := Send(ts, data, settings, senders)
if err != nil {
fmt.Printf(err.Error())
}
}
if redis_is_cluster {
r = rCluster.Cmd("RPOP", redis_list)
} else {
r = rClient.Cmd("RPOP", redis_list)
}
}
switch r.Type {
case redis.ErrorReply:
fmt.Printf("[%s] ERROR ErrorReply received: %s\n", ts, r.Err.Error())
case redis.NilReply:
if debug {
fmt.Printf("[%s] INFO NilReply reply received\n", ts)
}
case redis.StatusReply:
if debug {
fmt.Printf("[%s] INFO StatusReply reply received: not processing\n", ts)
}
case redis.MultiReply:
if debug {
fmt.Printf("[%s] INFO MultiReply reply received: not processing\n", ts)
}
case redis.IntegerReply:
if debug {
fmt.Printf("[%s] INFO IntegerReply reply received: not processing\n", ts)
}
default:
if debug {
fmt.Printf("[%s] INFO Unknown reply received: not processing\n", ts)
}
}
time.Sleep(time.Duration(redis_watch_interval) * time.Millisecond)
}
}
