func AllBuckets(client *redis.Client) ([]*Bucket, error) {
var ret []*Bucket
keys, err := client.Cmd("KEYS", "*").Array()
if err != nil {
return nil, err
}
for _, key := range keys {
keyval, err := key.Str()
if err != nil {
continue
}
b, err := GetBucket(client, keyval)
if err != nil {
continue
}
ret = append(ret, b)
}
return ret, nil
}
// NewCustom is like New except you can specify a DialFunc which will be
// used when creating new connections for the pool. The common use-case is to do
// authentication for new connections.
func NewCustom(network, addr string, size int, df DialFunc) (*Pool, error) {
var client *redis.Client
var err error
pool := make([]*redis.Client, 0, size)
for i := 0; i < size; i++ {
client, err = df(network, addr)
if err != nil {
for _, client = range pool {
client.Close()
}
pool = pool[0:]
break
}
pool = append(pool, client)
}
p := Pool{
Network: network,
Addr: addr,
pool: make(chan *redis.Client, len(pool)),
df: df,
}
for i := range pool {
p.pool <- pool[i]
}
return &p, err
}
func scanHelper(redisClient *redis.Client, pattern string, retCh chan string) error {
cursor := "0"
for {
r := redisClient.Cmd("SCAN", cursor, "MATCH", pattern)
if r.Err != nil {
return r.Err
}
elems, err := r.Array()
if err != nil {
return err
}
results, err := elems[1].List()
if err != nil {
return err
}
for i := range results {
retCh <- results[i]
}
if cursor, err = elems[0].Str(); err != nil {
return err
} else if cursor == "0" {
return nil
}
}
}
func luaHelper(
c *redis.Client, cmd string, numKeys int, args ...interface{},
) *redis.Resp {
cmd = strings.ToUpper(cmd)
l, ok := luaScripts[cmd]
if !ok {
return redis.NewResp(fmt.Errorf("unknown lua script: %s", cmd))
}
realArgs := make([]interface{}, 0, len(args)+2)
realArgs = append(realArgs, l.hash, numKeys)
realArgs = append(realArgs, args...)
r, notLoaded := luaEvalSha(c, realArgs)
if !notLoaded {
return r
}
if err := c.Cmd("SCRIPT", "LOAD", l.script).Err; err != nil {
return r
}
r, _ = luaEvalSha(c, realArgs)
return r
}
// Performs and EVALSHA with the given args, returning the reply and whether or
// not that reply is due to the script for that sha not being loaded yet
func luaEvalSha(c *redis.Client, args []interface{}) (*redis.Resp, bool) {
r := c.Cmd("EVALSHA", args...)
if r.Err != nil {
if r.IsType(redis.AppErr) {
return r, strings.HasPrefix(r.Err.Error(), "NOSCRIPT")
}
}
return r, false
}
// Put returns a client back to the pool. If the pool is full the client is
// closed instead. If the client is already closed (due to connection failure or
// what-have-you) it will not be put back in the pool
func (p *Pool) Put(conn *redis.Client) {
if conn.LastCritical == nil {
select {
case p.pool <- conn:
default:
conn.Close()
}
}
}
// Pings redis to check whether the connection works. A connectTo...() methods needs
// to be called before.
func (t *Transport) pingRedis(client *redis.Client) error {
resp := client.Cmd("PING")
if resp.Err != nil {
msg := fmt.Sprintf("Cannot connect to Redis host '%s': %s", t.Address, resp.Err)
log.Fatal(msg)
return resp.Err
}
return nil
}
// Put putss the connection back in its pool. To be used alongside any of the
// Get* methods once use of the redis.Client is done
func (c *Cluster) Put(conn *redis.Client) {
c.callCh <- func(c *Cluster) {
p := c.pools[conn.Addr]
if p == nil {
conn.Close()
return
}
p.Put(conn)
}
}
// Empty removes and calls Close() on all the connections currently in the pool.
// Assuming there are no other connections waiting to be Put back this method
// effectively closes and cleans up the pool.
func (p *Pool) Empty() {
var conn *redis.Client
for {
select {
case conn = <-p.pool:
conn.Close()
default:
return
}
}
}
func migrateTowns(townsDb *sql.DB, redisCli *redis.Client) {
var townsCount int
err := townsDb.QueryRow(`SELECT COUNT(*) FROM towns`).Scan(&townsCount)
if err != nil {
log.Fatalf("migrate: towns: %v\n", err)
}
rows, err := townsDb.Query(`SELECT id, name, name_tr, region_id,
regional_center, latitude,
longitude, zoom FROM towns`)
if err != nil {
log.Fatalf("migrate: towns: %v\n", err)
}
currentTownIdx := 1
for rows.Next() {
town := new(Town)
var regionId uint32 = 0
err = rows.Scan(&town.Id, &town.Name, &town.NameTr,
®ionId, &town.RegionalCenter,
&town.Latitude, &town.Longitude, &town.Zoom)
if err != nil {
log.Fatal(err)
}
if regionId != 0 {
town.RegionId = new(uint32)
*town.RegionId = regionId
}
jsonData, err := json.Marshal(town)
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SET", "town:"+strconv.FormatUint(uint64(town.Id), 10), string(jsonData)).Err
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("GEOADD", "towns", town.Longitude, town.Latitude, town.Id).Err
if err != nil {
log.Fatal(err)
}
currentTownIdx++
if currentTownIdx%500 == 0 {
log.Printf("[%d/%d] Towns processed\n", currentTownIdx, townsCount)
}
}
log.Printf("[%d/%d] Towns processed\n", townsCount, townsCount)
}
func migrateBanks(banksDb *sql.DB, redisCli *redis.Client) {
var banksCount int
err := banksDb.QueryRow(`SELECT COUNT(*) FROM banks`).Scan(&banksCount)
if err != nil {
log.Fatalf("migrate: banks: %v", err)
}
rows, err := banksDb.Query(`SELECT id, name, name_tr, name_tr_alt, town,
licence, rating, tel FROM banks`)
if err != nil {
log.Fatalf("migrate: banks: %v", err)
}
currentBankIdx := 1
for rows.Next() {
bank := new(Bank)
var nameTr sql.NullString
err = rows.Scan(&bank.Id, &bank.Name, &nameTr, &bank.NameTrAlt,
&bank.Town, &bank.Licence, &bank.Rating, &bank.Tel)
if err != nil {
log.Fatal(err)
}
if nameTr.Valid {
bank.NameTr = nameTr.String
} else {
bank.NameTr = ""
}
jsonData, err := json.Marshal(bank)
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SET", "bank:"+strconv.FormatUint(uint64(bank.Id), 10), string(jsonData)).Err
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SADD", "banks", bank.Id).Err
if err != nil {
log.Fatal(err)
}
currentBankIdx++
if currentBankIdx%100 == 0 {
log.Printf("[%d/%d] Banks processed\n", currentBankIdx, banksCount)
}
}
log.Printf("[%d/%d] Banks processed\n", banksCount, banksCount)
}
func CreateBucket(client *redis.Client, name string, length int) error {
exists, err := client.Cmd("EXISTS", name).Int()
if err != nil {
return err
}
if exists == 1 {
return errors.New("Bucket already exists")
} else {
return client.Cmd("SET", name, length, "NX").Err
}
}
func GetBucket(client *redis.Client, name string) (*Bucket, error) {
length, err := client.Cmd("GET", name).Str()
if err != nil {
return nil, err
}
b := new(Bucket)
*b = Bucket{
client,
name,
atoi(length),
}
return b, nil
}
func migrateRegions(townsDb *sql.DB, redisCli *redis.Client) {
var regionsCount int
err := townsDb.QueryRow(`SELECT COUNT(*) FROM regions`).Scan(®ionsCount)
if err != nil {
log.Fatalf("migrate: regions: %v\n", err)
}
rows, err := townsDb.Query(`SELECT id, name, name_tr,
latitude, longitude, zoom FROM regions`)
if err != nil {
log.Fatalf("migrate: regions: %v\n", err)
}
currentRegionIdx := 1
for rows.Next() {
region := new(Region)
err = rows.Scan(®ion.Id, ®ion.Name, ®ion.NameTr,
®ion.Latitude, ®ion.Longitude, ®ion.Zoom)
if err != nil {
log.Fatal(err)
}
jsonData, err := json.Marshal(region)
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SET", "region:"+strconv.FormatUint(uint64(region.Id), 10), string(jsonData)).Err
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("GEOADD", "regions", region.Longitude, region.Latitude, region.Id).Err
if err != nil {
log.Fatal(err)
}
currentRegionIdx++
if currentRegionIdx%500 == 0 {
log.Printf("[%d/%d] Regions processed\n", currentRegionIdx, regionsCount)
}
}
log.Printf("[%d/%d] Regions processed\n", regionsCount, regionsCount)
}
func preloadRedisScriptSrc(redisCli *redis.Client, srcFilePath string) string {
context := "preloadRedisScripts: " + srcFilePath
buf := bytes.NewBuffer(nil)
file, err := os.Open(srcFilePath)
if err != nil {
log.Fatalf("%s => %v\n", context, err)
}
io.Copy(buf, file)
file.Close()
src := string(buf.Bytes())
response := redisCli.Cmd("SCRIPT", "LOAD", src)
if response.Err != nil {
log.Fatalf("%s => %v\n", context, response.Err)
}
scriptSha, err := response.Str()
if err != nil {
log.Fatalf("%s => %v\n", context, err)
}
return scriptSha
}
func allElos(conn *redis.Client) map[string]float64 {
result := conn.Cmd("HGETALL", "elo")
if result.Err != nil {
return nil
}
elos, err := result.Map()
if err != nil {
return nil
}
retval := make(map[string]float64)
for player, elo := range elos {
f, err := strconv.ParseFloat(elo, 64)
if err != nil {
conn.Cmd("HDEL", "elo", player)
} else {
retval[player] = f
}
}
return retval
}
func processSubImages(conn *redis.Client, typeName string, key string) {
var typeKey = "image:" + typeName + ":" + key
keys, err := conn.Cmd("HKEYS", typeKey).List()
if err != nil {
log.Fatal("fetch all sub hash keys error")
}
for _, name := range keys {
data, _ := conn.Cmd("HGET", typeKey, name).Bytes()
fid := saveFileToFS(data)
conn.Cmd("HSET", typeKey, name, fid)
}
}
func (c *Cluster) clientCmd(
client *redis.Client, cmd string, args []interface{}, ask bool,
tried map[string]bool, haveReset bool,
) *redis.Resp {
var err error
var r *redis.Resp
defer c.Put(client)
if ask {
r = client.Cmd("ASKING")
ask = false
}
// If we asked and got an error, we continue on with error handling as we
// would normally do. If we didn't ask or the ask succeeded we do the
// command normally, and see how that goes
if r == nil || r.Err == nil {
r = client.Cmd(cmd, args...)
}
if err = r.Err; err == nil {
return r
}
// At this point we have some kind of error we have to deal with. The above
// code is what will be run 99% of the time and is pretty streamlined,
// everything after this point is allowed to be hairy and gross
haveTriedBefore := haveTried(tried, client.Addr)
tried = justTried(tried, client.Addr)
// Deal with network error
if r.IsType(redis.IOErr) {
// If this is the first time trying this node, try it again
if !haveTriedBefore {
if client, try2err := c.getConn("", client.Addr); try2err == nil {
return c.clientCmd(client, cmd, args, false, tried, haveReset)
}
}
// Otherwise try calling Reset() and getting a random client
if !haveReset {
if resetErr := c.Reset(); resetErr != nil {
return errorRespf("Could not get cluster info: %s", resetErr)
}
client, getErr := c.getConn("", "")
if getErr != nil {
return errorResp(getErr)
}
return c.clientCmd(client, cmd, args, false, tried, true)
}
// Otherwise give up and return the most recent error
return r
}
// Here we deal with application errors that are either MOVED or ASK
msg := err.Error()
moved := strings.HasPrefix(msg, "MOVED ")
ask = strings.HasPrefix(msg, "ASK ")
if moved || ask {
_, addr := redirectInfo(msg)
c.callCh <- func(c *Cluster) {
select {
case c.MissCh <- struct{}{}:
default:
}
}
// If we've already called Reset and we're getting MOVED again than the
// cluster is having problems, likely telling us to try a node which is
// not reachable. Not much which can be done at this point
if haveReset {
return errorRespf("Cluster doesn't make sense, %s might be gone", addr)
}
if resetErr := c.Reset(); resetErr != nil {
return errorRespf("Could not get cluster info: %s", resetErr)
}
haveReset = true
// At this point addr is whatever redis told us it should be. However,
// if we can't get a connection to it we'll never actually mark it as
// tried, resulting in an infinite loop. Here we mark it as tried
// regardless of if it actually was or not
tried = justTried(tried, addr)
client, getErr := c.getConn("", addr)
if getErr != nil {
return errorResp(getErr)
}
return c.clientCmd(client, cmd, args, ask, tried, haveReset)
}
// It's a normal application error (like WRONG KEY TYPE or whatever), return
// that to the client
return r
}
func migrateCashpoints(cpDb *sql.DB, redisCli *redis.Client) {
var cashpointsCount int
err := cpDb.QueryRow(`SELECT COUNT(*) FROM cashpoints`).Scan(&cashpointsCount)
if err != nil {
log.Fatalf("migrate: cashpoints: %v\n", err)
}
rows, err := cpDb.Query(`SELECT id, type, bank_id, town_id,
longitude, latitude,
address, address_comment,
metro_name, free_access,
main_office, without_weekend,
round_the_clock, works_as_shop,
schedule_general, tel, additional,
rub, usd, eur, cash_in FROM cashpoints`)
if err != nil {
log.Fatalf("migrate: cashpoints: %v\n", err)
}
currentCashpointIndex := 1
for rows.Next() {
cp := new(CashPoint)
err = rows.Scan(&cp.Id, &cp.Type, &cp.BankId, &cp.TownId,
&cp.Longitude, &cp.Latitude,
&cp.Address, &cp.AddressComment,
&cp.MetroName, &cp.FreeAccess,
&cp.MainOffice, &cp.WithoutWeekend,
&cp.RoundTheClock, &cp.WorksAsShop,
&cp.Schedule, &cp.Tel, &cp.Additional,
&cp.Rub, &cp.Usd, &cp.Eur, &cp.CashIn)
if err != nil {
log.Fatal(err)
}
cashpointIdStr := strconv.FormatUint(uint64(cp.Id), 10)
townIdStr := strconv.FormatUint(uint64(cp.TownId), 10)
bankIdStr := strconv.FormatUint(uint64(cp.BankId), 10)
jsonData, err := json.Marshal(cp)
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SET", "cp:"+cashpointIdStr, string(jsonData)).Err
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SET", "cp:"+cashpointIdStr+":version", 1).Err
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("GEOADD", "cashpoints", cp.Longitude, cp.Latitude, cp.Id).Err
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SADD", "town:"+townIdStr+":cp", cp.Id).Err
if err != nil {
log.Fatal(err)
}
err = redisCli.Cmd("SADD", "bank:"+bankIdStr+":cp", cp.Id).Err
if err != nil {
log.Fatal(err)
}
currentCashpointIndex++
if currentCashpointIndex%500 == 0 {
log.Printf("[%d/%d] Cashpoints processed\n", currentCashpointIndex, cashpointsCount)
}
}
log.Printf("[%d/%d] Cashpoints processed\n", cashpointsCount, cashpointsCount)
}
func cleanupRedis(client *redis.Client) {
if client != nil {
defer client.Close()
}
}