// Decrements all jobs' score by one
func (q *delayedQueue) TickTock(conn redis.Conn) error {
// if ticktock is running, don't run another one
if q.isTicking {
return nil
}
q.isTicking = true
// there are multiple returns, but whichever return
// gets hit, make sure isTicking is switched off
defer func() {
q.isTicking = false
}()
// get all job ids
//
// NOTE: loading all ids into memory might be an overhead
// if the set is big, ZSCAN might be able to help on this
idToScore := GetQueueScores(q.GetIndexName(), conn)
// decrement each job's score
conn.Send("MULTI")
for id, score := range idToScore {
// only decrement scores > zero so we
// don't ran into negative scores
if score > 0 {
conn.Send("ZINCRBY", q.GetIndexName(), -1, id)
}
}
if _, err := conn.Do("EXEC"); err != nil {
return err
}
return nil
}
// check - lock - check
func (provider *RedisProvider) startSession(sid string, conn redis.Conn, option *Options) (Sessioner, bool, error) {
// 检查随机生成的session是否被占用了
if exist, err := redis.Bool(conn.Do("EXISTS", getRedisKey(sid))); exist || err != nil {
return nil, false, err
}
// 观察sid
conn.Send("WATCH", getRedisKey(sid))
conn.Send("MULTI")
conn.Send("HSET", getRedisKey(sid), "createtime", time.Now().Unix())
if option.Cookielifttime > 0 {
conn.Send("EXPIRE", getRedisKey(sid), option.Cookielifttime)
} else {
conn.Send("EXPIRE", getRedisKey(sid), 30*24*60*60)
}
replay, err := conn.Do("EXEC")
if err != nil {
return nil, false, err
}
if replay == nil {
return nil, false, nil
}
sessioner, err := provider.GetSessioner(sid)
return sessioner, true, err
}
func createShortURL(url string, conn redis.Conn) (Data, error) {
var d Data
count, err := redis.Int(conn.Do("INCR", "global:size"))
if err != nil {
log.Print(err)
return d, err
}
log.Print("Total: ", count)
encodedVar := base62.EncodeInt(int64(count))
key := strings.Join([]string{encodedVar, url}, "||")
conn.Send("MULTI")
conn.Send("HSET", key, "count", 0)
_, err2 := conn.Do("EXEC")
if err2 != nil {
log.Print(err2)
return d, err2
}
d.Original = url
d.HitCount = 0
d.Short = encodedVar
d.FullShort = strings.Join([]string{*base, encodedVar}, "")
return d, err
}
// Checking rate limit 10 request / 1 sec.
func checkRateLimit(con redis.Conn, c *echo.Context) bool {
ip, _, err := net.SplitHostPort(c.Request().RemoteAddr)
if err != nil {
panic(err)
}
//If list of ip address's length is 10 retun false.
current, err := redis.Int(con.Do("LLEN", ip))
if err == nil && current > 10 {
return false
}
exists, err := redis.Bool(con.Do("EXISTS", ip))
if err != nil {
panic(err)
}
if !exists {
con.Send("MULTI")
con.Send("RPUSH", ip, ip)
con.Send("EXPIRE", ip, 1)
_, err := con.Do("EXEC")
if err != nil {
panic(err)
}
} else {
_, err := con.Do("RPUSHX", ip, ip)
if err != nil {
panic(err)
}
}
return true
}
func Get(conn redis.Conn) string {
conn.Send("GET", "fooo")
conn.Flush()
result, _ := redis.String(conn.Receive()) // TODO missing two err catch
return result
}
func Save(obj interface{}, conn redis.Conn) reflect.Type {
conn.Send("GET", "fooo")
conn.Flush()
//result, _ := redis.String(conn.Receive()) // TODO missing two err catch
return reflect.ValueOf(&obj).Type()
}
func writeBatch(list string, conn redis.Conn, values []string) {
conn.Send("MULTI")
for _, value := range values {
value = strings.TrimSpace(value)
if value != "" {
conn.Send("LPUSH", list, value)
}
}
_, err := redis.Values(conn.Do("EXEC"))
if err != nil {
log.Fatalf("Could not LPUSH: %s \"%s\"\n", list, err.Error())
}
}
func recordHitAtGranularity( granularity Granularity, time float64, key string, c redis.Conn ) bool{
size := granularity.size
factor := granularity.factor
ttl := granularity.ttl
name := granularity.name
tsround := getRoundedTimestamp(time, size * factor)
s := []string{"stats",key,name,strconv.FormatFloat(tsround, 'f', 0, 64)}
redis_key := strings.Join(s,":")
ts := strconv.FormatFloat(getFactoredTimestamp (time, factor), 'f', 0, 64)
ttl_str := strconv.FormatFloat(tsround+ttl, 'f', 0, 64)
c.Send("HINCRBY", redis_key, ts, 1)
c.Send("EXPIREAT", redis_key, ttl_str)
return true;
}
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
return func(key outil.Selector, data []outputs.Data) ([]outputs.Data, error) {
var okEvents []outputs.Data
serialized := make([]interface{}, 0, len(data))
okEvents, serialized = serializeEvents(serialized, 0, data)
if len(serialized) == 0 {
return nil, nil
}
data = okEvents[:0]
for i, serializedEvent := range serialized {
eventKey, err := key.Select(okEvents[i].Event)
if err != nil {
logp.Err("Failed to set redis key: %v", err)
continue
}
data = append(data, okEvents[i])
if err := conn.Send(command, eventKey, serializedEvent); err != nil {
logp.Err("Failed to execute %v: %v", command, err)
return okEvents, err
}
}
if err := conn.Flush(); err != nil {
return data, err
}
failed := data[:0]
var lastErr error
for i := range serialized {
_, err := conn.Receive()
if err != nil {
if _, ok := err.(redis.Error); ok {
logp.Err("Failed to %v event to list with %v",
command, err)
failed = append(failed, data[i])
lastErr = err
} else {
logp.Err("Failed to %v multiple events to list with %v",
command, err)
failed = append(failed, data[i:]...)
lastErr = err
break
}
}
}
return failed, lastErr
}
}
func (b *BGPDump) parseBGPCSV(r io.Reader, conn redis.Conn) (int, error) {
day := b.day()
s := bufio.NewScanner(r)
n := 0
var asn string
for s.Scan() {
cols := strings.Split(s.Text(), "|")
if len(cols) < 7 {
return n, ParseError{
Message: "too few columns",
Path: filepath.Base(b.Path()),
LineNum: n,
Line: s.Text(),
}
}
block := cols[5]
if _, ok := asn12654blocks[block]; ok {
asn = "12654"
} else {
asPath := cols[6]
asns := strings.Split(asPath, " ")
asn = asns[len(asns)-1]
if asn == "" {
return n, ParseError{
Message: "no ASPATH data",
Path: filepath.Base(b.Path()),
LineNum: n,
Line: s.Text(),
}
}
}
conn.Send("HSET", fmt.Sprintf("i2a:%s", block), day, asn)
n++
if n%10000 == 0 {
err := conn.Flush()
if err != nil {
return 0, err
}
}
}
conn.Send("SADD", "i2a:imported_dates", day)
err := conn.Flush()
if err != nil {
return 0, err
}
return n, nil
}
func read(list string, conn redis.Conn, count int) {
for i := 0; i < count; i++ {
conn.Send("LPOP", list)
}
conn.Flush()
for i := 0; i < count; i++ {
value, err := redis.String(conn.Receive())
if err != nil {
os.Exit(0)
}
if value != "" {
fmt.Println(value)
}
}
}
func pipelineScore(conn redis.Conn, keyMembers []common.KeyMember) (map[common.KeyMember]Presence, error) {
for _, keyMember := range keyMembers {
if err := conn.Send("ZSCORE", keyMember.Key+insertSuffix, keyMember.Member); err != nil {
return map[common.KeyMember]Presence{}, err
}
if err := conn.Send("ZSCORE", keyMember.Key+deleteSuffix, keyMember.Member); err != nil {
return map[common.KeyMember]Presence{}, err
}
}
if err := conn.Flush(); err != nil {
return map[common.KeyMember]Presence{}, err
}
m := map[common.KeyMember]Presence{}
for i := 0; i < len(keyMembers); i++ {
insertReply, insertErr := conn.Receive()
insertValue, insertErr := redis.Float64(insertReply, insertErr)
deleteReply, deleteErr := conn.Receive()
deleteValue, deleteErr := redis.Float64(deleteReply, deleteErr)
switch {
case insertErr == nil && deleteErr == redis.ErrNil:
m[keyMembers[i]] = Presence{
Present: true,
Inserted: true,
Score: insertValue,
}
case insertErr == redis.ErrNil && deleteErr == nil:
m[keyMembers[i]] = Presence{
Present: true,
Inserted: false,
Score: deleteValue,
}
case insertErr == redis.ErrNil && deleteErr == redis.ErrNil:
m[keyMembers[i]] = Presence{
Present: false,
}
default:
return map[common.KeyMember]Presence{}, fmt.Errorf(
"pipelineScore bad state for %v (%v/%v)",
keyMembers[i],
insertErr,
deleteErr,
)
}
}
return m, nil
}
func (w *worker) start(conn redis.Conn, job *job) error {
work := &work{
Queue: job.Queue,
RunAt: time.Now(),
Payload: job.Payload,
}
buffer, err := json.Marshal(work)
if err != nil {
return err
}
conn.Send("SET", fmt.Sprintf("%sworker:%s", namespace, w), buffer)
logger.Debugf("Processing %s since %s [%v]", work.Queue, work.RunAt, work.Payload.Class)
return w.process.start(conn)
}
func (w *worker) fail(conn redis.Conn, job *job, err error) error {
failure := &failure{
FailedAt: time.Now(),
Payload: job.Payload,
Exception: "Error",
Error: err.Error(),
Worker: w,
Queue: job.Queue,
}
buffer, err := json.Marshal(failure)
if err != nil {
return err
}
conn.Send("RPUSH", fmt.Sprintf("%sfailed", namespace), buffer)
return w.process.fail(conn)
}
func pipelineRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
if limit < 0 {
return map[string][]common.KeyScoreMember{}, fmt.Errorf("negative limit is invalid for offset-based select")
}
for _, key := range keys {
if err := conn.Send(
"ZREVRANGE",
key+insertSuffix,
offset,
offset+limit-1,
"WITHSCORES",
); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
}
if err := conn.Flush(); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
m := make(map[string][]common.KeyScoreMember, len(keys))
for _, key := range keys {
values, err := redis.Values(conn.Receive())
if err != nil {
return map[string][]common.KeyScoreMember{}, err
}
var (
ksm = common.KeyScoreMember{Key: key}
keyScoreMembers = make([]common.KeyScoreMember, 0, len(values))
)
for len(values) > 0 {
if values, err = redis.Scan(values, &ksm.Member, &ksm.Score); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
keyScoreMembers = append(keyScoreMembers, ksm)
}
m[key] = keyScoreMembers
}
return m, nil
}
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
return func(dest []byte, events []common.MapStr) ([]common.MapStr, error) {
var args [2]interface{}
args[0] = dest
serialized := make([]interface{}, 0, len(events))
events, serialized = serializeEvents(serialized, 0, events)
if len(serialized) == 0 {
return nil, nil
}
for _, event := range serialized {
args[1] = event
if err := conn.Send(command, args[:]...); err != nil {
logp.Err("Failed to execute %v: %v", command, err)
return events, err
}
}
if err := conn.Flush(); err != nil {
return events, err
}
failed := events[:0]
var lastErr error
for i := range serialized {
_, err := conn.Receive()
if err != nil {
if _, ok := err.(redis.Error); ok {
logp.Err("Failed to %v event to list (%v) with %v",
command, dest, err)
failed = append(failed, events[i])
lastErr = err
} else {
logp.Err("Failed to %v multiple events to list (%v) with %v",
command, dest, err)
failed = append(failed, events[i:]...)
lastErr = err
break
}
}
}
return failed, lastErr
}
}
func GetRepo(c redis.Conn, repo string) (string, bool, error) {
c.Send("GET", repo)
c.Send("TTL", repo)
if err := c.Flush(); err != nil {
return "", false, nil
}
content, err := redis.String(c.Receive())
if err == redis.ErrNil {
return "", false, nil
}
if err != nil {
return "", false, err
}
if ttl, err := redis.Int(c.Receive()); err == nil && ttl > FRESH_CACHE_LIMIT {
return content, true, nil
}
return content, false, err
}
func (s *SBFFrame) Check(conn redis.Conn, element []byte) bool {
var flag int = 1
hashes := murmur.Hashes(element, s.SliceCount, s.SliceSize)
// check bit val
conn.Send("MULTI")
for index, h := range hashes {
pos := uint32(index)*s.SliceSize + s.StartIndex + h + SBF_FRAME_HEADER_SIZE<<3
conn.Send("GETBIT", s.Refer, pos)
}
if data, err := redis.Ints(conn.Do("EXEC")); err == nil {
for _, f := range data {
flag = flag & f
if flag != 1 {
return false
}
}
return (flag == 1)
} else {
return false
}
}
func (j job) enqueueAt(client redis.Conn, t time.Time, queue string) error {
jsonString := j.encode()
queueKey := fmt.Sprintf("resque:delayed:%d", t.Unix())
timestampsValue := fmt.Sprintf("delayed:%d", t.Unix())
client.Send("MULTI")
client.Send("RPUSH", queueKey, jsonString)
client.Send("SADD", "resque:timestamps:"+jsonString, timestampsValue)
client.Send("ZADD", "resque:delayed_queue_schedule", t.Unix(), t.Unix())
_, err := client.Do("EXEC")
return err
}
func (p *process) close(conn redis.Conn) error {
logger.Infof("%v shutdown", p)
conn.Send("SREM", fmt.Sprintf("%sworkers", namespace), p)
conn.Send("DEL", fmt.Sprintf("%sstat:processed:%s", namespace, p))
conn.Send("DEL", fmt.Sprintf("%sstat:failed:%s", namespace, p))
return conn.Flush()
}
func hashm(c redis.Conn, m map[string]interface{}) {
args := []interface{}{"drugs:1"}
for key, value := range m {
args = append(args, key, value)
}
c.Send("MULTI")
c.Send("HMSET", args...)
c.Send("EXPIRE", "drugs:1", "5")
r, err := c.Do("EXEC")
if err != nil {
panic(err)
}
r, err = c.Do("HGETALL", "drugs:1")
v, err := redis.Values(r, err)
if err != nil {
panic(err)
}
m2 := make(map[string]interface{})
for x := 0; x < len(v); x += 2 {
key := fmt.Sprintf("%s", v[x])
value := v[x+1]
switch t := value.(type) {
case []byte:
m2[key] = string(t)
}
}
//fmt.Printf("%#+v", m2)
}
func kmsgpackm(c redis.Conn, m map[string]interface{}) {
mp := &kmsgpack.Coder{}
b, err := mp.Marshal(m)
if err != nil {
panic(err)
}
c.Send("MULTI")
c.Send("SET", "drugs:1", string(b))
c.Send("EXPIRE", "drugs:1", "5")
r, err := c.Do("EXEC")
if err != nil {
panic(err)
}
r, err = c.Do("GET", "drugs:1")
v, err := redis.Bytes(r, err)
if err != nil {
panic(err)
}
m2 := make(map[string]interface{})
err = mp.Unmarshal(v, &m2)
if err != nil {
panic(err)
}
// fmt.Printf("%#+v", m2)
}
func gobm(c redis.Conn, m map[string]interface{}) {
gob.Register(time.Time{})
var buffer bytes.Buffer
encoder := gob.NewEncoder(&buffer)
err := encoder.Encode(m)
if err != nil {
panic(err)
}
c.Send("MULTI")
c.Send("SET", "drugs:1", buffer.String())
c.Send("EXPIRE", "drugs:1", "5")
r, err := c.Do("EXEC")
if err != nil {
panic(err)
}
r, err = c.Do("GET", "drugs:1")
v, err := redis.Bytes(r, err)
if err != nil {
panic(err)
}
m2 := make(map[string]interface{})
buffer2 := bytes.NewBuffer(v)
decoder := gob.NewDecoder(buffer2)
err = decoder.Decode(&m2)
if err != nil {
panic(err)
}
// fmt.Printf("%#+v", m2)
}
func Zpop(c redis.Conn, key string) (result string, err error) {
defer func() {
//Return connection to normal state on error.
if err != nil {
c.Do("DISCARD")
}
}()
//Loop until transaction is successful.
for {
if _, err := c.Do("WATCH", key); err != nil {
return "", err
}
members, err := redis.Strings(c.Do("ZRANGE", key, 0, 0))
fmt.Println(members)
if err != nil {
return "", err
}
if len(members) != 1 {
return "", redis.ErrNil
}
c.Send("MULTI")
c.Send("ZREM", key, members[0])
queued, err := c.Do("EXEC")
if err != nil {
return "", err
}
if queued != nil {
result = members[0]
break
}
}
return result, nil
}
func newSession(w http.ResponseWriter, userid uint64, username string, appid uint32, rc redis.Conn) (usertoken string, err error) {
if rc == nil {
rc = redisPool.Get()
defer rc.Close()
}
usertoken = ""
usertokenRaw, err := rc.Do("get", fmt.Sprintf("usertokens/%d+%d", userid, appid))
lwutil.CheckError(err, "")
if usertokenRaw != nil {
if usertoken, err := redis.String(usertokenRaw, err); err != nil {
return usertoken, lwutil.NewErr(err)
}
rc.Do("del", fmt.Sprintf("sessions/%s", usertoken))
}
usertoken = lwutil.GenUUID()
session := Session{userid, username, time.Now(), appid}
jsonSession, err := json.Marshal(session)
if err != nil {
return usertoken, lwutil.NewErr(err)
}
rc.Send("setex", fmt.Sprintf("sessions/%s", usertoken), sessionLifeSecond, jsonSession)
rc.Send("setex", fmt.Sprintf("usertokens/%d+%d", userid, appid), sessionLifeSecond, usertoken)
rc.Flush()
for i := 0; i < 2; i++ {
if _, err = rc.Receive(); err != nil {
return usertoken, lwutil.NewErr(err)
}
}
// cookie
http.SetCookie(w, &http.Cookie{Name: "usertoken", Value: usertoken, MaxAge: sessionLifeSecond, Path: "/"})
return usertoken, err
}
func pipelineRevRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
for _, key := range keys {
if err := conn.Send(
"ZREVRANGE",
key+insertSuffix,
offset,
offset+limit-1,
"WITHSCORES",
); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
}
if err := conn.Flush(); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
m := make(map[string][]common.KeyScoreMember, len(keys))
for _, key := range keys {
values, err := redis.Values(conn.Receive())
if err != nil {
return map[string][]common.KeyScoreMember{}, err
}
keyScoreMembers := make([]common.KeyScoreMember, 0, len(values))
for len(values) > 0 {
var member string
var score float64
if values, err = redis.Scan(values, &member, &score); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
keyScoreMembers = append(keyScoreMembers, common.KeyScoreMember{Key: key, Score: score, Member: member})
}
m[key] = keyScoreMembers
}
return m, nil
}
// Import stores the contents of a downloaded BGP dump into a redis server.
// -1 is returned if the dump is alredy imported into redis.
func (b *CIDRReport) Import(conn redis.Conn) (int, error) {
alreadyImported, err := redis.Bool(conn.Do("SISMEMBER", "asd:imported_dates", b.day()))
if err != nil {
return 0, err
}
if alreadyImported {
return -1, nil
}
file, err := os.Open(b.Path())
if err != nil {
return 0, err
}
n := 0
day := b.day()
err = parseReport(file, func(asd *ASDescription) error {
conn.Send("HSET", fmt.Sprintf("asd:%d", asd.ASN), day,
fmt.Sprintf("%s, %s", asd.Description, asd.CountryCode))
n++
if n%10000 == 0 {
err := conn.Flush()
if err != nil {
return err
}
}
return nil
})
conn.Send("SADD", "asd:imported_dates", day)
err = conn.Flush()
if err != nil {
return 0, err
}
return n, nil
}
func RetrievePeople(conn redis.Conn, ids []string) ([]Person, error) {
var people []Person
for _, id := range ids {
conn.Send("HGETALL", fmt.Sprintf("person:%s", id))
}
for i := 0; i < len(ids); i++ {
values, err := redis.Values(conn.Receive())
if err != nil {
return nil, err
}
var person Person
err = redis.ScanStruct(values, &person)
if err != nil {
return nil, err
}
people = append(people, person)
}
return people, nil
}
func insertPairsIntoRedisQueue(pairs []lib.GameSessionPair, redisConnection redis.Conn) {
redisConnection.Send("MULTI")
for _, pair := range pairs {
redisConnection.Send("SADD", lib.UnprocessedSetName, pair.RedisQueueKey())
}
redisConnection.Send("SDIFFSTORE", lib.UnprocessedSetName, lib.UnprocessedSetName, lib.ProcessedSetName)
_, err := redisConnection.Do("EXEC")
if err != nil {
panic(err)
}
}
func NewFrame(conn redis.Conn, header *SBFHeader, id uint16) (*SBFFrame, error) {
key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, header.Refer, id)
frame := new(SBFFrame)
frame.Key = key
frame.Refer = header.Refer
frame.frameDataRange(header, id)
conn.Send("MULTI")
conn.Send("SETBIT", frame.Refer, frame.EndIndex, 0)
conn.Send("SET", frame.Key, 0)
_, err := conn.Do("EXEC")
if err == nil {
return frame, nil
} else {
return nil, err
}
}
