func (self *redis_gateway) execute(c redis.Conn, commands [][]string) error {
switch len(commands) {
case 0:
return nil
case 1:
e := self.redis_do(c, commands[0])
if nil != e {
return errors.New("execute `" + strings.Join(commands[0], " ") + "` failed, " + e.Error())
}
return nil
default:
for _, command := range commands {
e := self.redis_send(c, command)
if nil != e {
return errors.New("execute `" + strings.Join(command, " ") + "` failed, " + e.Error())
}
}
e := c.Flush()
if nil != e {
return e
}
for i := 0; i < len(commands); i++ {
_, e = c.Receive()
if nil != e {
return errors.New("execute `" + strings.Join(commands[i], " ") + "` failed, " + e.Error())
}
}
return nil
}
}
func pipelineDelete(conn redis.Conn, keyScoreMembers []common.KeyScoreMember, maxSize int) error {
for _, keyScoreMember := range keyScoreMembers {
if err := deleteScript.Send(
conn,
keyScoreMember.Key,
keyScoreMember.Score,
keyScoreMember.Member,
maxSize,
); err != nil {
return err
}
}
if err := conn.Flush(); err != nil {
return err
}
for _ = range keyScoreMembers {
// TODO actually count deletes
if _, err := conn.Receive(); err != nil {
return err
}
}
return nil
}
func pipelineInsert(conn redis.Conn, keyScoreMembers []common.KeyScoreMember, maxSize int) error {
for _, tuple := range keyScoreMembers {
if err := insertScript.Send(
conn,
tuple.Key,
tuple.Score,
tuple.Member,
maxSize,
); err != nil {
return err
}
}
if err := conn.Flush(); err != nil {
return err
}
for _ = range keyScoreMembers {
// TODO actually count inserts
if _, err := conn.Receive(); err != nil {
return err
}
}
return nil
}
func Get(conn redis.Conn) string {
conn.Send("GET", "fooo")
conn.Flush()
result, _ := redis.String(conn.Receive()) // TODO missing two err catch
return result
}
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
return func(key outil.Selector, data []outputs.Data) ([]outputs.Data, error) {
var okEvents []outputs.Data
serialized := make([]interface{}, 0, len(data))
okEvents, serialized = serializeEvents(serialized, 0, data)
if len(serialized) == 0 {
return nil, nil
}
data = okEvents[:0]
for i, serializedEvent := range serialized {
eventKey, err := key.Select(okEvents[i].Event)
if err != nil {
logp.Err("Failed to set redis key: %v", err)
continue
}
data = append(data, okEvents[i])
if err := conn.Send(command, eventKey, serializedEvent); err != nil {
logp.Err("Failed to execute %v: %v", command, err)
return okEvents, err
}
}
if err := conn.Flush(); err != nil {
return data, err
}
failed := data[:0]
var lastErr error
for i := range serialized {
_, err := conn.Receive()
if err != nil {
if _, ok := err.(redis.Error); ok {
logp.Err("Failed to %v event to list with %v",
command, err)
failed = append(failed, data[i])
lastErr = err
} else {
logp.Err("Failed to %v multiple events to list with %v",
command, err)
failed = append(failed, data[i:]...)
lastErr = err
break
}
}
}
return failed, lastErr
}
}
func read(list string, conn redis.Conn, count int) {
for i := 0; i < count; i++ {
conn.Send("LPOP", list)
}
conn.Flush()
for i := 0; i < count; i++ {
value, err := redis.String(conn.Receive())
if err != nil {
os.Exit(0)
}
if value != "" {
fmt.Println(value)
}
}
}
func pipelineScore(conn redis.Conn, keyMembers []common.KeyMember) (map[common.KeyMember]Presence, error) {
for _, keyMember := range keyMembers {
if err := conn.Send("ZSCORE", keyMember.Key+insertSuffix, keyMember.Member); err != nil {
return map[common.KeyMember]Presence{}, err
}
if err := conn.Send("ZSCORE", keyMember.Key+deleteSuffix, keyMember.Member); err != nil {
return map[common.KeyMember]Presence{}, err
}
}
if err := conn.Flush(); err != nil {
return map[common.KeyMember]Presence{}, err
}
m := map[common.KeyMember]Presence{}
for i := 0; i < len(keyMembers); i++ {
insertReply, insertErr := conn.Receive()
insertValue, insertErr := redis.Float64(insertReply, insertErr)
deleteReply, deleteErr := conn.Receive()
deleteValue, deleteErr := redis.Float64(deleteReply, deleteErr)
switch {
case insertErr == nil && deleteErr == redis.ErrNil:
m[keyMembers[i]] = Presence{
Present: true,
Inserted: true,
Score: insertValue,
}
case insertErr == redis.ErrNil && deleteErr == nil:
m[keyMembers[i]] = Presence{
Present: true,
Inserted: false,
Score: deleteValue,
}
case insertErr == redis.ErrNil && deleteErr == redis.ErrNil:
m[keyMembers[i]] = Presence{
Present: false,
}
default:
return map[common.KeyMember]Presence{}, fmt.Errorf(
"pipelineScore bad state for %v (%v/%v)",
keyMembers[i],
insertErr,
deleteErr,
)
}
}
return m, nil
}
func pipelineRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
if limit < 0 {
return map[string][]common.KeyScoreMember{}, fmt.Errorf("negative limit is invalid for offset-based select")
}
for _, key := range keys {
if err := conn.Send(
"ZREVRANGE",
key+insertSuffix,
offset,
offset+limit-1,
"WITHSCORES",
); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
}
if err := conn.Flush(); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
m := make(map[string][]common.KeyScoreMember, len(keys))
for _, key := range keys {
values, err := redis.Values(conn.Receive())
if err != nil {
return map[string][]common.KeyScoreMember{}, err
}
var (
ksm = common.KeyScoreMember{Key: key}
keyScoreMembers = make([]common.KeyScoreMember, 0, len(values))
)
for len(values) > 0 {
if values, err = redis.Scan(values, &ksm.Member, &ksm.Score); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
keyScoreMembers = append(keyScoreMembers, ksm)
}
m[key] = keyScoreMembers
}
return m, nil
}
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
return func(dest []byte, events []common.MapStr) ([]common.MapStr, error) {
var args [2]interface{}
args[0] = dest
serialized := make([]interface{}, 0, len(events))
events, serialized = serializeEvents(serialized, 0, events)
if len(serialized) == 0 {
return nil, nil
}
for _, event := range serialized {
args[1] = event
if err := conn.Send(command, args[:]...); err != nil {
logp.Err("Failed to execute %v: %v", command, err)
return events, err
}
}
if err := conn.Flush(); err != nil {
return events, err
}
failed := events[:0]
var lastErr error
for i := range serialized {
_, err := conn.Receive()
if err != nil {
if _, ok := err.(redis.Error); ok {
logp.Err("Failed to %v event to list (%v) with %v",
command, dest, err)
failed = append(failed, events[i])
lastErr = err
} else {
logp.Err("Failed to %v multiple events to list (%v) with %v",
command, dest, err)
failed = append(failed, events[i:]...)
lastErr = err
break
}
}
}
return failed, lastErr
}
}
func GetRepo(c redis.Conn, repo string) (string, bool, error) {
c.Send("GET", repo)
c.Send("TTL", repo)
if err := c.Flush(); err != nil {
return "", false, nil
}
content, err := redis.String(c.Receive())
if err == redis.ErrNil {
return "", false, nil
}
if err != nil {
return "", false, err
}
if ttl, err := redis.Int(c.Receive()); err == nil && ttl > FRESH_CACHE_LIMIT {
return content, true, nil
}
return content, false, err
}
func newSession(w http.ResponseWriter, userid uint64, username string, appid uint32, rc redis.Conn) (usertoken string, err error) {
if rc == nil {
rc = redisPool.Get()
defer rc.Close()
}
usertoken = ""
usertokenRaw, err := rc.Do("get", fmt.Sprintf("usertokens/%d+%d", userid, appid))
lwutil.CheckError(err, "")
if usertokenRaw != nil {
if usertoken, err := redis.String(usertokenRaw, err); err != nil {
return usertoken, lwutil.NewErr(err)
}
rc.Do("del", fmt.Sprintf("sessions/%s", usertoken))
}
usertoken = lwutil.GenUUID()
session := Session{userid, username, time.Now(), appid}
jsonSession, err := json.Marshal(session)
if err != nil {
return usertoken, lwutil.NewErr(err)
}
rc.Send("setex", fmt.Sprintf("sessions/%s", usertoken), sessionLifeSecond, jsonSession)
rc.Send("setex", fmt.Sprintf("usertokens/%d+%d", userid, appid), sessionLifeSecond, usertoken)
rc.Flush()
for i := 0; i < 2; i++ {
if _, err = rc.Receive(); err != nil {
return usertoken, lwutil.NewErr(err)
}
}
// cookie
http.SetCookie(w, &http.Cookie{Name: "usertoken", Value: usertoken, MaxAge: sessionLifeSecond, Path: "/"})
return usertoken, err
}
func pipelineRevRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
for _, key := range keys {
if err := conn.Send(
"ZREVRANGE",
key+insertSuffix,
offset,
offset+limit-1,
"WITHSCORES",
); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
}
if err := conn.Flush(); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
m := make(map[string][]common.KeyScoreMember, len(keys))
for _, key := range keys {
values, err := redis.Values(conn.Receive())
if err != nil {
return map[string][]common.KeyScoreMember{}, err
}
keyScoreMembers := make([]common.KeyScoreMember, 0, len(values))
for len(values) > 0 {
var member string
var score float64
if values, err = redis.Scan(values, &member, &score); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
keyScoreMembers = append(keyScoreMembers, common.KeyScoreMember{Key: key, Score: score, Member: member})
}
m[key] = keyScoreMembers
}
return m, nil
}
func RetrievePeople(conn redis.Conn, ids []string) ([]Person, error) {
var people []Person
for _, id := range ids {
conn.Send("HGETALL", fmt.Sprintf("person:%s", id))
}
for i := 0; i < len(ids); i++ {
values, err := redis.Values(conn.Receive())
if err != nil {
return nil, err
}
var person Person
err = redis.ScanStruct(values, &person)
if err != nil {
return nil, err
}
people = append(people, person)
}
return people, nil
}
func pipelineRangeByScore(conn redis.Conn, keys []string, cursor, stopcursor common.Cursor, limit int) (map[string][]common.KeyScoreMember, error) {
if limit < 0 {
// TODO maybe change that
return map[string][]common.KeyScoreMember{}, fmt.Errorf("negative limit is invalid for cursor-based select")
}
// pastCursor returns true when the score+member are "past" the cursor
// (smaller score, larger lexicographically) and can therefore be included
// in the resultset.
pastCursor := func(score float64, member string) bool {
if score < cursor.Score {
return true
}
if score == cursor.Score && bytes.Compare([]byte(member), []byte(cursor.Member)) < 0 {
return true
}
return false
}
// beforeStop returns true as long as the score+member are "before" the
// stopcursor (larger score, smaller lexicographically) and can therefore
// be included in the resultset.
beforeStop := func(score float64, member string) bool {
if score > stopcursor.Score {
return true
}
if score == stopcursor.Score && bytes.Compare([]byte(member), []byte(stopcursor.Member)) > 0 {
return true
}
return false
}
// An unlimited number of members may exist at cursor.Score. Luckily,
// they're in lexicographically stable order. Walk the elements we get
// back. For as long as element.Score == cursor.Score, and a
// lexicographical comparison of (element.Score, cursor.Score) < 0,
// discard the element. As soon as that condition fails, break the loop,
// and collect elements. If we run out of elements before collecting the
// user-requested limit, double the limit and try again, up to N times.
var (
keysToSelect = keys // start with all
selectLimit = limit // double every time
maxAttempts = 3 // up to this many times (TODO could be paramaterized)
results = map[string][]common.KeyScoreMember{}
)
for attempt := 0; len(keysToSelect) > 0 && attempt < maxAttempts; attempt++ {
for _, key := range keysToSelect {
if err := conn.Send(
"ZREVRANGEBYSCORE",
key+insertSuffix,
fmt.Sprint(cursor.Score), // max
"-inf", // min
"WITHSCORES",
"LIMIT",
0,
selectLimit,
); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
}
if err := conn.Flush(); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
m := make(map[string][]common.KeyScoreMember, len(keys))
for _, key := range keysToSelect {
values, err := redis.Values(conn.Receive())
if err != nil {
return map[string][]common.KeyScoreMember{}, err
}
var (
collected = 0
validated = make([]common.KeyScoreMember, 0, len(values))
hitStop = false
)
for len(values) > 0 {
var member string
var score float64
if values, err = redis.Scan(values, &member, &score); err != nil {
return map[string][]common.KeyScoreMember{}, err
}
collected++
if !pastCursor(score, member) {
continue // this element is still behind or at our cursor
}
if !beforeStop(score, member) {
hitStop = true
continue // this element is beyond our stop point
}
validated = append(validated, common.KeyScoreMember{Key: key, Score: score, Member: member})
}
// At this point, we know if we can use these elements, or need to
// go back for more.
var (
haveEnoughElements = len(validated) >= limit
exhaustedElements = collected < selectLimit
)
if haveEnoughElements || exhaustedElements || hitStop {
if len(validated) > limit {
validated = validated[:limit]
}
m[key] = validated
}
}
retryKeys := make([]string, 0, len(keysToSelect))
for _, key := range keysToSelect {
if a, ok := m[key]; ok {
results[key] = a // use it
} else {
retryKeys = append(retryKeys, key) // try again
}
}
keysToSelect = retryKeys
selectLimit *= 2
}
if n := len(keysToSelect); n > 0 {
return map[string][]common.KeyScoreMember{}, fmt.Errorf("%d key(s) failed to yield enough elements", n)
}
return results, nil
}
