func (rr *Redrec) getSuggestCandidates(user string, max int) ([]string, error) {
similarUsers, err := redis.Strings(rr.rconn.Do("ZRANGE", fmt.Sprintf("user:%s:similars", user), 0, max))
if err != nil {
return nil, err
}
max = len(similarUsers)
args := []interface{}{}
args = append(args, "ztmp", float64(max+1), fmt.Sprintf("user:%s:items", user))
weights := []interface{}{}
weights = append(weights, "WEIGHTS", -1.0)
for _, simuser := range similarUsers {
args = append(args, fmt.Sprintf("user:%s:items", simuser))
weights = append(weights, 1.0)
}
args = append(args, weights...)
args = append(args, "AGGREGATE", "MIN")
_, err = rr.rconn.Do("ZUNIONSTORE", args...)
if err != nil {
return nil, err
}
candidates, err := redis.Strings(rr.rconn.Do("ZRANGEBYSCORE", "ztmp", 0, "inf"))
if err != nil {
return nil, err
}
_, err = rr.rconn.Do("DEL", "ztmp")
if err != nil {
return nil, err
}
return candidates, nil
}
func getJobs() (jobs map[string]*CronJob, err error) {
jobs = make(map[string]*CronJob)
var jobKeys []string
var jobData []string
jobKeys, err = redis.Strings(RedisConn.Do("SMEMBERS", "cronstalk:jobs"))
if err != nil {
log.Println("error updating jobs from redis")
return
}
for _, key := range jobKeys {
jobData, err = redis.Strings(RedisConn.Do("HGETALL", key))
if err != nil {
log.Printf("error getting job \"%s\": %s\n", key, err)
continue
}
job, err := NewJob(key, jobData)
if err != nil {
log.Printf("error creating job %s\n", key)
continue
}
jobs[key] = job
}
return
}
func TestFlushDb(t *testing.T) {
c := NewFakeRedis()
redis.String(c.Do("SET", "foo", "bar"))
assertStrings(t, must(redis.Strings(c.Do("KEYS", "foo"))).([]string), []string{"foo"}, false)
c.Do("FLUSHDB")
assertStrings(t, must(redis.Strings(c.Do("KEYS", "foo"))).([]string), []string{}, false)
}
func (s Similars) update(user User) error {
items, err := redis.Strings(s.e.c.Do("SUNION", fmt.Sprintf("%s:%s:%s", s.e.class, user, s.e.Likes.kind), fmt.Sprintf("%s:%s:%s", s.e.class, user, s.e.Dislikes.kind)))
if err != nil && err != redis.ErrNil {
return err
}
args := []interface{}{}
for _, item := range items {
args = append(args, fmt.Sprintf("%s:%s:%s", s.e.class, item, s.e.Likes.kind))
args = append(args, fmt.Sprintf("%s:%s:%s", s.e.class, item, s.e.Dislikes.kind))
}
users, err := redis.Strings(s.e.c.Do("SUNION", args...))
if err != nil && err != redis.ErrNil {
return err
}
for _, other := range users {
if other != string(user) {
v, err := s.Jaccard(user, User(other))
if err != nil {
return err
}
_, err = s.e.c.Do("ZADD", fmt.Sprintf("%s:%s:similars", s.e.class, user), v, other)
if err != nil {
return err
}
}
}
return nil
}
func GetUserDevices(userId int64) ([]int64, error) {
r := Redix[_GetUserDevices]
RedixMu[_GetUserDevices].Lock()
defer RedixMu[_GetUserDevices].Unlock()
var idStrs []string //字串类型的设备数组
hostId, err := redis.String(r.Do("hget", "user:family", userId))
//hostId为空说明此用户为孤儿
if err != nil || hostId == "" {
idStrs, _ = redis.Strings(r.Do("smembers", fmt.Sprintf(RedisUserDevices, userId)))
} else {
mems, _ := redis.Strings(r.Do("smembers", fmt.Sprintf("family:%v", hostId)))
for _, m := range mems {
devs, err := redis.Strings(r.Do("smembers", fmt.Sprintf(RedisUserDevices, m)))
if err == nil && len(devs) > 0 {
idStrs = append(idStrs, devs...)
}
}
}
bindedIds := make([]int64, 0, len(idStrs))
for _, v := range idStrs {
id, err := strconv.ParseInt(v, 10, 64)
if err != nil {
continue
}
bindedIds = append(bindedIds, id)
}
return bindedIds, nil
}
func (d *dataAccess) GetFullErrorHistory() (map[string][]*models.AlertError, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetFullErrorHistory"})()
conn := d.GetConnection()
defer conn.Close()
alerts, err := redis.Strings(conn.Do("SMEMBERS", alertsWithErrors))
if err != nil {
return nil, err
}
results := make(map[string][]*models.AlertError, len(alerts))
for _, a := range alerts {
rows, err := redis.Strings(conn.Do("LRANGE", errorListKey(a), 0, -1))
if err != nil {
return nil, err
}
list := make([]*models.AlertError, len(rows))
for i, row := range rows {
ae := &models.AlertError{}
err = json.Unmarshal([]byte(row), ae)
if err != nil {
return nil, err
}
list[i] = ae
}
results[a] = list
}
return results, nil
}
// GetKeysAndValuesWithFilter will return all keys and their values with a filter
func (r *RedisClusterStorageManager) GetKeysAndValuesWithFilter(filter string) map[string]string {
if r.db == nil {
log.Info("Connection dropped, connecting..")
r.Connect()
return r.GetKeysAndValuesWithFilter(filter)
}
searchStr := r.KeyPrefix + r.hashKey(filter) + "*"
log.Debug("[STORE] Getting list by: ", searchStr)
sessionsInterface, err := r.db.Do("KEYS", searchStr)
if err != nil {
log.Error("Error trying to get filtered client keys:")
log.Error(err)
} else {
keys, _ := redis.Strings(sessionsInterface, err)
valueObj, err := r.db.Do("MGET", sessionsInterface.([]interface{})...)
values, err := redis.Strings(valueObj, err)
returnValues := make(map[string]string)
for i, v := range keys {
returnValues[r.cleanKey(v)] = values[i]
}
return returnValues
}
return map[string]string{}
}
// TODO: add timeouts
func newServerState() (state *ServerState, err error) {
conn, err := redis.Dial("tcp", "127.0.0.1:6379")
if err != nil {
return
}
defer conn.Close()
urls, err := redis.Strings(conn.Do("SMEMBERS", "urls"))
if err != nil {
return
}
profiles, err := redis.Strings(conn.Do("SMEMBERS", "profiles"))
if err != nil {
return
}
urlProfileWeights, err := getWeights(conn, urls, profiles)
if err != nil {
return
}
state = new(ServerState)
state.urls = urls
state.profiles = profiles
state.urlProfileWeights = urlProfileWeights
return
}
// Fetch Txos and Txins
func (tx *Tx) Build(rpool *redis.Pool) (err error) {
c := rpool.Get()
defer c.Close()
tx.TxIns = []*TxIn{}
tx.TxOuts = []*TxOut{}
txinskeys := []interface{}{}
for i := range iter.N(int(tx.TxInCnt)) {
txinskeys = append(txinskeys, fmt.Sprintf("txi:%v:%v", tx.Hash, i))
}
txinsjson, _ := redis.Strings(c.Do("MGET", txinskeys...))
for _, txinjson := range txinsjson {
ctxi := new(TxIn)
err = json.Unmarshal([]byte(txinjson), ctxi)
tx.TxIns = append(tx.TxIns, ctxi)
}
txoutskeys := []interface{}{}
txoutsspentkeys := []interface{}{}
for i := range iter.N(int(tx.TxOutCnt)) {
txoutskeys = append(txoutskeys, fmt.Sprintf("txo:%v:%v", tx.Hash, i))
txoutsspentkeys = append(txoutsspentkeys, fmt.Sprintf("txo:%v:%v:spent", tx.Hash, i))
}
txoutsjson, _ := redis.Strings(c.Do("MGET", txoutskeys...))
txoutsspentjson, _ := redis.Strings(c.Do("MGET", txoutsspentkeys...))
for txoindex, txoutjson := range txoutsjson {
ctxo := new(TxOut)
err = json.Unmarshal([]byte(txoutjson), ctxo)
if txoutsspentjson[txoindex] != "" {
cspent := new(TxoSpent)
err = json.Unmarshal([]byte(txoutsspentjson[txoindex]), cspent)
ctxo.Spent = cspent
}
tx.TxOuts = append(tx.TxOuts, ctxo)
}
return
}
func crawl(c *command) {
if len(c.flag.Args()) != 0 {
c.printUsage()
os.Exit(1)
}
db, err := database.New()
if err != nil {
log.Fatal(err)
}
conn := db.Pool.Get()
defer conn.Close()
paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
if err != nil {
log.Fatal(err)
}
fmt.Println("NEW")
for _, path := range paths {
fmt.Println(path)
}
paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
if err != nil {
log.Fatal(err)
}
fmt.Println("BAD")
for _, path := range paths {
fmt.Println(path)
}
}
// GetKeysAndValues will return all keys and their values - not to be used lightly
func (r *RedisClusterStorageManager) GetKeysAndValues() map[string]string {
if GetRelevantClusterReference(r.IsCache) == nil {
log.Info("Connection dropped, connecting..")
r.Connect()
return r.GetKeysAndValues()
}
searchStr := r.KeyPrefix + "*"
sessionsInterface, err := GetRelevantClusterReference(r.IsCache).Do("KEYS", searchStr)
if err != nil {
log.Error("Error trying to get all keys:")
log.Error(err)
} else {
keys, _ := redis.Strings(sessionsInterface, err)
valueObj, err := GetRelevantClusterReference(r.IsCache).Do("MGET", sessionsInterface.([]interface{})...)
values, err := redis.Strings(valueObj, err)
m := make(map[string]string)
for i, v := range keys {
m[r.cleanKey(v)] = values[i]
}
return m
}
return map[string]string{}
}
func TestHashValues(t *testing.T) {
s, err := Run()
ok(t, err)
defer s.Close()
c, err := redis.Dial("tcp", s.Addr())
ok(t, err)
s.HSet("wim", "zus", "jet")
s.HSet("wim", "teun", "vuur")
s.HSet("wim", "gijs", "lam")
s.HSet("wim", "kees", "bok")
v, err := redis.Strings(c.Do("HVALS", "wim"))
ok(t, err)
equals(t, 4, len(v))
sort.Strings(v)
equals(t, []string{
"bok",
"jet",
"lam",
"vuur",
}, v)
v, err = redis.Strings(c.Do("HVALS", "nosuch"))
ok(t, err)
equals(t, 0, len(v))
// Wrong key type
s.Set("foo", "bar")
_, err = redis.Int(c.Do("HVALS", "foo"))
assert(t, err != nil, "no HVALS error")
}
// GetKeysAndValues will return all keys and their values - not to be used lightly
func (r *RedisStorageManager) GetKeysAndValues() map[string]string {
db := r.pool.Get()
defer db.Close()
if db == nil {
log.Info("Connection dropped, connecting..")
r.Connect()
return r.GetKeysAndValues()
}
searchStr := r.KeyPrefix + "*"
sessionsInterface, err := db.Do("KEYS", searchStr)
if err != nil {
log.Error("Error trying to get all keys:")
log.Error(err)
} else {
keys, _ := redis.Strings(sessionsInterface, err)
valueObj, err := db.Do("MGET", sessionsInterface.([]interface{})...)
values, err := redis.Strings(valueObj, err)
returnValues := make(map[string]string)
for i, v := range keys {
returnValues[r.cleanKey(v)] = values[i]
}
return returnValues
}
return map[string]string{}
}
func TestHashGetall(t *testing.T) {
s, err := Run()
ok(t, err)
defer s.Close()
c, err := redis.Dial("tcp", s.Addr())
ok(t, err)
s.HSet("wim", "zus", "jet")
s.HSet("wim", "teun", "vuur")
s.HSet("wim", "gijs", "lam")
s.HSet("wim", "kees", "bok")
v, err := redis.Strings(c.Do("HGETALL", "wim"))
ok(t, err)
equals(t, 8, len(v))
d := map[string]string{}
for len(v) > 0 {
d[v[0]] = v[1]
v = v[2:]
}
equals(t, map[string]string{
"zus": "jet",
"teun": "vuur",
"gijs": "lam",
"kees": "bok",
}, d)
v, err = redis.Strings(c.Do("HGETALL", "nosuch"))
ok(t, err)
equals(t, 0, len(v))
// Wrong key type
s.Set("foo", "bar")
_, err = redis.Int(c.Do("HGETALL", "foo"))
assert(t, err != nil, "no HGETALL error")
}
func (s *AuthTest) TestAuthenticateExcistingUser() {
entities.Save(&entities.Planet{
Name: "GOP6720",
Position: &vec2d.Vector{2, 2},
})
entities.Save(&entities.Player{
Username: "******",
RaceID: 1,
TwitterID: "gop",
HomePlanet: "planet.GOP6720",
ScreenSize: []uint64{1, 1},
ScreenPosition: &vec2d.Vector{2, 2},
})
players_before, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
before := len(players_before)
assert.Nil(s.T(), err)
s.assertSend(&user)
s.assertReceive("server_params")
s.assertReceive("login_success")
players_after, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
after := len(players_after)
assert.Nil(s.T(), err)
assert.Equal(s.T(), before, after)
}
func findAllUsers() []User {
conn := getConn()
defer conn.Close()
keys, err := redis.Strings(conn.Do("KEYS", "*"))
if err != nil {
panic(err)
}
var users = []User{}
for _, key := range keys {
var f []string
f, err = redis.Strings(conn.Do("HGETALL", key))
if err != nil {
panic(err)
}
user := User{Email: key}
for i, v := range f {
if v == "last_assignment" {
user.LastAssignmentTime = f[i+1]
}
if v == "trello_username" {
user.TrelloUsername = f[i+1]
}
}
users = append(users, user)
}
return users
}
func TestPopNextJobsScript(t *testing.T) {
testingSetUp()
defer testingTeardown()
// Set up some time parameters
pastTime := time.Now().Add(-10 * time.Millisecond).UTC().UnixNano()
// Set up the database
tx0 := newTransaction()
// One set will mimic the ready and sorted jobs
tx0.command("ZADD", redis.Args{Keys.JobsTimeIndex, pastTime, "two", pastTime, "four"}, nil)
// One set will mimic the queued set
tx0.command("ZADD", redis.Args{StatusQueued.Key(), 1, "one", 2, "two", 3, "three", 4, "four"}, nil)
// One set will mimic the executing set
tx0.command("ZADD", redis.Args{StatusExecuting.Key(), 5, "five"}, nil)
if err := tx0.exec(); err != nil {
t.Errorf("Unexpected error executing transaction: %s", err.Error())
}
// Start a new transaction and execute the script
tx1 := newTransaction()
gotJobs := []*Job{}
testPoolId := "testPool"
tx1.popNextJobs(2, testPoolId, newScanJobsHandler(&gotJobs))
if err := tx1.exec(); err != nil {
t.Errorf("Unexpected error executing transaction: %s", err.Error())
}
gotIds := []string{}
for _, job := range gotJobs {
gotIds = append(gotIds, job.id)
}
// Check the results
expectedIds := []string{"four", "two"}
if !reflect.DeepEqual(expectedIds, gotIds) {
t.Errorf("Ids returned by script were incorrect.\n\tExpected: %v\n\tBut got: %v", expectedIds, gotIds)
}
conn := redisPool.Get()
defer conn.Close()
expectedExecuting := []string{"five", "four", "two"}
gotExecuting, err := redis.Strings(conn.Do("ZREVRANGE", StatusExecuting.Key(), 0, -1))
if err != nil {
t.Errorf("Unexpected error in ZREVRANGE: %s", err.Error())
}
if !reflect.DeepEqual(expectedExecuting, gotExecuting) {
t.Errorf("Ids in the executing set were incorrect.\n\tExpected: %v\n\tBut got: %v", expectedExecuting, gotExecuting)
}
expectedQueued := []string{"three", "one"}
gotQueued, err := redis.Strings(conn.Do("ZREVRANGE", StatusQueued.Key(), 0, -1))
if err != nil {
t.Errorf("Unexpected error in ZREVRANGE: %s", err.Error())
}
if !reflect.DeepEqual(expectedQueued, gotQueued) {
t.Errorf("Ids in the queued set were incorrect.\n\tExpected: %v\n\tBut got: %v", expectedQueued, gotQueued)
}
expectKeyNotExists(t, Keys.JobsTemp)
}
func TestZadd(t *testing.T) {
c := NewFakeRedis()
c.Do("ZADD", "foo", 4, "four")
c.Do("ZADD", "foo", 3, "three")
assertInt(t, must(redis.Int(c.Do("ZADD", "foo", 2, "two", 1, "one", 0, "zero"))), 3)
assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"zero", "one", "two", "three", "four"}, false)
assertInt(t, must(redis.Int(c.Do("ZADD", "foo", 7, "zero", 1, "one", 5, "five"))), 1)
assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"one", "two", "three", "four", "five", "zero"}, false)
}
func TestSUnion(t *testing.T) {
c := NewFakeRedis()
c.Do("SADD", "foo", "member1", "member2")
c.Do("SADD", "bar", "member2", "member3")
assertStrings(t, must(redis.Strings(c.Do("SUNION", "empty"))).([]string), []string{}, true)
assertStrings(t, must(redis.Strings(c.Do("SUNION", "foo"))).([]string), []string{"member1", "member2"}, true)
assertStrings(t, must(redis.Strings(c.Do("SUNION", "foo", "empty"))).([]string), []string{"member1", "member2"}, true)
assertStrings(t, must(redis.Strings(c.Do("SUNION", "foo", "bar"))).([]string), []string{"member1", "member2", "member3"}, true)
}
func TestSadd(t *testing.T) {
c := NewFakeRedis()
assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member1"))), 1)
assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member1"))), 0)
assertStrings(t, must(redis.Strings(c.Do("SMEMBERS", "foo"))).([]string), []string{"member1"}, true)
assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member2", "member3"))), 2)
assertStrings(t, must(redis.Strings(c.Do("SMEMBERS", "foo"))).([]string), []string{"member1", "member2", "member3"}, true)
assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member3", "member4"))), 1)
assertStrings(t, must(redis.Strings(c.Do("SMEMBERS", "foo"))).([]string), []string{"member1", "member2", "member3", "member4"}, true)
}
func TestZRangeByScoreExclusive(t *testing.T) {
c := NewFakeRedis()
c.Do("ZADD", "foo", 0, "zero")
c.Do("ZADD", "foo", 2, "two")
c.Do("ZADD", "foo", 4, "four")
c.Do("ZADD", "foo", 5, "five")
assertStrings(t, must(redis.Strings(c.Do("ZRANGEBYSCORE", "foo", "(0", 6))).([]string), []string{"two", "four", "five"}, false)
assertStrings(t, must(redis.Strings(c.Do("ZRANGEBYSCORE", "foo", "(2", "(5"))).([]string), []string{"four"}, false)
assertStrings(t, must(redis.Strings(c.Do("ZRANGEBYSCORE", "foo", 0, "(4"))).([]string), []string{"zero", "two"}, false)
}
// 删除名称为key的list的index在[start,end]区间的所有value, total为list的长度
// 绝大部分情况下start不会超过3， 那么先截取[end+1, total-1], 再往头部插入[0, start-1]
func (this *RedisListHelper) DelRangeValues(key string, start, end, total int) bool {
if key == "" || total < 1 || start < 0 || end < start || total-1 < end {
return false
}
values := make([]string, 0)
if atomic.CompareAndSwapInt32(&this.delMark1, 0, 1) {
if start >= 1 {
values, _ = redis.Strings(this.delCon1.Do("LRANGE", key, 0, start-1))
}
this.delCon1.Do("LTRIM", key, end+1, total-1)
for i := len(values) - 1; i >= 0; i-- {
this.delCon1.Do("LPUSH", key, values[i])
}
atomic.StoreInt32(&this.delMark1, 0)
return true
} else if atomic.CompareAndSwapInt32(&this.delMark2, 0, 1) {
if start >= 1 {
values, _ = redis.Strings(this.delCon2.Do("LRANGE", key, 0, start-1))
}
this.delCon2.Do("LTRIM", key, end+1, total-1)
for i := len(values) - 1; i >= 0; i-- {
this.delCon2.Do("LPUSH", key, values[i])
}
atomic.StoreInt32(&this.delMark2, 0)
return true
} else if atomic.CompareAndSwapInt32(&this.delMark3, 0, 1) {
if start >= 1 {
values, _ = redis.Strings(this.delCon3.Do("LRANGE", key, 0, start-1))
}
this.delCon3.Do("LTRIM", key, end+1, total-1)
for i := len(values) - 1; i >= 0; i-- {
this.delCon3.Do("LPUSH", key, values[i])
}
atomic.StoreInt32(&this.delMark3, 0)
return true
} else if atomic.CompareAndSwapInt32(&this.delMark4, 0, 1) {
if start >= 1 {
values, _ = redis.Strings(this.delCon4.Do("LRANGE", key, 0, start-1))
}
this.delCon4.Do("LTRIM", key, end+1, total-1)
for i := len(values) - 1; i >= 0; i-- {
this.delCon4.Do("LPUSH", key, values[i])
}
atomic.StoreInt32(&this.delMark4, 0)
return true
}
return false
}
func TestZRemRangeByScore(t *testing.T) {
c := NewFakeRedis()
c.Do("ZADD", "foo", 0, "zero")
c.Do("ZADD", "foo", 2, "two")
c.Do("ZADD", "foo", 4, "four")
assertInt(t, must(redis.Int(c.Do("ZREMRANGEBYSCORE", "foo", 5, 10))), 0)
assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"zero", "two", "four"}, false)
assertInt(t, must(redis.Int(c.Do("ZREMRANGEBYSCORE", "foo", 1, 3))), 1)
assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"zero", "four"}, false)
assertInt(t, must(redis.Int(c.Do("ZREMRANGEBYSCORE", "foo", 0, 4))), 2)
assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{}, false)
}
// Of course this is not the right way to do constant time polling.
func pollCalendars() {
for {
time.Sleep(15 * time.Second) // FIXME Minute
entries, err := redis.Strings(gRedis.Do("KEYS", "caldav-user:*"))
log.Println("Polling", len(entries), "users.")
if err != nil {
log.Println("Error fetching keys")
continue
}
for _, entry := range entries {
redisHash, getAllErr := redis.Strings(gRedis.Do("HGETALL", entry))
if getAllErr != nil {
log.Println("Could not get hash entry for %s", entry)
continue
}
hash := mapFromRedisHash(redisHash)
updateEndpoint := hash["updateEndpoint"]
errorEndpoint := hash["errorEndpoint"]
delete(hash, "updateEndpoint")
delete(hash, "errorEndpoint")
passwd, found := gCredentials[Username(entry)]
if !found {
log.Println("Password not found for", entry)
notifyEndpoint(errorEndpoint)
gRedis.Do("DEL", entry)
delete(gCredentials, Username(entry))
continue
}
usernameParts := strings.Split(entry, "::")
if len(usernameParts) != 3 {
log.Panic("Non len 3 %s", usernameParts)
}
x := AccountModel{
User: usernameParts[1],
Domain: usernameParts[2],
UpdateEndpoint: updateEndpoint,
ErrorEndpoint: errorEndpoint,
}
if passwd.IsOauth {
x.Oauth.AccessToken = passwd.Token
} else {
x.Password = passwd.Token
}
go checkCalendars(x, hash)
}
}
}
func (d *dataAccess) GetTagMetadata(tags opentsdb.TagSet, name string) ([]*TagMetadata, error) {
defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetTagMeta"})()
conn := d.GetConnection()
defer conn.Close()
args := []interface{}{}
for tagK, tagV := range tags {
args = append(args, tagMetaIdxKey(tagK, tagV))
}
keys, err := redis.Strings(conn.Do("SINTER", args...))
if err != nil {
return nil, err
}
args = []interface{}{}
for _, key := range keys {
if name == "" || strings.HasSuffix(key, ":"+name) {
args = append(args, key)
}
}
results, err := redis.Strings(conn.Do("MGET", args...))
data := []*TagMetadata{}
for i := range args {
// break up key to get tags and name
key := args[i].(string)[len("tmeta:"):]
sepIdx := strings.LastIndex(key, ":")
tags := key[:sepIdx]
name := key[sepIdx+1:]
tagSet, err := opentsdb.ParseTags(tags)
if err != nil {
return nil, err
}
// break up response to get time and value
parts := strings.SplitN(results[i], ":", 2)
if len(parts) != 2 {
return nil, fmt.Errorf("Expect metadata value to be `time:value`")
}
val := parts[1]
time, err := strconv.ParseInt(parts[0], 10, 64)
if err != nil {
return nil, err
}
obj := &TagMetadata{
Tags: tagSet,
Name: name,
Value: val,
LastTouched: time,
}
data = append(data, obj)
}
return data, nil
}
func (s *AuthTest) TestAuthenticateUserWithIncompleteData() {
players_before, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
before := len(players_before)
assert.Nil(s.T(), err)
s.assertSend(&incompleteUser)
s.assertReceive("login_failed")
players_after, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
after := len(players_after)
assert.Nil(s.T(), err)
assert.Equal(s.T(), before, after)
}
func (rli *RedisInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
fmt.Println("Addr", rli.conf.Address)
fmt.Println("key", rli.conf.Key)
fmt.Println("batch_count:", rli.conf.Batch_count)
fmt.Println("decoder:", rli.conf.Decoder)
var (
dRunner pipeline.DecoderRunner
decoder pipeline.Decoder
ok bool
e error
reply interface{}
vals []string
msg string
)
if rli.conf.Decoder != "" {
if dRunner, ok = h.DecoderRunner(rli.conf.Decoder, fmt.Sprintf("%s-%s", ir.Name(), rli.conf.Decoder)); !ok {
return fmt.Errorf("Decoder not found: %s", rli.conf.Decoder)
}
decoder = dRunner.Decoder()
}
for {
reply, e = rli.conn.Do("BLPOP", rli.conf.Key, "0")
if e == nil {
vals, e = redis.Strings(reply, nil)
msg = vals[1]
if e == nil {
rli.InsertMessage(ir, decoder, msg)
}
}
reply, e = rli.batchlpop.Do(rli.conn, rli.conf.Key, rli.conf.Batch_count)
if e == nil {
vals, e = redis.Strings(reply, nil)
if e == nil {
for _, msg = range vals {
rli.InsertMessage(ir, decoder, msg)
}
} else {
fmt.Printf("err: %v\n", e)
}
} else {
fmt.Printf("type: %T, error: %v\n", reply, e)
return e
}
}
return nil
}
func TestHashKeys(t *testing.T) {
s, err := Run()
ok(t, err)
defer s.Close()
c, err := redis.Dial("tcp", s.Addr())
ok(t, err)
s.HSet("wim", "zus", "jet")
s.HSet("wim", "teun", "vuur")
s.HSet("wim", "gijs", "lam")
s.HSet("wim", "kees", "bok")
{
v, err := redis.Strings(c.Do("HKEYS", "wim"))
ok(t, err)
equals(t, 4, len(v))
sort.Strings(v)
equals(t, []string{
"gijs",
"kees",
"teun",
"zus",
}, v)
}
// Direct command
{
direct, err := s.HKeys("wim")
ok(t, err)
sort.Strings(direct)
equals(t, []string{
"gijs",
"kees",
"teun",
"zus",
}, direct)
_, err = s.HKeys("nosuch")
equals(t, err, ErrKeyNotFound)
}
v, err := redis.Strings(c.Do("HKEYS", "nosuch"))
ok(t, err)
equals(t, 0, len(v))
// Wrong key type
s.Set("foo", "bar")
_, err = redis.Int(c.Do("HKEYS", "foo"))
assert(t, err != nil, "no HKEYS error")
}
// checkRedis check status and add current ts to redis
func checkRedis(redisPool *redis.Pool, key string, before, now int64, interval time.Duration) (int, int64, int64, error) {
c := redisPool.Get()
defer func() {
if c != nil {
c.Close()
}
}()
c.Send("MULTI")
c.Send("ZREMRANGEBYSCORE", key, 0, before)
c.Send("ZCARD", key)
c.Send("ZRANGEBYSCORE", key, "-inf", "+inf", "LIMIT", 0, 1)
c.Send("ZREVRANGEBYSCORE", key, "+inf", "-inf", "LIMIT", 0, 1)
c.Send("ZADD", key, now, now)
c.Send("EXPIRE", key, interval.Seconds())
r, err := c.Do("EXEC")
if err != nil {
return 0, 0, 0, err
}
rs, cast := r.([]interface{})
if !cast {
return 0, 0, 0, fmt.Errorf("cast error")
}
total, err := redis.Int(rs[1], nil)
if err != nil {
return 0, 0, 0, err
}
fa, err := redis.Strings(rs[2], nil)
if err != nil {
return 0, 0, 0, err
}
la, err := redis.Strings(rs[3], nil)
if err != nil {
return 0, 0, 0, err
}
var (
firstReq int64
lastReq int64
)
firstReq, err = parseFirst(fa)
if err != nil {
return 0, 0, 0, err
}
lastReq, err = parseFirst(la)
if err != nil {
return 0, 0, 0, err
}
return total, firstReq, lastReq, nil
}
func (s *scheduled) poll(continuing bool) {
if s.closed {
return
}
conn := Config.Pool.Get()
defer conn.Close()
now := time.Now().Unix()
for _, key := range s.keys {
for {
messages, _ := redis.Strings(conn.Do("zrangebyscore", key, "-inf", now, "limit", 0, 1))
if len(messages) == 0 {
break
}
message, _ := NewMsg(messages[0])
if removed, _ := redis.Bool(conn.Do("zrem", key, messages[0])); removed {
queue, _ := message.Get("queue").String()
conn.Do("lpush", fmt.Sprint("queue:", queue), message.ToJson())
}
}
}
if continuing {
time.Sleep(POLL_INTERVAL * time.Second)
s.poll(true)
}
}