Example #1
0
func (rr *Redrec) getSuggestCandidates(user string, max int) ([]string, error) {
	similarUsers, err := redis.Strings(rr.rconn.Do("ZRANGE", fmt.Sprintf("user:%s:similars", user), 0, max))
	if err != nil {
		return nil, err
	}

	max = len(similarUsers)
	args := []interface{}{}
	args = append(args, "ztmp", float64(max+1), fmt.Sprintf("user:%s:items", user))
	weights := []interface{}{}
	weights = append(weights, "WEIGHTS", -1.0)
	for _, simuser := range similarUsers {
		args = append(args, fmt.Sprintf("user:%s:items", simuser))
		weights = append(weights, 1.0)
	}

	args = append(args, weights...)
	args = append(args, "AGGREGATE", "MIN")
	_, err = rr.rconn.Do("ZUNIONSTORE", args...)
	if err != nil {
		return nil, err
	}

	candidates, err := redis.Strings(rr.rconn.Do("ZRANGEBYSCORE", "ztmp", 0, "inf"))
	if err != nil {
		return nil, err
	}

	_, err = rr.rconn.Do("DEL", "ztmp")
	if err != nil {
		return nil, err
	}

	return candidates, nil
}
Example #2
0
func getJobs() (jobs map[string]*CronJob, err error) {
	jobs = make(map[string]*CronJob)
	var jobKeys []string
	var jobData []string

	jobKeys, err = redis.Strings(RedisConn.Do("SMEMBERS", "cronstalk:jobs"))
	if err != nil {
		log.Println("error updating jobs from redis")
		return
	}

	for _, key := range jobKeys {
		jobData, err = redis.Strings(RedisConn.Do("HGETALL", key))
		if err != nil {
			log.Printf("error getting job \"%s\": %s\n", key, err)
			continue
		}

		job, err := NewJob(key, jobData)
		if err != nil {
			log.Printf("error creating job %s\n", key)
			continue
		}

		jobs[key] = job
	}
	return
}
Example #3
0
func TestFlushDb(t *testing.T) {
	c := NewFakeRedis()
	redis.String(c.Do("SET", "foo", "bar"))
	assertStrings(t, must(redis.Strings(c.Do("KEYS", "foo"))).([]string), []string{"foo"}, false)
	c.Do("FLUSHDB")
	assertStrings(t, must(redis.Strings(c.Do("KEYS", "foo"))).([]string), []string{}, false)
}
Example #4
0
func (s Similars) update(user User) error {
	items, err := redis.Strings(s.e.c.Do("SUNION", fmt.Sprintf("%s:%s:%s", s.e.class, user, s.e.Likes.kind), fmt.Sprintf("%s:%s:%s", s.e.class, user, s.e.Dislikes.kind)))
	if err != nil && err != redis.ErrNil {
		return err
	}

	args := []interface{}{}
	for _, item := range items {
		args = append(args, fmt.Sprintf("%s:%s:%s", s.e.class, item, s.e.Likes.kind))
		args = append(args, fmt.Sprintf("%s:%s:%s", s.e.class, item, s.e.Dislikes.kind))
	}
	users, err := redis.Strings(s.e.c.Do("SUNION", args...))
	if err != nil && err != redis.ErrNil {
		return err
	}

	for _, other := range users {
		if other != string(user) {
			v, err := s.Jaccard(user, User(other))
			if err != nil {
				return err
			}

			_, err = s.e.c.Do("ZADD", fmt.Sprintf("%s:%s:similars", s.e.class, user), v, other)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
Example #5
0
func GetUserDevices(userId int64) ([]int64, error) {
	r := Redix[_GetUserDevices]
	RedixMu[_GetUserDevices].Lock()
	defer RedixMu[_GetUserDevices].Unlock()
	var idStrs []string //字串类型的设备数组
	hostId, err := redis.String(r.Do("hget", "user:family", userId))
	//hostId为空说明此用户为孤儿
	if err != nil || hostId == "" {
		idStrs, _ = redis.Strings(r.Do("smembers", fmt.Sprintf(RedisUserDevices, userId)))
	} else {
		mems, _ := redis.Strings(r.Do("smembers", fmt.Sprintf("family:%v", hostId)))
		for _, m := range mems {
			devs, err := redis.Strings(r.Do("smembers", fmt.Sprintf(RedisUserDevices, m)))
			if err == nil && len(devs) > 0 {
				idStrs = append(idStrs, devs...)
			}
		}
	}
	bindedIds := make([]int64, 0, len(idStrs))
	for _, v := range idStrs {
		id, err := strconv.ParseInt(v, 10, 64)
		if err != nil {
			continue
		}
		bindedIds = append(bindedIds, id)
	}
	return bindedIds, nil
}
Example #6
0
func (d *dataAccess) GetFullErrorHistory() (map[string][]*models.AlertError, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetFullErrorHistory"})()
	conn := d.GetConnection()
	defer conn.Close()

	alerts, err := redis.Strings(conn.Do("SMEMBERS", alertsWithErrors))
	if err != nil {
		return nil, err
	}
	results := make(map[string][]*models.AlertError, len(alerts))
	for _, a := range alerts {
		rows, err := redis.Strings(conn.Do("LRANGE", errorListKey(a), 0, -1))
		if err != nil {
			return nil, err
		}
		list := make([]*models.AlertError, len(rows))
		for i, row := range rows {
			ae := &models.AlertError{}
			err = json.Unmarshal([]byte(row), ae)
			if err != nil {
				return nil, err
			}
			list[i] = ae
		}
		results[a] = list
	}
	return results, nil
}
// GetKeysAndValuesWithFilter will return all keys and their values with a filter
func (r *RedisClusterStorageManager) GetKeysAndValuesWithFilter(filter string) map[string]string {

	if r.db == nil {
		log.Info("Connection dropped, connecting..")
		r.Connect()
		return r.GetKeysAndValuesWithFilter(filter)
	}

	searchStr := r.KeyPrefix + r.hashKey(filter) + "*"
	log.Debug("[STORE] Getting list by: ", searchStr)
	sessionsInterface, err := r.db.Do("KEYS", searchStr)
	if err != nil {
		log.Error("Error trying to get filtered client keys:")
		log.Error(err)

	} else {
		keys, _ := redis.Strings(sessionsInterface, err)
		valueObj, err := r.db.Do("MGET", sessionsInterface.([]interface{})...)
		values, err := redis.Strings(valueObj, err)

		returnValues := make(map[string]string)
		for i, v := range keys {
			returnValues[r.cleanKey(v)] = values[i]
		}

		return returnValues
	}

	return map[string]string{}
}
Example #8
0
File: rcm.go Project: ifn/rcm
// TODO: add timeouts
func newServerState() (state *ServerState, err error) {
	conn, err := redis.Dial("tcp", "127.0.0.1:6379")
	if err != nil {
		return
	}
	defer conn.Close()

	urls, err := redis.Strings(conn.Do("SMEMBERS", "urls"))
	if err != nil {
		return
	}

	profiles, err := redis.Strings(conn.Do("SMEMBERS", "profiles"))
	if err != nil {
		return
	}

	urlProfileWeights, err := getWeights(conn, urls, profiles)
	if err != nil {
		return
	}

	state = new(ServerState)
	state.urls = urls
	state.profiles = profiles
	state.urlProfileWeights = urlProfileWeights

	return
}
Example #9
0
// Fetch Txos and Txins
func (tx *Tx) Build(rpool *redis.Pool) (err error) {
	c := rpool.Get()
	defer c.Close()
	tx.TxIns = []*TxIn{}
	tx.TxOuts = []*TxOut{}
	txinskeys := []interface{}{}
	for i := range iter.N(int(tx.TxInCnt)) {
		txinskeys = append(txinskeys, fmt.Sprintf("txi:%v:%v", tx.Hash, i))
	}
	txinsjson, _ := redis.Strings(c.Do("MGET", txinskeys...))
	for _, txinjson := range txinsjson {
		ctxi := new(TxIn)
		err = json.Unmarshal([]byte(txinjson), ctxi)
		tx.TxIns = append(tx.TxIns, ctxi)
	}
	txoutskeys := []interface{}{}
	txoutsspentkeys := []interface{}{}
	for i := range iter.N(int(tx.TxOutCnt)) {
		txoutskeys = append(txoutskeys, fmt.Sprintf("txo:%v:%v", tx.Hash, i))
		txoutsspentkeys = append(txoutsspentkeys, fmt.Sprintf("txo:%v:%v:spent", tx.Hash, i))
	}
	txoutsjson, _ := redis.Strings(c.Do("MGET", txoutskeys...))
	txoutsspentjson, _ := redis.Strings(c.Do("MGET", txoutsspentkeys...))
	for txoindex, txoutjson := range txoutsjson {
		ctxo := new(TxOut)
		err = json.Unmarshal([]byte(txoutjson), ctxo)
		if txoutsspentjson[txoindex] != "" {
			cspent := new(TxoSpent)
			err = json.Unmarshal([]byte(txoutsspentjson[txoindex]), cspent)
			ctxo.Spent = cspent
		}
		tx.TxOuts = append(tx.TxOuts, ctxo)
	}
	return
}
Example #10
0
File: crawl.go Project: AlekSi/gddo
func crawl(c *command) {
	if len(c.flag.Args()) != 0 {
		c.printUsage()
		os.Exit(1)
	}
	db, err := database.New()
	if err != nil {
		log.Fatal(err)
	}
	conn := db.Pool.Get()
	defer conn.Close()

	paths, err := redis.Strings(conn.Do("SMEMBERS", "newCrawl"))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("NEW")
	for _, path := range paths {
		fmt.Println(path)
	}

	paths, err = redis.Strings(conn.Do("SMEMBERS", "badCrawl"))
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println("BAD")
	for _, path := range paths {
		fmt.Println(path)
	}
}
// GetKeysAndValues will return all keys and their values - not to be used lightly
func (r *RedisClusterStorageManager) GetKeysAndValues() map[string]string {

	if GetRelevantClusterReference(r.IsCache) == nil {
		log.Info("Connection dropped, connecting..")
		r.Connect()
		return r.GetKeysAndValues()
	}

	searchStr := r.KeyPrefix + "*"
	sessionsInterface, err := GetRelevantClusterReference(r.IsCache).Do("KEYS", searchStr)
	if err != nil {
		log.Error("Error trying to get all keys:")
		log.Error(err)

	} else {
		keys, _ := redis.Strings(sessionsInterface, err)
		valueObj, err := GetRelevantClusterReference(r.IsCache).Do("MGET", sessionsInterface.([]interface{})...)
		values, err := redis.Strings(valueObj, err)

		m := make(map[string]string)
		for i, v := range keys {
			m[r.cleanKey(v)] = values[i]
		}
		return m
	}

	return map[string]string{}
}
Example #12
0
func TestHashValues(t *testing.T) {
	s, err := Run()
	ok(t, err)
	defer s.Close()
	c, err := redis.Dial("tcp", s.Addr())
	ok(t, err)

	s.HSet("wim", "zus", "jet")
	s.HSet("wim", "teun", "vuur")
	s.HSet("wim", "gijs", "lam")
	s.HSet("wim", "kees", "bok")
	v, err := redis.Strings(c.Do("HVALS", "wim"))
	ok(t, err)
	equals(t, 4, len(v))
	sort.Strings(v)
	equals(t, []string{
		"bok",
		"jet",
		"lam",
		"vuur",
	}, v)

	v, err = redis.Strings(c.Do("HVALS", "nosuch"))
	ok(t, err)
	equals(t, 0, len(v))

	// Wrong key type
	s.Set("foo", "bar")
	_, err = redis.Int(c.Do("HVALS", "foo"))
	assert(t, err != nil, "no HVALS error")
}
Example #13
0
// GetKeysAndValues will return all keys and their values - not to be used lightly
func (r *RedisStorageManager) GetKeysAndValues() map[string]string {
	db := r.pool.Get()
	defer db.Close()
	if db == nil {
		log.Info("Connection dropped, connecting..")
		r.Connect()
		return r.GetKeysAndValues()
	}

	searchStr := r.KeyPrefix + "*"
	sessionsInterface, err := db.Do("KEYS", searchStr)
	if err != nil {
		log.Error("Error trying to get all keys:")
		log.Error(err)

	} else {
		keys, _ := redis.Strings(sessionsInterface, err)
		valueObj, err := db.Do("MGET", sessionsInterface.([]interface{})...)
		values, err := redis.Strings(valueObj, err)

		returnValues := make(map[string]string)
		for i, v := range keys {
			returnValues[r.cleanKey(v)] = values[i]
		}

		return returnValues
	}

	return map[string]string{}
}
Example #14
0
func TestHashGetall(t *testing.T) {
	s, err := Run()
	ok(t, err)
	defer s.Close()
	c, err := redis.Dial("tcp", s.Addr())
	ok(t, err)

	s.HSet("wim", "zus", "jet")
	s.HSet("wim", "teun", "vuur")
	s.HSet("wim", "gijs", "lam")
	s.HSet("wim", "kees", "bok")
	v, err := redis.Strings(c.Do("HGETALL", "wim"))
	ok(t, err)
	equals(t, 8, len(v))
	d := map[string]string{}
	for len(v) > 0 {
		d[v[0]] = v[1]
		v = v[2:]
	}
	equals(t, map[string]string{
		"zus":  "jet",
		"teun": "vuur",
		"gijs": "lam",
		"kees": "bok",
	}, d)

	v, err = redis.Strings(c.Do("HGETALL", "nosuch"))
	ok(t, err)
	equals(t, 0, len(v))

	// Wrong key type
	s.Set("foo", "bar")
	_, err = redis.Int(c.Do("HGETALL", "foo"))
	assert(t, err != nil, "no HGETALL error")
}
Example #15
0
func (s *AuthTest) TestAuthenticateExcistingUser() {
	entities.Save(&entities.Planet{
		Name:     "GOP6720",
		Position: &vec2d.Vector{2, 2},
	})
	entities.Save(&entities.Player{
		Username:       "******",
		RaceID:         1,
		TwitterID:      "gop",
		HomePlanet:     "planet.GOP6720",
		ScreenSize:     []uint64{1, 1},
		ScreenPosition: &vec2d.Vector{2, 2},
	})

	players_before, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
	before := len(players_before)
	assert.Nil(s.T(), err)

	s.assertSend(&user)
	s.assertReceive("server_params")
	s.assertReceive("login_success")

	players_after, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
	after := len(players_after)
	assert.Nil(s.T(), err)

	assert.Equal(s.T(), before, after)
}
Example #16
0
func findAllUsers() []User {
	conn := getConn()
	defer conn.Close()
	keys, err := redis.Strings(conn.Do("KEYS", "*"))
	if err != nil {
		panic(err)
	}
	var users = []User{}
	for _, key := range keys {
		var f []string
		f, err = redis.Strings(conn.Do("HGETALL", key))
		if err != nil {
			panic(err)
		}
		user := User{Email: key}
		for i, v := range f {
			if v == "last_assignment" {
				user.LastAssignmentTime = f[i+1]
			}
			if v == "trello_username" {
				user.TrelloUsername = f[i+1]
			}
		}
		users = append(users, user)
	}
	return users
}
Example #17
0
func TestPopNextJobsScript(t *testing.T) {
	testingSetUp()
	defer testingTeardown()

	// Set up some time parameters
	pastTime := time.Now().Add(-10 * time.Millisecond).UTC().UnixNano()

	// Set up the database
	tx0 := newTransaction()
	// One set will mimic the ready and sorted jobs
	tx0.command("ZADD", redis.Args{Keys.JobsTimeIndex, pastTime, "two", pastTime, "four"}, nil)
	// One set will mimic the queued set
	tx0.command("ZADD", redis.Args{StatusQueued.Key(), 1, "one", 2, "two", 3, "three", 4, "four"}, nil)
	// One set will mimic the executing set
	tx0.command("ZADD", redis.Args{StatusExecuting.Key(), 5, "five"}, nil)
	if err := tx0.exec(); err != nil {
		t.Errorf("Unexpected error executing transaction: %s", err.Error())
	}

	// Start a new transaction and execute the script
	tx1 := newTransaction()
	gotJobs := []*Job{}
	testPoolId := "testPool"
	tx1.popNextJobs(2, testPoolId, newScanJobsHandler(&gotJobs))
	if err := tx1.exec(); err != nil {
		t.Errorf("Unexpected error executing transaction: %s", err.Error())
	}

	gotIds := []string{}
	for _, job := range gotJobs {
		gotIds = append(gotIds, job.id)
	}

	// Check the results
	expectedIds := []string{"four", "two"}
	if !reflect.DeepEqual(expectedIds, gotIds) {
		t.Errorf("Ids returned by script were incorrect.\n\tExpected: %v\n\tBut got:  %v", expectedIds, gotIds)
	}
	conn := redisPool.Get()
	defer conn.Close()
	expectedExecuting := []string{"five", "four", "two"}
	gotExecuting, err := redis.Strings(conn.Do("ZREVRANGE", StatusExecuting.Key(), 0, -1))
	if err != nil {
		t.Errorf("Unexpected error in ZREVRANGE: %s", err.Error())
	}
	if !reflect.DeepEqual(expectedExecuting, gotExecuting) {
		t.Errorf("Ids in the executing set were incorrect.\n\tExpected: %v\n\tBut got:  %v", expectedExecuting, gotExecuting)
	}
	expectedQueued := []string{"three", "one"}
	gotQueued, err := redis.Strings(conn.Do("ZREVRANGE", StatusQueued.Key(), 0, -1))
	if err != nil {
		t.Errorf("Unexpected error in ZREVRANGE: %s", err.Error())
	}
	if !reflect.DeepEqual(expectedQueued, gotQueued) {
		t.Errorf("Ids in the queued set were incorrect.\n\tExpected: %v\n\tBut got:  %v", expectedQueued, gotQueued)
	}
	expectKeyNotExists(t, Keys.JobsTemp)
}
Example #18
0
func TestZadd(t *testing.T) {
	c := NewFakeRedis()
	c.Do("ZADD", "foo", 4, "four")
	c.Do("ZADD", "foo", 3, "three")
	assertInt(t, must(redis.Int(c.Do("ZADD", "foo", 2, "two", 1, "one", 0, "zero"))), 3)
	assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"zero", "one", "two", "three", "four"}, false)
	assertInt(t, must(redis.Int(c.Do("ZADD", "foo", 7, "zero", 1, "one", 5, "five"))), 1)
	assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"one", "two", "three", "four", "five", "zero"}, false)
}
Example #19
0
func TestSUnion(t *testing.T) {
	c := NewFakeRedis()
	c.Do("SADD", "foo", "member1", "member2")
	c.Do("SADD", "bar", "member2", "member3")
	assertStrings(t, must(redis.Strings(c.Do("SUNION", "empty"))).([]string), []string{}, true)
	assertStrings(t, must(redis.Strings(c.Do("SUNION", "foo"))).([]string), []string{"member1", "member2"}, true)
	assertStrings(t, must(redis.Strings(c.Do("SUNION", "foo", "empty"))).([]string), []string{"member1", "member2"}, true)
	assertStrings(t, must(redis.Strings(c.Do("SUNION", "foo", "bar"))).([]string), []string{"member1", "member2", "member3"}, true)
}
Example #20
0
func TestSadd(t *testing.T) {
	c := NewFakeRedis()
	assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member1"))), 1)
	assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member1"))), 0)
	assertStrings(t, must(redis.Strings(c.Do("SMEMBERS", "foo"))).([]string), []string{"member1"}, true)
	assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member2", "member3"))), 2)
	assertStrings(t, must(redis.Strings(c.Do("SMEMBERS", "foo"))).([]string), []string{"member1", "member2", "member3"}, true)
	assertInt(t, must(redis.Int(c.Do("SADD", "foo", "member3", "member4"))), 1)
	assertStrings(t, must(redis.Strings(c.Do("SMEMBERS", "foo"))).([]string), []string{"member1", "member2", "member3", "member4"}, true)
}
Example #21
0
func TestZRangeByScoreExclusive(t *testing.T) {
	c := NewFakeRedis()
	c.Do("ZADD", "foo", 0, "zero")
	c.Do("ZADD", "foo", 2, "two")
	c.Do("ZADD", "foo", 4, "four")
	c.Do("ZADD", "foo", 5, "five")
	assertStrings(t, must(redis.Strings(c.Do("ZRANGEBYSCORE", "foo", "(0", 6))).([]string), []string{"two", "four", "five"}, false)
	assertStrings(t, must(redis.Strings(c.Do("ZRANGEBYSCORE", "foo", "(2", "(5"))).([]string), []string{"four"}, false)
	assertStrings(t, must(redis.Strings(c.Do("ZRANGEBYSCORE", "foo", 0, "(4"))).([]string), []string{"zero", "two"}, false)
}
Example #22
0
// 删除名称为key的list的index在[start,end]区间的所有value, total为list的长度
// 绝大部分情况下start不会超过3, 那么先截取[end+1, total-1], 再往头部插入[0, start-1]
func (this *RedisListHelper) DelRangeValues(key string, start, end, total int) bool {
	if key == "" || total < 1 || start < 0 || end < start || total-1 < end {
		return false
	}

	values := make([]string, 0)

	if atomic.CompareAndSwapInt32(&this.delMark1, 0, 1) {
		if start >= 1 {
			values, _ = redis.Strings(this.delCon1.Do("LRANGE", key, 0, start-1))
		}
		this.delCon1.Do("LTRIM", key, end+1, total-1)
		for i := len(values) - 1; i >= 0; i-- {
			this.delCon1.Do("LPUSH", key, values[i])
		}
		atomic.StoreInt32(&this.delMark1, 0)
		return true

	} else if atomic.CompareAndSwapInt32(&this.delMark2, 0, 1) {
		if start >= 1 {
			values, _ = redis.Strings(this.delCon2.Do("LRANGE", key, 0, start-1))
		}
		this.delCon2.Do("LTRIM", key, end+1, total-1)
		for i := len(values) - 1; i >= 0; i-- {
			this.delCon2.Do("LPUSH", key, values[i])
		}
		atomic.StoreInt32(&this.delMark2, 0)
		return true

	} else if atomic.CompareAndSwapInt32(&this.delMark3, 0, 1) {
		if start >= 1 {
			values, _ = redis.Strings(this.delCon3.Do("LRANGE", key, 0, start-1))
		}
		this.delCon3.Do("LTRIM", key, end+1, total-1)
		for i := len(values) - 1; i >= 0; i-- {
			this.delCon3.Do("LPUSH", key, values[i])
		}
		atomic.StoreInt32(&this.delMark3, 0)
		return true

	} else if atomic.CompareAndSwapInt32(&this.delMark4, 0, 1) {
		if start >= 1 {
			values, _ = redis.Strings(this.delCon4.Do("LRANGE", key, 0, start-1))
		}
		this.delCon4.Do("LTRIM", key, end+1, total-1)
		for i := len(values) - 1; i >= 0; i-- {
			this.delCon4.Do("LPUSH", key, values[i])
		}
		atomic.StoreInt32(&this.delMark4, 0)
		return true
	}

	return false
}
Example #23
0
func TestZRemRangeByScore(t *testing.T) {
	c := NewFakeRedis()
	c.Do("ZADD", "foo", 0, "zero")
	c.Do("ZADD", "foo", 2, "two")
	c.Do("ZADD", "foo", 4, "four")
	assertInt(t, must(redis.Int(c.Do("ZREMRANGEBYSCORE", "foo", 5, 10))), 0)
	assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"zero", "two", "four"}, false)
	assertInt(t, must(redis.Int(c.Do("ZREMRANGEBYSCORE", "foo", 1, 3))), 1)
	assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{"zero", "four"}, false)
	assertInt(t, must(redis.Int(c.Do("ZREMRANGEBYSCORE", "foo", 0, 4))), 2)
	assertStrings(t, must(redis.Strings(c.Do("ZRANGE", "foo", 0, -1))).([]string), []string{}, false)
}
Example #24
0
// Of course this is not the right way to do constant time polling.
func pollCalendars() {
	for {
		time.Sleep(15 * time.Second) // FIXME Minute
		entries, err := redis.Strings(gRedis.Do("KEYS", "caldav-user:*"))
		log.Println("Polling", len(entries), "users.")
		if err != nil {
			log.Println("Error fetching keys")
			continue
		}

		for _, entry := range entries {
			redisHash, getAllErr := redis.Strings(gRedis.Do("HGETALL", entry))
			if getAllErr != nil {
				log.Println("Could not get hash entry for %s", entry)
				continue
			}

			hash := mapFromRedisHash(redisHash)
			updateEndpoint := hash["updateEndpoint"]
			errorEndpoint := hash["errorEndpoint"]
			delete(hash, "updateEndpoint")
			delete(hash, "errorEndpoint")

			passwd, found := gCredentials[Username(entry)]
			if !found {
				log.Println("Password not found for", entry)
				notifyEndpoint(errorEndpoint)
				gRedis.Do("DEL", entry)
				delete(gCredentials, Username(entry))
				continue
			}

			usernameParts := strings.Split(entry, "::")
			if len(usernameParts) != 3 {
				log.Panic("Non len 3 %s", usernameParts)
			}
			x := AccountModel{
				User:           usernameParts[1],
				Domain:         usernameParts[2],
				UpdateEndpoint: updateEndpoint,
				ErrorEndpoint:  errorEndpoint,
			}
			if passwd.IsOauth {
				x.Oauth.AccessToken = passwd.Token
			} else {
				x.Password = passwd.Token
			}
			go checkCalendars(x, hash)
		}
	}
}
Example #25
0
func (d *dataAccess) GetTagMetadata(tags opentsdb.TagSet, name string) ([]*TagMetadata, error) {
	defer collect.StartTimer("redis", opentsdb.TagSet{"op": "GetTagMeta"})()
	conn := d.GetConnection()
	defer conn.Close()
	args := []interface{}{}
	for tagK, tagV := range tags {
		args = append(args, tagMetaIdxKey(tagK, tagV))
	}
	keys, err := redis.Strings(conn.Do("SINTER", args...))
	if err != nil {
		return nil, err
	}
	args = []interface{}{}
	for _, key := range keys {
		if name == "" || strings.HasSuffix(key, ":"+name) {
			args = append(args, key)
		}
	}
	results, err := redis.Strings(conn.Do("MGET", args...))
	data := []*TagMetadata{}
	for i := range args {
		// break up key to get tags and name
		key := args[i].(string)[len("tmeta:"):]
		sepIdx := strings.LastIndex(key, ":")
		tags := key[:sepIdx]
		name := key[sepIdx+1:]
		tagSet, err := opentsdb.ParseTags(tags)
		if err != nil {
			return nil, err
		}
		// break up response to get time and value
		parts := strings.SplitN(results[i], ":", 2)
		if len(parts) != 2 {
			return nil, fmt.Errorf("Expect metadata value to be `time:value`")
		}
		val := parts[1]
		time, err := strconv.ParseInt(parts[0], 10, 64)
		if err != nil {
			return nil, err
		}
		obj := &TagMetadata{
			Tags:        tagSet,
			Name:        name,
			Value:       val,
			LastTouched: time,
		}
		data = append(data, obj)
	}
	return data, nil
}
Example #26
0
func (s *AuthTest) TestAuthenticateUserWithIncompleteData() {
	players_before, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
	before := len(players_before)
	assert.Nil(s.T(), err)

	s.assertSend(&incompleteUser)
	s.assertReceive("login_failed")

	players_after, err := redis.Strings(s.conn.Do("KEYS", "player.*"))
	after := len(players_after)
	assert.Nil(s.T(), err)

	assert.Equal(s.T(), before, after)
}
Example #27
0
func (rli *RedisInput) Run(ir pipeline.InputRunner, h pipeline.PluginHelper) error {
	fmt.Println("Addr", rli.conf.Address)
	fmt.Println("key", rli.conf.Key)
	fmt.Println("batch_count:", rli.conf.Batch_count)
	fmt.Println("decoder:", rli.conf.Decoder)
	var (
		dRunner pipeline.DecoderRunner
		decoder pipeline.Decoder
		ok      bool
		e       error
		reply   interface{}
		vals    []string
		msg     string
	)

	if rli.conf.Decoder != "" {
		if dRunner, ok = h.DecoderRunner(rli.conf.Decoder, fmt.Sprintf("%s-%s", ir.Name(), rli.conf.Decoder)); !ok {
			return fmt.Errorf("Decoder not found: %s", rli.conf.Decoder)
		}
		decoder = dRunner.Decoder()
	}

	for {
		reply, e = rli.conn.Do("BLPOP", rli.conf.Key, "0")
		if e == nil {
			vals, e = redis.Strings(reply, nil)
			msg = vals[1]
			if e == nil {
				rli.InsertMessage(ir, decoder, msg)
			}
		}
		reply, e = rli.batchlpop.Do(rli.conn, rli.conf.Key, rli.conf.Batch_count)
		if e == nil {
			vals, e = redis.Strings(reply, nil)
			if e == nil {
				for _, msg = range vals {
					rli.InsertMessage(ir, decoder, msg)
				}
			} else {
				fmt.Printf("err: %v\n", e)
			}
		} else {
			fmt.Printf("type: %T, error: %v\n", reply, e)
			return e
		}
	}

	return nil
}
Example #28
0
func TestHashKeys(t *testing.T) {
	s, err := Run()
	ok(t, err)
	defer s.Close()
	c, err := redis.Dial("tcp", s.Addr())
	ok(t, err)

	s.HSet("wim", "zus", "jet")
	s.HSet("wim", "teun", "vuur")
	s.HSet("wim", "gijs", "lam")
	s.HSet("wim", "kees", "bok")
	{
		v, err := redis.Strings(c.Do("HKEYS", "wim"))
		ok(t, err)
		equals(t, 4, len(v))
		sort.Strings(v)
		equals(t, []string{
			"gijs",
			"kees",
			"teun",
			"zus",
		}, v)
	}

	// Direct command
	{
		direct, err := s.HKeys("wim")
		ok(t, err)
		sort.Strings(direct)
		equals(t, []string{
			"gijs",
			"kees",
			"teun",
			"zus",
		}, direct)
		_, err = s.HKeys("nosuch")
		equals(t, err, ErrKeyNotFound)
	}

	v, err := redis.Strings(c.Do("HKEYS", "nosuch"))
	ok(t, err)
	equals(t, 0, len(v))

	// Wrong key type
	s.Set("foo", "bar")
	_, err = redis.Int(c.Do("HKEYS", "foo"))
	assert(t, err != nil, "no HKEYS error")
}
Example #29
0
// checkRedis check status and add current ts to redis
func checkRedis(redisPool *redis.Pool, key string, before, now int64, interval time.Duration) (int, int64, int64, error) {
	c := redisPool.Get()
	defer func() {
		if c != nil {
			c.Close()
		}
	}()
	c.Send("MULTI")
	c.Send("ZREMRANGEBYSCORE", key, 0, before)
	c.Send("ZCARD", key)
	c.Send("ZRANGEBYSCORE", key, "-inf", "+inf", "LIMIT", 0, 1)
	c.Send("ZREVRANGEBYSCORE", key, "+inf", "-inf", "LIMIT", 0, 1)
	c.Send("ZADD", key, now, now)
	c.Send("EXPIRE", key, interval.Seconds())
	r, err := c.Do("EXEC")
	if err != nil {
		return 0, 0, 0, err
	}
	rs, cast := r.([]interface{})
	if !cast {
		return 0, 0, 0, fmt.Errorf("cast error")
	}
	total, err := redis.Int(rs[1], nil)
	if err != nil {
		return 0, 0, 0, err
	}
	fa, err := redis.Strings(rs[2], nil)
	if err != nil {
		return 0, 0, 0, err
	}
	la, err := redis.Strings(rs[3], nil)
	if err != nil {
		return 0, 0, 0, err
	}
	var (
		firstReq int64
		lastReq  int64
	)
	firstReq, err = parseFirst(fa)
	if err != nil {
		return 0, 0, 0, err
	}
	lastReq, err = parseFirst(la)
	if err != nil {
		return 0, 0, 0, err
	}
	return total, firstReq, lastReq, nil
}
Example #30
0
func (s *scheduled) poll(continuing bool) {
	if s.closed {
		return
	}

	conn := Config.Pool.Get()
	defer conn.Close()

	now := time.Now().Unix()

	for _, key := range s.keys {
		for {
			messages, _ := redis.Strings(conn.Do("zrangebyscore", key, "-inf", now, "limit", 0, 1))

			if len(messages) == 0 {
				break
			}

			message, _ := NewMsg(messages[0])

			if removed, _ := redis.Bool(conn.Do("zrem", key, messages[0])); removed {
				queue, _ := message.Get("queue").String()
				conn.Do("lpush", fmt.Sprint("queue:", queue), message.ToJson())
			}
		}
	}

	if continuing {
		time.Sleep(POLL_INTERVAL * time.Second)
		s.poll(true)
	}
}