Esempio n. 1
0
// Decrements all jobs' score by one
func (q *delayedQueue) TickTock(conn redis.Conn) error {
	// if ticktock is running, don't run another one
	if q.isTicking {
		return nil
	}

	q.isTicking = true

	// there are multiple returns, but whichever return
	// gets hit, make sure isTicking is switched off
	defer func() {
		q.isTicking = false
	}()

	// get all job ids
	//
	// NOTE: loading all ids into memory might be an overhead
	// if the set is big, ZSCAN might be able to help on this
	idToScore := GetQueueScores(q.GetIndexName(), conn)

	// decrement each job's score
	conn.Send("MULTI")
	for id, score := range idToScore {
		// only decrement scores > zero so we
		// don't ran into negative scores
		if score > 0 {
			conn.Send("ZINCRBY", q.GetIndexName(), -1, id)
		}
	}
	if _, err := conn.Do("EXEC"); err != nil {
		return err
	}

	return nil
}
Esempio n. 2
0
// check - lock - check
func (provider *RedisProvider) startSession(sid string, conn redis.Conn, option *Options) (Sessioner, bool, error) {
	// 检查随机生成的session是否被占用了
	if exist, err := redis.Bool(conn.Do("EXISTS", getRedisKey(sid))); exist || err != nil {
		return nil, false, err
	}
	// 观察sid
	conn.Send("WATCH", getRedisKey(sid))
	conn.Send("MULTI")
	conn.Send("HSET", getRedisKey(sid), "createtime", time.Now().Unix())
	if option.Cookielifttime > 0 {
		conn.Send("EXPIRE", getRedisKey(sid), option.Cookielifttime)
	} else {
		conn.Send("EXPIRE", getRedisKey(sid), 30*24*60*60)
	}

	replay, err := conn.Do("EXEC")
	if err != nil {
		return nil, false, err
	}
	if replay == nil {
		return nil, false, nil
	}
	sessioner, err := provider.GetSessioner(sid)
	return sessioner, true, err
}
Esempio n. 3
0
func createShortURL(url string, conn redis.Conn) (Data, error) {
	var d Data
	count, err := redis.Int(conn.Do("INCR", "global:size"))
	if err != nil {
		log.Print(err)
		return d, err
	}
	log.Print("Total: ", count)
	encodedVar := base62.EncodeInt(int64(count))
	key := strings.Join([]string{encodedVar, url}, "||")
	conn.Send("MULTI")
	conn.Send("HSET", key, "count", 0)
	_, err2 := conn.Do("EXEC")

	if err2 != nil {
		log.Print(err2)
		return d, err2
	}

	d.Original = url
	d.HitCount = 0
	d.Short = encodedVar
	d.FullShort = strings.Join([]string{*base, encodedVar}, "")

	return d, err
}
Esempio n. 4
0
// Checking rate limit 10 request / 1 sec.
func checkRateLimit(con redis.Conn, c *echo.Context) bool {
	ip, _, err := net.SplitHostPort(c.Request().RemoteAddr)
	if err != nil {
		panic(err)
	}

	//If list of ip address's length is 10 retun false.
	current, err := redis.Int(con.Do("LLEN", ip))
	if err == nil && current > 10 {
		return false
	}
	exists, err := redis.Bool(con.Do("EXISTS", ip))
	if err != nil {
		panic(err)
	}
	if !exists {
		con.Send("MULTI")
		con.Send("RPUSH", ip, ip)
		con.Send("EXPIRE", ip, 1)
		_, err := con.Do("EXEC")
		if err != nil {
			panic(err)
		}
	} else {
		_, err := con.Do("RPUSHX", ip, ip)
		if err != nil {
			panic(err)
		}
	}
	return true
}
Esempio n. 5
0
func Get(conn redis.Conn) string {

	conn.Send("GET", "fooo")
	conn.Flush()

	result, _ := redis.String(conn.Receive()) // TODO missing two err catch

	return result
}
Esempio n. 6
0
func Save(obj interface{}, conn redis.Conn) reflect.Type {

	conn.Send("GET", "fooo")
	conn.Flush()

	//result, _ := redis.String(conn.Receive()) // TODO missing two err catch

	return reflect.ValueOf(&obj).Type()

}
Esempio n. 7
0
func writeBatch(list string, conn redis.Conn, values []string) {
	conn.Send("MULTI")
	for _, value := range values {
		value = strings.TrimSpace(value)
		if value != "" {
			conn.Send("LPUSH", list, value)
		}
	}
	_, err := redis.Values(conn.Do("EXEC"))
	if err != nil {
		log.Fatalf("Could not LPUSH: %s \"%s\"\n", list, err.Error())
	}
}
Esempio n. 8
0
func recordHitAtGranularity( granularity Granularity, time float64, key string, c redis.Conn ) bool{
	size := granularity.size
	factor := granularity.factor
	ttl := granularity.ttl
	name := granularity.name
	tsround := getRoundedTimestamp(time, size * factor)
	s := []string{"stats",key,name,strconv.FormatFloat(tsround, 'f', 0, 64)}
	redis_key := strings.Join(s,":")
	ts := strconv.FormatFloat(getFactoredTimestamp (time, factor), 'f', 0, 64)    	
	ttl_str := strconv.FormatFloat(tsround+ttl, 'f', 0, 64)
	c.Send("HINCRBY", redis_key, ts, 1)	
	c.Send("EXPIREAT", redis_key, ttl_str)	
	return true;
}
Esempio n. 9
0
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
	return func(key outil.Selector, data []outputs.Data) ([]outputs.Data, error) {
		var okEvents []outputs.Data
		serialized := make([]interface{}, 0, len(data))
		okEvents, serialized = serializeEvents(serialized, 0, data)
		if len(serialized) == 0 {
			return nil, nil
		}

		data = okEvents[:0]
		for i, serializedEvent := range serialized {
			eventKey, err := key.Select(okEvents[i].Event)
			if err != nil {
				logp.Err("Failed to set redis key: %v", err)
				continue
			}

			data = append(data, okEvents[i])
			if err := conn.Send(command, eventKey, serializedEvent); err != nil {
				logp.Err("Failed to execute %v: %v", command, err)
				return okEvents, err
			}
		}

		if err := conn.Flush(); err != nil {
			return data, err
		}

		failed := data[:0]
		var lastErr error
		for i := range serialized {
			_, err := conn.Receive()
			if err != nil {
				if _, ok := err.(redis.Error); ok {
					logp.Err("Failed to %v event to list with %v",
						command, err)
					failed = append(failed, data[i])
					lastErr = err
				} else {
					logp.Err("Failed to %v multiple events to list with %v",
						command, err)
					failed = append(failed, data[i:]...)
					lastErr = err
					break
				}
			}
		}
		return failed, lastErr
	}
}
Esempio n. 10
0
func (b *BGPDump) parseBGPCSV(r io.Reader, conn redis.Conn) (int, error) {
	day := b.day()
	s := bufio.NewScanner(r)
	n := 0
	var asn string
	for s.Scan() {
		cols := strings.Split(s.Text(), "|")
		if len(cols) < 7 {
			return n, ParseError{
				Message: "too few columns",
				Path:    filepath.Base(b.Path()),
				LineNum: n,
				Line:    s.Text(),
			}
		}
		block := cols[5]

		if _, ok := asn12654blocks[block]; ok {
			asn = "12654"
		} else {
			asPath := cols[6]
			asns := strings.Split(asPath, " ")
			asn = asns[len(asns)-1]
			if asn == "" {
				return n, ParseError{
					Message: "no ASPATH data",
					Path:    filepath.Base(b.Path()),
					LineNum: n,
					Line:    s.Text(),
				}
			}
		}
		conn.Send("HSET", fmt.Sprintf("i2a:%s", block), day, asn)
		n++
		if n%10000 == 0 {
			err := conn.Flush()
			if err != nil {
				return 0, err
			}
		}
	}
	conn.Send("SADD", "i2a:imported_dates", day)
	err := conn.Flush()
	if err != nil {
		return 0, err
	}
	return n, nil
}
Esempio n. 11
0
func read(list string, conn redis.Conn, count int) {
	for i := 0; i < count; i++ {
		conn.Send("LPOP", list)
	}
	conn.Flush()
	for i := 0; i < count; i++ {
		value, err := redis.String(conn.Receive())

		if err != nil {
			os.Exit(0)
		}
		if value != "" {
			fmt.Println(value)
		}
	}
}
Esempio n. 12
0
func pipelineScore(conn redis.Conn, keyMembers []common.KeyMember) (map[common.KeyMember]Presence, error) {
	for _, keyMember := range keyMembers {
		if err := conn.Send("ZSCORE", keyMember.Key+insertSuffix, keyMember.Member); err != nil {
			return map[common.KeyMember]Presence{}, err
		}
		if err := conn.Send("ZSCORE", keyMember.Key+deleteSuffix, keyMember.Member); err != nil {
			return map[common.KeyMember]Presence{}, err
		}
	}
	if err := conn.Flush(); err != nil {
		return map[common.KeyMember]Presence{}, err
	}

	m := map[common.KeyMember]Presence{}
	for i := 0; i < len(keyMembers); i++ {
		insertReply, insertErr := conn.Receive()
		insertValue, insertErr := redis.Float64(insertReply, insertErr)
		deleteReply, deleteErr := conn.Receive()
		deleteValue, deleteErr := redis.Float64(deleteReply, deleteErr)
		switch {
		case insertErr == nil && deleteErr == redis.ErrNil:
			m[keyMembers[i]] = Presence{
				Present:  true,
				Inserted: true,
				Score:    insertValue,
			}
		case insertErr == redis.ErrNil && deleteErr == nil:
			m[keyMembers[i]] = Presence{
				Present:  true,
				Inserted: false,
				Score:    deleteValue,
			}
		case insertErr == redis.ErrNil && deleteErr == redis.ErrNil:
			m[keyMembers[i]] = Presence{
				Present: false,
			}
		default:
			return map[common.KeyMember]Presence{}, fmt.Errorf(
				"pipelineScore bad state for %v (%v/%v)",
				keyMembers[i],
				insertErr,
				deleteErr,
			)
		}
	}
	return m, nil
}
Esempio n. 13
0
func (w *worker) start(conn redis.Conn, job *job) error {
	work := &work{
		Queue:   job.Queue,
		RunAt:   time.Now(),
		Payload: job.Payload,
	}

	buffer, err := json.Marshal(work)
	if err != nil {
		return err
	}

	conn.Send("SET", fmt.Sprintf("%sworker:%s", namespace, w), buffer)
	logger.Debugf("Processing %s since %s [%v]", work.Queue, work.RunAt, work.Payload.Class)

	return w.process.start(conn)
}
Esempio n. 14
0
func (w *worker) fail(conn redis.Conn, job *job, err error) error {
	failure := &failure{
		FailedAt:  time.Now(),
		Payload:   job.Payload,
		Exception: "Error",
		Error:     err.Error(),
		Worker:    w,
		Queue:     job.Queue,
	}
	buffer, err := json.Marshal(failure)
	if err != nil {
		return err
	}
	conn.Send("RPUSH", fmt.Sprintf("%sfailed", namespace), buffer)

	return w.process.fail(conn)
}
Esempio n. 15
0
func pipelineRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
	if limit < 0 {
		return map[string][]common.KeyScoreMember{}, fmt.Errorf("negative limit is invalid for offset-based select")
	}
	for _, key := range keys {
		if err := conn.Send(
			"ZREVRANGE",
			key+insertSuffix,
			offset,
			offset+limit-1,
			"WITHSCORES",
		); err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}
	}

	if err := conn.Flush(); err != nil {
		return map[string][]common.KeyScoreMember{}, err
	}

	m := make(map[string][]common.KeyScoreMember, len(keys))

	for _, key := range keys {
		values, err := redis.Values(conn.Receive())
		if err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}

		var (
			ksm             = common.KeyScoreMember{Key: key}
			keyScoreMembers = make([]common.KeyScoreMember, 0, len(values))
		)

		for len(values) > 0 {
			if values, err = redis.Scan(values, &ksm.Member, &ksm.Score); err != nil {
				return map[string][]common.KeyScoreMember{}, err
			}

			keyScoreMembers = append(keyScoreMembers, ksm)
		}

		m[key] = keyScoreMembers
	}

	return m, nil
}
Esempio n. 16
0
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
	return func(dest []byte, events []common.MapStr) ([]common.MapStr, error) {
		var args [2]interface{}
		args[0] = dest

		serialized := make([]interface{}, 0, len(events))
		events, serialized = serializeEvents(serialized, 0, events)
		if len(serialized) == 0 {
			return nil, nil
		}

		for _, event := range serialized {
			args[1] = event
			if err := conn.Send(command, args[:]...); err != nil {
				logp.Err("Failed to execute %v: %v", command, err)
				return events, err
			}
		}

		if err := conn.Flush(); err != nil {
			return events, err
		}

		failed := events[:0]
		var lastErr error
		for i := range serialized {
			_, err := conn.Receive()
			if err != nil {
				if _, ok := err.(redis.Error); ok {
					logp.Err("Failed to %v event to list (%v) with %v",
						command, dest, err)
					failed = append(failed, events[i])
					lastErr = err
				} else {
					logp.Err("Failed to %v multiple events to list (%v) with %v",
						command, dest, err)
					failed = append(failed, events[i:]...)
					lastErr = err
					break
				}
			}
		}
		return failed, lastErr
	}
}
Esempio n. 17
0
func GetRepo(c redis.Conn, repo string) (string, bool, error) {
	c.Send("GET", repo)
	c.Send("TTL", repo)
	if err := c.Flush(); err != nil {
		return "", false, nil
	}
	content, err := redis.String(c.Receive())
	if err == redis.ErrNil {
		return "", false, nil
	}
	if err != nil {
		return "", false, err
	}

	if ttl, err := redis.Int(c.Receive()); err == nil && ttl > FRESH_CACHE_LIMIT {
		return content, true, nil
	}

	return content, false, err
}
Esempio n. 18
0
func (s *SBFFrame) Check(conn redis.Conn, element []byte) bool {
	var flag int = 1
	hashes := murmur.Hashes(element, s.SliceCount, s.SliceSize)
	// check bit val
	conn.Send("MULTI")
	for index, h := range hashes {
		pos := uint32(index)*s.SliceSize + s.StartIndex + h + SBF_FRAME_HEADER_SIZE<<3
		conn.Send("GETBIT", s.Refer, pos)
	}
	if data, err := redis.Ints(conn.Do("EXEC")); err == nil {
		for _, f := range data {
			flag = flag & f
			if flag != 1 {
				return false
			}
		}
		return (flag == 1)
	} else {
		return false
	}
}
Esempio n. 19
0
func (j job) enqueueAt(client redis.Conn, t time.Time, queue string) error {
	jsonString := j.encode()

	queueKey := fmt.Sprintf("resque:delayed:%d", t.Unix())
	timestampsValue := fmt.Sprintf("delayed:%d", t.Unix())

	client.Send("MULTI")
	client.Send("RPUSH", queueKey, jsonString)
	client.Send("SADD", "resque:timestamps:"+jsonString, timestampsValue)
	client.Send("ZADD", "resque:delayed_queue_schedule", t.Unix(), t.Unix())
	_, err := client.Do("EXEC")

	return err
}
Esempio n. 20
0
func (p *process) close(conn redis.Conn) error {
	logger.Infof("%v shutdown", p)
	conn.Send("SREM", fmt.Sprintf("%sworkers", namespace), p)
	conn.Send("DEL", fmt.Sprintf("%sstat:processed:%s", namespace, p))
	conn.Send("DEL", fmt.Sprintf("%sstat:failed:%s", namespace, p))
	return conn.Flush()
}
Esempio n. 21
0
func hashm(c redis.Conn, m map[string]interface{}) {
	args := []interface{}{"drugs:1"}
	for key, value := range m {
		args = append(args, key, value)
	}

	c.Send("MULTI")
	c.Send("HMSET", args...)
	c.Send("EXPIRE", "drugs:1", "5")
	r, err := c.Do("EXEC")
	if err != nil {
		panic(err)
	}

	r, err = c.Do("HGETALL", "drugs:1")
	v, err := redis.Values(r, err)
	if err != nil {
		panic(err)
	}

	m2 := make(map[string]interface{})
	for x := 0; x < len(v); x += 2 {
		key := fmt.Sprintf("%s", v[x])
		value := v[x+1]

		switch t := value.(type) {
		case []byte:
			m2[key] = string(t)
		}
	}

	//fmt.Printf("%#+v", m2)
}
Esempio n. 22
0
func kmsgpackm(c redis.Conn, m map[string]interface{}) {
	mp := &kmsgpack.Coder{}

	b, err := mp.Marshal(m)
	if err != nil {
		panic(err)
	}

	c.Send("MULTI")
	c.Send("SET", "drugs:1", string(b))
	c.Send("EXPIRE", "drugs:1", "5")
	r, err := c.Do("EXEC")
	if err != nil {
		panic(err)
	}

	r, err = c.Do("GET", "drugs:1")
	v, err := redis.Bytes(r, err)
	if err != nil {
		panic(err)
	}

	m2 := make(map[string]interface{})
	err = mp.Unmarshal(v, &m2)
	if err != nil {
		panic(err)
	}

	// fmt.Printf("%#+v", m2)
}
Esempio n. 23
0
func gobm(c redis.Conn, m map[string]interface{}) {
	gob.Register(time.Time{})

	var buffer bytes.Buffer
	encoder := gob.NewEncoder(&buffer)

	err := encoder.Encode(m)
	if err != nil {
		panic(err)
	}

	c.Send("MULTI")
	c.Send("SET", "drugs:1", buffer.String())
	c.Send("EXPIRE", "drugs:1", "5")
	r, err := c.Do("EXEC")
	if err != nil {
		panic(err)
	}

	r, err = c.Do("GET", "drugs:1")
	v, err := redis.Bytes(r, err)
	if err != nil {
		panic(err)
	}

	m2 := make(map[string]interface{})
	buffer2 := bytes.NewBuffer(v)
	decoder := gob.NewDecoder(buffer2)
	err = decoder.Decode(&m2)
	if err != nil {
		panic(err)
	}

	// fmt.Printf("%#+v", m2)
}
Esempio n. 24
0
File: redis.go Progetto: sangba/im
func Zpop(c redis.Conn, key string) (result string, err error) {

	defer func() {
		//Return connection to normal state on error.
		if err != nil {
			c.Do("DISCARD")
		}
	}()

	//Loop until transaction is successful.
	for {
		if _, err := c.Do("WATCH", key); err != nil {
			return "", err
		}

		members, err := redis.Strings(c.Do("ZRANGE", key, 0, 0))
		fmt.Println(members)
		if err != nil {
			return "", err
		}
		if len(members) != 1 {
			return "", redis.ErrNil
		}

		c.Send("MULTI")
		c.Send("ZREM", key, members[0])
		queued, err := c.Do("EXEC")
		if err != nil {
			return "", err
		}

		if queued != nil {
			result = members[0]
			break
		}
	}

	return result, nil
}
Esempio n. 25
0
func newSession(w http.ResponseWriter, userid uint64, username string, appid uint32, rc redis.Conn) (usertoken string, err error) {
	if rc == nil {
		rc = redisPool.Get()
		defer rc.Close()
	}
	usertoken = ""
	usertokenRaw, err := rc.Do("get", fmt.Sprintf("usertokens/%d+%d", userid, appid))
	lwutil.CheckError(err, "")
	if usertokenRaw != nil {
		if usertoken, err := redis.String(usertokenRaw, err); err != nil {
			return usertoken, lwutil.NewErr(err)
		}
		rc.Do("del", fmt.Sprintf("sessions/%s", usertoken))
	}

	usertoken = lwutil.GenUUID()

	session := Session{userid, username, time.Now(), appid}
	jsonSession, err := json.Marshal(session)
	if err != nil {
		return usertoken, lwutil.NewErr(err)
	}

	rc.Send("setex", fmt.Sprintf("sessions/%s", usertoken), sessionLifeSecond, jsonSession)
	rc.Send("setex", fmt.Sprintf("usertokens/%d+%d", userid, appid), sessionLifeSecond, usertoken)
	rc.Flush()
	for i := 0; i < 2; i++ {
		if _, err = rc.Receive(); err != nil {
			return usertoken, lwutil.NewErr(err)
		}
	}

	// cookie
	http.SetCookie(w, &http.Cookie{Name: "usertoken", Value: usertoken, MaxAge: sessionLifeSecond, Path: "/"})

	return usertoken, err
}
Esempio n. 26
0
func pipelineRevRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
	for _, key := range keys {
		if err := conn.Send(
			"ZREVRANGE",
			key+insertSuffix,
			offset,
			offset+limit-1,
			"WITHSCORES",
		); err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}
	}

	if err := conn.Flush(); err != nil {
		return map[string][]common.KeyScoreMember{}, err
	}

	m := make(map[string][]common.KeyScoreMember, len(keys))
	for _, key := range keys {
		values, err := redis.Values(conn.Receive())
		if err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}

		keyScoreMembers := make([]common.KeyScoreMember, 0, len(values))
		for len(values) > 0 {
			var member string
			var score float64
			if values, err = redis.Scan(values, &member, &score); err != nil {
				return map[string][]common.KeyScoreMember{}, err
			}
			keyScoreMembers = append(keyScoreMembers, common.KeyScoreMember{Key: key, Score: score, Member: member})
		}
		m[key] = keyScoreMembers
	}
	return m, nil
}
Esempio n. 27
0
// Import stores the contents of a downloaded BGP dump into a redis server.
// -1 is returned if the dump is alredy imported into redis.
func (b *CIDRReport) Import(conn redis.Conn) (int, error) {

	alreadyImported, err := redis.Bool(conn.Do("SISMEMBER", "asd:imported_dates", b.day()))
	if err != nil {
		return 0, err
	}
	if alreadyImported {
		return -1, nil
	}

	file, err := os.Open(b.Path())
	if err != nil {
		return 0, err
	}
	n := 0
	day := b.day()
	err = parseReport(file, func(asd *ASDescription) error {
		conn.Send("HSET", fmt.Sprintf("asd:%d", asd.ASN), day,
			fmt.Sprintf("%s, %s", asd.Description, asd.CountryCode))
		n++
		if n%10000 == 0 {
			err := conn.Flush()
			if err != nil {
				return err

			}
		}
		return nil
	})
	conn.Send("SADD", "asd:imported_dates", day)
	err = conn.Flush()
	if err != nil {
		return 0, err
	}
	return n, nil
}
Esempio n. 28
0
func RetrievePeople(conn redis.Conn, ids []string) ([]Person, error) {
	var people []Person

	for _, id := range ids {
		conn.Send("HGETALL", fmt.Sprintf("person:%s", id))
	}

	for i := 0; i < len(ids); i++ {
		values, err := redis.Values(conn.Receive())
		if err != nil {
			return nil, err
		}

		var person Person
		err = redis.ScanStruct(values, &person)
		if err != nil {
			return nil, err
		}

		people = append(people, person)
	}

	return people, nil
}
Esempio n. 29
0
func insertPairsIntoRedisQueue(pairs []lib.GameSessionPair, redisConnection redis.Conn) {
	redisConnection.Send("MULTI")
	for _, pair := range pairs {
		redisConnection.Send("SADD", lib.UnprocessedSetName, pair.RedisQueueKey())
	}
	redisConnection.Send("SDIFFSTORE", lib.UnprocessedSetName, lib.UnprocessedSetName, lib.ProcessedSetName)
	_, err := redisConnection.Do("EXEC")
	if err != nil {
		panic(err)
	}
}
Esempio n. 30
0
func NewFrame(conn redis.Conn, header *SBFHeader, id uint16) (*SBFFrame, error) {
	key := fmt.Sprintf("%s:count:%s:%d", SBF_NAME, header.Refer, id)
	frame := new(SBFFrame)
	frame.Key = key
	frame.Refer = header.Refer
	frame.frameDataRange(header, id)
	conn.Send("MULTI")
	conn.Send("SETBIT", frame.Refer, frame.EndIndex, 0)
	conn.Send("SET", frame.Key, 0)
	_, err := conn.Do("EXEC")
	if err == nil {
		return frame, nil
	} else {
		return nil, err
	}
}