예제 #1
0
func pipelineInsert(conn redis.Conn, keyScoreMembers []common.KeyScoreMember, maxSize int) error {
	for _, tuple := range keyScoreMembers {
		if err := insertScript.Send(
			conn,
			tuple.Key,
			tuple.Score,
			tuple.Member,
			maxSize,
		); err != nil {
			return err
		}
	}

	if err := conn.Flush(); err != nil {
		return err
	}

	for _ = range keyScoreMembers {
		// TODO actually count inserts
		if _, err := conn.Receive(); err != nil {
			return err
		}
	}

	return nil
}
예제 #2
0
파일: process.go 프로젝트: Hebo/goworker
func (p *process) close(conn redis.Conn) error {
	logger.Infof("%v shutdown", p)
	conn.Send("SREM", fmt.Sprintf("%sworkers", namespace), p)
	conn.Send("DEL", fmt.Sprintf("%sstat:processed:%s", namespace, p))
	conn.Send("DEL", fmt.Sprintf("%sstat:failed:%s", namespace, p))
	return conn.Flush()
}
예제 #3
0
func pipelineDelete(conn redis.Conn, keyScoreMembers []common.KeyScoreMember, maxSize int) error {
	for _, keyScoreMember := range keyScoreMembers {
		if err := deleteScript.Send(
			conn,
			keyScoreMember.Key,
			keyScoreMember.Score,
			keyScoreMember.Member,
			maxSize,
		); err != nil {
			return err
		}
	}

	if err := conn.Flush(); err != nil {
		return err
	}

	for _ = range keyScoreMembers {
		// TODO actually count deletes
		if _, err := conn.Receive(); err != nil {
			return err
		}
	}

	return nil
}
예제 #4
0
func (self *redis_gateway) execute(c redis.Conn, commands [][]string) error {
	switch len(commands) {
	case 0:
		return nil
	case 1:
		e := self.redis_do(c, commands[0])
		if nil != e {
			return errors.New("execute `" + strings.Join(commands[0], " ") + "` failed, " + e.Error())
		}
		return nil
	default:
		for _, command := range commands {
			e := self.redis_send(c, command)
			if nil != e {
				return errors.New("execute `" + strings.Join(command, " ") + "` failed, " + e.Error())
			}
		}

		e := c.Flush()
		if nil != e {
			return e
		}

		for i := 0; i < len(commands); i++ {
			_, e = c.Receive()
			if nil != e {
				return errors.New("execute `" + strings.Join(commands[i], " ") + "` failed, " + e.Error())
			}
		}
		return nil
	}
}
예제 #5
0
파일: goorm.go 프로젝트: aleksclark/vigia
func Get(conn redis.Conn) string {

	conn.Send("GET", "fooo")
	conn.Flush()

	result, _ := redis.String(conn.Receive()) // TODO missing two err catch

	return result
}
예제 #6
0
파일: goorm.go 프로젝트: aleksclark/vigia
func Save(obj interface{}, conn redis.Conn) reflect.Type {

	conn.Send("GET", "fooo")
	conn.Flush()

	//result, _ := redis.String(conn.Receive()) // TODO missing two err catch

	return reflect.ValueOf(&obj).Type()

}
예제 #7
0
파일: client.go 프로젝트: YaSuenag/hsbeat
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
	return func(key outil.Selector, data []outputs.Data) ([]outputs.Data, error) {
		var okEvents []outputs.Data
		serialized := make([]interface{}, 0, len(data))
		okEvents, serialized = serializeEvents(serialized, 0, data)
		if len(serialized) == 0 {
			return nil, nil
		}

		data = okEvents[:0]
		for i, serializedEvent := range serialized {
			eventKey, err := key.Select(okEvents[i].Event)
			if err != nil {
				logp.Err("Failed to set redis key: %v", err)
				continue
			}

			data = append(data, okEvents[i])
			if err := conn.Send(command, eventKey, serializedEvent); err != nil {
				logp.Err("Failed to execute %v: %v", command, err)
				return okEvents, err
			}
		}

		if err := conn.Flush(); err != nil {
			return data, err
		}

		failed := data[:0]
		var lastErr error
		for i := range serialized {
			_, err := conn.Receive()
			if err != nil {
				if _, ok := err.(redis.Error); ok {
					logp.Err("Failed to %v event to list with %v",
						command, err)
					failed = append(failed, data[i])
					lastErr = err
				} else {
					logp.Err("Failed to %v multiple events to list with %v",
						command, err)
					failed = append(failed, data[i:]...)
					lastErr = err
					break
				}
			}
		}
		return failed, lastErr
	}
}
예제 #8
0
파일: bgpdump.go 프로젝트: thomasf/alkasir
func (b *BGPDump) parseBGPCSV(r io.Reader, conn redis.Conn) (int, error) {
	day := b.day()
	s := bufio.NewScanner(r)
	n := 0
	var asn string
	for s.Scan() {
		cols := strings.Split(s.Text(), "|")
		if len(cols) < 7 {
			return n, ParseError{
				Message: "too few columns",
				Path:    filepath.Base(b.Path()),
				LineNum: n,
				Line:    s.Text(),
			}
		}
		block := cols[5]

		if _, ok := asn12654blocks[block]; ok {
			asn = "12654"
		} else {
			asPath := cols[6]
			asns := strings.Split(asPath, " ")
			asn = asns[len(asns)-1]
			if asn == "" {
				return n, ParseError{
					Message: "no ASPATH data",
					Path:    filepath.Base(b.Path()),
					LineNum: n,
					Line:    s.Text(),
				}
			}
		}
		conn.Send("HSET", fmt.Sprintf("i2a:%s", block), day, asn)
		n++
		if n%10000 == 0 {
			err := conn.Flush()
			if err != nil {
				return 0, err
			}
		}
	}
	conn.Send("SADD", "i2a:imported_dates", day)
	err := conn.Flush()
	if err != nil {
		return 0, err
	}
	return n, nil
}
예제 #9
0
func read(list string, conn redis.Conn, count int) {
	for i := 0; i < count; i++ {
		conn.Send("LPOP", list)
	}
	conn.Flush()
	for i := 0; i < count; i++ {
		value, err := redis.String(conn.Receive())

		if err != nil {
			os.Exit(0)
		}
		if value != "" {
			fmt.Println(value)
		}
	}
}
예제 #10
0
func pipelineScore(conn redis.Conn, keyMembers []common.KeyMember) (map[common.KeyMember]Presence, error) {
	for _, keyMember := range keyMembers {
		if err := conn.Send("ZSCORE", keyMember.Key+insertSuffix, keyMember.Member); err != nil {
			return map[common.KeyMember]Presence{}, err
		}
		if err := conn.Send("ZSCORE", keyMember.Key+deleteSuffix, keyMember.Member); err != nil {
			return map[common.KeyMember]Presence{}, err
		}
	}
	if err := conn.Flush(); err != nil {
		return map[common.KeyMember]Presence{}, err
	}

	m := map[common.KeyMember]Presence{}
	for i := 0; i < len(keyMembers); i++ {
		insertReply, insertErr := conn.Receive()
		insertValue, insertErr := redis.Float64(insertReply, insertErr)
		deleteReply, deleteErr := conn.Receive()
		deleteValue, deleteErr := redis.Float64(deleteReply, deleteErr)
		switch {
		case insertErr == nil && deleteErr == redis.ErrNil:
			m[keyMembers[i]] = Presence{
				Present:  true,
				Inserted: true,
				Score:    insertValue,
			}
		case insertErr == redis.ErrNil && deleteErr == nil:
			m[keyMembers[i]] = Presence{
				Present:  true,
				Inserted: false,
				Score:    deleteValue,
			}
		case insertErr == redis.ErrNil && deleteErr == redis.ErrNil:
			m[keyMembers[i]] = Presence{
				Present: false,
			}
		default:
			return map[common.KeyMember]Presence{}, fmt.Errorf(
				"pipelineScore bad state for %v (%v/%v)",
				keyMembers[i],
				insertErr,
				deleteErr,
			)
		}
	}
	return m, nil
}
예제 #11
0
파일: cluster.go 프로젝트: Greentor/roshi
func pipelineRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
	if limit < 0 {
		return map[string][]common.KeyScoreMember{}, fmt.Errorf("negative limit is invalid for offset-based select")
	}
	for _, key := range keys {
		if err := conn.Send(
			"ZREVRANGE",
			key+insertSuffix,
			offset,
			offset+limit-1,
			"WITHSCORES",
		); err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}
	}

	if err := conn.Flush(); err != nil {
		return map[string][]common.KeyScoreMember{}, err
	}

	m := make(map[string][]common.KeyScoreMember, len(keys))

	for _, key := range keys {
		values, err := redis.Values(conn.Receive())
		if err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}

		var (
			ksm             = common.KeyScoreMember{Key: key}
			keyScoreMembers = make([]common.KeyScoreMember, 0, len(values))
		)

		for len(values) > 0 {
			if values, err = redis.Scan(values, &ksm.Member, &ksm.Score); err != nil {
				return map[string][]common.KeyScoreMember{}, err
			}

			keyScoreMembers = append(keyScoreMembers, ksm)
		}

		m[key] = keyScoreMembers
	}

	return m, nil
}
예제 #12
0
파일: client.go 프로젝트: ChongFeng/beats
func publishEventsPipeline(conn redis.Conn, command string) publishFn {
	return func(dest []byte, events []common.MapStr) ([]common.MapStr, error) {
		var args [2]interface{}
		args[0] = dest

		serialized := make([]interface{}, 0, len(events))
		events, serialized = serializeEvents(serialized, 0, events)
		if len(serialized) == 0 {
			return nil, nil
		}

		for _, event := range serialized {
			args[1] = event
			if err := conn.Send(command, args[:]...); err != nil {
				logp.Err("Failed to execute %v: %v", command, err)
				return events, err
			}
		}

		if err := conn.Flush(); err != nil {
			return events, err
		}

		failed := events[:0]
		var lastErr error
		for i := range serialized {
			_, err := conn.Receive()
			if err != nil {
				if _, ok := err.(redis.Error); ok {
					logp.Err("Failed to %v event to list (%v) with %v",
						command, dest, err)
					failed = append(failed, events[i])
					lastErr = err
				} else {
					logp.Err("Failed to %v multiple events to list (%v) with %v",
						command, dest, err)
					failed = append(failed, events[i:]...)
					lastErr = err
					break
				}
			}
		}
		return failed, lastErr
	}
}
예제 #13
0
func GetRepo(c redis.Conn, repo string) (string, bool, error) {
	c.Send("GET", repo)
	c.Send("TTL", repo)
	if err := c.Flush(); err != nil {
		return "", false, nil
	}
	content, err := redis.String(c.Receive())
	if err == redis.ErrNil {
		return "", false, nil
	}
	if err != nil {
		return "", false, err
	}

	if ttl, err := redis.Int(c.Receive()); err == nil && ttl > FRESH_CACHE_LIMIT {
		return content, true, nil
	}

	return content, false, err
}
예제 #14
0
파일: auth.go 프로젝트: henyouqian/grid
func newSession(w http.ResponseWriter, userid uint64, username string, appid uint32, rc redis.Conn) (usertoken string, err error) {
	if rc == nil {
		rc = redisPool.Get()
		defer rc.Close()
	}
	usertoken = ""
	usertokenRaw, err := rc.Do("get", fmt.Sprintf("usertokens/%d+%d", userid, appid))
	lwutil.CheckError(err, "")
	if usertokenRaw != nil {
		if usertoken, err := redis.String(usertokenRaw, err); err != nil {
			return usertoken, lwutil.NewErr(err)
		}
		rc.Do("del", fmt.Sprintf("sessions/%s", usertoken))
	}

	usertoken = lwutil.GenUUID()

	session := Session{userid, username, time.Now(), appid}
	jsonSession, err := json.Marshal(session)
	if err != nil {
		return usertoken, lwutil.NewErr(err)
	}

	rc.Send("setex", fmt.Sprintf("sessions/%s", usertoken), sessionLifeSecond, jsonSession)
	rc.Send("setex", fmt.Sprintf("usertokens/%d+%d", userid, appid), sessionLifeSecond, usertoken)
	rc.Flush()
	for i := 0; i < 2; i++ {
		if _, err = rc.Receive(); err != nil {
			return usertoken, lwutil.NewErr(err)
		}
	}

	// cookie
	http.SetCookie(w, &http.Cookie{Name: "usertoken", Value: usertoken, MaxAge: sessionLifeSecond, Path: "/"})

	return usertoken, err
}
예제 #15
0
파일: cluster.go 프로젝트: neurodrone/roshi
func pipelineRevRange(conn redis.Conn, keys []string, offset, limit int) (map[string][]common.KeyScoreMember, error) {
	for _, key := range keys {
		if err := conn.Send(
			"ZREVRANGE",
			key+insertSuffix,
			offset,
			offset+limit-1,
			"WITHSCORES",
		); err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}
	}

	if err := conn.Flush(); err != nil {
		return map[string][]common.KeyScoreMember{}, err
	}

	m := make(map[string][]common.KeyScoreMember, len(keys))
	for _, key := range keys {
		values, err := redis.Values(conn.Receive())
		if err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}

		keyScoreMembers := make([]common.KeyScoreMember, 0, len(values))
		for len(values) > 0 {
			var member string
			var score float64
			if values, err = redis.Scan(values, &member, &score); err != nil {
				return map[string][]common.KeyScoreMember{}, err
			}
			keyScoreMembers = append(keyScoreMembers, common.KeyScoreMember{Key: key, Score: score, Member: member})
		}
		m[key] = keyScoreMembers
	}
	return m, nil
}
예제 #16
0
// Import stores the contents of a downloaded BGP dump into a redis server.
// -1 is returned if the dump is alredy imported into redis.
func (b *CIDRReport) Import(conn redis.Conn) (int, error) {

	alreadyImported, err := redis.Bool(conn.Do("SISMEMBER", "asd:imported_dates", b.day()))
	if err != nil {
		return 0, err
	}
	if alreadyImported {
		return -1, nil
	}

	file, err := os.Open(b.Path())
	if err != nil {
		return 0, err
	}
	n := 0
	day := b.day()
	err = parseReport(file, func(asd *ASDescription) error {
		conn.Send("HSET", fmt.Sprintf("asd:%d", asd.ASN), day,
			fmt.Sprintf("%s, %s", asd.Description, asd.CountryCode))
		n++
		if n%10000 == 0 {
			err := conn.Flush()
			if err != nil {
				return err

			}
		}
		return nil
	})
	conn.Send("SADD", "asd:imported_dates", day)
	err = conn.Flush()
	if err != nil {
		return 0, err
	}
	return n, nil
}
예제 #17
0
func SetCache(c redis.Conn, repo, content, coverage string) error {
	c.Send("SETEX", repo, CACHE_EXPIRE_TIME, content)
	c.Send("SET", repo+".coverage", coverage)
	return c.Flush()
}
예제 #18
0
func SetStats(c redis.Conn, repo string) error {
	c.Send("ZINCRBY", "top", 1, repo)
	c.Send("ZADD", "last", time.Now().Unix(), repo)
	return c.Flush()
}
예제 #19
0
func pipelineRangeByScore(conn redis.Conn, keys []string, cursor, stopcursor common.Cursor, limit int) (map[string][]common.KeyScoreMember, error) {
	if limit < 0 {
		// TODO maybe change that
		return map[string][]common.KeyScoreMember{}, fmt.Errorf("negative limit is invalid for cursor-based select")
	}

	// pastCursor returns true when the score+member are "past" the cursor
	// (smaller score, larger lexicographically) and can therefore be included
	// in the resultset.
	pastCursor := func(score float64, member string) bool {
		if score < cursor.Score {
			return true
		}
		if score == cursor.Score && bytes.Compare([]byte(member), []byte(cursor.Member)) < 0 {
			return true
		}
		return false
	}

	// beforeStop returns true as long as the score+member are "before" the
	// stopcursor (larger score, smaller lexicographically) and can therefore
	// be included in the resultset.
	beforeStop := func(score float64, member string) bool {
		if score > stopcursor.Score {
			return true
		}
		if score == stopcursor.Score && bytes.Compare([]byte(member), []byte(stopcursor.Member)) > 0 {
			return true
		}
		return false
	}

	// An unlimited number of members may exist at cursor.Score. Luckily,
	// they're in lexicographically stable order. Walk the elements we get
	// back. For as long as element.Score == cursor.Score, and a
	// lexicographical comparison of (element.Score, cursor.Score) < 0,
	// discard the element. As soon as that condition fails, break the loop,
	// and collect elements. If we run out of elements before collecting the
	// user-requested limit, double the limit and try again, up to N times.

	var (
		keysToSelect = keys  // start with all
		selectLimit  = limit // double every time
		maxAttempts  = 3     // up to this many times (TODO could be paramaterized)
		results      = map[string][]common.KeyScoreMember{}
	)
	for attempt := 0; len(keysToSelect) > 0 && attempt < maxAttempts; attempt++ {
		for _, key := range keysToSelect {
			if err := conn.Send(
				"ZREVRANGEBYSCORE",
				key+insertSuffix,
				fmt.Sprint(cursor.Score), // max
				"-inf", // min
				"WITHSCORES",
				"LIMIT",
				0,
				selectLimit,
			); err != nil {
				return map[string][]common.KeyScoreMember{}, err
			}
		}

		if err := conn.Flush(); err != nil {
			return map[string][]common.KeyScoreMember{}, err
		}

		m := make(map[string][]common.KeyScoreMember, len(keys))
		for _, key := range keysToSelect {
			values, err := redis.Values(conn.Receive())
			if err != nil {
				return map[string][]common.KeyScoreMember{}, err
			}

			var (
				collected = 0
				validated = make([]common.KeyScoreMember, 0, len(values))
				hitStop   = false
			)

			for len(values) > 0 {
				var member string
				var score float64
				if values, err = redis.Scan(values, &member, &score); err != nil {
					return map[string][]common.KeyScoreMember{}, err
				}

				collected++

				if !pastCursor(score, member) {
					continue // this element is still behind or at our cursor
				}
				if !beforeStop(score, member) {
					hitStop = true
					continue // this element is beyond our stop point
				}

				validated = append(validated, common.KeyScoreMember{Key: key, Score: score, Member: member})
			}

			// At this point, we know if we can use these elements, or need to
			// go back for more.
			var (
				haveEnoughElements = len(validated) >= limit
				exhaustedElements  = collected < selectLimit
			)
			if haveEnoughElements || exhaustedElements || hitStop {
				if len(validated) > limit {
					validated = validated[:limit]
				}
				m[key] = validated
			}
		}

		retryKeys := make([]string, 0, len(keysToSelect))
		for _, key := range keysToSelect {
			if a, ok := m[key]; ok {
				results[key] = a // use it
			} else {
				retryKeys = append(retryKeys, key) // try again
			}
		}
		keysToSelect = retryKeys
		selectLimit *= 2
	}

	if n := len(keysToSelect); n > 0 {
		return map[string][]common.KeyScoreMember{}, fmt.Errorf("%d key(s) failed to yield enough elements", n)
	}

	return results, nil
}
예제 #20
0
파일: process.go 프로젝트: Hebo/goworker
func (p *process) open(conn redis.Conn) error {
	conn.Send("SADD", fmt.Sprintf("%sworkers", namespace), p)
	conn.Send("SET", fmt.Sprintf("%sstat:processed:%v", namespace, p), "0")
	conn.Send("SET", fmt.Sprintf("%sstat:failed:%v", namespace, p), "0")
	return conn.Flush()
}
예제 #21
0
파일: process.go 프로젝트: Hebo/goworker
func (p *process) start(conn redis.Conn) error {
	conn.Send("SET", fmt.Sprintf("%sworker:%s:started", namespace, p), time.Now().String())
	return conn.Flush()
}
예제 #22
0
파일: bgpdump.go 프로젝트: thomasf/alkasir
func (b *BGPDump) parseBGPDump(conn redis.Conn) (int, error) {
	day := b.day()
	n := 0

	f, err := os.Open(b.Path())
	if err != nil {
		return 0, err
	}

	gzipReader, err := gzip.NewReader(f)
	if err != nil {
		return n, fmt.Errorf("couldn't create gzip reader: %v", err)
	}
	scanner := bufio.NewScanner(gzipReader)
	scanner.Split(mrt.SplitMrt)
	count := 0

	indexTableCount := 0
entries:
	for scanner.Scan() {
		count++
		data := scanner.Bytes()

		hdr := &mrt.MRTHeader{}
		errh := hdr.DecodeFromBytes(data[:mrt.MRT_COMMON_HEADER_LEN])
		if err != nil {
			return 0, errh
		}

		msg, err := mrt.ParseMRTBody(hdr, data[mrt.MRT_COMMON_HEADER_LEN:])
		if err != nil {
			log.Printf("could not parse mrt body: %v", err)
			continue entries
		}

		if msg.Header.Type != mrt.TABLE_DUMPv2 {
			return 0, fmt.Errorf("unexpected message type: %d", msg.Header.Type)
		}

		switch mtrBody := msg.Body.(type) {
		case *mrt.PeerIndexTable:
			indexTableCount++
			if indexTableCount != 1 {
				return 0, fmt.Errorf("got >1 PeerIndexTable")
			}

		case *mrt.Rib:
			prefix := mtrBody.Prefix
			if len(mtrBody.Entries) < 0 {
				return 0, fmt.Errorf("no entries")
			}

			for _, entry := range mtrBody.Entries {
			attrs:
				for _, attr := range entry.PathAttributes {
					switch attr := attr.(type) {
					case *bgp.PathAttributeAsPath:
						if len(attr.Value) < 1 {
							continue attrs
						}
						if v, ok := attr.Value[0].(*bgp.As4PathParam); ok {
							if len(v.AS) < 0 {
								continue attrs
							}
							conn.Send("HSET", fmt.Sprintf("i2a:%s", prefix), day, v.AS[len(v.AS)-1])
							n++
							if n%10000 == 0 {
								err := conn.Flush()
								if err != nil {
									return 0, err
								}
							}
							continue entries
						}
					}
				}
			}
		default:
			return 0, fmt.Errorf("unsupported message %v %s", mtrBody, spew.Sdump(msg))
		}
	}
	conn.Send("SADD", "i2a:imported_dates", day)
	err = conn.Flush()
	if err != nil {
		return 0, err
	}

	return n, nil
}
예제 #23
0
파일: process.go 프로젝트: Hebo/goworker
func (p *process) fail(conn redis.Conn) error {
	conn.Send("INCR", fmt.Sprintf("%sstat:failed", namespace))
	conn.Send("INCR", fmt.Sprintf("%sstat:failed:%s", namespace, p))
	return conn.Flush()
}
예제 #24
0
파일: process.go 프로젝트: Hebo/goworker
func (p *process) finish(conn redis.Conn) error {
	conn.Send("DEL", fmt.Sprintf("%sworker:%s", namespace, p))
	conn.Send("DEL", fmt.Sprintf("%sworker:%s:started", namespace, p))
	return conn.Flush()
}