func (sr *StoredRow) ParseKey(key []byte) error {
buf := bytes.NewBuffer(key)
_, err := buf.ReadByte() // type
if err != nil {
return err
}
sr.docID, err = buf.ReadBytes(ByteSeparator)
if len(sr.docID) < 2 { // 1 for min doc id length, 1 for separator
err = fmt.Errorf("invalid doc length 0")
return err
}
sr.docID = sr.docID[:len(sr.docID)-1] // trim off separator byte
sr.docNum, err = binary.ReadUvarint(buf)
if err != nil {
return err
}
err = binary.Read(buf, binary.LittleEndian, &sr.field)
if err != nil {
return err
}
sr.arrayPositions = make([]uint64, 0)
nextArrayPos, err := binary.ReadUvarint(buf)
for err == nil {
sr.arrayPositions = append(sr.arrayPositions, nextArrayPos)
nextArrayPos, err = binary.ReadUvarint(buf)
}
return nil
}
func NewStoredRowK(key []byte) (*StoredRow, error) {
rv := StoredRow{}
buf := bytes.NewBuffer(key)
_, err := buf.ReadByte() // type
if err != nil {
return nil, err
}
rv.doc, err = buf.ReadBytes(ByteSeparator)
if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator
err = fmt.Errorf("invalid doc length 0")
return nil, err
}
rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte
err = binary.Read(buf, binary.LittleEndian, &rv.field)
if err != nil {
return nil, err
}
rv.arrayPositions = make([]uint64, 0)
nextArrayPos, err := binary.ReadUvarint(buf)
for err == nil {
rv.arrayPositions = append(rv.arrayPositions, nextArrayPos)
nextArrayPos, err = binary.ReadUvarint(buf)
}
return &rv, nil
}
func (p *Player) recv() {
for {
length, err := ReadVarint(p.conn)
if err != nil {
p.Game.RemovePlayer(p.Name)
fmt.Println("Player disconnected")
return
}
buf := make([]byte, length)
io.ReadFull(p.conn, buf)
r := bytes.NewReader(buf)
binary.ReadUvarint(r)
id, _ := binary.ReadUvarint(r)
switch id {
case 0x01:
message := ReadString(r)
if strings.HasPrefix(message, "/") {
// Commands
} else {
WriteChatMessage(p.Game, fmt.Sprintf("<%s> %s", p.Name, message))
fmt.Printf("<%s> %s\n", p.Name, message)
}
}
}
}
func readEntry(reader *dirIndexReader, offset uint64) (*indexEntry, error) {
seekErr := seekIndex(reader, offset)
if seekErr != nil {
return nil, seekErr
}
n, err := binary.ReadUvarint(reader.indexReader)
if err != nil {
return nil, err
}
entries := []indexEntryElement{}
for i := uint64(0); i < n; i++ {
docId, derr := binary.ReadUvarint(reader.indexReader)
if derr != nil {
return nil, derr
}
var freq float64
ferr := binary.Read(reader.indexReader, binary.LittleEndian, &freq)
if ferr != nil {
return nil, ferr
}
entries = append(entries, indexEntryElement{docId, freq})
}
tail, terr := binary.ReadUvarint(reader.indexReader)
if terr != nil {
return nil, terr
}
return &indexEntry{entries, tail}, nil
}
// Format of a simple task:
// 2 bytes : magic number
// 8 bytes : id length : l
// 8 bytes : data length : n
// 32 bytes : target
// 32 bytes : action
// l bytes : id
// n bytes : data
func UnserializeSimpleTask(data []byte) *SimpleTask {
// Magic number 01
if data[0] != 0 || data[1] != 1 {
return nil
}
// ID Length
idLength, err := binary.ReadUvarint(bytes.NewBuffer(data[2:10]))
if err != nil {
return nil
}
// Data Length
length, err := binary.ReadUvarint(bytes.NewBuffer(data[10:18]))
if err != nil {
return nil
}
// Target
target := string(data[18:50])
// Action
action := string(data[50:82])
// ID
readId := string(data[82 : 82+idLength])
// Data
readData := data[82+idLength : 82+idLength+length]
return NewSimpleTask(readId, target, action, readData)
}
func (bin *mBinaryProto) readUvarint(reader io.Reader) (uint64, error) {
if br, ok := reader.(io.ByteReader); ok {
return binary.ReadUvarint(br)
}
br := newByteReader(reader, bin.readBuffer)
return binary.ReadUvarint(br)
}
func ReadHandshake(r *bytes.Reader) Handshake {
handshake := Handshake{}
handshake.Version, _ = binary.ReadUvarint(r)
handshake.Address = ReadString(r)
binary.Read(r, binary.BigEndian, &handshake.Port)
handshake.State, _ = binary.ReadUvarint(r)
return handshake
}
func readMsgDial(r io.ByteReader) (*msgDial, error) {
id, err := binary.ReadUvarint(r)
if err != nil {
return nil, err
}
seqno, err := binary.ReadUvarint(r)
if err != nil {
return nil, err
}
return &msgDial{ID: chainID(id), SeqNo: SeqNo(seqno)}, nil
}
func ReadDHCPOptions(payload []byte) ([]DHCPOption, error) {
magicCookieStart := 236
magicCookieEnd := 240
if len(payload) < magicCookieEnd {
return []DHCPOption{}, PayloadError
}
magicCookie := []byte{byte(99), byte(130), byte(83), byte(99)}
if !bytes.Equal(payload[magicCookieStart:magicCookieEnd], magicCookie) {
return []DHCPOption{}, errors.New("Cannot find magic cookie")
}
opts := []DHCPOption{}
pR := bytes.NewReader(payload[magicCookieEnd:])
// End Loop when only 255 end DHCP option is left
for {
buf := make([]byte, 1)
if _, err := pR.Read(buf); err != nil {
return []DHCPOption{}, err
}
// break if the end is reached
if bytes.Equal(buf, []byte{byte(255)}) {
break
}
code, err := binary.ReadUvarint(bytes.NewReader(buf))
if err != nil {
return []DHCPOption{}, err
}
buf = make([]byte, 1)
if _, err := pR.Read(buf); err != nil {
return []DHCPOption{}, err
}
size, err := binary.ReadUvarint(bytes.NewReader(buf))
if err != nil {
return []DHCPOption{}, err
}
buf = make([]byte, size)
if _, err := pR.Read(buf); err != nil {
return []DHCPOption{}, err
}
opts = append(opts, DHCPOption{code, buf, size})
}
return opts, nil
}
func (r *restartRange) next() (err error) {
if r.cached && len(r.cache) > r.pos {
r.kv = r.cache[r.pos]
r.pos++
return
}
if r.buf.Len() == 0 {
return io.EOF
}
var nkey []byte
// Read header
var shared, nonShared, valueLen uint64
shared, err = binary.ReadUvarint(r.buf)
if err != nil || shared > uint64(len(r.kv.key)) {
goto corrupt
}
nonShared, err = binary.ReadUvarint(r.buf)
if err != nil {
goto corrupt
}
valueLen, err = binary.ReadUvarint(r.buf)
if err != nil {
goto corrupt
}
if nonShared+valueLen > uint64(r.buf.Len()) {
goto corrupt
}
if r.cached && r.pos > 0 {
r.cache = append(r.cache, r.kv)
}
// Read content
nkey = r.buf.Next(int(nonShared))
if shared == 0 {
r.kv.key = nkey
} else {
pkey := r.kv.key[:shared]
key := make([]byte, shared+nonShared)
copy(key, pkey)
copy(key[shared:], nkey)
r.kv.key = key
}
r.kv.value = r.buf.Next(int(valueLen))
r.pos++
return
corrupt:
return errors.ErrCorrupt("bad entry in block")
}
func (conn *conn) readHeader() (uint64, uint64, error) {
seq, err := binary.ReadUvarint(conn.buf)
if err != nil {
return 0, 0, err
}
size, err := binary.ReadUvarint(conn.buf)
if err != nil {
return 0, 0, err
}
return seq, size, nil
}
func readUint64(r Reader) uint64 {
x, err := binary.ReadUvarint(r)
if err != nil {
log.Fatal(err)
}
return x
}
func decodeUint(buf *bytes.Reader) (uint32, error) {
v, err := binary.ReadUvarint(buf)
if v > 0xffffffff {
return 0, errors.New("Something wrong, this number looks too big")
}
return uint32(v), err
}
// NewStreamReader returns a reader that will decode the supplied data stream.
//
// This is compatible content from the NewStreamWriter function.
//
// When you are done with the Reader, use Close to release resources.
func NewStreamReader(in io.Reader) (Reader, error) {
f := &streamReader{
ready: make(chan *rblock, 8), // Read up to 8 blocks ahead
closeReader: make(chan struct{}, 0),
readerClosed: make(chan struct{}, 0),
curBlock: 0,
}
br := bufio.NewReader(in)
format, err := binary.ReadUvarint(br)
if err != nil {
return nil, err
}
switch format {
case 2:
err = f.readFormat2(br)
if err != nil {
return nil, err
}
default:
return nil, ErrUnknownFormat
}
go f.streamReader(br)
return f, nil
}
// NewSeekRead returns a reader that will decode the supplied index and data stream.
//
// This is compatible content from the NewWriter function.
//
// No blocks will be kept in memory, but the block data input must be seekable.
// The function will decode the index before returning.
//
// When you are done with the Reader, use Close to release resources.
func NewSeekReader(index io.Reader, blocks io.ReadSeeker) (IndexedReader, error) {
f := &reader{streamReader: streamReader{
ready: make(chan *rblock, 8), // Read up to 8 blocks ahead
closeReader: make(chan struct{}, 0),
readerClosed: make(chan struct{}, 0),
curBlock: 0,
maxLength: 8, // We have 8 blocks readahead.
}}
idx := bufio.NewReader(index)
format, err := binary.ReadUvarint(idx)
if err != nil {
return nil, err
}
switch format {
case 1:
err = f.readFormat1(idx)
default:
err = ErrUnknownFormat
}
go f.seekReader(blocks)
return f, err
}
// Read a message from io.ByteReader by first reading a varint size,
// and then reading and decoding the message object.
// If buf is not big enough a new buffer will be allocated to replace buf.
func ReadMessage(buf *[]byte, r ByteReadReader, msg proto.Message) error {
size, err := binary.ReadUvarint(r)
if err != nil {
return err
}
if cap(*buf) < int(size) {
*buf = make([]byte, size)
}
b := (*buf)[:size]
read := uint64(0)
for read != size {
n, err := r.Read(b[read:])
if err == io.EOF {
return fmt.Errorf("unexpected EOF, expected %d more bytes", size)
}
if err != nil {
return err
}
read += uint64(n)
}
err = proto.Unmarshal(b, msg)
if err != nil {
return err
}
return nil
}
func (c *baseConn) recvProto(m proto.Message,
uncompressedSize uint32, decompressor decompressFunc) error {
size, err := binary.ReadUvarint(c.r)
if err != nil {
return err
}
if size == 0 {
return nil
}
if c.r.Buffered() >= int(size) {
// Parse proto directly from the buffered data.
data, err := c.r.Peek(int(size))
if err != nil {
return err
}
if err := decompressor(data, uncompressedSize, m); err != nil {
return err
}
// TODO(pmattis): This is a hack to advance the bufio pointer by
// reading into the same slice that bufio.Reader.Peek
// returned. In Go 1.5 we'll be able to use
// bufio.Reader.Discard.
_, err = io.ReadFull(c.r, data)
return err
}
data := make([]byte, size)
if _, err := io.ReadFull(c.r, data); err != nil {
return err
}
return decompressor(data, uncompressedSize, m)
}
func ReadNextToken(rw io.ReadWriter) (string, error) {
br := &byteReader{rw}
length, err := binary.ReadUvarint(br)
if err != nil {
return "", err
}
if length > 64*1024 {
err := delimWrite(rw, []byte("messages over 64k are not allowed"))
if err != nil {
return "", err
}
return "", ErrTooLarge
}
buf := make([]byte, length)
_, err = io.ReadFull(rw, buf)
if err != nil {
return "", err
}
if len(buf) == 0 || buf[length-1] != '\n' {
return "", errors.New("message did not have trailing newline")
}
// slice off the trailing newline
buf = buf[:length-1]
return string(buf), nil
}
func (t *Transport) decodeArrayish(r reader, val reflect.Value) error {
usize, err := binary.ReadUvarint(r)
if err != nil {
return err
}
size := int(usize)
// Special cases: []byte, string
isByteArr := val.Kind() == reflect.Array && val.Type().Elem().Kind() == reflect.Uint8
isString := val.Kind() == reflect.String
if isByteArr || isString {
raw := make([]byte, size)
if _, err := io.ReadFull(r, raw); err != nil {
return err
}
switch {
case isString:
val.SetString(string(raw))
case isByteArr:
val.SetBytes(raw)
}
return nil
}
slice := reflect.MakeSlice(val.Type(), size, size)
for i := 0; i < size; i++ {
if err := t.decodeValue(r, slice.Index(i)); err != nil {
return err
}
}
val.Set(slice)
return nil
}
func NewLevelDbDatastore(dbDir string) (Datastore, error) {
opts := levigo.NewOptions()
opts.SetCache(levigo.NewLRUCache(ONE_GIGABYTE))
opts.SetCreateIfMissing(true)
opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES)
filter := levigo.NewBloomFilter(BLOOM_FILTER_BITS_PER_KEY)
opts.SetFilterPolicy(filter)
db, err := levigo.Open(dbDir, opts)
if err != nil {
return nil, err
}
ro := levigo.NewReadOptions()
lastIdBytes, err2 := db.Get(ro, NEXT_ID_KEY)
if err2 != nil {
return nil, err2
}
lastId := uint64(0)
if lastIdBytes != nil {
lastId, err2 = binary.ReadUvarint(bytes.NewBuffer(lastIdBytes))
if err2 != nil {
return nil, err2
}
}
wo := levigo.NewWriteOptions()
return &LevelDbDatastore{db: db, lastIdUsed: lastId, readOptions: ro, writeOptions: wo}, nil
}
func readUVarintAsInt(r *ReaderByteReader) (int, error) {
n, err := binary.ReadUvarint(r)
if err != nil {
return 0, err
}
return int(n), nil
}
func ReadRLE(reader io.ByteReader) (j *Judy1, err error) {
j = NewJudy1()
var idx uint64
ones := false
for {
var width uint64
width, err = binary.ReadUvarint(reader)
if err == io.EOF {
err = nil
return
} else if err != nil {
j.Free()
j = nil
return
}
if !ones {
idx += width
} else {
var i uint64
for i = 0; i < width; i = i + 1 {
j.Set(idx)
idx = idx + 1
}
}
ones = !ones
}
return
}
// Split splits datagram of a Packet from Reader.
func Split(r Reader) ([]byte, error) {
buf := &bytes.Buffer{}
// read header: message type
c, err := r.ReadByte()
if err != nil {
return nil, err
}
err = buf.WriteByte(c)
if err != nil {
return nil, err
}
// read length of payload
l, err := binary.ReadUvarint(r)
if err != nil {
return nil, err
}
// FIXME: should be check l is too long.
b := make([]byte, binary.MaxVarintLen32)
n := binary.PutUvarint(b, l)
buf.Write(b[0:n])
// read whole payload to buf.
_, err = io.CopyN(buf, r, int64(l))
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func newDecoder(b []byte, t Type) (*decoder, error) {
if len(b) < 2 {
return nil, errInvalidPacketLength
}
b0 := b[0]
h := header{
Type: decodeType(b0),
Dup: b0&0x08 != 0,
QoS: QoS(b0 >> 1 & 0x3),
Retain: b0&0x01 != 0,
}
if h.Type != t {
return nil, errTypeMismatch
}
b = b[1:]
r := bytes.NewReader(b)
u, err := binary.ReadUvarint(r)
if err != nil {
return nil, err
}
if len(b)-r.Len() > 4 || r.Len() != int(u) {
return nil, errInvalidRemainLength
}
return &decoder{header: h, r: r}, nil
}
func (c *baseConn) recvProto(m proto.Message,
uncompressedSize uint32, decompressor decompressFunc) error {
size, err := binary.ReadUvarint(c.r)
if err != nil {
return err
}
if size == 0 {
return nil
}
if c.r.Buffered() >= int(size) {
// Parse proto directly from the buffered data.
data, err := c.r.Peek(int(size))
if err != nil {
return err
}
if err := decompressor(data, uncompressedSize, m); err != nil {
return err
}
_, err = c.r.Discard(int(size))
return err
}
data := make([]byte, size)
if _, err := io.ReadFull(c.r, data); err != nil {
return err
}
return decompressor(data, uncompressedSize, m)
}
// ReadVarIntToUint reads a variable length integer from the input buffer.
// The inflated integer is written is returned as a uint64 value.
// This method only "inflates" the varint into a uint64; it does NOT
// zigzag decode it.
func (r *Reader) ReadUvarint() uint64 {
v, err := binary.ReadUvarint(r.br)
if err != nil {
r.err = err
}
return v
}
func (space *Space) request(requestId int32, body *bytes.Buffer) (tuples [][][]byte, err error) {
var (
returnCode int32
tuplesCount int32
tuplesSize int32
cardinality int32
size uint64
response *iproto.Response
)
response, err = space.conn.Request(requestId, body)
if err != nil {
return
}
// Ping has no Body
if requestId == PingOp {
tuples = [][][]byte{}
return
}
err = binary.Read(response.Body, binary.LittleEndian, &returnCode)
if err != nil {
return
}
if returnCode != 0 {
err = fmt.Errorf("Return code is not 0, but %d; Error message: %s", returnCode, response.Body.String())
return
}
err = binary.Read(response.Body, binary.LittleEndian, &tuplesCount)
if err != nil {
return
}
tuples = make([][][]byte, tuplesCount)
for i := int32(0); i < tuplesCount && response.Body.Len() > 0; i++ {
err = binary.Read(response.Body, binary.LittleEndian, &tuplesSize)
if err != nil {
return
}
err = binary.Read(response.Body, binary.LittleEndian, &cardinality)
if err != nil {
return
}
tuples[i] = make([][]byte, cardinality)
for j := int32(0); j < cardinality; j++ {
size, err = binary.ReadUvarint(response.Body)
if err != nil {
return
}
tuples[i][j] = make([]byte, size)
_, err = response.Body.Read(tuples[i][j])
if err != nil {
return
}
}
}
return
}
// Reads encoded Properties from a stream.
func (properties *Properties) ReadFrom(reader io.Reader) error {
length, err := binary.ReadUvarint(byteReader{reader})
if err != nil {
return err
}
if length == 0 {
return nil
}
body := make([]byte, length)
if _, err := io.ReadFull(reader, body); err != nil {
return err
}
if body[length-1] != 0 {
return fmt.Errorf("Invalid properties (not NUL-terminated)")
}
eachProp := bytes.Split(body[0:length-1], []byte{0})
nProps := len(eachProp) / 2
if nProps*2 != len(eachProp) {
return fmt.Errorf("Odd number of strings in properties")
}
*properties = Properties{}
for i := 0; i < len(eachProp); i += 2 {
key := decodeProperty(eachProp[i])
value := decodeProperty(eachProp[i+1])
if _, exists := (*properties)[key]; exists {
return fmt.Errorf("Duplicate property name %q", key)
}
(*properties)[key] = value
}
return nil
}
// stdDistribution distributes the keyspace evenly.
func stdDistribution(key string, n uint64) uint64 {
h := sha1.New()
io.WriteString(h, key)
buf := bytes.NewBuffer(h.Sum(nil)[:8])
i, _ := binary.ReadUvarint(buf)
return i % n
}
func readSequence(r io.ByteReader) uint64 {
seq, err := binary.ReadUvarint(r)
if err != nil {
panic("readSequence failed")
}
return seq
}
