func (sr *StoredRow) ParseKey(key []byte) error { buf := bytes.NewBuffer(key) _, err := buf.ReadByte() // type if err != nil { return err } sr.docID, err = buf.ReadBytes(ByteSeparator) if len(sr.docID) < 2 { // 1 for min doc id length, 1 for separator err = fmt.Errorf("invalid doc length 0") return err } sr.docID = sr.docID[:len(sr.docID)-1] // trim off separator byte sr.docNum, err = binary.ReadUvarint(buf) if err != nil { return err } err = binary.Read(buf, binary.LittleEndian, &sr.field) if err != nil { return err } sr.arrayPositions = make([]uint64, 0) nextArrayPos, err := binary.ReadUvarint(buf) for err == nil { sr.arrayPositions = append(sr.arrayPositions, nextArrayPos) nextArrayPos, err = binary.ReadUvarint(buf) } return nil }
func NewStoredRowK(key []byte) (*StoredRow, error) { rv := StoredRow{} buf := bytes.NewBuffer(key) _, err := buf.ReadByte() // type if err != nil { return nil, err } rv.doc, err = buf.ReadBytes(ByteSeparator) if len(rv.doc) < 2 { // 1 for min doc id length, 1 for separator err = fmt.Errorf("invalid doc length 0") return nil, err } rv.doc = rv.doc[:len(rv.doc)-1] // trim off separator byte err = binary.Read(buf, binary.LittleEndian, &rv.field) if err != nil { return nil, err } rv.arrayPositions = make([]uint64, 0) nextArrayPos, err := binary.ReadUvarint(buf) for err == nil { rv.arrayPositions = append(rv.arrayPositions, nextArrayPos) nextArrayPos, err = binary.ReadUvarint(buf) } return &rv, nil }
func (p *Player) recv() { for { length, err := ReadVarint(p.conn) if err != nil { p.Game.RemovePlayer(p.Name) fmt.Println("Player disconnected") return } buf := make([]byte, length) io.ReadFull(p.conn, buf) r := bytes.NewReader(buf) binary.ReadUvarint(r) id, _ := binary.ReadUvarint(r) switch id { case 0x01: message := ReadString(r) if strings.HasPrefix(message, "/") { // Commands } else { WriteChatMessage(p.Game, fmt.Sprintf("<%s> %s", p.Name, message)) fmt.Printf("<%s> %s\n", p.Name, message) } } } }
func readEntry(reader *dirIndexReader, offset uint64) (*indexEntry, error) { seekErr := seekIndex(reader, offset) if seekErr != nil { return nil, seekErr } n, err := binary.ReadUvarint(reader.indexReader) if err != nil { return nil, err } entries := []indexEntryElement{} for i := uint64(0); i < n; i++ { docId, derr := binary.ReadUvarint(reader.indexReader) if derr != nil { return nil, derr } var freq float64 ferr := binary.Read(reader.indexReader, binary.LittleEndian, &freq) if ferr != nil { return nil, ferr } entries = append(entries, indexEntryElement{docId, freq}) } tail, terr := binary.ReadUvarint(reader.indexReader) if terr != nil { return nil, terr } return &indexEntry{entries, tail}, nil }
// Format of a simple task: // 2 bytes : magic number // 8 bytes : id length : l // 8 bytes : data length : n // 32 bytes : target // 32 bytes : action // l bytes : id // n bytes : data func UnserializeSimpleTask(data []byte) *SimpleTask { // Magic number 01 if data[0] != 0 || data[1] != 1 { return nil } // ID Length idLength, err := binary.ReadUvarint(bytes.NewBuffer(data[2:10])) if err != nil { return nil } // Data Length length, err := binary.ReadUvarint(bytes.NewBuffer(data[10:18])) if err != nil { return nil } // Target target := string(data[18:50]) // Action action := string(data[50:82]) // ID readId := string(data[82 : 82+idLength]) // Data readData := data[82+idLength : 82+idLength+length] return NewSimpleTask(readId, target, action, readData) }
func (bin *mBinaryProto) readUvarint(reader io.Reader) (uint64, error) { if br, ok := reader.(io.ByteReader); ok { return binary.ReadUvarint(br) } br := newByteReader(reader, bin.readBuffer) return binary.ReadUvarint(br) }
func ReadHandshake(r *bytes.Reader) Handshake { handshake := Handshake{} handshake.Version, _ = binary.ReadUvarint(r) handshake.Address = ReadString(r) binary.Read(r, binary.BigEndian, &handshake.Port) handshake.State, _ = binary.ReadUvarint(r) return handshake }
func readMsgDial(r io.ByteReader) (*msgDial, error) { id, err := binary.ReadUvarint(r) if err != nil { return nil, err } seqno, err := binary.ReadUvarint(r) if err != nil { return nil, err } return &msgDial{ID: chainID(id), SeqNo: SeqNo(seqno)}, nil }
func ReadDHCPOptions(payload []byte) ([]DHCPOption, error) { magicCookieStart := 236 magicCookieEnd := 240 if len(payload) < magicCookieEnd { return []DHCPOption{}, PayloadError } magicCookie := []byte{byte(99), byte(130), byte(83), byte(99)} if !bytes.Equal(payload[magicCookieStart:magicCookieEnd], magicCookie) { return []DHCPOption{}, errors.New("Cannot find magic cookie") } opts := []DHCPOption{} pR := bytes.NewReader(payload[magicCookieEnd:]) // End Loop when only 255 end DHCP option is left for { buf := make([]byte, 1) if _, err := pR.Read(buf); err != nil { return []DHCPOption{}, err } // break if the end is reached if bytes.Equal(buf, []byte{byte(255)}) { break } code, err := binary.ReadUvarint(bytes.NewReader(buf)) if err != nil { return []DHCPOption{}, err } buf = make([]byte, 1) if _, err := pR.Read(buf); err != nil { return []DHCPOption{}, err } size, err := binary.ReadUvarint(bytes.NewReader(buf)) if err != nil { return []DHCPOption{}, err } buf = make([]byte, size) if _, err := pR.Read(buf); err != nil { return []DHCPOption{}, err } opts = append(opts, DHCPOption{code, buf, size}) } return opts, nil }
func (r *restartRange) next() (err error) { if r.cached && len(r.cache) > r.pos { r.kv = r.cache[r.pos] r.pos++ return } if r.buf.Len() == 0 { return io.EOF } var nkey []byte // Read header var shared, nonShared, valueLen uint64 shared, err = binary.ReadUvarint(r.buf) if err != nil || shared > uint64(len(r.kv.key)) { goto corrupt } nonShared, err = binary.ReadUvarint(r.buf) if err != nil { goto corrupt } valueLen, err = binary.ReadUvarint(r.buf) if err != nil { goto corrupt } if nonShared+valueLen > uint64(r.buf.Len()) { goto corrupt } if r.cached && r.pos > 0 { r.cache = append(r.cache, r.kv) } // Read content nkey = r.buf.Next(int(nonShared)) if shared == 0 { r.kv.key = nkey } else { pkey := r.kv.key[:shared] key := make([]byte, shared+nonShared) copy(key, pkey) copy(key[shared:], nkey) r.kv.key = key } r.kv.value = r.buf.Next(int(valueLen)) r.pos++ return corrupt: return errors.ErrCorrupt("bad entry in block") }
func (conn *conn) readHeader() (uint64, uint64, error) { seq, err := binary.ReadUvarint(conn.buf) if err != nil { return 0, 0, err } size, err := binary.ReadUvarint(conn.buf) if err != nil { return 0, 0, err } return seq, size, nil }
func readUint64(r Reader) uint64 { x, err := binary.ReadUvarint(r) if err != nil { log.Fatal(err) } return x }
func decodeUint(buf *bytes.Reader) (uint32, error) { v, err := binary.ReadUvarint(buf) if v > 0xffffffff { return 0, errors.New("Something wrong, this number looks too big") } return uint32(v), err }
// NewStreamReader returns a reader that will decode the supplied data stream. // // This is compatible content from the NewStreamWriter function. // // When you are done with the Reader, use Close to release resources. func NewStreamReader(in io.Reader) (Reader, error) { f := &streamReader{ ready: make(chan *rblock, 8), // Read up to 8 blocks ahead closeReader: make(chan struct{}, 0), readerClosed: make(chan struct{}, 0), curBlock: 0, } br := bufio.NewReader(in) format, err := binary.ReadUvarint(br) if err != nil { return nil, err } switch format { case 2: err = f.readFormat2(br) if err != nil { return nil, err } default: return nil, ErrUnknownFormat } go f.streamReader(br) return f, nil }
// NewSeekRead returns a reader that will decode the supplied index and data stream. // // This is compatible content from the NewWriter function. // // No blocks will be kept in memory, but the block data input must be seekable. // The function will decode the index before returning. // // When you are done with the Reader, use Close to release resources. func NewSeekReader(index io.Reader, blocks io.ReadSeeker) (IndexedReader, error) { f := &reader{streamReader: streamReader{ ready: make(chan *rblock, 8), // Read up to 8 blocks ahead closeReader: make(chan struct{}, 0), readerClosed: make(chan struct{}, 0), curBlock: 0, maxLength: 8, // We have 8 blocks readahead. }} idx := bufio.NewReader(index) format, err := binary.ReadUvarint(idx) if err != nil { return nil, err } switch format { case 1: err = f.readFormat1(idx) default: err = ErrUnknownFormat } go f.seekReader(blocks) return f, err }
// Read a message from io.ByteReader by first reading a varint size, // and then reading and decoding the message object. // If buf is not big enough a new buffer will be allocated to replace buf. func ReadMessage(buf *[]byte, r ByteReadReader, msg proto.Message) error { size, err := binary.ReadUvarint(r) if err != nil { return err } if cap(*buf) < int(size) { *buf = make([]byte, size) } b := (*buf)[:size] read := uint64(0) for read != size { n, err := r.Read(b[read:]) if err == io.EOF { return fmt.Errorf("unexpected EOF, expected %d more bytes", size) } if err != nil { return err } read += uint64(n) } err = proto.Unmarshal(b, msg) if err != nil { return err } return nil }
func (c *baseConn) recvProto(m proto.Message, uncompressedSize uint32, decompressor decompressFunc) error { size, err := binary.ReadUvarint(c.r) if err != nil { return err } if size == 0 { return nil } if c.r.Buffered() >= int(size) { // Parse proto directly from the buffered data. data, err := c.r.Peek(int(size)) if err != nil { return err } if err := decompressor(data, uncompressedSize, m); err != nil { return err } // TODO(pmattis): This is a hack to advance the bufio pointer by // reading into the same slice that bufio.Reader.Peek // returned. In Go 1.5 we'll be able to use // bufio.Reader.Discard. _, err = io.ReadFull(c.r, data) return err } data := make([]byte, size) if _, err := io.ReadFull(c.r, data); err != nil { return err } return decompressor(data, uncompressedSize, m) }
func ReadNextToken(rw io.ReadWriter) (string, error) { br := &byteReader{rw} length, err := binary.ReadUvarint(br) if err != nil { return "", err } if length > 64*1024 { err := delimWrite(rw, []byte("messages over 64k are not allowed")) if err != nil { return "", err } return "", ErrTooLarge } buf := make([]byte, length) _, err = io.ReadFull(rw, buf) if err != nil { return "", err } if len(buf) == 0 || buf[length-1] != '\n' { return "", errors.New("message did not have trailing newline") } // slice off the trailing newline buf = buf[:length-1] return string(buf), nil }
func (t *Transport) decodeArrayish(r reader, val reflect.Value) error { usize, err := binary.ReadUvarint(r) if err != nil { return err } size := int(usize) // Special cases: []byte, string isByteArr := val.Kind() == reflect.Array && val.Type().Elem().Kind() == reflect.Uint8 isString := val.Kind() == reflect.String if isByteArr || isString { raw := make([]byte, size) if _, err := io.ReadFull(r, raw); err != nil { return err } switch { case isString: val.SetString(string(raw)) case isByteArr: val.SetBytes(raw) } return nil } slice := reflect.MakeSlice(val.Type(), size, size) for i := 0; i < size; i++ { if err := t.decodeValue(r, slice.Index(i)); err != nil { return err } } val.Set(slice) return nil }
func NewLevelDbDatastore(dbDir string) (Datastore, error) { opts := levigo.NewOptions() opts.SetCache(levigo.NewLRUCache(ONE_GIGABYTE)) opts.SetCreateIfMissing(true) opts.SetBlockSize(TWO_FIFTY_SIX_KILOBYTES) filter := levigo.NewBloomFilter(BLOOM_FILTER_BITS_PER_KEY) opts.SetFilterPolicy(filter) db, err := levigo.Open(dbDir, opts) if err != nil { return nil, err } ro := levigo.NewReadOptions() lastIdBytes, err2 := db.Get(ro, NEXT_ID_KEY) if err2 != nil { return nil, err2 } lastId := uint64(0) if lastIdBytes != nil { lastId, err2 = binary.ReadUvarint(bytes.NewBuffer(lastIdBytes)) if err2 != nil { return nil, err2 } } wo := levigo.NewWriteOptions() return &LevelDbDatastore{db: db, lastIdUsed: lastId, readOptions: ro, writeOptions: wo}, nil }
func readUVarintAsInt(r *ReaderByteReader) (int, error) { n, err := binary.ReadUvarint(r) if err != nil { return 0, err } return int(n), nil }
func ReadRLE(reader io.ByteReader) (j *Judy1, err error) { j = NewJudy1() var idx uint64 ones := false for { var width uint64 width, err = binary.ReadUvarint(reader) if err == io.EOF { err = nil return } else if err != nil { j.Free() j = nil return } if !ones { idx += width } else { var i uint64 for i = 0; i < width; i = i + 1 { j.Set(idx) idx = idx + 1 } } ones = !ones } return }
// Split splits datagram of a Packet from Reader. func Split(r Reader) ([]byte, error) { buf := &bytes.Buffer{} // read header: message type c, err := r.ReadByte() if err != nil { return nil, err } err = buf.WriteByte(c) if err != nil { return nil, err } // read length of payload l, err := binary.ReadUvarint(r) if err != nil { return nil, err } // FIXME: should be check l is too long. b := make([]byte, binary.MaxVarintLen32) n := binary.PutUvarint(b, l) buf.Write(b[0:n]) // read whole payload to buf. _, err = io.CopyN(buf, r, int64(l)) if err != nil { return nil, err } return buf.Bytes(), nil }
func newDecoder(b []byte, t Type) (*decoder, error) { if len(b) < 2 { return nil, errInvalidPacketLength } b0 := b[0] h := header{ Type: decodeType(b0), Dup: b0&0x08 != 0, QoS: QoS(b0 >> 1 & 0x3), Retain: b0&0x01 != 0, } if h.Type != t { return nil, errTypeMismatch } b = b[1:] r := bytes.NewReader(b) u, err := binary.ReadUvarint(r) if err != nil { return nil, err } if len(b)-r.Len() > 4 || r.Len() != int(u) { return nil, errInvalidRemainLength } return &decoder{header: h, r: r}, nil }
func (c *baseConn) recvProto(m proto.Message, uncompressedSize uint32, decompressor decompressFunc) error { size, err := binary.ReadUvarint(c.r) if err != nil { return err } if size == 0 { return nil } if c.r.Buffered() >= int(size) { // Parse proto directly from the buffered data. data, err := c.r.Peek(int(size)) if err != nil { return err } if err := decompressor(data, uncompressedSize, m); err != nil { return err } _, err = c.r.Discard(int(size)) return err } data := make([]byte, size) if _, err := io.ReadFull(c.r, data); err != nil { return err } return decompressor(data, uncompressedSize, m) }
// ReadVarIntToUint reads a variable length integer from the input buffer. // The inflated integer is written is returned as a uint64 value. // This method only "inflates" the varint into a uint64; it does NOT // zigzag decode it. func (r *Reader) ReadUvarint() uint64 { v, err := binary.ReadUvarint(r.br) if err != nil { r.err = err } return v }
func (space *Space) request(requestId int32, body *bytes.Buffer) (tuples [][][]byte, err error) { var ( returnCode int32 tuplesCount int32 tuplesSize int32 cardinality int32 size uint64 response *iproto.Response ) response, err = space.conn.Request(requestId, body) if err != nil { return } // Ping has no Body if requestId == PingOp { tuples = [][][]byte{} return } err = binary.Read(response.Body, binary.LittleEndian, &returnCode) if err != nil { return } if returnCode != 0 { err = fmt.Errorf("Return code is not 0, but %d; Error message: %s", returnCode, response.Body.String()) return } err = binary.Read(response.Body, binary.LittleEndian, &tuplesCount) if err != nil { return } tuples = make([][][]byte, tuplesCount) for i := int32(0); i < tuplesCount && response.Body.Len() > 0; i++ { err = binary.Read(response.Body, binary.LittleEndian, &tuplesSize) if err != nil { return } err = binary.Read(response.Body, binary.LittleEndian, &cardinality) if err != nil { return } tuples[i] = make([][]byte, cardinality) for j := int32(0); j < cardinality; j++ { size, err = binary.ReadUvarint(response.Body) if err != nil { return } tuples[i][j] = make([]byte, size) _, err = response.Body.Read(tuples[i][j]) if err != nil { return } } } return }
// Reads encoded Properties from a stream. func (properties *Properties) ReadFrom(reader io.Reader) error { length, err := binary.ReadUvarint(byteReader{reader}) if err != nil { return err } if length == 0 { return nil } body := make([]byte, length) if _, err := io.ReadFull(reader, body); err != nil { return err } if body[length-1] != 0 { return fmt.Errorf("Invalid properties (not NUL-terminated)") } eachProp := bytes.Split(body[0:length-1], []byte{0}) nProps := len(eachProp) / 2 if nProps*2 != len(eachProp) { return fmt.Errorf("Odd number of strings in properties") } *properties = Properties{} for i := 0; i < len(eachProp); i += 2 { key := decodeProperty(eachProp[i]) value := decodeProperty(eachProp[i+1]) if _, exists := (*properties)[key]; exists { return fmt.Errorf("Duplicate property name %q", key) } (*properties)[key] = value } return nil }
// stdDistribution distributes the keyspace evenly. func stdDistribution(key string, n uint64) uint64 { h := sha1.New() io.WriteString(h, key) buf := bytes.NewBuffer(h.Sum(nil)[:8]) i, _ := binary.ReadUvarint(buf) return i % n }
func readSequence(r io.ByteReader) uint64 { seq, err := binary.ReadUvarint(r) if err != nil { panic("readSequence failed") } return seq }