func encodeListTxIndexes(listTx []uint64) []byte {
	b := proto.NewBuffer([]byte{})
	for i := range listTx {
		b.EncodeVarint(listTx[i])
	}
	return b.Bytes()
}
Esempio n. 2
0
func (trieNode *trieNode) marshal() ([]byte, error) {
	buffer := proto.NewBuffer([]byte{})

	// write value
	err := buffer.EncodeRawBytes(trieNode.value)
	if err != nil {
		return nil, err
	}

	numCryptoHashes := trieNode.getNumChildren()

	//write number of crypto-hashes
	err = buffer.EncodeVarint(uint64(numCryptoHashes))
	if err != nil {
		return nil, err
	}

	if numCryptoHashes == 0 {
		return buffer.Bytes(), nil
	}

	for i, cryptoHash := range trieNode.childrenCryptoHashes {
		//write crypto-hash Index
		err = buffer.EncodeVarint(uint64(i))
		if err != nil {
			return nil, err
		}
		// write crypto-hash
		err = buffer.EncodeRawBytes(cryptoHash)
		if err != nil {
			return nil, err
		}
	}
	return buffer.Bytes(), nil
}
Esempio n. 3
0
func TestMsg(t *testing.T) {
	recMsg := &OSMsg{Fromu: proto.String("jack"), Tou: proto.String("tom"), Content: proto.String("first")}
	b, err := proto.Marshal(recMsg)
	if checkerr(err) {
		return
	}

	buf := proto.NewBuffer(b)
	err = proto.Unmarshal(b, &msg)
	if checkerr(err) {
		return
	}
	fmt.Println(msg)
	fmt.Println(msg.String())

	err = proto.Unmarshal(buf.Bytes(), &osmsg)
	if checkerr(err) {
		return
	}

	fmt.Println(osmsg)
	fmt.Println(osmsg.String())

	a := []int{1, 2}
	a1 := a[:1]
	fmt.Println(a, a1)
	a1 = append(a1, 3)
	fmt.Println(a, a1)
	a1 = append(a1, 4)
	fmt.Println(a, a1)
	a1[1] = 10
	fmt.Println(a, a1)
}
Esempio n. 4
0
// BenchmarkVarint64ArrayMixed shows the performance of lots of small messages, each
// containing a small number of large (3, 4, and 5 byte) repeated int64s.
func BenchmarkVarint64ArrayMixed(b *testing.B) {
	for i := uint(1); i <= 1<<5; i <<= 1 {
		dist := genUint64Dist([11]int{0, 0, 0, 4, 6, 4, 0, 0, 0, 0, 0}, int(i))
		// number of sub fields
		for k := uint(1); k <= 1<<10; k <<= 2 {
			msg := &tpb.Message{}
			for m := uint(0); m < k; m++ {
				msg.Children = append(msg.Children, &tpb.Message{
					Key: dist,
				})
			}
			raw, err := proto.Marshal(msg)
			if err != nil {
				b.Error("wrong encode", err)
			}
			b.Run(fmt.Sprintf("Fields%vLen%v", k, i), func(b *testing.B) {
				scratchBuf := proto.NewBuffer(nil)
				b.ResetTimer()
				for k := 0; k < b.N; k++ {
					scratchBuf.SetBuf(raw)
					msgBlackhole.Reset()
					if err := scratchBuf.Unmarshal(msgBlackhole); err != nil {
						b.Error("wrong decode", err)
					}
				}
			})
		}
	}
}
Esempio n. 5
0
func unmarshalTrieNode(key *trieKey, serializedContent []byte) (*trieNode, error) {
	trieNode := newTrieNode(key, nil, false)
	buffer := proto.NewBuffer(serializedContent)
	value, err := buffer.DecodeRawBytes(false)
	if err != nil {
		return nil, err
	}
	trieNode.value = value

	numCryptoHashes, err := buffer.DecodeVarint()
	if err != nil {
		return nil, err
	}

	for i := uint64(0); i < numCryptoHashes; i++ {
		index, err := buffer.DecodeVarint()
		if err != nil {
			return nil, err
		}
		cryptoHash, err := buffer.DecodeRawBytes(false)
		if err != nil {
			return nil, err
		}
		trieNode.childrenCryptoHashes[int(index)] = cryptoHash
	}
	return trieNode, nil
}
Esempio n. 6
0
func (bucketNode *bucketNode) marshal() []byte {
	buffer := proto.NewBuffer([]byte{})
	for i := 0; i < conf.getMaxGroupingAtEachLevel(); i++ {
		buffer.EncodeRawBytes(bucketNode.childrenCryptoHash[i])
	}
	return buffer.Bytes()
}
Esempio n. 7
0
func unmarshalTrieNode(key *trieKey, serializedContent []byte) (*trieNode, error) {
	stateTrieLogger.Debug("key = [%s], len(serializedContent) = %d", key, len(serializedContent))
	trieNode := newTrieNode(key, nil, false)
	buffer := proto.NewBuffer(serializedContent)
	trieNode.value = unmarshalTrieNodeValueFromBuffer(buffer)

	numCryptoHashes, err := buffer.DecodeVarint()
	stateTrieLogger.Debug("numCryptoHashes = [%d]", numCryptoHashes)
	if err != nil {
		return nil, err
	}
	for i := uint64(0); i < numCryptoHashes; i++ {
		index, err := buffer.DecodeVarint()
		if err != nil {
			return nil, err
		}
		cryptoHash, err := buffer.DecodeRawBytes(false)
		if err != nil {
			return nil, err
		}
		trieNode.childrenCryptoHashes[int(index)] = cryptoHash
	}
	stateTrieLogger.Debug("unmarshalled trieNode = [%s]", trieNode)
	return trieNode, nil
}
Esempio n. 8
0
func (c *codec) Encoder(w io.Writer) mc.Encoder {
	return &encoder{
		w:   w,
		buf: proto.NewBuffer(nil),
		c:   c,
	}
}
Esempio n. 9
0
func (s *Server) writeResponse(c *Client, cmd int, seq uint32, pb proto.Message) error {
	var err error

	c.sendlock.Lock()
	defer c.sendlock.Unlock()

	// cmd
	binary.LittleEndian.PutUint16(c.wb[0:2], uint16(cmd+1))
	// seq
	binary.LittleEndian.PutUint32(c.wb[4:8], seq)
	logger.Println("send header:", cmd+1, seq)
	// pb
	if pb != nil {
		buf := proto.NewBuffer(c.wb[0:8])
		buf.Marshal(pb)
		respb := buf.Bytes()
		// len
		binary.LittleEndian.PutUint16(respb[2:4], uint16(len(respb)))
		_, err = c.w.Write(respb)
	} else {
		// len
		binary.LittleEndian.PutUint16(c.wb[2:4], uint16(HeaderSize))
		_, err = c.w.Write(c.wb[0:8])
	}
	return err
}
Esempio n. 10
0
func (s *Server) NotifyClient(c *Client, cmd int, pb proto.Message, cb NotifyCallback) error {
	var err error

	seq := c.addPendingRequest(cb)

	c.sendlock.Lock()
	defer c.sendlock.Unlock()

	// cmd
	binary.LittleEndian.PutUint16(c.wb[0:2], uint16(cmd))
	// seq
	binary.LittleEndian.PutUint32(c.wb[4:8], seq)
	// pb
	if pb != nil {
		buf := proto.NewBuffer(c.wb[0:8])
		buf.Marshal(pb)
		respb := buf.Bytes()
		// len
		binary.LittleEndian.PutUint16(respb[2:4], uint16(len(respb)))
		_, err = c.w.Write(respb)
	} else {
		// len
		binary.LittleEndian.PutUint16(c.wb[2:4], uint16(HeaderSize))
		_, err = c.w.Write(c.wb[0:8])
	}

	return err
}
Esempio n. 11
0
// dataAsString returns the given byte array as a string; handles detecting
// protocol buffers.
func dataAsString(data []byte) string {
	dataString := string(data)
	if !strings.HasPrefix(dataString, "{") {
		dataString = "\n" + hex.Dump(data)
		proto.NewBuffer(make([]byte, 0, 1024)).DebugPrint("decoded object", data)
	}
	return dataString
}
Esempio n. 12
0
func unmarshalTrieNodeValue(serializedContent []byte) []byte {
	buffer := proto.NewBuffer(serializedContent)
	value, err := buffer.DecodeRawBytes(false)
	if err != nil {
		panic(fmt.Errorf("This error is not excpected: %s", err))
	}
	return value
}
Esempio n. 13
0
func (self *Shard) Write(database string, series []*protocol.Series) error {
	self.closeLock.RLock()
	defer self.closeLock.RUnlock()
	if self.closed {
		return fmt.Errorf("Shard is closed")
	}

	wb := make([]storage.Write, 0)

	for _, s := range series {
		if len(s.Points) == 0 {
			return errors.New("Unable to write no data. Series was nil or had no points.")
		}
		if len(s.FieldIds) == 0 {
			return errors.New("Unable to write points without fields")
		}

		count := 0
		for fieldIndex, id := range s.FieldIds {
			for _, point := range s.Points {
				// keyBuffer and dataBuffer have to be recreated since we are
				// batching the writes, otherwise new writes will override the
				// old writes that are still in memory
				dataBuffer := proto.NewBuffer(nil)
				var err error

				sk := newStorageKey(id, point.GetTimestamp(), point.GetSequenceNumber())
				if point.Values[fieldIndex].GetIsNull() {
					wb = append(wb, storage.Write{Key: sk.bytes(), Value: nil})
					goto check
				}

				err = dataBuffer.Marshal(point.Values[fieldIndex])
				if err != nil {
					return err
				}
				wb = append(wb, storage.Write{Key: sk.bytes(), Value: dataBuffer.Bytes()})
			check:
				count++
				if count >= self.writeBatchSize {
					err = self.db.BatchPut(wb)
					if err != nil {
						return err
					}
					count = 0
					wb = make([]storage.Write, 0, self.writeBatchSize)
				}
			}
		}
	}

	return self.db.BatchPut(wb)
}
Esempio n. 14
0
func decodeBlockNumTxIndex(bytes []byte) (blockNum uint64, txIndex uint64, err error) {
	b := proto.NewBuffer(bytes)
	blockNum, err = b.DecodeVarint()
	if err != nil {
		return
	}
	txIndex, err = b.DecodeVarint()
	if err != nil {
		return
	}
	return
}
Esempio n. 15
0
// Marshal serializes a `TxReadWriteSet`
func (txRW *TxReadWriteSet) Marshal() ([]byte, error) {
	buf := proto.NewBuffer(nil)
	var err error
	if err = buf.EncodeVarint(uint64(len(txRW.NsRWs))); err != nil {
		return nil, err
	}
	for i := 0; i < len(txRW.NsRWs); i++ {
		if err = txRW.NsRWs[i].Marshal(buf); err != nil {
			return nil, err
		}
	}
	return buf.Bytes(), nil
}
Esempio n. 16
0
func writeFromPointers(queryid string, f io.Writer, pointers []resultPointer) error {
	stateMu.RLock()
	s := state[queryid]
	stateMu.RUnlock()
	firstPathRank := s.FirstPathRank

	s.tempFilesMu.Lock()
	defer s.tempFilesMu.Unlock()

	if _, err := f.Write([]byte("[")); err != nil {
		return err
	}
	var msg pb.SearchReply
	buf := proto.NewBuffer(nil)
	for idx, pointer := range pointers {
		src := s.perBackend[pointer.backendidx].tempFile
		if _, err := src.Seek(pointer.offset, os.SEEK_SET); err != nil {
			return err
		}
		// TODO: Avoid the allocations by using a slice and only allocate a new buffer when pointer.length > cap(rdbuf)
		rdbuf := make([]byte, pointer.length)
		if _, err := src.Read(rdbuf); err != nil {
			return err
		}
		if idx > 0 {
			if _, err := f.Write([]byte(",")); err != nil {
				return err
			}
		}
		buf.SetBuf(rdbuf)
		msg.Reset()
		if err := buf.Unmarshal(&msg); err != nil {
			return err
		}
		if msg.Type != pb.SearchReply_MATCH {
			return fmt.Errorf("Expected to find a pb.SearchReply_MATCH, instead got %d", msg.Type)
		}
		match := msg.Match
		// We need to fix the ranking here because we persist raw results from
		// the dcs-source-backend in queryBackend(), but then modify the
		// ranking in storeResult().
		match.Ranking = match.Pathrank + ((firstPathRank * 0.1) * match.Ranking)
		if err := WriteMatchJSON(match, f); err != nil {
			return err
		}
	}
	if _, err := f.Write([]byte("]\n")); err != nil {
		return err
	}
	return nil
}
Esempio n. 17
0
func unmarshalBucketNode(bucketKey *bucketKey, serializedBytes []byte) *bucketNode {
	bucketNode := newBucketNode(bucketKey)
	buffer := proto.NewBuffer(serializedBytes)
	for i := 0; i < conf.getMaxGroupingAtEachLevel(); i++ {
		childCryptoHash, err := buffer.DecodeRawBytes(false)
		if err != nil {
			panic(fmt.Errorf("this error should not occur: %s", err))
		}
		if !util.IsNil(childCryptoHash) {
			bucketNode.childrenCryptoHash[i] = childCryptoHash
		}
	}
	return bucketNode
}
Esempio n. 18
0
func (i *checkpointInfo) marshal() ([]byte, error) {
	buffer := proto.NewBuffer([]byte{})
	var err error
	if err = buffer.EncodeVarint(uint64(i.latestFileChunkSuffixNum)); err != nil {
		return nil, err
	}
	if err = buffer.EncodeVarint(uint64(i.latestFileChunksize)); err != nil {
		return nil, err
	}
	if err = buffer.EncodeVarint(i.lastBlockNumber); err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}
Esempio n. 19
0
// Marshal serializes the StateDelta
func (stateDelta *StateDelta) Marshal() (b []byte) {
	buffer := proto.NewBuffer([]byte{})
	err := buffer.EncodeVarint(uint64(len(stateDelta.chaincodeStateDeltas)))
	if err != nil {
		// in protobuf code the error return is always nil
		panic(fmt.Errorf("This error should not occure: %s", err))
	}
	for chaincodeID, chaincodeStateDelta := range stateDelta.chaincodeStateDeltas {
		buffer.EncodeStringBytes(chaincodeID)
		chaincodeStateDelta.marshal(buffer)
	}
	b = buffer.Bytes()
	return
}
Esempio n. 20
0
File: conn.go Progetto: yzl11/vessel
func processMessage(msg []byte) [][]byte {
	buf := pb.NewBuffer(msg)
	payloads := make([][]byte, 0)

	for {
		hbytes, err := buf.DecodeRawBytes(true)
		if err != nil {
			break
		}

		payloads = append(payloads, hbytes)
	}

	return payloads
}
Esempio n. 21
0
func unmarshalBucketNode(bucketKey *bucketKey, serializedBytes []byte) *bucketNode {
	bucketNode := newBucketNode(bucketKey)
	buffer := proto.NewBuffer(serializedBytes)
	for i := 0; i < conf.getMaxGroupingAtEachLevel(); i++ {
		childCryptoHash, err := buffer.DecodeRawBytes(false)
		if err != nil {
			panic(fmt.Errorf("this error should not occur: %s", err))
		}
		//protobuf's buffer.EncodeRawBytes/buffer.DecodeRawBytes convert a nil into a zero length byte-array, so nil check would not work
		if len(childCryptoHash) != 0 {
			bucketNode.childrenCryptoHash[i] = childCryptoHash
		}
	}
	return bucketNode
}
Esempio n. 22
0
func serializeBlock(block *common.Block) ([]byte, *serializedBlockInfo, error) {
	buf := proto.NewBuffer(nil)
	var err error
	info := &serializedBlockInfo{}
	info.blockHeader = block.Header
	if err = addHeaderBytes(block.Header, buf); err != nil {
		return nil, nil, err
	}
	if info.txOffsets, err = addDataBytes(block.Data, buf); err != nil {
		return nil, nil, err
	}
	if err = addMetadataBytes(block.Metadata, buf); err != nil {
		return nil, nil, err
	}
	return buf.Bytes(), info, nil
}
Esempio n. 23
0
func (flp *fileLocPointer) marshal() ([]byte, error) {
	buffer := proto.NewBuffer([]byte{})
	e := buffer.EncodeVarint(uint64(flp.fileSuffixNum))
	if e != nil {
		return nil, e
	}
	e = buffer.EncodeVarint(uint64(flp.offset))
	if e != nil {
		return nil, e
	}
	e = buffer.EncodeVarint(uint64(flp.bytesLength))
	if e != nil {
		return nil, e
	}
	return buffer.Bytes(), nil
}
Esempio n. 24
0
// Unmarshal deserializes a `TxReadWriteSet`
func (txRW *TxReadWriteSet) Unmarshal(b []byte) error {
	buf := proto.NewBuffer(b)
	var err error
	var numEntries uint64
	if numEntries, err = buf.DecodeVarint(); err != nil {
		return err
	}
	for i := 0; i < int(numEntries); i++ {
		nsRW := &NsReadWriteSet{}
		if err = nsRW.Unmarshal(buf); err != nil {
			return err
		}
		txRW.NsRWs = append(txRW.NsRWs, nsRW)
	}
	return nil
}
Esempio n. 25
0
func (trieNode *trieNode) marshal() ([]byte, error) {
	buffer := proto.NewBuffer([]byte{})

	// write value marker explicitly because rocksdb apis convertes a nil into an empty array and protobuf does it other-way around
	var valueMarker uint64 = 0 // ignore golint warning. Dropping '= 0' makes assignment less clear
	if trieNode.value != nil {
		valueMarker = 1
	}
	err := buffer.EncodeVarint(valueMarker)
	if err != nil {
		return nil, err
	}
	if trieNode.value != nil {
		// write value
		err = buffer.EncodeRawBytes(trieNode.value)
		if err != nil {
			return nil, err
		}
	}
	//write number of crypto-hashes
	numCryptoHashes := trieNode.getNumChildren()
	err = buffer.EncodeVarint(uint64(numCryptoHashes))
	if err != nil {
		return nil, err
	}

	if numCryptoHashes == 0 {
		return buffer.Bytes(), nil
	}

	for i, cryptoHash := range trieNode.childrenCryptoHashes {
		//write crypto-hash Index
		err = buffer.EncodeVarint(uint64(i))
		if err != nil {
			return nil, err
		}
		// write crypto-hash
		err = buffer.EncodeRawBytes(cryptoHash)
		if err != nil {
			return nil, err
		}
	}
	serializedBytes := buffer.Bytes()
	stateTrieLogger.Debug("Marshalled trieNode [%s]. Serialized bytes size = %d", trieNode.trieKey, len(serializedBytes))
	return serializedBytes, nil
}
Esempio n. 26
0
func (in *inputStream) processMessage(msg []byte) [][]byte {
	buf := pb.NewBuffer(msg)
	payloads := make([][]byte, 0)

	for {
		hbytes, err := buf.DecodeRawBytes(true)
		if err != nil {
			break
		}

		payloads = append(payloads, hbytes)
	}

	log.Debug("Messages processed [n=%d]", len(payloads))

	return payloads
}
Esempio n. 27
0
func readMetaData(conn net.Conn) (payloadSize int64, err error) {
	metadata := make([]byte, METADATA_LENGTH_BYTES)
	_, err = conn.Read(metadata)
	if err != nil {
		return
	}
	log.Debug("metadata dump from %s :%v", conn.RemoteAddr().String(), metadata)
	buf := proto.NewBuffer(metadata)
	uintSize, err := buf.DecodeFixed32()
	if err != nil {
		return 0, err
	}
	payloadSize = int64(uintSize)
	if payloadSize < 0 {
		return payloadSize, errors.New(fmt.Sprintf("payload length is negative %s: %d", conn.RemoteAddr().String(), payloadSize))
	}
	return payloadSize, nil
}
Esempio n. 28
0
func main() {
	var sizeBuf [4]byte
	inbuf := make([]byte, 0, 4096)
	outbuf := proto.NewBuffer(nil)
	for {
		if _, err := io.ReadFull(os.Stdin, sizeBuf[:]); err == io.EOF {
			break
		} else if err != nil {
			fmt.Fprintln(os.Stderr, "go conformance: read request:", err)
			os.Exit(1)
		}
		size := binary.LittleEndian.Uint32(sizeBuf[:])
		if int(size) > cap(inbuf) {
			inbuf = make([]byte, size)
		}
		inbuf = inbuf[:size]
		if _, err := io.ReadFull(os.Stdin, inbuf); err != nil {
			fmt.Fprintln(os.Stderr, "go conformance: read request:", err)
			os.Exit(1)
		}

		req := new(pb.ConformanceRequest)
		if err := proto.Unmarshal(inbuf, req); err != nil {
			fmt.Fprintln(os.Stderr, "go conformance: parse request:", err)
			os.Exit(1)
		}
		res := handle(req)

		if err := outbuf.Marshal(res); err != nil {
			fmt.Fprintln(os.Stderr, "go conformance: marshal response:", err)
			os.Exit(1)
		}
		binary.LittleEndian.PutUint32(sizeBuf[:], uint32(len(outbuf.Bytes())))
		if _, err := os.Stdout.Write(sizeBuf[:]); err != nil {
			fmt.Fprintln(os.Stderr, "go conformance: write response:", err)
			os.Exit(1)
		}
		if _, err := os.Stdout.Write(outbuf.Bytes()); err != nil {
			fmt.Fprintln(os.Stderr, "go conformance: write response:", err)
			os.Exit(1)
		}
		outbuf.Reset()
	}
}
Esempio n. 29
0
func (flp *fileLocPointer) unmarshal(b []byte) error {
	buffer := proto.NewBuffer(b)
	i, e := buffer.DecodeVarint()
	if e != nil {
		return e
	}
	flp.fileSuffixNum = int(i)

	i, e = buffer.DecodeVarint()
	if e != nil {
		return e
	}
	flp.offset = int(i)
	i, e = buffer.DecodeVarint()
	if e != nil {
		return e
	}
	flp.bytesLength = int(i)
	return nil
}
Esempio n. 30
0
// Unmarshal deserializes StateDelta
func (stateDelta *StateDelta) Unmarshal(bytes []byte) {
	buffer := proto.NewBuffer(bytes)
	size, err := buffer.DecodeVarint()
	if err != nil {
		panic(fmt.Errorf("This error should not occure: %s", err))
	}
	stateDelta.chaincodeStateDeltas = make(map[string]*chaincodeStateDelta, size)
	for i := uint64(0); i < size; i++ {
		chaincodeID, err := buffer.DecodeStringBytes()
		if err != nil {
			panic(fmt.Errorf("This error should not occure: %s", err))
		}
		chaincodeStateDelta := newChaincodeStateDelta(chaincodeID)
		err = chaincodeStateDelta.unmarshal(buffer)
		if err != nil {
			panic(fmt.Errorf("This error should not occure: %s", err))
		}
		stateDelta.chaincodeStateDeltas[chaincodeID] = chaincodeStateDelta
	}
}