Esempio n. 1
0
func decodeValue(encodedValue []byte) ([]byte, uint64) {
	version, len1 := proto.DecodeVarint(encodedValue)
	deleteMarker, len2 := proto.DecodeVarint(encodedValue[len1:])
	if deleteMarker == 1 {
		return nil, version
	}
	value := encodedValue[len1+len2:]
	return value, version
}
Esempio n. 2
0
func (gaugeListener *GaugeListener) processMessages(buffer *bytes.Buffer) {
	for {
		messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())
		if messageLength > 0 && messageLength < uint64(buffer.Len()) {
			message := &gauge_messages.Message{}
			messageBoundary := int(messageLength) + bytesRead
			err := proto.Unmarshal(buffer.Bytes()[bytesRead:messageBoundary], message)
			if err != nil {
				log.Printf("Failed to read proto message: %s\n", err.Error())
			} else {
				if *message.MessageType == gauge_messages.Message_KillProcessRequest {
					gaugeListener.connection.Close()
					os.Exit(0)
				}
				if *message.MessageType == gauge_messages.Message_SuiteExecutionResult {
					result := message.GetSuiteExecutionResult()
					gaugeListener.onResultHandler(result)
				}
				buffer.Next(messageBoundary)
				if buffer.Len() == 0 {
					return
				}
			}
		} else {
			return
		}
	}
}
Esempio n. 3
0
func (client *Client) listen() {
	buffer := new(bytes.Buffer)
	data := make([]byte, 8192)

	for {
		n, err := client.Conn.Read(data)

		if err != nil {
			if err == io.EOF {
				log.Println("Client exited")
				return
			}
			log.Println(err.Error())
		}

		buffer.Write(data[0:n])

		messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())

		if messageLength > 0 && messageLength < uint64(buffer.Len()) {
			response := &limitd.Response{}

			err = proto.Unmarshal(buffer.Bytes()[bytesRead:messageLength+uint64(bytesRead)], response)
			if err != nil {
				log.Printf("Failed to read proto response: %s\n", err.Error())
			} else {
				responseChannel := client.PendingRequests[*response.RequestId]
				responseChannel <- response
				delete(client.PendingRequests, *response.RequestId)
				buffer.Reset()
			}
		}
	}
}
Esempio n. 4
0
// DecodeOrderPreservingVarUint64 decodes the number from the bytes obtained from method 'EncodeOrderPreservingVarUint64'.
// Also, returns the number of bytes that are consumed in the process
func DecodeOrderPreservingVarUint64(bytes []byte) (uint64, int) {
	s, _ := proto.DecodeVarint(bytes)
	size := int(s)
	decodedBytes := make([]byte, 8)
	copy(decodedBytes[8-size:], bytes[1:size+1])
	numBytesConsumed := size + 1
	return binary.BigEndian.Uint64(decodedBytes), numBytesConsumed
}
Esempio n. 5
0
func newDataKeyFromEncodedBytes(encodedBytes []byte) *dataKey {
	bucketNum, l := proto.DecodeVarint(encodedBytes)
	if !bytes.Equal(encodedBytes[l:l+1], []byte{byte(0)}) {
		panic(fmt.Errorf("[%#v] is not a valid data key", encodedBytes))
	}
	compositeKey := encodedBytes[l+1:]
	return &dataKey{newBucketKeyAtLowestLevel(int(bucketNum)), compositeKey}
}
Esempio n. 6
0
func (mgr *blockfileMgr) fetchTransaction(lp *fileLocPointer) (*pb.Transaction, error) {
	var err error
	var txEnvelopeBytes []byte
	if txEnvelopeBytes, err = mgr.fetchRawBytes(lp); err != nil {
		return nil, err
	}
	_, n := proto.DecodeVarint(txEnvelopeBytes)
	return extractTransaction(txEnvelopeBytes[n:])
}
Esempio n. 7
0
// readDelimited decodes a message from the provided length-delimited stream,
// where the length is encoded as 32-bit varint prefix to the message body.
// It returns the total number of bytes read and any applicable error.  This is
// roughly equivalent to the companion Java API's
// MessageLite#parseDelimitedFrom.  As per the reader contract, this function
// calls r.Read repeatedly as required until exactly one message including its
// prefix is read and decoded (or an error has occurred).  The function never
// reads more bytes from the stream than required.  The function never returns
// an error if a message has been read and decoded correctly, even if the end
// of the stream has been reached in doing so.  In that case, any subsequent
// calls return (0, io.EOF).
func readDelimited(r io.Reader, m proto.Message, base64Decode bool, newline bool) (int, error) {
	// Per AbstractParser#parsePartialDelimitedFrom with
	// CodedInputStream#readRawVarint32.
	headerBuf := make([]byte, binary.MaxVarintLen32)
	var bytesRead, varIntBytes int
	var messageLength uint64
	for varIntBytes == 0 { // i.e. no varint has been decoded yet.
		if bytesRead >= len(headerBuf) {
			return bytesRead, errInvalidVarint
		}
		// We have to read byte by byte here to avoid reading more bytes
		// than required. Each read byte is appended to what we have
		// read before.
		newBytesRead, err := r.Read(headerBuf[bytesRead : bytesRead+1])
		if newBytesRead == 0 {
			if err != nil {
				return bytesRead, err
			}
			// A Reader should not return (0, nil), but if it does,
			// it should be treated as no-op (according to the
			// Reader contract). So let's go on...
			continue
		}
		bytesRead += newBytesRead
		// Now present everything read so far to the varint decoder and
		// see if a varint can be decoded already.
		messageLength, varIntBytes = proto.DecodeVarint(headerBuf[:bytesRead])
	}

	messageBuf := make([]byte, messageLength)
	newBytesRead, err := io.ReadFull(r, messageBuf)
	bytesRead += newBytesRead
	if err != nil {
		return bytesRead, err
	}
	if base64Decode {
		messageBuf, err = base64.StdEncoding.DecodeString(string(messageBuf))
		if err != nil {
			return bytesRead, err
		}
	}
	if newline {
		newlineBuf := make([]byte, 1)
		n, err := io.ReadFull(r, newlineBuf)
		bytesRead += n
		if err != nil {
			return bytesRead, err
		}
	}
	return bytesRead, proto.Unmarshal(messageBuf, m)
}
Esempio n. 8
0
func (decoder *ProtobufDecoderV1) Read(ctx context.Context, req *http.Request) error {
	body := req.Body
	bufferedBody := bufio.NewReaderSize(body, 32768)
	for {
		log.WithField("body", body).Debug("Starting protobuf loop")
		buf, err := bufferedBody.Peek(1)
		if err == io.EOF {
			log.Debug("EOF")
			return nil
		}
		buf, err = bufferedBody.Peek(4) // should be big enough for any varint

		if err != nil {
			log.WithField("err", err).Info("peek error")
			return err
		}
		num, bytesRead := proto.DecodeVarint(buf)
		if bytesRead == 0 {
			// Invalid varint?
			return errInvalidProtobufVarint
		}
		if num > 32768 {
			// Sanity check
			return errProtobufTooLarge
		}
		// Get the varint out
		buf = make([]byte, bytesRead)
		io.ReadFull(bufferedBody, buf)

		// Get the structure out
		buf = make([]byte, num)
		_, err = io.ReadFull(bufferedBody, buf)
		if err != nil {
			return fmt.Errorf("unable to fully read protobuf message: %s", err)
		}
		var msg com_signalfx_metrics_protobuf.DataPoint
		err = proto.Unmarshal(buf, &msg)
		if err != nil {
			return err
		}
		if datapointProtobufIsInvalidForV1(&msg) {
			return errInvalidProtobuf
		}
		mt := decoder.TypeGetter.GetMetricTypeFromMap(msg.GetMetric())
		if dp, err := NewProtobufDataPointWithType(&msg, mt); err == nil {
			decoder.Sink.AddDatapoints(ctx, []*datapoint.Datapoint{dp})
		}
	}
}
Esempio n. 9
0
func (connectionHandler *GaugeConnectionHandler) processMessage(buffer *bytes.Buffer, conn net.Conn) {
	for {
		messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())
		if messageLength > 0 && messageLength < uint64(buffer.Len()) {
			messageBoundary := int(messageLength) + bytesRead
			receivedBytes := buffer.Bytes()[bytesRead : messageLength+uint64(bytesRead)]
			connectionHandler.messageHandler.MessageBytesReceived(receivedBytes, conn)
			buffer.Next(messageBoundary)
			if buffer.Len() == 0 {
				return
			}
		} else {
			return
		}
	}
}
Esempio n. 10
0
func readMessageBytes(conn net.Conn) ([]byte, error) {
	buffer := new(bytes.Buffer)
	data := make([]byte, c.MaxMessageSize)
	for {
		n, err := conn.Read(data)
		if err != nil {
			conn.Close()
			return nil, fmt.Errorf("Connection closed [%s] cause: %s", conn.RemoteAddr(), err.Error())
		}

		buffer.Write(data[0:n])

		messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())
		if messageLength > 0 && messageLength < uint64(buffer.Len()) {
			return buffer.Bytes()[bytesRead : messageLength+uint64(bytesRead)], nil
		}
	}
}
func TestSerializedBlockInfo(t *testing.T) {
	block := testutil.ConstructTestBlock(t, 10, 100)
	bb, info, err := serializeBlock(block)
	testutil.AssertNoError(t, err, "")
	infoFromBB, err := extractSerializedBlockInfo(bb)
	testutil.AssertNoError(t, err, "")
	testutil.AssertEquals(t, infoFromBB, info)
	testutil.AssertEquals(t, len(info.txOffsets), len(block.Data.Data))
	for txIndex, txEnvBytes := range block.Data.Data {
		txid, err := extractTxID(txEnvBytes)
		testutil.AssertNoError(t, err, "")

		indexInfo := info.txOffsets[txIndex]
		indexTxID := indexInfo.txID
		indexOffset := indexInfo.loc

		testutil.AssertEquals(t, txid, indexTxID)
		b := bb[indexOffset.offset:]
		len, num := proto.DecodeVarint(b)
		txEnvBytesFromBB := b[num : num+int(len)]
		testutil.AssertEquals(t, txEnvBytesFromBB, txEnvBytes)
	}
}
Esempio n. 12
0
func (c *Client) receiveRpcs() {
	var sz [4]byte
	for {
		err := c.readFully(sz[:])
		if err != nil {
			c.setSendErr(err)
			c.errorEncountered()
			return
		}

		buf := make([]byte, binary.BigEndian.Uint32(sz[:]))
		err = c.readFully(buf)
		if err != nil {
			c.setSendErr(err)
			c.errorEncountered()
			return
		}

		resp := &pb.ResponseHeader{}
		respLen, nb := proto.DecodeVarint(buf)
		buf = buf[nb:]
		err = proto.UnmarshalMerge(buf[:respLen], resp)
		buf = buf[respLen:]
		if err != nil {
			// Failed to deserialize the response header
			c.setSendErr(err)
			c.errorEncountered()
			return
		}
		if resp.CallId == nil {
			// Response doesn't have a call ID
			log.Error("Response doesn't have a call ID!")
			c.setSendErr(ErrMissingCallID)
			c.errorEncountered()
			return
		}

		c.sentRPCsMutex.Lock()
		rpc, ok := c.sentRPCs[*resp.CallId]
		c.sentRPCsMutex.Unlock()

		if !ok {
			log.WithFields(log.Fields{
				"CallId": *resp.CallId,
			}).Error("Received a response with an unexpected call ID")

			log.Error("Waiting for responses to the following calls:")
			c.sentRPCsMutex.Lock()
			for id, call := range c.sentRPCs {
				log.Errorf("\t\t%d: %v", id, call)
			}
			c.sentRPCsMutex.Unlock()

			c.setSendErr(fmt.Errorf("HBase sent a response with an unexpected call ID: %d",
				resp.CallId))
			c.errorEncountered()
			return
		}

		var rpcResp proto.Message
		if resp.Exception == nil {
			respLen, nb = proto.DecodeVarint(buf)
			buf = buf[nb:]
			rpcResp = rpc.NewResponse()
			err = proto.UnmarshalMerge(buf, rpcResp)
			buf = buf[respLen:]
		} else {
			javaClass := *resp.Exception.ExceptionClassName
			err = fmt.Errorf("HBase Java exception %s: \n%s", javaClass,
				*resp.Exception.StackTrace)
			if _, ok := javaRetryableExceptions[javaClass]; ok {
				// This is a recoverable error. The client should retry.
				err = RetryableError{err}
			}
		}
		rpc.GetResultChan() <- hrpc.RPCResult{Msg: rpcResp, Error: err}

		c.sentRPCsMutex.Lock()
		delete(c.sentRPCs, *resp.CallId)
		c.sentRPCsMutex.Unlock()
	}
}
Esempio n. 13
0
func decodeBlockNum(blockNumBytes []byte) uint64 {
	blockNum, _ := proto.DecodeVarint(blockNumBytes)
	return blockNum
}
Esempio n. 14
0
func toInt(balanceBytes []byte) int {
	v, _ := proto.DecodeVarint(balanceBytes)
	return int(v)
}
Esempio n. 15
0
func decodeBlockNumber(blockNumberBytes []byte) (blockNumber uint64) {
	blockNumber, _ = proto.DecodeVarint(blockNumberBytes)
	return
}