func decodeValue(encodedValue []byte) ([]byte, uint64) {
version, len1 := proto.DecodeVarint(encodedValue)
deleteMarker, len2 := proto.DecodeVarint(encodedValue[len1:])
if deleteMarker == 1 {
return nil, version
}
value := encodedValue[len1+len2:]
return value, version
}
func (gaugeListener *GaugeListener) processMessages(buffer *bytes.Buffer) {
for {
messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())
if messageLength > 0 && messageLength < uint64(buffer.Len()) {
message := &gauge_messages.Message{}
messageBoundary := int(messageLength) + bytesRead
err := proto.Unmarshal(buffer.Bytes()[bytesRead:messageBoundary], message)
if err != nil {
log.Printf("Failed to read proto message: %s\n", err.Error())
} else {
if *message.MessageType == gauge_messages.Message_KillProcessRequest {
gaugeListener.connection.Close()
os.Exit(0)
}
if *message.MessageType == gauge_messages.Message_SuiteExecutionResult {
result := message.GetSuiteExecutionResult()
gaugeListener.onResultHandler(result)
}
buffer.Next(messageBoundary)
if buffer.Len() == 0 {
return
}
}
} else {
return
}
}
}
func (client *Client) listen() {
buffer := new(bytes.Buffer)
data := make([]byte, 8192)
for {
n, err := client.Conn.Read(data)
if err != nil {
if err == io.EOF {
log.Println("Client exited")
return
}
log.Println(err.Error())
}
buffer.Write(data[0:n])
messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())
if messageLength > 0 && messageLength < uint64(buffer.Len()) {
response := &limitd.Response{}
err = proto.Unmarshal(buffer.Bytes()[bytesRead:messageLength+uint64(bytesRead)], response)
if err != nil {
log.Printf("Failed to read proto response: %s\n", err.Error())
} else {
responseChannel := client.PendingRequests[*response.RequestId]
responseChannel <- response
delete(client.PendingRequests, *response.RequestId)
buffer.Reset()
}
}
}
}
// DecodeOrderPreservingVarUint64 decodes the number from the bytes obtained from method 'EncodeOrderPreservingVarUint64'.
// Also, returns the number of bytes that are consumed in the process
func DecodeOrderPreservingVarUint64(bytes []byte) (uint64, int) {
s, _ := proto.DecodeVarint(bytes)
size := int(s)
decodedBytes := make([]byte, 8)
copy(decodedBytes[8-size:], bytes[1:size+1])
numBytesConsumed := size + 1
return binary.BigEndian.Uint64(decodedBytes), numBytesConsumed
}
func newDataKeyFromEncodedBytes(encodedBytes []byte) *dataKey {
bucketNum, l := proto.DecodeVarint(encodedBytes)
if !bytes.Equal(encodedBytes[l:l+1], []byte{byte(0)}) {
panic(fmt.Errorf("[%#v] is not a valid data key", encodedBytes))
}
compositeKey := encodedBytes[l+1:]
return &dataKey{newBucketKeyAtLowestLevel(int(bucketNum)), compositeKey}
}
func (mgr *blockfileMgr) fetchTransaction(lp *fileLocPointer) (*pb.Transaction, error) {
var err error
var txEnvelopeBytes []byte
if txEnvelopeBytes, err = mgr.fetchRawBytes(lp); err != nil {
return nil, err
}
_, n := proto.DecodeVarint(txEnvelopeBytes)
return extractTransaction(txEnvelopeBytes[n:])
}
// readDelimited decodes a message from the provided length-delimited stream,
// where the length is encoded as 32-bit varint prefix to the message body.
// It returns the total number of bytes read and any applicable error. This is
// roughly equivalent to the companion Java API's
// MessageLite#parseDelimitedFrom. As per the reader contract, this function
// calls r.Read repeatedly as required until exactly one message including its
// prefix is read and decoded (or an error has occurred). The function never
// reads more bytes from the stream than required. The function never returns
// an error if a message has been read and decoded correctly, even if the end
// of the stream has been reached in doing so. In that case, any subsequent
// calls return (0, io.EOF).
func readDelimited(r io.Reader, m proto.Message, base64Decode bool, newline bool) (int, error) {
// Per AbstractParser#parsePartialDelimitedFrom with
// CodedInputStream#readRawVarint32.
headerBuf := make([]byte, binary.MaxVarintLen32)
var bytesRead, varIntBytes int
var messageLength uint64
for varIntBytes == 0 { // i.e. no varint has been decoded yet.
if bytesRead >= len(headerBuf) {
return bytesRead, errInvalidVarint
}
// We have to read byte by byte here to avoid reading more bytes
// than required. Each read byte is appended to what we have
// read before.
newBytesRead, err := r.Read(headerBuf[bytesRead : bytesRead+1])
if newBytesRead == 0 {
if err != nil {
return bytesRead, err
}
// A Reader should not return (0, nil), but if it does,
// it should be treated as no-op (according to the
// Reader contract). So let's go on...
continue
}
bytesRead += newBytesRead
// Now present everything read so far to the varint decoder and
// see if a varint can be decoded already.
messageLength, varIntBytes = proto.DecodeVarint(headerBuf[:bytesRead])
}
messageBuf := make([]byte, messageLength)
newBytesRead, err := io.ReadFull(r, messageBuf)
bytesRead += newBytesRead
if err != nil {
return bytesRead, err
}
if base64Decode {
messageBuf, err = base64.StdEncoding.DecodeString(string(messageBuf))
if err != nil {
return bytesRead, err
}
}
if newline {
newlineBuf := make([]byte, 1)
n, err := io.ReadFull(r, newlineBuf)
bytesRead += n
if err != nil {
return bytesRead, err
}
}
return bytesRead, proto.Unmarshal(messageBuf, m)
}
func (decoder *ProtobufDecoderV1) Read(ctx context.Context, req *http.Request) error {
body := req.Body
bufferedBody := bufio.NewReaderSize(body, 32768)
for {
log.WithField("body", body).Debug("Starting protobuf loop")
buf, err := bufferedBody.Peek(1)
if err == io.EOF {
log.Debug("EOF")
return nil
}
buf, err = bufferedBody.Peek(4) // should be big enough for any varint
if err != nil {
log.WithField("err", err).Info("peek error")
return err
}
num, bytesRead := proto.DecodeVarint(buf)
if bytesRead == 0 {
// Invalid varint?
return errInvalidProtobufVarint
}
if num > 32768 {
// Sanity check
return errProtobufTooLarge
}
// Get the varint out
buf = make([]byte, bytesRead)
io.ReadFull(bufferedBody, buf)
// Get the structure out
buf = make([]byte, num)
_, err = io.ReadFull(bufferedBody, buf)
if err != nil {
return fmt.Errorf("unable to fully read protobuf message: %s", err)
}
var msg com_signalfx_metrics_protobuf.DataPoint
err = proto.Unmarshal(buf, &msg)
if err != nil {
return err
}
if datapointProtobufIsInvalidForV1(&msg) {
return errInvalidProtobuf
}
mt := decoder.TypeGetter.GetMetricTypeFromMap(msg.GetMetric())
if dp, err := NewProtobufDataPointWithType(&msg, mt); err == nil {
decoder.Sink.AddDatapoints(ctx, []*datapoint.Datapoint{dp})
}
}
}
func (connectionHandler *GaugeConnectionHandler) processMessage(buffer *bytes.Buffer, conn net.Conn) {
for {
messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())
if messageLength > 0 && messageLength < uint64(buffer.Len()) {
messageBoundary := int(messageLength) + bytesRead
receivedBytes := buffer.Bytes()[bytesRead : messageLength+uint64(bytesRead)]
connectionHandler.messageHandler.MessageBytesReceived(receivedBytes, conn)
buffer.Next(messageBoundary)
if buffer.Len() == 0 {
return
}
} else {
return
}
}
}
func readMessageBytes(conn net.Conn) ([]byte, error) {
buffer := new(bytes.Buffer)
data := make([]byte, c.MaxMessageSize)
for {
n, err := conn.Read(data)
if err != nil {
conn.Close()
return nil, fmt.Errorf("Connection closed [%s] cause: %s", conn.RemoteAddr(), err.Error())
}
buffer.Write(data[0:n])
messageLength, bytesRead := proto.DecodeVarint(buffer.Bytes())
if messageLength > 0 && messageLength < uint64(buffer.Len()) {
return buffer.Bytes()[bytesRead : messageLength+uint64(bytesRead)], nil
}
}
}
func TestSerializedBlockInfo(t *testing.T) {
block := testutil.ConstructTestBlock(t, 10, 100)
bb, info, err := serializeBlock(block)
testutil.AssertNoError(t, err, "")
infoFromBB, err := extractSerializedBlockInfo(bb)
testutil.AssertNoError(t, err, "")
testutil.AssertEquals(t, infoFromBB, info)
testutil.AssertEquals(t, len(info.txOffsets), len(block.Data.Data))
for txIndex, txEnvBytes := range block.Data.Data {
txid, err := extractTxID(txEnvBytes)
testutil.AssertNoError(t, err, "")
indexInfo := info.txOffsets[txIndex]
indexTxID := indexInfo.txID
indexOffset := indexInfo.loc
testutil.AssertEquals(t, txid, indexTxID)
b := bb[indexOffset.offset:]
len, num := proto.DecodeVarint(b)
txEnvBytesFromBB := b[num : num+int(len)]
testutil.AssertEquals(t, txEnvBytesFromBB, txEnvBytes)
}
}
func (c *Client) receiveRpcs() {
var sz [4]byte
for {
err := c.readFully(sz[:])
if err != nil {
c.setSendErr(err)
c.errorEncountered()
return
}
buf := make([]byte, binary.BigEndian.Uint32(sz[:]))
err = c.readFully(buf)
if err != nil {
c.setSendErr(err)
c.errorEncountered()
return
}
resp := &pb.ResponseHeader{}
respLen, nb := proto.DecodeVarint(buf)
buf = buf[nb:]
err = proto.UnmarshalMerge(buf[:respLen], resp)
buf = buf[respLen:]
if err != nil {
// Failed to deserialize the response header
c.setSendErr(err)
c.errorEncountered()
return
}
if resp.CallId == nil {
// Response doesn't have a call ID
log.Error("Response doesn't have a call ID!")
c.setSendErr(ErrMissingCallID)
c.errorEncountered()
return
}
c.sentRPCsMutex.Lock()
rpc, ok := c.sentRPCs[*resp.CallId]
c.sentRPCsMutex.Unlock()
if !ok {
log.WithFields(log.Fields{
"CallId": *resp.CallId,
}).Error("Received a response with an unexpected call ID")
log.Error("Waiting for responses to the following calls:")
c.sentRPCsMutex.Lock()
for id, call := range c.sentRPCs {
log.Errorf("\t\t%d: %v", id, call)
}
c.sentRPCsMutex.Unlock()
c.setSendErr(fmt.Errorf("HBase sent a response with an unexpected call ID: %d",
resp.CallId))
c.errorEncountered()
return
}
var rpcResp proto.Message
if resp.Exception == nil {
respLen, nb = proto.DecodeVarint(buf)
buf = buf[nb:]
rpcResp = rpc.NewResponse()
err = proto.UnmarshalMerge(buf, rpcResp)
buf = buf[respLen:]
} else {
javaClass := *resp.Exception.ExceptionClassName
err = fmt.Errorf("HBase Java exception %s: \n%s", javaClass,
*resp.Exception.StackTrace)
if _, ok := javaRetryableExceptions[javaClass]; ok {
// This is a recoverable error. The client should retry.
err = RetryableError{err}
}
}
rpc.GetResultChan() <- hrpc.RPCResult{Msg: rpcResp, Error: err}
c.sentRPCsMutex.Lock()
delete(c.sentRPCs, *resp.CallId)
c.sentRPCsMutex.Unlock()
}
}
func decodeBlockNum(blockNumBytes []byte) uint64 {
blockNum, _ := proto.DecodeVarint(blockNumBytes)
return blockNum
}
func toInt(balanceBytes []byte) int {
v, _ := proto.DecodeVarint(balanceBytes)
return int(v)
}
func decodeBlockNumber(blockNumberBytes []byte) (blockNumber uint64) {
blockNumber, _ = proto.DecodeVarint(blockNumberBytes)
return
}