func isSpecialPgsqlCommand(data []byte) (bool, int) {
if len(data) < 8 {
// 8 bytes required
return false, 0
}
// read length
length := int(common.Bytes_Ntohl(data[0:4]))
// read command identifier
code := int(common.Bytes_Ntohl(data[4:8]))
if length == 16 && code == 80877102 {
// Cancel Request
logp.Debug("pgsqldetailed", "Cancel Request, length=%d", length)
return true, CancelRequest
} else if length == 8 && code == 80877103 {
// SSL Request
logp.Debug("pgsqldetailed", "SSL Request, length=%d", length)
return true, SSLRequest
} else if code == 196608 {
// Startup Message
logp.Debug("pgsqldetailed", "Startup Message, length=%d", length)
return true, StartupMessage
}
return false, 0
}
func (pgsql *Pgsql) pgsqlRowsParser(s *PgsqlStream) error {
m := s.message
// read field count (int16)
field_count := int(common.Bytes_Ntohs(s.data[s.parseOffset : s.parseOffset+2]))
s.parseOffset += 2
logp.Debug("pgsqldetailed", "DataRow field count=%d", field_count)
row := []string{}
var row_len int
for i := 0; i < field_count; i++ {
// read column length (int32)
column_length := int32(common.Bytes_Ntohl(s.data[s.parseOffset : s.parseOffset+4]))
s.parseOffset += 4
if column_length > 0 && int(column_length) > len(s.data[s.parseOffset:]) {
logp.Err("Pgsql invalid column_length=%v, buffer_length=%v, i=%v",
column_length, len(s.data[s.parseOffset:]), i)
return errInvalidLength
}
// read column value (byten)
column_value := []byte{}
if m.FieldsFormat[i] == 0 {
// field value in text format
if column_length > 0 {
column_value = s.data[s.parseOffset : s.parseOffset+int(column_length)]
} else if column_length == -1 {
column_value = nil
}
}
if row_len < pgsql.maxRowLength {
if row_len+len(column_value) > pgsql.maxRowLength {
column_value = column_value[:pgsql.maxRowLength-row_len]
}
row = append(row, string(column_value))
row_len += len(column_value)
}
if column_length > 0 {
s.parseOffset += int(column_length)
}
logp.Debug("pgsqldetailed", "Value %s, length=%d", string(column_value), column_length)
}
m.NumberOfRows += 1
if len(m.Rows) < pgsql.maxStoreRows {
m.Rows = append(m.Rows, row)
}
return nil
}
func (thrift *Thrift) readI32(data []byte) (value string, ok bool, complete bool, off int) {
if len(data) < 4 {
return "", true, false, 0
}
i32 := common.Bytes_Ntohl(data[:4])
value = strconv.Itoa(int(i32))
return value, true, true, 4
}
func (thrift *Thrift) readMap(data []byte) (value string, ok bool, complete bool, off int) {
if len(data) < 6 {
return "", true, false, 0
}
type_key := data[0]
type_value := data[1]
funcReaderKey, typeFound := thrift.funcReadersByType(type_key)
if !typeFound {
logp.Debug("thrift", "Field type %d not known", type_key)
return "", false, false, 0
}
funcReaderValue, typeFound := thrift.funcReadersByType(type_value)
if !typeFound {
logp.Debug("thrift", "Field type %d not known", type_value)
return "", false, false, 0
}
sz := int(common.Bytes_Ntohl(data[2:6]))
if sz < 0 {
logp.Debug("thrift", "Map too big: %d", sz)
return "", false, false, 0
}
fields := []string{}
offset := 6
for i := 0; i < sz; i++ {
key, ok, complete, bytesRead := funcReaderKey(data[offset:])
if !ok {
return "", false, false, 0
}
if !complete {
return "", true, false, 0
}
offset += bytesRead
value, ok, complete, bytesRead := funcReaderValue(data[offset:])
if !ok {
return "", false, false, 0
}
if !complete {
return "", true, false, 0
}
offset += bytesRead
if i < thrift.CollectionMaxSize {
fields = append(fields, key+": "+value)
} else if i == thrift.CollectionMaxSize {
fields = append(fields, "...")
}
}
return "{" + strings.Join(fields, ", ") + "}", true, true, offset
}
// Parse 32bit binary value from the buffer at index. Will not advance the read
// buffer
func (b *Buffer) ReadNetUint32At(index int) (uint32, error) {
if b.Failed() {
return 0, b.err
}
if !b.Avail(4 + index) {
return 0, b.bufferEndError()
}
return common.Bytes_Ntohl(b.data[index+b.mark:]), nil
}
// Parse 32bit binary value in network byte order from Buffer
// (converted to Host order).
func (b *Buffer) ReadNetUint32() (uint32, error) {
if b.Failed() {
return 0, b.err
}
tmp := b.data[b.mark:]
if err := b.Advance(4); err != nil {
return 0, err
}
value := common.Bytes_Ntohl(tmp)
return value, nil
}
// Common implementation for lists and sets (they share the same binary repr).
func (thrift *Thrift) readListOrSet(data []byte) (value string, ok bool, complete bool, off int) {
if len(data) < 5 {
return "", true, false, 0
}
type_ := data[0]
funcReader, typeFound := thrift.funcReadersByType(type_)
if !typeFound {
logp.Debug("thrift", "Field type %d not known", type_)
return "", false, false, 0
}
sz := int(common.Bytes_Ntohl(data[1:5]))
if sz < 0 {
logp.Debug("thrift", "List/Set too big: %d", sz)
return "", false, false, 0
}
fields := []string{}
offset := 5
for i := 0; i < sz; i++ {
value, ok, complete, bytesRead := funcReader(data[offset:])
if !ok {
return "", false, false, 0
}
if !complete {
return "", true, false, 0
}
if i < thrift.CollectionMaxSize {
fields = append(fields, value)
} else if i == thrift.CollectionMaxSize {
fields = append(fields, "...")
}
offset += bytesRead
}
return strings.Join(fields, ", "), true, true, offset
}
// thriftReadString caps the returned value to ThriftStringMaxSize but returns the
// off to the end of it.
func (thrift *Thrift) readString(data []byte) (value string, ok bool, complete bool, off int) {
if len(data) < 4 {
return "", true, false, 0 // ok, not complete
}
sz := int(common.Bytes_Ntohl(data[:4]))
if int32(sz) < 0 {
return "", false, false, 0 // not ok
}
if len(data[4:]) < sz {
return "", true, false, 0 // ok, not complete
}
if sz > thrift.StringMaxSize {
value = string(data[4 : 4+thrift.StringMaxSize])
value += "..."
} else {
value = string(data[4 : 4+sz])
}
off = 4 + sz
return value, true, true, off // all good
}
func (pgsql *Pgsql) pgsqlMessageParser(s *PgsqlStream) (bool, bool) {
m := s.message
for s.parseOffset < len(s.data) {
switch s.parseState {
case PgsqlStartState:
if len(s.data[s.parseOffset:]) < 5 {
logp.Warn("Postgresql Message too short. %X (length=%d). Wait for more.", s.data[s.parseOffset:], len(s.data[s.parseOffset:]))
return true, false
}
is_special, command := isSpecialPgsqlCommand(s.data[s.parseOffset:])
if is_special {
// In case of Commands: StartupMessage, SSLRequest, CancelRequest that don't have
// their type in the first byte
// read length
length := int(common.Bytes_Ntohl(s.data[s.parseOffset : s.parseOffset+4]))
// ignore command
if len(s.data[s.parseOffset:]) >= length {
if command == SSLRequest {
// if SSLRequest is received, expect for one byte reply (S or N)
m.start = s.parseOffset
s.parseOffset += length
m.end = s.parseOffset
m.isSSLRequest = true
m.Size = uint64(m.end - m.start)
return true, true
}
s.parseOffset += length
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 1")
return true, false
}
} else {
// In case of Commands that have their type in the first byte
// read type
typ := byte(s.data[s.parseOffset])
if s.expectSSLResponse {
// SSLRequest was received in the other stream
if typ == 'N' || typ == 'S' {
// one byte reply to SSLRequest
logp.Debug("pgsqldetailed", "Reply for SSLRequest %c", typ)
m.start = s.parseOffset
s.parseOffset += 1
m.end = s.parseOffset
m.isSSLResponse = true
m.Size = uint64(m.end - m.start)
return true, true
}
}
// read length
length := int(common.Bytes_Ntohl(s.data[s.parseOffset+1 : s.parseOffset+5]))
if length < 4 {
// length should include the size of itself (int32)
logp.Debug("pgsqldetailed", "Invalid pgsql command length.")
return false, false
}
logp.Debug("pgsqldetailed", "Pgsql type %c, length=%d", typ, length)
if typ == 'Q' {
// SimpleQuery
m.start = s.parseOffset
m.IsRequest = true
if len(s.data[s.parseOffset:]) >= length+1 {
s.parseOffset += 1 //type
s.parseOffset += length
m.end = s.parseOffset
m.Size = uint64(m.end - m.start)
m.Query = string(s.data[m.start+5 : m.end-1]) //without string termination
m.toExport = true
logp.Debug("pgsqldetailed", "Simple Query: %s", m.Query)
return true, true
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 2")
return true, false
}
} else if typ == 'T' {
// RowDescription
m.start = s.parseOffset
m.IsRequest = false
m.IsOK = true
m.toExport = true
if len(s.data[s.parseOffset:]) >= length+1 {
s.parseOffset += 1 //type
s.parseOffset += 4 //length
pgsqlFieldsParser(s)
logp.Debug("pgsqldetailed", "Fields: %s", m.Fields)
s.parseState = PgsqlGetDataState
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 3")
return true, false
}
} else if typ == 'I' {
// EmptyQueryResponse, appears as a response for empty queries
// substitutes CommandComplete
logp.Debug("pgsqldetailed", "EmptyQueryResponse")
m.start = s.parseOffset
m.IsOK = true
m.IsRequest = false
m.toExport = true
s.parseOffset += 5 // type + length
m.end = s.parseOffset
m.Size = uint64(m.end - m.start)
return true, true
} else if typ == 'E' {
// ErrorResponse
logp.Debug("pgsqldetailed", "ErrorResponse")
m.start = s.parseOffset
m.IsRequest = false
m.IsError = true
m.toExport = true
if len(s.data[s.parseOffset:]) >= length+1 {
s.parseOffset += 1 //type
s.parseOffset += 4 //length
pgsqlErrorParser(s)
m.end = s.parseOffset
m.Size = uint64(m.end - m.start)
return true, true
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 4")
return true, false
}
} else if typ == 'C' {
// CommandComplete -> Successful response
m.start = s.parseOffset
m.IsRequest = false
m.IsOK = true
m.toExport = true
if len(s.data[s.parseOffset:]) >= length+1 {
s.parseOffset += 1 //type
name := string(s.data[s.parseOffset+4 : s.parseOffset+length-1]) //without \0
logp.Debug("pgsqldetailed", "CommandComplete length=%d, tag=%s", length, name)
s.parseOffset += length
m.end = s.parseOffset
m.Size = uint64(m.end - m.start)
return true, true
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 5")
return true, false
}
} else if typ == 'Z' {
// ReadyForQuery -> backend ready for a new query cycle
if len(s.data[s.parseOffset:]) >= length+1 {
m.start = s.parseOffset
s.parseOffset += 1 // type
s.parseOffset += length
m.end = s.parseOffset
m.Size = uint64(m.end - m.start)
return true, true
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more 5b")
return true, false
}
} else {
// TODO: add info from NoticeResponse in case there are warning messages for a query
// ignore command
if len(s.data[s.parseOffset:]) >= length+1 {
s.parseOffset += 1 //type
s.parseOffset += length
m.end = s.parseOffset
m.Size = uint64(m.end - m.start)
// ok and complete, but ignore
m.toExport = false
return true, true
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 6")
return true, false
}
}
}
break
case PgsqlGetDataState:
// The response to queries that return row sets contains:
// RowDescription
// zero or more DataRow
// CommandComplete
// ReadyForQuery
if len(s.data[s.parseOffset:]) < 5 {
logp.Warn("Postgresql Message too short (length=%d). Wait for more.", len(s.data[s.parseOffset:]))
return true, false
}
// read type
typ := byte(s.data[s.parseOffset])
// read message length
length := int(common.Bytes_Ntohl(s.data[s.parseOffset+1 : s.parseOffset+5]))
if typ == 'D' {
// DataRow
if len(s.data[s.parseOffset:]) >= length+1 {
// skip type
s.parseOffset += 1
// skip length size
s.parseOffset += 4
pgsql.pgsqlRowsParser(s)
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 7")
return true, false
}
} else if typ == 'C' {
// CommandComplete
if len(s.data[s.parseOffset:]) >= length+1 {
// skip type
s.parseOffset += 1
name := string(s.data[s.parseOffset+4 : s.parseOffset+length-1]) //without \0
logp.Debug("pgsqldetailed", "CommandComplete length=%d, tag=%s", length, name)
s.parseOffset += length
m.end = s.parseOffset
m.Size = uint64(m.end - m.start)
s.parseState = PgsqlStartState
logp.Debug("pgsqldetailed", "Rows: %s", m.Rows)
return true, true
} else {
// wait for more
logp.Debug("pgsqldetailed", "Wait for more data 8")
return true, false
}
} else {
// shouldn't happen
logp.Debug("pgsqldetailed", "Skip command of type %c", typ)
s.parseState = PgsqlStartState
}
break
}
}
return true, false
}
// length field in pgsql counts total length of length field + payload, not
// including the message identifier. => Always check buffer size >= length + 1
func readLength(b []byte) int {
return int(common.Bytes_Ntohl(b))
}
func (thrift *Thrift) messageParser(s *ThriftStream) (bool, bool) {
var ok, complete bool
var m = s.message
logp.Debug("thriftdetailed", "messageParser called parseState=%v offset=%v",
s.parseState, s.parseOffset)
for s.parseOffset < len(s.data) {
switch s.parseState {
case ThriftStartState:
m.start = s.parseOffset
if thrift.TransportType == ThriftTFramed {
// read I32
if len(s.data) < 4 {
return true, false
}
m.FrameSize = common.Bytes_Ntohl(s.data[:4])
s.parseOffset = 4
}
ok, complete = thrift.readMessageBegin(s)
logp.Debug("thriftdetailed", "readMessageBegin returned: %v %v", ok, complete)
if !ok {
return false, false
}
if !complete {
return true, false
}
if !m.IsRequest && !thrift.CaptureReply {
// don't actually read the result
logp.Debug("thrift", "Don't capture reply")
m.ReturnValue = ""
m.Exceptions = ""
return true, true
}
s.parseState = ThriftFieldState
case ThriftFieldState:
ok, complete, field := thrift.readField(s)
logp.Debug("thriftdetailed", "readField returned: %v %v", ok, complete)
if !ok {
return false, false
}
if complete {
// done
var method *ThriftIdlMethod = nil
if thrift.Idl != nil {
method = thrift.Idl.FindMethod(m.Method)
}
if m.IsRequest {
if method != nil {
m.Params = thrift.formatStruct(m.fields, true, method.Params)
m.Service = method.Service.Name
} else {
m.Params = thrift.formatStruct(m.fields, false, nil)
}
} else {
if len(m.fields) > 1 {
logp.Warn("Thrift RPC response with more than field. Ignoring all but first")
}
if len(m.fields) > 0 {
field := m.fields[0]
if field.Id == 0 {
m.ReturnValue = field.Value
m.Exceptions = ""
} else {
m.ReturnValue = ""
if method != nil {
m.Exceptions = thrift.formatStruct(m.fields, true, method.Exceptions)
} else {
m.Exceptions = thrift.formatStruct(m.fields, false, nil)
}
m.HasException = true
}
}
}
return true, true
}
if field == nil {
return true, false // ok, not complete
}
m.fields = append(m.fields, *field)
}
}
return true, false
}
func (thrift *Thrift) readMessageBegin(s *ThriftStream) (bool, bool) {
var ok, complete bool
var offset, off int
m := s.message
if len(s.data[s.parseOffset:]) < 9 {
return true, false // ok, not complete
}
sz := common.Bytes_Ntohl(s.data[s.parseOffset : s.parseOffset+4])
if int32(sz) < 0 {
m.Version = sz & ThriftVersionMask
if m.Version != ThriftVersion1 {
logp.Debug("thrift", "Unexpected version: %d", m.Version)
}
logp.Debug("thriftdetailed", "version = %d", m.Version)
offset = s.parseOffset + 4
logp.Debug("thriftdetailed", "offset = %d", offset)
m.Type = sz & ThriftTypeMask
m.Method, ok, complete, off = thrift.readString(s.data[offset:])
if !ok {
return false, false // not ok, not complete
}
if !complete {
logp.Debug("thriftdetailed", "Method name not complete")
return true, false // ok, not complete
}
offset += off
logp.Debug("thriftdetailed", "method = %s", m.Method)
logp.Debug("thriftdetailed", "offset = %d", offset)
if len(s.data[offset:]) < 4 {
logp.Debug("thriftdetailed", "Less then 4 bytes remaining")
return true, false // ok, not complete
}
m.SeqId = common.Bytes_Ntohl(s.data[offset : offset+4])
s.parseOffset = offset + 4
} else {
// no version mode
offset = s.parseOffset
m.Method, ok, complete, off = thrift.readString(s.data[offset:])
if !ok {
return false, false // not ok, not complete
}
if !complete {
logp.Debug("thriftdetailed", "Method name not complete")
return true, false // ok, not complete
}
offset += off
logp.Debug("thriftdetailed", "method = %s", m.Method)
logp.Debug("thriftdetailed", "offset = %d", offset)
if len(s.data[offset:]) < 5 {
return true, false // ok, not complete
}
m.Type = uint32(s.data[offset])
offset += 1
m.SeqId = common.Bytes_Ntohl(s.data[offset : offset+4])
s.parseOffset = offset + 4
}
if m.Type == ThriftMsgTypeCall || m.Type == ThriftMsgTypeOneway {
m.IsRequest = true
} else {
m.IsRequest = false
}
return true, true
}
