func (s *FormatDescriptionEventSuite) Test56FDE(c *C) {
// This FDE entry was copied from a 5.6 HDB shard.
s.WriteEvent(
mysql_proto.LogEventType_FORMAT_DESCRIPTION_EVENT,
uint16(0),
[]byte{
// binlog version
4, 0,
// server version
53, 46, 54, 46, 49, 53, 45, 54, 51, 46,
48, 45, 108, 111, 103, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// created timestamp
0, 0, 0, 0,
// total header size
19,
// fixed length data size per event type
56, 13, 0, 8, 0, 18, 0, 4, 4, 4, 4, 18, 0, 0, 92, 0, 4, 26,
8, 0, 0, 0, 8, 8, 8, 2, 0, 0, 0, 10, 10, 10, 25, 25, 0,
// checksum algorithm
1,
// checksum
40, 216, 52, 169}) // 0x28 0xd8 0x34 0xa9
event, err := s.NextEvent()
c.Assert(err, IsNil)
c.Assert(event, NotNil)
fde, ok := event.(*FormatDescriptionEvent)
c.Assert(ok, IsTrue)
c.Check(fde.BinlogVersion(), Equals, uint16(4))
c.Check(string(fde.ServerVersion()), Equals, "5.6.15-63.0-log")
c.Check(fde.CreatedTimestamp(), Equals, uint32(0))
c.Check(fde.ExtraHeadersSize(), Equals, 0)
for i := 0; i < int(mysql_proto.LogEventType_PREVIOUS_GTIDS_LOG_EVENT); i++ {
c.Log(mysql_proto.LogEventType_Type(i).String())
_, inMap := fde.fixedLengthSizes[mysql_proto.LogEventType_Type(i)]
c.Check(inMap, IsTrue)
}
c.Check(fde.ChecksumAlgorithm(), Equals, mysql_proto.ChecksumAlgorithm_CRC32)
c.Check(fde.Checksum(), DeepEquals, []byte{40, 216, 52, 169})
// extra sanity checks
c.Check(fde.SourceName(), Equals, testSourceName)
c.Check(
fde.EventType(),
Equals,
mysql_proto.LogEventType_FORMAT_DESCRIPTION_EVENT)
_, ok = event.(*RawV4Event)
c.Check(ok, IsFalse)
}
func (s *FormatDescriptionEventSuite) Test55FDE(c *C) {
// This FDE entry was copied from a 5.5 SFJ shard.
s.WriteEvent(
mysql_proto.LogEventType_FORMAT_DESCRIPTION_EVENT,
uint16(0),
[]byte{
// binlog version
4, 0,
// server version
53, 46, 53, 46, 51, 52, 45, 51, 50, 46,
48, 45, 108, 111, 103, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
// created timestamp
0, 0, 0, 0,
// total header size
19,
// fixed length data size per event type
56, 13, 0, 8, 0, 18, 0, 4, 4, 4, 4, 18, 0, 0, 84, 0, 4,
26, 8, 0, 0, 0, 8, 8, 8, 2, 0})
event, err := s.NextEvent()
c.Assert(err, IsNil)
c.Assert(event, NotNil)
fde, ok := event.(*FormatDescriptionEvent)
c.Assert(ok, IsTrue)
c.Check(fde.BinlogVersion(), Equals, uint16(4))
c.Check(string(fde.ServerVersion()), Equals, "5.5.34-32.0-log")
c.Check(fde.CreatedTimestamp(), Equals, uint32(0))
c.Check(fde.ExtraHeadersSize(), Equals, 0)
for i := 0; i < int(mysql_proto.LogEventType_HEARTBEAT_LOG_EVENT); i++ {
c.Log(mysql_proto.LogEventType_Type(i).String())
_, inMap := fde.fixedLengthSizes[mysql_proto.LogEventType_Type(i)]
c.Check(inMap, IsTrue)
}
for i := int(mysql_proto.LogEventType_IGNORABLE_LOG_EVENT); i < int(mysql_proto.LogEventType_PREVIOUS_GTIDS_LOG_EVENT); i++ {
c.Log(mysql_proto.LogEventType_Type(i).String())
_, inMap := fde.fixedLengthSizes[mysql_proto.LogEventType_Type(i)]
c.Check(inMap, IsFalse)
}
c.Check(fde.ChecksumAlgorithm(), Equals, mysql_proto.ChecksumAlgorithm_OFF)
c.Check(fde.Checksum(), DeepEquals, []byte{})
}
func (h *formatVersionHeader) version() int {
eventType := mysql_proto.LogEventType_Type(h.EventType)
if eventType == mysql_proto.LogEventType_FORMAT_DESCRIPTION_EVENT {
return 4
} else if eventType == mysql_proto.LogEventType_START_EVENT_V3 {
if h.EventLength >= maxFirstEventLengthForV1 {
return 3
}
return 1
}
return 3
}
func (s *RawV4EventReaderSuite) TestEOFMidHeader(c *C) {
eventBytes := s.GenerateEvent(
1, // timestamp
2, // event type
3, // server id
4, // next position
5, // flags
2) // data length
_, err := s.src.Write(eventBytes[:5])
c.Assert(err, IsNil)
event, err := s.reader.NextEvent()
c.Assert(err, Equals, io.EOF)
c.Check(event, IsNil)
_, err = s.src.Write(eventBytes[5:13])
c.Assert(err, IsNil)
event, err = s.reader.NextEvent()
c.Assert(err, Equals, io.EOF)
c.Check(event, IsNil)
c.Check(s.reader.nextEventEndPosition(), Equals, int64(19))
// Finish writing the entire event.
_, err = s.src.Write(eventBytes[13:])
c.Assert(err, IsNil)
event, err = s.reader.NextEvent()
c.Assert(err, IsNil)
c.Check(event.SourceName(), Equals, testSourceName)
c.Check(event.SourcePosition(), Equals, int64(0))
c.Check(event.Timestamp(), Equals, uint32(1))
c.Check(event.EventType(), Equals, mysql_proto.LogEventType_Type(2))
c.Check(event.ServerId(), Equals, uint32(3))
c.Check(event.EventLength(), Equals, uint32(len(eventBytes)))
c.Check(event.NextPosition(), Equals, uint32(4))
c.Check(event.Flags(), Equals, uint16(5))
c.Check(event.ExtraHeaders(), DeepEquals, []byte{})
c.Check(event.FixedLengthData(), DeepEquals, []byte{})
c.Check(event.VariableLengthData(), DeepEquals, []byte("\xfe\xfe"))
c.Check(
s.reader.nextEventEndPosition(),
Equals,
int64(event.EventLength())+int64(19))
}
if len(raw.FixedLengthData()) != 4 {
return raw, errors.New("Parse error")
}
err := binary.Read(
bytes.NewReader(raw.FixedLengthData()),
binary.BigEndian,
&event.value)
if err != nil {
return raw, err
}
return event, nil
}
const testRegisteredEventType = mysql_proto.LogEventType_Type(1)
const testParseErrorEventType = mysql_proto.LogEventType_Type(2)
const testInvalidFixedLengthDataSizeEventType = mysql_proto.LogEventType_Type(3)
const testNotRegisteredEventType = mysql_proto.LogEventType_Type(4)
const testFDEEventType = mysql_proto.LogEventType_FORMAT_DESCRIPTION_EVENT // 15
type ParsedV4EventReaderSuite struct {
src *bytes.Buffer
rawReader EventReader
parsers V4EventParserMap
reader EventReader
}
var _ = Suite(&ParsedV4EventReaderSuite{})
func (s *ParsedV4EventReaderSuite) SetUpTest(c *C) {
// FormatDecriptionEventParser's Parse processes a raw FDE event into a
// FormatDescriptionEvent.
func (p *FormatDescriptionEventParser) Parse(raw *RawV4Event) (Event, error) {
fde := &FormatDescriptionEvent{
Event: raw,
fixedLengthSizes: make(map[mysql_proto.LogEventType_Type]int),
}
data := raw.VariableLengthData()
data, err := readLittleEndian(data, &fde.binlogVersion)
if err != nil {
return raw, errors.Wrap(err, "Failed to read binlog verison")
}
serverVersion, data, err := readSlice(data, 50)
if err != nil {
return raw, errors.Wrap(err, "Failed to read server version")
}
if idx := bytes.IndexByte(serverVersion, byte(0)); idx > -1 {
serverVersion = serverVersion[:idx]
}
fde.serverVersion = serverVersion
data, err = readLittleEndian(data, &fde.createdTimestamp)
if err != nil {
return raw, errors.Wrap(err, "Failed to read created timestamp")
}
var totalHeaderSize uint8
data, err = readLittleEndian(data, &totalHeaderSize)
if err != nil {
return raw, errors.Wrap(err, "Failed to read total header size")
}
fde.extraHeadersSize = int(totalHeaderSize) - sizeOfBasicV4EventHeader
numEvents := len(mysql_proto.LogEventType_Type_value)
hasChecksum := true
if len(data) == 27 { // mysql 5.5(.37)
numEvents = 28
hasChecksum = false
} else if len(data) == 40 { // mysql 5.6(.17)
// This is a relay log where the master is 5.5 and slave is 5.6
if data[int(mysql_proto.LogEventType_WRITE_ROWS_EVENT)-1] == 0 {
numEvents = 28
}
} else {
return raw, errors.Newf(
"Unable to parse FDE for mysql variant: %s",
fde.serverVersion)
}
// unknown event's fixed length is implicit.
fde.fixedLengthSizes[mysql_proto.LogEventType_UNKNOWN_EVENT] = 0
for i := 1; i < numEvents; i++ {
fde.fixedLengthSizes[mysql_proto.LogEventType_Type(i)] = int(data[i-1])
}
if hasChecksum {
fde.checksumAlgorithm = mysql_proto.ChecksumAlgorithm_Type(
data[len(data)-5])
raw.SetChecksumSize(4)
}
return fde, nil
}
// EventType returns the event's type.
func (e *RawV4Event) EventType() mysql_proto.LogEventType_Type {
return mysql_proto.LogEventType_Type(e.header.EventType)
}
func (r *logFileV4EventReader) checkFDE(fde *FormatDescriptionEvent) error {
if fde.BinlogVersion() != 4 {
return errors.Newf(
"Invalid binlog format version: %d",
fde.BinlogVersion())
}
if fde.ExtraHeadersSize() != 0 {
return errors.Newf(
"Invalid extra headers size: %d",
fde.ExtraHeadersSize())
}
alg := fde.ChecksumAlgorithm()
if alg != mysql_proto.ChecksumAlgorithm_OFF &&
alg != mysql_proto.ChecksumAlgorithm_CRC32 {
return errors.Newf(
"Invalid checksum algorithm: %d (%s)",
alg,
alg.String())
}
errMsg := ""
for i := 0; i < fde.NumKnownEventTypes(); i++ {
t := mysql_proto.LogEventType_Type(i)
if t == mysql_proto.LogEventType_FORMAT_DESCRIPTION_EVENT {
actual := fde.FixedLengthDataSizeForType(t)
if actual != FDEFixedLengthDataSizeFor56 &&
actual != FDEFixedLengthDataSizeFor55 {
errMsg += fmt.Sprintf(
"%s (expected: %d (5.6) or %d (5.5) actual: %d); ",
t.String(),
FDEFixedLengthDataSizeFor56,
FDEFixedLengthDataSizeFor55,
actual)
}
} else {
parser := r.parsers.Get(t)
if parser == nil {
continue
}
expected := parser.FixedLengthDataSize()
actual := fde.FixedLengthDataSizeForType(t)
if expected != actual {
errMsg += fmt.Sprintf(
"%s (expected: %d actual: %d); ",
t.String(),
expected,
actual)
}
}
}
if errMsg != "" {
return errors.New("Invalid fixed length data size: " + errMsg)
}
return nil
}
func (s *RawV4EventReaderSuite) TestParseSimpleEvents(c *C) {
// hand code a single event just to be sure
eventBytes1 := []byte(
"\x04\x03\x02\x01" + // timestamp
"\x12" + // event type
"\x0f\x0e\x0e\x0b" + // server id
"\x16\x00\x00\x00" + // event length (19 header + 3 data)
"\xf4\xf3\xf2\xf1" + // next position
"\xad\xde" + // flags
"\x0a\x0b\x0c") // data
_, err := s.src.Write(eventBytes1)
c.Assert(err, IsNil)
eventBytes2 := s.GenerateEvent(
0xdeadbeef, // timestamp
0x69, // event type
0xdecafbad, // server id
0x12345678, // next position
0xf00d, // flags
7) // data length
_, err = s.src.Write(eventBytes2)
c.Assert(err, IsNil)
eventBytes3 := s.GenerateEvent(
0x1aaaaaaa, // timestamp
0x2b, // event type
0x3ccccccc, // server id
0x4ddddddd, // next position
0x5eee, // flags
0) // data length
_, err = s.src.Write(eventBytes3)
c.Assert(err, IsNil)
event1, err := s.reader.NextEvent()
c.Assert(err, IsNil)
c.Check(event1.SourceName(), Equals, testSourceName)
c.Check(event1.SourcePosition(), Equals, int64(0))
c.Check(event1.Timestamp(), Equals, uint32(0x01020304))
c.Check(event1.EventType(), Equals, mysql_proto.LogEventType_Type(0x12))
c.Check(event1.ServerId(), Equals, uint32(0x0b0e0e0f))
c.Check(event1.EventLength(), Equals, uint32(len(eventBytes1)))
c.Check(event1.NextPosition(), Equals, uint32(0xf1f2f3f4))
c.Check(event1.Flags(), Equals, uint16(0xdead))
c.Check(event1.Bytes(), DeepEquals, eventBytes1)
c.Check(
event1.BasicHeader(),
DeepEquals,
eventBytes1[:sizeOfBasicV4EventHeader])
c.Check(event1.ExtraHeaders(), DeepEquals, []byte{})
c.Check(event1.FixedLengthData(), DeepEquals, []byte{})
c.Check(event1.VariableLengthData(), DeepEquals, []byte("\x0a\x0b\x0c"))
event2, err := s.reader.NextEvent()
c.Assert(err, IsNil)
c.Check(event2.SourceName(), Equals, testSourceName)
c.Check(event2.SourcePosition(), Equals, int64(len(eventBytes1)))
c.Check(event2.Timestamp(), Equals, uint32(0xdeadbeef))
c.Check(event2.EventType(), Equals, mysql_proto.LogEventType_Type(0x69))
c.Check(event2.ServerId(), Equals, uint32(0xdecafbad))
c.Check(event2.EventLength(), Equals, uint32(len(eventBytes2)))
c.Check(event2.NextPosition(), Equals, uint32(0x12345678))
c.Check(event2.Flags(), Equals, uint16(0xf00d))
c.Check(event2.ExtraHeaders(), DeepEquals, []byte{})
c.Check(event2.FixedLengthData(), DeepEquals, []byte{})
c.Check(
event2.VariableLengthData(),
DeepEquals,
[]byte("\xfe\xfe\xfe\xfe\xfe\xfe\xfe"))
event3, err := s.reader.NextEvent()
c.Assert(err, IsNil)
c.Check(event3.SourceName(), Equals, testSourceName)
c.Check(
event3.SourcePosition(),
Equals,
int64(len(eventBytes1)+len(eventBytes2)))
c.Check(event3.Timestamp(), Equals, uint32(0x1aaaaaaa))
c.Check(event3.EventType(), Equals, mysql_proto.LogEventType_Type(0x2b))
c.Check(event3.ServerId(), Equals, uint32(0x3ccccccc))
c.Check(event3.EventLength(), Equals, uint32(len(eventBytes3)))
c.Check(event3.NextPosition(), Equals, uint32(0x4ddddddd))
c.Check(event3.Flags(), Equals, uint16(0x5eee))
c.Check(event3.ExtraHeaders(), DeepEquals, []byte{})
c.Check(event3.FixedLengthData(), DeepEquals, []byte{})
c.Check(event3.VariableLengthData(), DeepEquals, []byte{})
_, err = s.src.ReadByte()
c.Assert(err, Equals, io.EOF)
event4, err := s.reader.NextEvent()
c.Check(event4, IsNil)
c.Check(err, Equals, io.EOF)
}