// Read (as much as possible of) a chunk of binary log events starting the given startingCoordinates
func readBinlogEventsChunk(instanceKey *InstanceKey, startingCoordinates BinlogCoordinates) ([]BinlogEvent, error) {
events := []BinlogEvent{}
db, err := db.OpenTopology(instanceKey.Hostname, instanceKey.Port)
if err != nil {
return events, err
}
commandToken := math.TernaryString(startingCoordinates.Type == BinaryLog, "binlog", "relaylog")
if startingCoordinates.LogFile == "" {
return events, log.Errorf("readBinlogEventsChunk: empty binlog file name for %+v.", *instanceKey)
}
query := fmt.Sprintf("show %s events in '%s' FROM %d LIMIT %d", commandToken, startingCoordinates.LogFile, startingCoordinates.LogPos, config.Config.BinlogEventsChunkSize)
err = sqlutils.QueryRowsMap(db, query, func(m sqlutils.RowMap) error {
binlogEvent := BinlogEvent{}
binlogEvent.Coordinates.LogFile = m.GetString("Log_name")
binlogEvent.Coordinates.LogPos = m.GetInt64("Pos")
binlogEvent.Coordinates.Type = startingCoordinates.Type
binlogEvent.NextEventPos = m.GetInt64("End_log_pos")
binlogEvent.EventType = m.GetString("Event_type")
binlogEvent.Info = m.GetString("Info")
events = append(events, binlogEvent)
return nil
})
return events, err
}
// ScanInstanceRow executes a read-a-single-row query on a given MySQL topology instance
func ScanInstanceRow(instanceKey *InstanceKey, query string, dest ...interface{}) error {
db, err := db.OpenTopology(instanceKey.Hostname, instanceKey.Port)
if err != nil {
return err
}
err = db.QueryRow(query).Scan(dest...)
return err
}
// ExecInstanceNoPrepare executes a given query on the given MySQL topology instance, without using prepared statements
func ExecInstanceNoPrepare(instanceKey *InstanceKey, query string, args ...interface{}) (sql.Result, error) {
db, err := db.OpenTopology(instanceKey.Hostname, instanceKey.Port)
if err != nil {
return nil, err
}
res, err := sqlutils.ExecNoPrepare(db, query, args...)
return res, err
}
// Try and find the last position of a pseudo GTID query entry in the given binary log.
// Also return the full text of that entry.
// maxCoordinates is the position beyond which we should not read. This is relevant when reading relay logs; in particular,
// the last relay log. We must be careful not to scan for Pseudo-GTID entries past the position executed by the SQL thread.
// maxCoordinates == nil means no limit.
func getLastPseudoGTIDEntryInBinlog(instanceKey *InstanceKey, binlog string, binlogType BinlogType, maxCoordinates *BinlogCoordinates) (*BinlogCoordinates, string, error) {
if binlog == "" {
return nil, "", log.Errorf("getLastPseudoGTIDEntryInBinlog: empty binlog file name for %+v. maxCoordinates = %+v", *instanceKey, maxCoordinates)
}
binlogCoordinates := BinlogCoordinates{LogFile: binlog, LogPos: 0, Type: binlogType}
db, err := db.OpenTopology(instanceKey.Hostname, instanceKey.Port)
if err != nil {
return nil, "", err
}
moreRowsExpected := true
var nextPos int64 = 0
step := 0
entryText := ""
for moreRowsExpected {
query := ""
if binlogCoordinates.Type == BinaryLog {
query = fmt.Sprintf("show binlog events in '%s' FROM %d LIMIT %d", binlog, nextPos, config.Config.BinlogEventsChunkSize)
} else {
query = fmt.Sprintf("show relaylog events in '%s' LIMIT %d,%d", binlog, (step * config.Config.BinlogEventsChunkSize), config.Config.BinlogEventsChunkSize)
}
moreRowsExpected = false
queryRowsFunc := sqlutils.QueryRowsMap
if config.Config.BufferBinlogEvents {
queryRowsFunc = sqlutils.QueryRowsMapBuffered
}
err = queryRowsFunc(db, query, func(m sqlutils.RowMap) error {
moreRowsExpected = true
nextPos = m.GetInt64("End_log_pos")
binlogEntryInfo := m.GetString("Info")
if matched, _ := regexp.MatchString(config.Config.PseudoGTIDPattern, binlogEntryInfo); matched {
if maxCoordinates != nil && maxCoordinates.SmallerThan(&BinlogCoordinates{LogFile: binlog, LogPos: m.GetInt64("Pos")}) {
// past the limitation
moreRowsExpected = false
return nil
}
binlogCoordinates.LogPos = m.GetInt64("Pos")
entryText = binlogEntryInfo
// Found a match. But we keep searching: we're interested in the LAST entry, and, alas,
// we can only search in ASCENDING order...
}
return nil
})
if err != nil {
return nil, "", err
}
step++
}
// Not found? return nil. an error is reserved to SQL problems.
if binlogCoordinates.LogPos == 0 {
return nil, "", nil
}
return &binlogCoordinates, entryText, err
}
// EmptyCommitInstance issues an empty COMMIT on a given instance
func EmptyCommitInstance(instanceKey *InstanceKey) error {
db, err := db.OpenTopology(instanceKey.Hostname, instanceKey.Port)
if err != nil {
return err
}
tx, err := db.Begin()
if err != nil {
return err
}
err = tx.Commit()
if err != nil {
return err
}
return err
}
// Given a binlog entry text (query), search it in the given binary log of a given instance
func SearchPseudoGTIDEntryInBinlog(instanceKey *InstanceKey, binlog string, entryText string, entriesMonotonic bool) (BinlogCoordinates, bool, error) {
binlogCoordinates := BinlogCoordinates{LogFile: binlog, LogPos: 0, Type: BinaryLog}
if binlog == "" {
return binlogCoordinates, false, log.Errorf("SearchPseudoGTIDEntryInBinlog: empty binlog file name for %+v", *instanceKey)
}
db, err := db.OpenTopology(instanceKey.Hostname, instanceKey.Port)
if err != nil {
return binlogCoordinates, false, err
}
moreRowsExpected := true
skipRestOfBinlog := false
alreadyMatchedAscendingPseudoGTID := false
var nextPos int64 = 0
// commandToken := math.TernaryString(binlogCoordinates.Type == BinaryLog, "binlog", "relaylog")
for moreRowsExpected {
query := fmt.Sprintf("show binlog events in '%s' FROM %d LIMIT %d", binlog, nextPos, config.Config.BinlogEventsChunkSize)
// We don't know in advance when we will hit the end of the binlog. We will implicitly understand it when our
// `show binlog events` query does not return any row.
moreRowsExpected = false
queryRowsFunc := sqlutils.QueryRowsMap
if config.Config.BufferBinlogEvents {
queryRowsFunc = sqlutils.QueryRowsMapBuffered
}
err = queryRowsFunc(db, query, func(m sqlutils.RowMap) error {
if binlogCoordinates.LogPos != 0 {
// Entry found!
skipRestOfBinlog = true
return nil
}
if skipRestOfBinlog {
return nil
}
moreRowsExpected = true
nextPos = m.GetInt64("End_log_pos")
binlogEntryInfo := m.GetString("Info")
//
if binlogEntryInfo == entryText {
// found it!
binlogCoordinates.LogPos = m.GetInt64("Pos")
} else if entriesMonotonic && !alreadyMatchedAscendingPseudoGTID {
// More heavyweight computation here. Need to verify whether the binlog entry we have is a pseudo-gtid entry
// We only want to check for ASCENDING once in the top of the binary log.
// If we find the first entry to be higher than the searched one, clearly we are done.
// If not, then by virtue of binary logs, we still have to full-scan the entrie binlog sequentially; we
// do not check again for ASCENDING (no point), so we save up CPU energy wasted in regexp.
if matched, _ := regexp.MatchString(config.Config.PseudoGTIDPattern, binlogEntryInfo); matched {
alreadyMatchedAscendingPseudoGTID = true
log.Debugf("Matched ascending Pseudo-GTID entry in %+v", binlog)
if binlogEntryInfo > entryText {
// Entries ascending, and current entry is larger than the one we are searching for.
// There is no need to scan further on. We can skip the entire binlog
log.Debugf(`Pseudo GTID entries are monotonic and we hit "%+v" > "%+v"; skipping binlog %+v`, m.GetString("Info"), entryText, binlogCoordinates.LogFile)
skipRestOfBinlog = true
return nil
}
}
}
return nil
})
if err != nil {
return binlogCoordinates, (binlogCoordinates.LogPos != 0), err
}
if skipRestOfBinlog {
return binlogCoordinates, (binlogCoordinates.LogPos != 0), err
}
}
return binlogCoordinates, (binlogCoordinates.LogPos != 0), err
}