// writeRecoveryPosition will write the current GTID as the recovery position // for the next transaction. // We will also try to get the timestamp for the transaction. Two cases: // - we have statements, and they start with a SET TIMESTAMP that we // can parse: then we update transaction_timestamp in blp_checkpoint // with it, and set SecondsBehindMaster to now() - transaction_timestamp // - otherwise (the statements are probably filtered out), we leave // transaction_timestamp alone (keeping the old value), and we don't // change SecondsBehindMaster func (blp *BinlogPlayer) writeRecoveryPosition(tx *proto.BinlogTransaction) error { now := time.Now().Unix() blp.blpPos.Position = myproto.AppendGTID(blp.blpPos.Position, tx.GTIDField.Value) updateRecovery := UpdateBlpCheckpoint(blp.blpPos.Uid, blp.blpPos.Position, now, tx.Timestamp) qr, err := blp.exec(updateRecovery) if err != nil { return fmt.Errorf("Error %v in writing recovery info %v", err, updateRecovery) } if qr.RowsAffected != 1 { return fmt.Errorf("Cannot update blp_recovery table, affected %v rows", qr.RowsAffected) } blp.blplStats.SetLastPosition(blp.blpPos.Position) if tx.Timestamp != 0 { blp.blplStats.SecondsBehindMaster.Set(now - tx.Timestamp) } return nil }
// AppendGTID updates the current replication position by appending a GTID to // the set of transactions that have been processed. func (rci *RowcacheInvalidator) AppendGTID(gtid myproto.GTID) { rci.posMutex.Lock() defer rci.posMutex.Unlock() rci.pos = myproto.AppendGTID(rci.pos, gtid) }
// parseEvents processes the raw binlog dump stream from the server, one event // at a time, and groups them into transactions. It is called from within the // service function launched by Stream(). // // If the sendTransaction func returns io.EOF, parseEvents returns ClientEOF. // If the events channel is closed, parseEvents returns ServerEOF. func (bls *BinlogStreamer) parseEvents(ctx *sync2.ServiceContext, events <-chan proto.BinlogEvent) (myproto.ReplicationPosition, error) { var statements []proto.Statement var format proto.BinlogFormat var gtid myproto.GTID var pos = bls.startPos var autocommit = true var err error // A begin can be triggered either by a BEGIN query, or by a GTID_EVENT. begin := func() { if statements != nil { // If this happened, it would be a legitimate error. log.Errorf("BEGIN in binlog stream while still in another transaction; dropping %d statements: %v", len(statements), statements) binlogStreamerErrors.Add("ParseEvents", 1) } statements = make([]proto.Statement, 0, 10) autocommit = false } // A commit can be triggered either by a COMMIT query, or by an XID_EVENT. // Statements that aren't wrapped in BEGIN/COMMIT are committed immediately. commit := func(timestamp uint32) error { trans := &proto.BinlogTransaction{ Statements: statements, Timestamp: int64(timestamp), GTIDField: myproto.GTIDField{Value: gtid}, } if err = bls.sendTransaction(trans); err != nil { if err == io.EOF { return ClientEOF } return fmt.Errorf("send reply error: %v", err) } statements = nil autocommit = true return nil } // Parse events. for ctx.IsRunning() { var ev proto.BinlogEvent var ok bool select { case ev, ok = <-events: if !ok { // events channel has been closed, which means the connection died. log.Infof("reached end of binlog event stream") return pos, ServerEOF } case <-ctx.ShuttingDown: log.Infof("stopping early due to BinlogStreamer service shutdown") return pos, nil } // Validate the buffer before reading fields from it. if !ev.IsValid() { return pos, fmt.Errorf("can't parse binlog event, invalid data: %#v", ev) } // We need to keep checking for FORMAT_DESCRIPTION_EVENT even after we've // seen one, because another one might come along (e.g. on log rotate due to // binlog settings change) that changes the format. if ev.IsFormatDescription() { format, err = ev.Format() if err != nil { return pos, fmt.Errorf("can't parse FORMAT_DESCRIPTION_EVENT: %v, event data: %#v", err, ev) } continue } // We can't parse anything until we get a FORMAT_DESCRIPTION_EVENT that // tells us the size of the event header. if format.IsZero() { // The only thing that should come before the FORMAT_DESCRIPTION_EVENT // is a fake ROTATE_EVENT, which the master sends to tell us the name // of the current log file. if ev.IsRotate() { continue } return pos, fmt.Errorf("got a real event before FORMAT_DESCRIPTION_EVENT: %#v", ev) } // Strip the checksum, if any. We don't actually verify the checksum, so discard it. ev, _ = ev.StripChecksum(format) // Update the GTID if the event has one. The actual event type could be // something special like GTID_EVENT (MariaDB, MySQL 5.6), or it could be // an arbitrary event with a GTID in the header (Google MySQL). if ev.HasGTID(format) { gtid, err = ev.GTID(format) if err != nil { return pos, fmt.Errorf("can't get GTID from binlog event: %v, event data: %#v", err, ev) } pos = myproto.AppendGTID(pos, gtid) } switch { case ev.IsGTID(): // GTID_EVENT if ev.IsBeginGTID(format) { begin() } case ev.IsXID(): // XID_EVENT (equivalent to COMMIT) if err = commit(ev.Timestamp()); err != nil { return pos, err } case ev.IsIntVar(): // INTVAR_EVENT name, value, err := ev.IntVar(format) if err != nil { return pos, fmt.Errorf("can't parse INTVAR_EVENT: %v, event data: %#v", err, ev) } statements = append(statements, proto.Statement{ Category: proto.BL_SET, Sql: []byte(fmt.Sprintf("SET %s=%d", name, value)), }) case ev.IsRand(): // RAND_EVENT seed1, seed2, err := ev.Rand(format) if err != nil { return pos, fmt.Errorf("can't parse RAND_EVENT: %v, event data: %#v", err, ev) } statements = append(statements, proto.Statement{ Category: proto.BL_SET, Sql: []byte(fmt.Sprintf("SET @@RAND_SEED1=%d, @@RAND_SEED2=%d", seed1, seed2)), }) case ev.IsQuery(): // QUERY_EVENT // Extract the query string and group into transactions. q, err := ev.Query(format) if err != nil { return pos, fmt.Errorf("can't get query from binlog event: %v, event data: %#v", err, ev) } switch cat := getStatementCategory(q.Sql); cat { case proto.BL_BEGIN: begin() case proto.BL_ROLLBACK: // Rollbacks are possible under some circumstances. Since the stream // client keeps track of its replication position by updating the set // of GTIDs it's seen, we must commit an empty transaction so the client // can update its position. statements = nil fallthrough case proto.BL_COMMIT: if err = commit(ev.Timestamp()); err != nil { return pos, err } default: // BL_DDL, BL_DML, BL_SET, BL_UNRECOGNIZED if q.Database != "" && q.Database != bls.dbname { // Skip cross-db statements. continue } setTimestamp := proto.Statement{ Category: proto.BL_SET, Sql: []byte(fmt.Sprintf("SET TIMESTAMP=%d", ev.Timestamp())), } statement := proto.Statement{Category: cat, Sql: q.Sql} // If the statement has a charset and it's different than our client's // default charset, send it along with the statement. // If our client hasn't told us its charset, always send it. if bls.clientCharset == nil || (q.Charset != nil && *q.Charset != *bls.clientCharset) { setTimestamp.Charset = q.Charset statement.Charset = q.Charset } statements = append(statements, setTimestamp, statement) if autocommit { if err = commit(ev.Timestamp()); err != nil { return pos, err } } } } } return pos, nil }