// Stream starts streaming binlog events using the settings from NewStreamer().
func (bls *Streamer) Stream(ctx context.Context) (err error) {
stopPos := bls.startPos
defer func() {
if err != nil && err != mysqlctl.ErrBinlogUnavailable {
err = fmt.Errorf("stream error @ %v: %v", stopPos, err)
}
log.Infof("stream ended @ %v, err = %v", stopPos, err)
}()
if bls.conn, err = bls.mysqld.NewSlaveConnection(); err != nil {
return err
}
defer bls.conn.Close()
// Check that the default charsets match, if the client specified one.
// Note that Streamer uses the settings for the 'dba' user, while
// BinlogPlayer uses the 'filtered' user, so those are the ones whose charset
// must match. Filtered replication should still succeed even with a default
// mismatch, since we pass per-statement charset info. However, Vitess in
// general doesn't support servers with different default charsets, so we
// treat it as a configuration error.
if bls.clientCharset != nil {
cs, err := sqldb.GetCharset(bls.conn)
if err != nil {
return fmt.Errorf("can't get charset to check binlog stream: %v", err)
}
log.Infof("binlog stream client charset = %v, server charset = %v", *bls.clientCharset, cs)
if *cs != *bls.clientCharset {
return fmt.Errorf("binlog stream client charset (%v) doesn't match server (%v)", bls.clientCharset, cs)
}
}
var events <-chan replication.BinlogEvent
if bls.timestamp != 0 {
// MySQL 5.6 only: We are going to start reading the
// logs from the beginning of a binlog file. That is
// going to send us the PREVIOUS_GTIDS_EVENT that
// contains the starting GTIDSet, and we will save
// that as the current position.
bls.usePreviousGTIDs = true
events, err = bls.conn.StartBinlogDumpFromBinlogBeforeTimestamp(ctx, bls.timestamp)
} else if !bls.startPos.IsZero() {
// MySQL 5.6 only: we are starting from a random
// binlog position. It turns out we will receive a
// PREVIOUS_GTIDS_EVENT event, that has a GTIDSet
// extracted from the binlogs. It is not related to
// the starting position we pass in, it seems it is
// just the PREVIOUS_GTIDS_EVENT from the file we're reading.
// So we have to skip it.
events, err = bls.conn.StartBinlogDumpFromPosition(ctx, bls.startPos)
} else {
bls.startPos, events, err = bls.conn.StartBinlogDumpFromCurrent(ctx)
}
if err != nil {
return err
}
// parseEvents will loop until the events channel is closed, the
// service enters the SHUTTING_DOWN state, or an error occurs.
stopPos, err = bls.parseEvents(ctx, events)
return err
}
// ApplyBinlogEvents makes an RPC request to BinlogServer
// and processes the events. It will return nil if the provided context
// was canceled, or if we reached the stopping point.
// It will return io.EOF if the server stops sending us updates.
// It may return any other error it encounters.
func (blp *BinlogPlayer) ApplyBinlogEvents(ctx context.Context) error {
// Instantiate the throttler based on the configuration stored in the db.
maxTPS, maxReplicationLag, err := blp.readThrottlerSettings()
if err != nil {
log.Error(err)
return err
}
t, err := throttler.NewThrottler(
fmt.Sprintf("BinlogPlayer/%d", blp.uid), "transactions", 1 /* threadCount */, maxTPS, maxReplicationLag)
if err != nil {
err := fmt.Errorf("failed to instantiate throttler: %v", err)
log.Error(err)
return err
}
defer t.Close()
// Log the mode of operation and when the player stops.
if len(blp.tables) > 0 {
log.Infof("BinlogPlayer client %v for tables %v starting @ '%v', server: %v",
blp.uid,
blp.tables,
blp.position,
blp.tablet,
)
} else {
log.Infof("BinlogPlayer client %v for keyrange '%v-%v' starting @ '%v', server: %v",
blp.uid,
hex.EncodeToString(blp.keyRange.Start),
hex.EncodeToString(blp.keyRange.End),
blp.position,
blp.tablet,
)
}
if !blp.stopPosition.IsZero() {
// We need to stop at some point. Sanity check the point.
switch {
case blp.position.Equal(blp.stopPosition):
log.Infof("Not starting BinlogPlayer, we're already at the desired position %v", blp.stopPosition)
return nil
case blp.position.AtLeast(blp.stopPosition):
return fmt.Errorf("starting point %v greater than stopping point %v", blp.position, blp.stopPosition)
default:
log.Infof("Will stop player when reaching %v", blp.stopPosition)
}
}
clientFactory, ok := clientFactories[*binlogPlayerProtocol]
if !ok {
return fmt.Errorf("no binlog player client factory named %v", *binlogPlayerProtocol)
}
blplClient := clientFactory()
err = blplClient.Dial(blp.tablet, *BinlogPlayerConnTimeout)
if err != nil {
err := fmt.Errorf("error dialing binlog server: %v", err)
log.Error(err)
return err
}
defer blplClient.Close()
// Get the current charset of our connection, so we can ask the stream server
// to check that they match. The streamer will also only send per-statement
// charset data if that statement's charset is different from what we specify.
if dbClient, ok := blp.dbClient.(*DBClient); ok {
blp.defaultCharset, err = sqldb.GetCharset(dbClient.dbConn)
if err != nil {
return fmt.Errorf("can't get charset to request binlog stream: %v", err)
}
log.Infof("original charset: %v", blp.defaultCharset)
blp.currentCharset = blp.defaultCharset
// Restore original charset when we're done.
defer func() {
// If the connection has been closed, there's no need to restore
// this connection-specific setting.
if dbClient.dbConn == nil {
return
}
log.Infof("restoring original charset %v", blp.defaultCharset)
if csErr := sqldb.SetCharset(dbClient.dbConn, blp.defaultCharset); csErr != nil {
log.Errorf("can't restore original charset %v: %v", blp.defaultCharset, csErr)
}
}()
}
var stream BinlogTransactionStream
if len(blp.tables) > 0 {
stream, err = blplClient.StreamTables(ctx, replication.EncodePosition(blp.position), blp.tables, blp.defaultCharset)
} else {
stream, err = blplClient.StreamKeyRange(ctx, replication.EncodePosition(blp.position), blp.keyRange, blp.defaultCharset)
}
if err != nil {
err := fmt.Errorf("error sending streaming query to binlog server: %v", err)
log.Error(err)
return err
}
for {
// Block if we are throttled.
for {
backoff := t.Throttle(0 /* threadID */)
if backoff == throttler.NotThrottled {
break
}
// We don't bother checking for context cancellation here because the
// sleep will block only up to 1 second. (Usually, backoff is 1s / rate
// e.g. a rate of 1000 TPS results into a backoff of 1 ms.)
time.Sleep(backoff)
}
// get the response
response, err := stream.Recv()
if err != nil {
switch err {
case context.Canceled:
return nil
default:
// if the context is canceled, we
// return nil (some RPC
// implementations will remap the
// context error to their own errors)
select {
case <-ctx.Done():
if ctx.Err() == context.Canceled {
return nil
}
default:
}
return fmt.Errorf("Error received from Stream %v", err)
}
}
// process the transaction
for {
ok, err = blp.processTransaction(response)
if err != nil {
return fmt.Errorf("Error in processing binlog event %v", err)
}
if ok {
if !blp.stopPosition.IsZero() {
if blp.position.AtLeast(blp.stopPosition) {
log.Infof("Reached stopping position, done playing logs")
return nil
}
}
break
}
log.Infof("Retrying txn")
time.Sleep(1 * time.Second)
}
}
}