// Stream starts streaming binlog events using the settings from NewStreamer(). func (bls *Streamer) Stream(ctx context.Context) (err error) { stopPos := bls.startPos defer func() { if err != nil && err != mysqlctl.ErrBinlogUnavailable { err = fmt.Errorf("stream error @ %v: %v", stopPos, err) } log.Infof("stream ended @ %v, err = %v", stopPos, err) }() if bls.conn, err = bls.mysqld.NewSlaveConnection(); err != nil { return err } defer bls.conn.Close() // Check that the default charsets match, if the client specified one. // Note that Streamer uses the settings for the 'dba' user, while // BinlogPlayer uses the 'filtered' user, so those are the ones whose charset // must match. Filtered replication should still succeed even with a default // mismatch, since we pass per-statement charset info. However, Vitess in // general doesn't support servers with different default charsets, so we // treat it as a configuration error. if bls.clientCharset != nil { cs, err := sqldb.GetCharset(bls.conn) if err != nil { return fmt.Errorf("can't get charset to check binlog stream: %v", err) } log.Infof("binlog stream client charset = %v, server charset = %v", *bls.clientCharset, cs) if *cs != *bls.clientCharset { return fmt.Errorf("binlog stream client charset (%v) doesn't match server (%v)", bls.clientCharset, cs) } } var events <-chan replication.BinlogEvent if bls.timestamp != 0 { // MySQL 5.6 only: We are going to start reading the // logs from the beginning of a binlog file. That is // going to send us the PREVIOUS_GTIDS_EVENT that // contains the starting GTIDSet, and we will save // that as the current position. bls.usePreviousGTIDs = true events, err = bls.conn.StartBinlogDumpFromBinlogBeforeTimestamp(ctx, bls.timestamp) } else if !bls.startPos.IsZero() { // MySQL 5.6 only: we are starting from a random // binlog position. It turns out we will receive a // PREVIOUS_GTIDS_EVENT event, that has a GTIDSet // extracted from the binlogs. It is not related to // the starting position we pass in, it seems it is // just the PREVIOUS_GTIDS_EVENT from the file we're reading. // So we have to skip it. events, err = bls.conn.StartBinlogDumpFromPosition(ctx, bls.startPos) } else { bls.startPos, events, err = bls.conn.StartBinlogDumpFromCurrent(ctx) } if err != nil { return err } // parseEvents will loop until the events channel is closed, the // service enters the SHUTTING_DOWN state, or an error occurs. stopPos, err = bls.parseEvents(ctx, events) return err }
// ApplyBinlogEvents makes an RPC request to BinlogServer // and processes the events. It will return nil if the provided context // was canceled, or if we reached the stopping point. // It will return io.EOF if the server stops sending us updates. // It may return any other error it encounters. func (blp *BinlogPlayer) ApplyBinlogEvents(ctx context.Context) error { // Instantiate the throttler based on the configuration stored in the db. maxTPS, maxReplicationLag, err := blp.readThrottlerSettings() if err != nil { log.Error(err) return err } t, err := throttler.NewThrottler( fmt.Sprintf("BinlogPlayer/%d", blp.uid), "transactions", 1 /* threadCount */, maxTPS, maxReplicationLag) if err != nil { err := fmt.Errorf("failed to instantiate throttler: %v", err) log.Error(err) return err } defer t.Close() // Log the mode of operation and when the player stops. if len(blp.tables) > 0 { log.Infof("BinlogPlayer client %v for tables %v starting @ '%v', server: %v", blp.uid, blp.tables, blp.position, blp.tablet, ) } else { log.Infof("BinlogPlayer client %v for keyrange '%v-%v' starting @ '%v', server: %v", blp.uid, hex.EncodeToString(blp.keyRange.Start), hex.EncodeToString(blp.keyRange.End), blp.position, blp.tablet, ) } if !blp.stopPosition.IsZero() { // We need to stop at some point. Sanity check the point. switch { case blp.position.Equal(blp.stopPosition): log.Infof("Not starting BinlogPlayer, we're already at the desired position %v", blp.stopPosition) return nil case blp.position.AtLeast(blp.stopPosition): return fmt.Errorf("starting point %v greater than stopping point %v", blp.position, blp.stopPosition) default: log.Infof("Will stop player when reaching %v", blp.stopPosition) } } clientFactory, ok := clientFactories[*binlogPlayerProtocol] if !ok { return fmt.Errorf("no binlog player client factory named %v", *binlogPlayerProtocol) } blplClient := clientFactory() err = blplClient.Dial(blp.tablet, *BinlogPlayerConnTimeout) if err != nil { err := fmt.Errorf("error dialing binlog server: %v", err) log.Error(err) return err } defer blplClient.Close() // Get the current charset of our connection, so we can ask the stream server // to check that they match. The streamer will also only send per-statement // charset data if that statement's charset is different from what we specify. if dbClient, ok := blp.dbClient.(*DBClient); ok { blp.defaultCharset, err = sqldb.GetCharset(dbClient.dbConn) if err != nil { return fmt.Errorf("can't get charset to request binlog stream: %v", err) } log.Infof("original charset: %v", blp.defaultCharset) blp.currentCharset = blp.defaultCharset // Restore original charset when we're done. defer func() { // If the connection has been closed, there's no need to restore // this connection-specific setting. if dbClient.dbConn == nil { return } log.Infof("restoring original charset %v", blp.defaultCharset) if csErr := sqldb.SetCharset(dbClient.dbConn, blp.defaultCharset); csErr != nil { log.Errorf("can't restore original charset %v: %v", blp.defaultCharset, csErr) } }() } var stream BinlogTransactionStream if len(blp.tables) > 0 { stream, err = blplClient.StreamTables(ctx, replication.EncodePosition(blp.position), blp.tables, blp.defaultCharset) } else { stream, err = blplClient.StreamKeyRange(ctx, replication.EncodePosition(blp.position), blp.keyRange, blp.defaultCharset) } if err != nil { err := fmt.Errorf("error sending streaming query to binlog server: %v", err) log.Error(err) return err } for { // Block if we are throttled. for { backoff := t.Throttle(0 /* threadID */) if backoff == throttler.NotThrottled { break } // We don't bother checking for context cancellation here because the // sleep will block only up to 1 second. (Usually, backoff is 1s / rate // e.g. a rate of 1000 TPS results into a backoff of 1 ms.) time.Sleep(backoff) } // get the response response, err := stream.Recv() if err != nil { switch err { case context.Canceled: return nil default: // if the context is canceled, we // return nil (some RPC // implementations will remap the // context error to their own errors) select { case <-ctx.Done(): if ctx.Err() == context.Canceled { return nil } default: } return fmt.Errorf("Error received from Stream %v", err) } } // process the transaction for { ok, err = blp.processTransaction(response) if err != nil { return fmt.Errorf("Error in processing binlog event %v", err) } if ok { if !blp.stopPosition.IsZero() { if blp.position.AtLeast(blp.stopPosition) { log.Infof("Reached stopping position, done playing logs") return nil } } break } log.Infof("Retrying txn") time.Sleep(1 * time.Second) } } }