// Open runs the invalidation loop.
func (rci *RowcacheInvalidator) Open(dbname string, mysqld *mysqlctl.Mysqld) {
rp, err := mysqld.MasterStatus()
if err != nil {
panic(NewTabletError(FATAL, "Rowcache invalidator aborting: cannot determine replication position: %v", err))
}
if mysqld.Cnf().BinLogPath == "" {
panic(NewTabletError(FATAL, "Rowcache invalidator aborting: binlog path not specified"))
}
ok := rci.svm.Go(func(_ *sync2.ServiceManager) {
rci.mu.Lock()
rci.dbname = dbname
rci.mysqld = mysqld
rci.evs = binlog.NewEventStreamer(dbname, mysqld)
rci.SetGTID(rp.MasterLogGTIDField.Value)
rci.mu.Unlock()
rci.run()
rci.mu.Lock()
rci.evs = nil
rci.mu.Unlock()
})
if ok {
log.Infof("Rowcache invalidator starting, dbname: %s, path: %s, logfile: %s, position: %d", dbname, mysqld.Cnf().BinLogPath, rp.MasterLogFile, rp.MasterLogPosition)
} else {
log.Infof("Rowcache invalidator already running")
}
}
// newBinlogFileStreamer creates a BinlogStreamer. dbname specifes
// the db to stream events for, and binlogPrefix is as defined
// by the mycnf variable.
func newBinlogFileStreamer(dbname string, mysqld *mysqlctl.Mysqld) BinlogStreamer {
return &binlogFileStreamer{
dbname: dbname,
dir: path.Dir(mysqld.Cnf().BinLogPath),
mysqld: mysqld,
}
}
func multisnapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
concurrency := subFlags.Int("concurrency", 8, "how many compression jobs to run simultaneously")
spec := subFlags.String("spec", "-", "shard specification")
tablesString := subFlags.String("tables", "", "dump only this comma separated list of tables")
skipSlaveRestart := subFlags.Bool("skip-slave-restart", false, "after the snapshot is done, do not restart slave replication")
maximumFilesize := subFlags.Uint64("maximum-file-size", 128*1024*1024, "the maximum size for an uncompressed data file")
subFlags.Parse(args)
if subFlags.NArg() != 2 {
relog.Fatal("action multisnapshot requires <db name> <key name>")
}
shards, err := key.ParseShardingSpec(*spec)
if err != nil {
relog.Fatal("multisnapshot failed: %v", err)
}
var tables []string
if *tablesString != "" {
tables = strings.Split(*tablesString, ",")
}
filenames, err := mysqld.CreateMultiSnapshot(shards, subFlags.Arg(0), subFlags.Arg(1), tabletAddr, false, *concurrency, tables, *skipSlaveRestart, *maximumFilesize, nil)
if err != nil {
relog.Fatal("multisnapshot failed: %v", err)
} else {
relog.Info("manifest locations: %v", filenames)
}
}
func teardownCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
force := subFlags.Bool("force", false, "will remove the root directory even if mysqld shutdown fails")
subFlags.Parse(args)
if err := mysqld.Teardown(*force); err != nil {
log.Fatalf("failed teardown mysql (forced? %v): %v", *force, err)
}
}
func shutdownCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
waitTime := subFlags.Duration("wait_time", mysqlctl.MysqlWaitTime, "how long to wait for shutdown")
subFlags.Parse(args)
if mysqlErr := mysqld.Shutdown(true, *waitTime); mysqlErr != nil {
log.Fatalf("failed shutdown mysql: %v", mysqlErr)
}
}
func startCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
waitTime := subFlags.Duration("wait_time", mysqlctl.MysqlWaitTime, "how long to wait for startup")
subFlags.Parse(args)
if err := mysqld.Start(*waitTime); err != nil {
log.Fatalf("failed start mysql: %v", err)
}
}
// newBinlogFileStreamer creates a BinlogStreamer.
//
// dbname specifes the db to stream events for.
func newBinlogFileStreamer(dbname string, mysqld *mysqlctl.Mysqld, startPos myproto.ReplicationPosition, sendTransaction sendTransactionFunc) BinlogStreamer {
return &binlogFileStreamer{
dbname: dbname,
dir: path.Dir(mysqld.Cnf().BinLogPath),
mysqld: mysqld,
startPos: startPos,
sendTransaction: sendTransaction,
}
}
func initCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
waitTime := subFlags.Duration("wait_time", mysqlctl.MysqlWaitTime, "how long to wait for startup")
bootstrapArchive := subFlags.String("bootstrap_archive", "mysql-db-dir.tbz", "name of bootstrap archive within vitess/data/bootstrap directory")
skipSchema := subFlags.Bool("skip_schema", false, "don't apply initial schema")
subFlags.Parse(args)
if err := mysqld.Init(*waitTime, *bootstrapArchive, *skipSchema); err != nil {
log.Fatalf("failed init mysql: %v", err)
}
}
func snapshotSourceEndCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
slaveStartRequired := subFlags.Bool("slave-start", false, "will restart replication")
readWrite := subFlags.Bool("read-write", false, "will make the server read-write")
subFlags.Parse(args)
err := mysqld.SnapshotSourceEnd(*slaveStartRequired, !(*readWrite), true)
if err != nil {
relog.Fatal("snapshotsourceend failed: %v", err)
}
}
func startCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) error {
waitTime := subFlags.Duration("wait_time", 2*time.Minute, "how long to wait for startup")
subFlags.Parse(args)
ctx, cancel := context.WithTimeout(context.Background(), *waitTime)
defer cancel()
if err := mysqld.Start(ctx); err != nil {
return fmt.Errorf("failed start mysql: %v", err)
}
return nil
}
func initCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) error {
waitTime := subFlags.Duration("wait_time", 2*time.Minute, "how long to wait for startup")
initDBSQLFile := subFlags.String("init_db_sql_file", "", "path to .sql file to run after mysql_install_db")
subFlags.Parse(args)
ctx, cancel := context.WithTimeout(context.Background(), *waitTime)
defer cancel()
if err := mysqld.Init(ctx, *initDBSQLFile); err != nil {
return fmt.Errorf("failed init mysql: %v", err)
}
return nil
}
func teardownCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) error {
waitTime := subFlags.Duration("wait_time", 2*time.Minute, "how long to wait for shutdown")
force := subFlags.Bool("force", false, "will remove the root directory even if mysqld shutdown fails")
subFlags.Parse(args)
ctx, cancel := context.WithTimeout(context.Background(), *waitTime)
defer cancel()
if err := mysqld.Teardown(ctx, *force); err != nil {
return fmt.Errorf("failed teardown mysql (forced? %v): %v", *force, err)
}
return nil
}
func initCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) error {
waitTime := subFlags.Duration("wait_time", 2*time.Minute, "how long to wait for startup")
bootstrapArchive := subFlags.String("bootstrap_archive", "mysql-db-dir.tbz", "name of bootstrap archive within vitess/data/bootstrap directory")
subFlags.Parse(args)
ctx, cancel := context.WithTimeout(context.Background(), *waitTime)
defer cancel()
if err := mysqld.Init(ctx, *bootstrapArchive); err != nil {
return fmt.Errorf("failed init mysql: %v", err)
}
return nil
}
func snapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
concurrency := subFlags.Int("concurrency", 4, "how many compression jobs to run simultaneously")
subFlags.Parse(args)
if subFlags.NArg() != 1 {
relog.Fatal("Command snapshot requires <db name>")
}
filename, _, _, err := mysqld.CreateSnapshot(subFlags.Arg(0), tabletAddr, false, *concurrency, false, nil)
if err != nil {
relog.Fatal("snapshot failed: %v", err)
} else {
relog.Info("manifest location: %v", filename)
}
}
func multiRestoreCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
starts := subFlags.String("starts", "", "starts of the key range")
ends := subFlags.String("ends", "", "ends of the key range")
fetchRetryCount := subFlags.Int("fetch_retry_count", 3, "how many times to retry a failed transfer")
concurrency := subFlags.Int("concurrency", 8, "how many concurrent db inserts to run simultaneously")
fetchConcurrency := subFlags.Int("fetch_concurrency", 4, "how many files to fetch simultaneously")
insertTableConcurrency := subFlags.Int("insert_table_concurrency", 4, "how many myisam tables to load into a single destination table simultaneously")
strategy := subFlags.String("strategy", "", "which strategy to use for restore, can contain:\n"+
" skipAutoIncrement(TTT): we won't add the AUTO_INCREMENT back to that table\n"+
" delayPrimaryKey: we won't add the primary key until after the table is populated\n"+
" delaySecondaryIndexes: we won't add the secondary indexes until after the table is populated\n"+
" useMyIsam: create the table as MyISAM, then convert it to InnoDB after population\n"+
" writeBinLogs: write all operations to the binlogs")
subFlags.Parse(args)
if subFlags.NArg() < 2 {
log.Fatalf("multirestore requires <destination_dbname> <source_host>[/<source_dbname>]... %v", args)
}
startArray := strings.Split(*starts, ",")
endArray := strings.Split(*ends, ",")
if len(startArray) != len(endArray) || len(startArray) != subFlags.NArg()-1 {
log.Fatalf("Need as many starts and ends as source URLs")
}
keyRanges := make([]key.KeyRange, len(startArray))
for i, s := range startArray {
var err error
keyRanges[i], err = key.ParseKeyRangeParts(s, endArray[i])
if err != nil {
log.Fatalf("Invalid start or end: %v", err)
}
}
dbName, dbis := subFlags.Arg(0), subFlags.Args()[1:]
sources := make([]*url.URL, len(dbis))
for i, dbi := range dbis {
if !strings.HasPrefix(dbi, "vttp://") && !strings.HasPrefix(dbi, "http://") {
dbi = "vttp://" + dbi
}
dbUrl, err := url.Parse(dbi)
if err != nil {
log.Fatalf("incorrect source url: %v", err)
}
sources[i] = dbUrl
}
if err := mysqld.MultiRestore(dbName, keyRanges, sources, nil, *concurrency, *fetchConcurrency, *insertTableConcurrency, *fetchRetryCount, *strategy); err != nil {
log.Fatalf("multirestore failed: %v", err)
}
}
func partialRestoreCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
fetchConcurrency := subFlags.Int("fetch-concurrency", 3, "how many files to fetch simultaneously")
fetchRetryCount := subFlags.Int("fetch-retry-count", 3, "how many times to retry a failed transfer")
subFlags.Parse(args)
if subFlags.NArg() != 1 {
relog.Fatal("Command partialrestore requires <split snapshot manifest file>")
}
rs, err := mysqlctl.ReadSplitSnapshotManifest(subFlags.Arg(0))
if err == nil {
err = mysqld.RestoreFromPartialSnapshot(rs, *fetchConcurrency, *fetchRetryCount)
}
if err != nil {
relog.Fatal("partialrestore failed: %v", err)
}
}
func snapshotSourceStartCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
concurrency := subFlags.Int("concurrency", 4, "how many checksum jobs to run simultaneously")
subFlags.Parse(args)
if subFlags.NArg() != 1 {
relog.Fatal("Command snapshotsourcestart requires <db name>")
}
filename, slaveStartRequired, readOnly, err := mysqld.CreateSnapshot(subFlags.Arg(0), tabletAddr, false, *concurrency, true, nil)
if err != nil {
relog.Fatal("snapshot failed: %v", err)
} else {
relog.Info("manifest location: %v", filename)
relog.Info("slave start required: %v", slaveStartRequired)
relog.Info("read only: %v", readOnly)
}
}
func partialSnapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
start := subFlags.String("start", "", "start of the key range")
end := subFlags.String("end", "", "end of the key range")
concurrency := subFlags.Int("concurrency", 4, "how many compression jobs to run simultaneously")
subFlags.Parse(args)
if subFlags.NArg() != 2 {
relog.Fatal("action partialsnapshot requires <db name> <key name>")
}
filename, err := mysqld.CreateSplitSnapshot(subFlags.Arg(0), subFlags.Arg(1), key.HexKeyspaceId(*start), key.HexKeyspaceId(*end), tabletAddr, false, *concurrency, nil)
if err != nil {
relog.Fatal("partialsnapshot failed: %v", err)
} else {
relog.Info("manifest location: %v", filename)
}
}
func multiRestoreCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
start := subFlags.String("start", "", "start of the key range")
end := subFlags.String("end", "", "end of the key range")
fetchRetryCount := subFlags.Int("fetch-retry-count", 3, "how many times to retry a failed transfer")
concurrency := subFlags.Int("concurrency", 8, "how many concurrent db inserts to run simultaneously")
fetchConcurrency := subFlags.Int("fetch-concurrency", 4, "how many files to fetch simultaneously")
insertTableConcurrency := subFlags.Int("insert-table-concurrency", 4, "how many myisam tables to load into a single destination table simultaneously")
strategy := subFlags.String("strategy", "", "which strategy to use for restore, can contain:\n"+
" skipAutoIncrement(TTT): we won't add the AUTO_INCREMENT back to that table\n"+
" delayPrimaryKey: we won't add the primary key until after the table is populated\n"+
" delaySecondaryIndexes: we won't add the secondary indexes until after the table is populated\n"+
" useMyIsam: create the table as MyISAM, then convert it to InnoDB after population\n"+
" writeBinLogs: write all operations to the binlogs")
subFlags.Parse(args)
s, err := key.HexKeyspaceId(*start).Unhex()
if err != nil {
relog.Fatal("Invalid start key %v: %v", *start, err)
}
e, err := key.HexKeyspaceId(*end).Unhex()
if err != nil {
relog.Fatal("Invalid end key %v: %v", *end, err)
}
keyRange := key.KeyRange{Start: s, End: e}
if subFlags.NArg() < 2 {
relog.Fatal("multirestore requires <destination_dbname> <source_host>[/<source_dbname>]... %v", args)
}
dbName, dbis := subFlags.Arg(0), subFlags.Args()[1:]
sources := make([]*url.URL, len(dbis))
uids := make([]uint32, len(dbis))
for i, dbi := range dbis {
if !strings.HasPrefix(dbi, "vttp://") && !strings.HasPrefix(dbi, "http://") {
dbi = "vttp://" + dbi
}
dbUrl, err := url.Parse(dbi)
if err != nil {
relog.Fatal("incorrect source url: %v", err)
}
sources[i] = dbUrl
uids[i] = uint32(i)
}
if err := mysqld.RestoreFromMultiSnapshot(dbName, keyRange, sources, uids, *concurrency, *fetchConcurrency, *insertTableConcurrency, *fetchRetryCount, *strategy); err != nil {
relog.Fatal("multirestore failed: %v", err)
}
}
func restoreCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
dontWaitForSlaveStart := subFlags.Bool("dont-wait-for-slave-start", false, "won't wait for replication to start (useful when restoring from master server)")
fetchConcurrency := subFlags.Int("fetch-concurrency", 3, "how many files to fetch simultaneously")
fetchRetryCount := subFlags.Int("fetch-retry-count", 3, "how many times to retry a failed transfer")
subFlags.Parse(args)
if subFlags.NArg() != 1 {
relog.Fatal("Command restore requires <snapshot manifest file>")
}
rs, err := mysqlctl.ReadSnapshotManifest(subFlags.Arg(0))
if err == nil {
err = mysqld.RestoreFromSnapshot(rs, *fetchConcurrency, *fetchRetryCount, *dontWaitForSlaveStart, nil)
}
if err != nil {
relog.Fatal("restore failed: %v", err)
}
}
// Open runs the invalidation loop.
func (rci *RowcacheInvalidator) Open(dbname string, mysqld *mysqlctl.Mysqld) {
rp, err := mysqld.MasterPosition()
if err != nil {
panic(NewTabletError(FATAL, "Rowcache invalidator aborting: cannot determine replication position: %v", err))
}
if mysqld.Cnf().BinLogPath == "" {
panic(NewTabletError(FATAL, "Rowcache invalidator aborting: binlog path not specified"))
}
rci.dbname = dbname
rci.mysqld = mysqld
rci.SetPosition(rp)
ok := rci.svm.Go(rci.run)
if ok {
log.Infof("Rowcache invalidator starting, dbname: %s, path: %s, position: %v", dbname, mysqld.Cnf().BinLogPath, rp)
} else {
log.Infof("Rowcache invalidator already running")
}
}
func multisnapshotCmd(mysqld *mysqlctl.Mysqld, subFlags *flag.FlagSet, args []string) {
concurrency := subFlags.Int("concurrency", 8, "how many compression jobs to run simultaneously")
spec := subFlags.String("spec", "-", "shard specification")
tablesString := subFlags.String("tables", "", "dump only this comma separated list of regexp for tables")
excludeTablesString := subFlags.String("exclude_tables", "", "do not dump this comma separated list of regexp for tables")
skipSlaveRestart := subFlags.Bool("skip_slave_restart", false, "after the snapshot is done, do not restart slave replication")
maximumFilesize := subFlags.Uint64("maximum_file_size", 128*1024*1024, "the maximum size for an uncompressed data file")
keyType := subFlags.String("key_type", "uint64", "type of the key column")
subFlags.Parse(args)
if subFlags.NArg() != 2 {
log.Fatalf("action multisnapshot requires <db name> <key name>")
}
shards, err := key.ParseShardingSpec(*spec)
if err != nil {
log.Fatalf("multisnapshot failed: %v", err)
}
var tables []string
if *tablesString != "" {
tables = strings.Split(*tablesString, ",")
}
var excludedTables []string
if *excludeTablesString != "" {
excludedTables = strings.Split(*excludeTablesString, ",")
}
kit := key.KeyspaceIdType(*keyType)
if !key.IsKeyspaceIdTypeInList(kit, key.AllKeyspaceIdTypes) {
log.Fatalf("invalid key_type")
}
filenames, err := mysqld.CreateMultiSnapshot(shards, subFlags.Arg(0), subFlags.Arg(1), kit, tabletAddr, false, *concurrency, tables, excludedTables, *skipSlaveRestart, *maximumFilesize, nil)
if err != nil {
log.Fatalf("multisnapshot failed: %v", err)
} else {
log.Infof("manifest locations: %v", filenames)
}
}