func getClusterName(clusterAlias string, instanceKey *inst.InstanceKey) (clusterName string) {
var err error
if clusterAlias != "" {
clusterName, err = inst.ReadClusterByAlias(clusterAlias)
if err != nil {
log.Fatale(err)
}
} else {
// deduce cluster by instance
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatalf("Unable to get cluster instances: unresolved instance")
}
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
clusterName = instance.ClusterName
}
if clusterName == "" {
log.Fatalf("Unable to determine cluster name")
}
return clusterName
}
// getClusterName will make a best effort to deduce a cluster name using either a given alias
// or an instanceKey. First attempt is at alias, and if that doesn't work, we try instanceKey.
func getClusterName(clusterAlias string, instanceKey *inst.InstanceKey) (clusterName string) {
var err error
if clusterAlias != "" {
if clusterName, err = inst.ReadClusterNameByAlias(clusterAlias); err == nil && clusterName != "" {
return clusterName
}
}
if clusterAlias != "" {
// We're still here? So this wasn't an exact cluster name. Let's cehck if it's fuzzy:
fuzzyInstanceKey, err := inst.ParseRawInstanceKeyLoose(clusterAlias)
if err != nil {
log.Fatale(err)
}
if clusterName, err = inst.FindClusterNameByFuzzyInstanceKey(fuzzyInstanceKey); clusterName == "" {
log.Fatalf("Unable to determine cluster name by alias %+v", clusterAlias)
}
return clusterName
}
// So there is no alias. Let's check by instance key
instanceKey = inst.ReadFuzzyInstanceKeyIfPossible(instanceKey)
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unable to get cluster instances: unresolved instance")
}
instance := validateInstanceIsFound(instanceKey)
clusterName = instance.ClusterName
if clusterName == "" {
log.Fatalf("Unable to determine cluster name")
}
return clusterName
}
// deployIfNotAlreadyDeployed will issue given sql queries that are not already known to be deployed.
// This iterates both lists (to-run and already-deployed) and also verifies no contraditions.
func deployIfNotAlreadyDeployed(db *sql.DB, queries []string, deployedQueries []string, deploymentType string, fatalOnError bool) error {
for i, query := range queries {
queryAlreadyExecuted := false
// While iterating 'queries', also iterate 'deployedQueries'. Expect identity
if len(deployedQueries) > i {
if deployedQueries[i] != query {
log.Fatalf("initOrchestratorDB() PANIC: non matching %s queries between deployment requests and _orchestrator_db_deployment. Please execute 'orchestrator -c reset-internal-db-deployment'", deploymentType)
}
queryAlreadyExecuted = true
}
if queryAlreadyExecuted {
continue
}
if config.Config.SmartOrchestratorDatabaseUpdate {
log.Debugf("initOrchestratorDB executing: %.80s", strings.TrimSpace(strings.Replace(query, "\n", "", -1)))
}
if fatalOnError {
if _, err := execInternal(db, query); err != nil {
return log.Fatalf("Cannot initiate orchestrator: %+v", err)
}
} else {
execInternalSilently(db, query)
}
writeInternalDeployment(db, deploymentType, query, i)
}
return nil
}
// deployIfNotAlreadyDeployed will issue given sql queries that are not already known to be deployed.
// This iterates both lists (to-run and already-deployed) and also verifies no contraditions.
func deployIfNotAlreadyDeployed(db *sql.DB, queries []string, deployedQueries []string, deploymentType string, fatalOnError bool) error {
tx, err := db.Begin()
if err != nil {
log.Fatale(err)
}
// Ugly workaround ahead.
// Origin of this workaround is the existence of some "timestamp NOT NULL," column definitions,
// where in NO_ZERO_IN_DATE,NO_ZERO_DATE sql_mode are invalid (since default is implicitly "0")
// This means installation of orchestrator fails on such configured servers, and in particular on 5.7
// where this setting is the dfault.
// For purpose of backwards compatability, what we do is force sql_mode to be more relaxed, create the schemas
// along with the "invalid" definition, and then go ahead and fix those definitions via following ALTER statements.
// My bad.
originalSqlMode := ""
err = tx.QueryRow(`select @@session.sql_mode`).Scan(&originalSqlMode)
if _, err := tx.Exec(`set @@session.sql_mode=REPLACE(@@session.sql_mode, 'NO_ZERO_DATE', '')`); err != nil {
log.Fatale(err)
}
if _, err := tx.Exec(`set @@session.sql_mode=REPLACE(@@session.sql_mode, 'NO_ZERO_IN_DATE', '')`); err != nil {
log.Fatale(err)
}
for i, query := range queries {
queryAlreadyExecuted := false
// While iterating 'queries', also iterate 'deployedQueries'. Expect identity
if len(deployedQueries) > i {
if deployedQueries[i] != query {
log.Fatalf("initOrchestratorDB() PANIC: non matching %s queries between deployment requests and _orchestrator_db_deployment. Please execute 'orchestrator -c reset-internal-db-deployment'", deploymentType)
}
queryAlreadyExecuted = true
}
if queryAlreadyExecuted {
continue
}
if i == 0 {
log.Debugf("sql_mode is: %+v", originalSqlMode)
}
if config.Config.SmartOrchestratorDatabaseUpdate {
log.Debugf("initOrchestratorDB executing: %.80s", strings.TrimSpace(strings.Replace(query, "\n", "", -1)))
}
if fatalOnError {
if _, err := tx.Exec(query); err != nil {
return log.Fatalf("Cannot initiate orchestrator: %+v", err)
}
} else {
tx.Exec(query)
// And ignore any error
}
writeInternalDeployment(db, deploymentType, query, i)
}
if _, err := tx.Exec(`set session sql_mode=?`, originalSqlMode); err != nil {
log.Fatale(err)
}
if err := tx.Commit(); err != nil {
log.Fatale(err)
}
return nil
}
func postReadAdjustments() {
if Config.MySQLOrchestratorCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, Config.MySQLOrchestratorCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed orchestrator credentials from %s", Config.MySQLOrchestratorCredentialsConfigFile)
Config.MySQLOrchestratorUser = mySQLConfig.Client.User
Config.MySQLOrchestratorPassword = mySQLConfig.Client.Password
}
}
{
// We accept password in the form "${SOME_ENV_VARIABLE}" in which case we pull
// the given variable from os env
submatch := envVariableRegexp.FindStringSubmatch(Config.MySQLOrchestratorPassword)
if len(submatch) > 1 {
Config.MySQLOrchestratorPassword = os.Getenv(submatch[1])
}
}
if Config.MySQLTopologyCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, Config.MySQLTopologyCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed topology credentials from %s", Config.MySQLTopologyCredentialsConfigFile)
Config.MySQLTopologyUser = mySQLConfig.Client.User
Config.MySQLTopologyPassword = mySQLConfig.Client.Password
}
}
{
// We accept password in the form "${SOME_ENV_VARIABLE}" in which case we pull
// the given variable from os env
submatch := envVariableRegexp.FindStringSubmatch(Config.MySQLTopologyPassword)
if len(submatch) > 1 {
Config.MySQLTopologyPassword = os.Getenv(submatch[1])
}
}
if Config.RecoveryPeriodBlockSeconds == 0 && Config.RecoveryPeriodBlockMinutes > 0 {
// RecoveryPeriodBlockSeconds is a newer addition that overrides RecoveryPeriodBlockMinutes
// The code does not consider RecoveryPeriodBlockMinutes anymore, but RecoveryPeriodBlockMinutes
// still supported in config file for backwards compatibility
Config.RecoveryPeriodBlockSeconds = Config.RecoveryPeriodBlockMinutes * 60
}
}
// ResetInternalDeployment will clear existing deployment history (and the effect will be a complete re-deployment
// the next run). This is made available for possible operational errors, a red button
func ResetInternalDeployment() error {
db, err := OpenOrchestrator()
if err != nil {
log.Fatalf("Cannot reset orchestrator internal deployment: %+v", err)
}
if _, err := execInternal(db, `delete from _orchestrator_db_deployment`); err != nil {
log.Fatalf("Unable to clear _orchestrator_db_deployment: %+v", err)
}
return nil
}
func init() {
var err error
ThisHostname, err = os.Hostname()
if err != nil {
log.Fatalf("Cannot resolve self hostname; required. Aborting. %+v", err)
}
}
// read reads configuration from given file, or silently skips if the file does not exist.
// If the file does exist, then it is expected to be in valid JSON format or the function bails out.
func read(file_name string) (*Configuration, error) {
file, err := os.Open(file_name)
if err == nil {
decoder := json.NewDecoder(file)
err := decoder.Decode(Config)
if err == nil {
log.Infof("Read config: %s", file_name)
} else {
log.Fatal("Cannot read config file:", file_name, err)
}
if Config.MySQLOrchestratorCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, Config.MySQLOrchestratorCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed orchestrator credentials from %s", Config.MySQLOrchestratorCredentialsConfigFile)
Config.MySQLOrchestratorUser = mySQLConfig.Client.User
Config.MySQLOrchestratorPassword = mySQLConfig.Client.Password
}
}
if Config.MySQLTopologyCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, Config.MySQLTopologyCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed topology credentials from %s", Config.MySQLTopologyCredentialsConfigFile)
Config.MySQLTopologyUser = mySQLConfig.Client.User
Config.MySQLTopologyPassword = mySQLConfig.Client.Password
}
}
}
return Config, err
}
// Create a TLS configuration from the config supplied CA, Certificate, and Private key.
// Register the TLS config with the mysql drivers as the "orchestrator" config
// Modify the supplied URI to call the TLS config
func SetupMySQLOrchestratorTLS(uri string) (string, error) {
if !orchestratorTLSConfigured {
tlsConfig, err := ssl.NewTLSConfig(config.Config.MySQLOrchestratorSSLCAFile, true)
if err != nil {
return "", log.Fatalf("Can't create TLS configuration for Orchestrator connection %s: %s", uri, err)
}
tlsConfig.InsecureSkipVerify = config.Config.MySQLOrchestratorSSLSkipVerify
if err = ssl.AppendKeyPair(tlsConfig, config.Config.MySQLOrchestratorSSLCertFile, config.Config.MySQLOrchestratorSSLPrivateKeyFile); err != nil {
return "", log.Fatalf("Can't setup TLS key pairs for %s: %s", uri, err)
}
if err = mysql.RegisterTLSConfig("orchestrator", tlsConfig); err != nil {
return "", log.Fatalf("Can't register mysql TLS config for orchestrator: %s", err)
}
orchestratorTLSConfigured = true
}
return fmt.Sprintf("%s&tls=orchestrator", uri), nil
}
func acceptSignal() {
c := make(chan os.Signal, 1)
signal.Notify(c, os.Interrupt, os.Kill, syscall.SIGHUP)
// Block until a signal is received.
sig := <-c
log.Fatalf("Got signal: %+v", sig)
}
func validateInstanceIsFound(instanceKey *inst.InstanceKey) (instance *inst.Instance) {
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
return instance
}
// Create a TLS configuration from the config supplied CA, Certificate, and Private key.
// Register the TLS config with the mysql drivers as the "topology" config
// Modify the supplied URI to call the TLS config
// TODO: Way to have password mixed with TLS for various nodes in the topology. Currently everything is TLS or everything is password
func SetupMySQLTopologyTLS(uri string) (string, error) {
if !topologyTLSConfigured {
tlsConfig, err := ssl.NewTLSConfig(config.Config.MySQLTopologySSLCAFile, !config.Config.MySQLTopologySSLSkipVerify)
// Drop to TLS 1.0 for talking to MySQL
tlsConfig.MinVersion = tls.VersionTLS10
if err != nil {
return "", log.Fatalf("Can't create TLS configuration for Topology connection %s: %s", uri, err)
}
tlsConfig.InsecureSkipVerify = config.Config.MySQLTopologySSLSkipVerify
if err = ssl.AppendKeyPair(tlsConfig, config.Config.MySQLTopologySSLCertFile, config.Config.MySQLTopologySSLPrivateKeyFile); err != nil {
return "", log.Fatalf("Can't setup TLS key pairs for %s: %s", uri, err)
}
if err = mysql.RegisterTLSConfig("topology", tlsConfig); err != nil {
return "", log.Fatalf("Can't register mysql TLS config for topology: %s", err)
}
topologyTLSConfigured = true
}
return fmt.Sprintf("%s&tls=topology", uri), nil
}
// readInternalDeployments reads orchestrator db deployment statements that are known to have been executed
func readInternalDeployments() (baseDeployments []string, patchDeployments []string, err error) {
if !config.Config.SmartOrchestratorDatabaseUpdate {
return baseDeployments, patchDeployments, nil
}
query := fmt.Sprintf(`
select
deployment_type,
sql_statement
from
_orchestrator_db_deployment
order by
deployment_id
`)
db, err := OpenOrchestrator()
if err != nil {
log.Fatalf("Cannot initiate orchestrator internal deployment: %+v", err)
}
err = sqlutils.QueryRowsMap(db, query, func(m sqlutils.RowMap) error {
deploymentType := m.GetString("deployment_type")
sqlStatement := m.GetString("sql_statement")
if deploymentType == "base" {
baseDeployments = append(baseDeployments, sqlStatement)
} else if deploymentType == "patch" {
patchDeployments = append(patchDeployments, sqlStatement)
} else {
log.Fatalf("Unknown deployment type (%+v) encountered in _orchestrator_db_deployment", deploymentType)
}
return nil
})
if err != nil {
log.Debugf("Deploying internal orchestrator tables to fix the above; this is a one time operation")
// Table does not exist? Create it for first time
for _, query := range internalDBDeploymentSQL {
if _, err = execInternal(db, query); err != nil {
log.Fatalf("Cannot initiate orchestrator internal deployment: %+v", err)
}
}
}
return baseDeployments, patchDeployments, nil
}
func postReadAdjustments() {
if Config.MySQLOrchestratorCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, Config.MySQLOrchestratorCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed orchestrator credentials from %s", Config.MySQLOrchestratorCredentialsConfigFile)
Config.MySQLOrchestratorUser = mySQLConfig.Client.User
Config.MySQLOrchestratorPassword = mySQLConfig.Client.Password
}
}
if Config.MySQLTopologyCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, Config.MySQLTopologyCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed topology credentials from %s", Config.MySQLTopologyCredentialsConfigFile)
Config.MySQLTopologyUser = mySQLConfig.Client.User
Config.MySQLTopologyPassword = mySQLConfig.Client.Password
}
}
if Config.RecoveryPeriodBlockSeconds == 0 && Config.RecoveryPeriodBlockMinutes > 0 {
// RecoveryPeriodBlockSeconds is a newer addition that overrides RecoveryPeriodBlockMinutes
// The code does not consider RecoveryPeriodBlockMinutes anymore, but RecoveryPeriodBlockMinutes
// still supported in config file for backwards compatibility
Config.RecoveryPeriodBlockSeconds = Config.RecoveryPeriodBlockMinutes * 60
}
}
// writeInternalDeployment will persist a successful deployment
func writeInternalDeployment(db *sql.DB, deploymentType string, sqlStatement string, statementIndex int) error {
if !config.Config.SmartOrchestratorDatabaseUpdate {
return nil
}
query := `
insert ignore into _orchestrator_db_deployment (
deployment_type, sql_statement, statement_digest, statement_index) VALUES (
?, ?, CONCAT(SHA1(?), ':', LEFT(REPLACE(REPLACE(REPLACE(?, ' ', ''), '\n', ' '), '\t', ''), 60)), ?)
`
if _, err := execInternal(db, query, deploymentType, sqlStatement, sqlStatement, sqlStatement, statementIndex); err != nil {
log.Fatalf("Unable to write to _orchestrator_db_deployment: %+v", err)
}
return nil
}
// registerOrchestratorDeployment updates the orchestrator_metadata table upon successful deployment
func registerOrchestratorDeployment(db *sql.DB) error {
query := `
replace into orchestrator_db_deployments (
deployed_version, deployed_timestamp
) values (
?, NOW()
)
`
if _, err := execInternal(db, query, config.RuntimeCLIFlags.ConfiguredVersion); err != nil {
log.Fatalf("Unable to write to orchestrator_metadata: %+v", err)
}
log.Debugf("Migrated database schema to version [%+v]", config.RuntimeCLIFlags.ConfiguredVersion)
return nil
}
// initOrchestratorDB attempts to create/upgrade the orchestrator backend database. It is created once in the
// application's lifetime.
func initOrchestratorDB(db *sql.DB) error {
log.Debug("Initializing orchestrator")
for _, query := range generateSQL {
_, err := execInternal(db, query)
if err != nil {
return log.Fatalf("Cannot initiate orchestrator: %+v", err)
}
}
for _, query := range generateSQLPatches {
// Patches are allowed to fail.
_, _ = execInternalSilently(db, query)
}
return nil
}
// deployStatements will issue given sql queries that are not already known to be deployed.
// This iterates both lists (to-run and already-deployed) and also verifies no contraditions.
func deployStatements(db *sql.DB, queries []string, fatalOnError bool) error {
tx, err := db.Begin()
if err != nil {
log.Fatale(err)
}
// Ugly workaround ahead.
// Origin of this workaround is the existence of some "timestamp NOT NULL," column definitions,
// where in NO_ZERO_IN_DATE,NO_ZERO_DATE sql_mode are invalid (since default is implicitly "0")
// This means installation of orchestrator fails on such configured servers, and in particular on 5.7
// where this setting is the dfault.
// For purpose of backwards compatability, what we do is force sql_mode to be more relaxed, create the schemas
// along with the "invalid" definition, and then go ahead and fix those definitions via following ALTER statements.
// My bad.
originalSqlMode := ""
err = tx.QueryRow(`select @@session.sql_mode`).Scan(&originalSqlMode)
if _, err := tx.Exec(`set @@session.sql_mode=REPLACE(@@session.sql_mode, 'NO_ZERO_DATE', '')`); err != nil {
log.Fatale(err)
}
if _, err := tx.Exec(`set @@session.sql_mode=REPLACE(@@session.sql_mode, 'NO_ZERO_IN_DATE', '')`); err != nil {
log.Fatale(err)
}
for i, query := range queries {
if i == 0 {
//log.Debugf("sql_mode is: %+v", originalSqlMode)
}
if fatalOnError {
if _, err := tx.Exec(query); err != nil {
return log.Fatalf("Cannot initiate orchestrator: %+v", err)
}
} else {
tx.Exec(query)
// And ignore any error
}
}
if _, err := tx.Exec(`set session sql_mode=?`, originalSqlMode); err != nil {
log.Fatale(err)
}
if err := tx.Commit(); err != nil {
log.Fatale(err)
}
return nil
}
func (this *Configuration) postReadAdjustments() error {
if this.MySQLOrchestratorCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, this.MySQLOrchestratorCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed orchestrator credentials from %s", this.MySQLOrchestratorCredentialsConfigFile)
this.MySQLOrchestratorUser = mySQLConfig.Client.User
this.MySQLOrchestratorPassword = mySQLConfig.Client.Password
}
}
{
// We accept password in the form "${SOME_ENV_VARIABLE}" in which case we pull
// the given variable from os env
submatch := envVariableRegexp.FindStringSubmatch(this.MySQLOrchestratorPassword)
if len(submatch) > 1 {
this.MySQLOrchestratorPassword = os.Getenv(submatch[1])
}
}
if this.MySQLTopologyCredentialsConfigFile != "" {
mySQLConfig := struct {
Client struct {
User string
Password string
}
}{}
err := gcfg.ReadFileInto(&mySQLConfig, this.MySQLTopologyCredentialsConfigFile)
if err != nil {
log.Fatalf("Failed to parse gcfg data from file: %+v", err)
} else {
log.Debugf("Parsed topology credentials from %s", this.MySQLTopologyCredentialsConfigFile)
this.MySQLTopologyUser = mySQLConfig.Client.User
this.MySQLTopologyPassword = mySQLConfig.Client.Password
}
}
{
// We accept password in the form "${SOME_ENV_VARIABLE}" in which case we pull
// the given variable from os env
submatch := envVariableRegexp.FindStringSubmatch(this.MySQLTopologyPassword)
if len(submatch) > 1 {
this.MySQLTopologyPassword = os.Getenv(submatch[1])
}
}
if this.RecoveryPeriodBlockSeconds == 0 && this.RecoveryPeriodBlockMinutes > 0 {
// RecoveryPeriodBlockSeconds is a newer addition that overrides RecoveryPeriodBlockMinutes
// The code does not consider RecoveryPeriodBlockMinutes anymore, but RecoveryPeriodBlockMinutes
// still supported in config file for backwards compatibility
this.RecoveryPeriodBlockSeconds = this.RecoveryPeriodBlockMinutes * 60
}
{
if this.ReplicationLagQuery != "" && this.SlaveLagQuery != "" && this.ReplicationLagQuery != this.SlaveLagQuery {
return fmt.Errorf("config's ReplicationLagQuery and SlaveLagQuery are synonyms and cannot both be defined")
}
if this.SlaveLagQuery != "" {
this.ReplicationLagQuery = this.SlaveLagQuery
}
}
{
if this.DetachLostSlavesAfterMasterFailover {
this.DetachLostReplicasAfterMasterFailover = true
}
}
{
if this.MasterFailoverDetachSlaveMasterHost {
this.MasterFailoverDetachReplicaMasterHost = true
}
}
{
if this.PostponeReplicaRecoveryOnLagMinutes != 0 && this.PostponeSlaveRecoveryOnLagMinutes != 0 &&
this.PostponeReplicaRecoveryOnLagMinutes != this.PostponeSlaveRecoveryOnLagMinutes {
return fmt.Errorf("config's PostponeReplicaRecoveryOnLagMinutes and PostponeSlaveRecoveryOnLagMinutes are synonyms and cannot both be defined")
}
if this.PostponeSlaveRecoveryOnLagMinutes != 0 {
this.PostponeReplicaRecoveryOnLagMinutes = this.PostponeSlaveRecoveryOnLagMinutes
}
}
if this.URLPrefix != "" {
// Ensure the prefix starts with "/" and has no trailing one.
this.URLPrefix = strings.TrimLeft(this.URLPrefix, "/")
this.URLPrefix = strings.TrimRight(this.URLPrefix, "/")
this.URLPrefix = "/" + this.URLPrefix
}
return nil
}
// main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces.
func main() {
migrationContext := base.GetMigrationContext()
flag.StringVar(&migrationContext.InspectorConnectionConfig.Key.Hostname, "host", "127.0.0.1", "MySQL hostname (preferably a replica, not the master)")
flag.IntVar(&migrationContext.InspectorConnectionConfig.Key.Port, "port", 3306, "MySQL port (preferably a replica, not the master)")
flag.StringVar(&migrationContext.CliUser, "user", "", "MySQL user")
flag.StringVar(&migrationContext.CliPassword, "password", "", "MySQL password")
flag.StringVar(&migrationContext.ConfigFile, "conf", "", "Config file")
flag.StringVar(&migrationContext.DatabaseName, "database", "", "database name (mandatory)")
flag.StringVar(&migrationContext.OriginalTableName, "table", "", "table name (mandatory)")
flag.StringVar(&migrationContext.AlterStatement, "alter", "", "alter statement (mandatory)")
flag.BoolVar(&migrationContext.CountTableRows, "exact-rowcount", false, "actually count table rows as opposed to estimate them (results in more accurate progress estimation)")
flag.BoolVar(&migrationContext.AllowedRunningOnMaster, "allow-on-master", false, "allow this migration to run directly on master. Preferably it would run on a replica")
flag.BoolVar(&migrationContext.AllowedMasterMaster, "allow-master-master", false, "explicitly allow running in a master-master setup")
flag.BoolVar(&migrationContext.NullableUniqueKeyAllowed, "allow-nullable-unique-key", false, "allow gh-ost to migrate based on a unique key with nullable columns. As long as no NULL values exist, this should be OK. If NULL values exist in chosen key, data may be corrupted. Use at your own risk!")
flag.BoolVar(&migrationContext.ApproveRenamedColumns, "approve-renamed-columns", false, "in case your `ALTER` statement renames columns, gh-ost will note that and offer its interpretation of the rename. By default gh-ost does not proceed to execute. This flag approves that gh-ost's interpretation si correct")
flag.BoolVar(&migrationContext.SkipRenamedColumns, "skip-renamed-columns", false, "in case your `ALTER` statement renames columns, gh-ost will note that and offer its interpretation of the rename. By default gh-ost does not proceed to execute. This flag tells gh-ost to skip the renamed columns, i.e. to treat what gh-ost thinks are renamed columns as unrelated columns. NOTE: you may lose column data")
executeFlag := flag.Bool("execute", false, "actually execute the alter & migrate the table. Default is noop: do some tests and exit")
flag.BoolVar(&migrationContext.TestOnReplica, "test-on-replica", false, "Have the migration run on a replica, not on the master. At the end of migration replication is stopped, and tables are swapped and immediately swap-revert. Replication remains stopped and you can compare the two tables for building trust")
flag.BoolVar(&migrationContext.MigrateOnReplica, "migrate-on-replica", false, "Have the migration run on a replica, not on the master. This will do the full migration on the replica including cut-over (as opposed to --test-on-replica)")
flag.BoolVar(&migrationContext.OkToDropTable, "ok-to-drop-table", false, "Shall the tool drop the old table at end of operation. DROPping tables can be a long locking operation, which is why I'm not doing it by default. I'm an online tool, yes?")
flag.BoolVar(&migrationContext.InitiallyDropOldTable, "initially-drop-old-table", false, "Drop a possibly existing OLD table (remains from a previous run?) before beginning operation. Default is to panic and abort if such table exists")
flag.BoolVar(&migrationContext.InitiallyDropGhostTable, "initially-drop-ghost-table", false, "Drop a possibly existing Ghost table (remains from a previous run?) before beginning operation. Default is to panic and abort if such table exists")
cutOver := flag.String("cut-over", "atomic", "choose cut-over type (default|atomic, two-step)")
flag.BoolVar(&migrationContext.SwitchToRowBinlogFormat, "switch-to-rbr", false, "let this tool automatically switch binary log format to 'ROW' on the replica, if needed. The format will NOT be switched back. I'm too scared to do that, and wish to protect you if you happen to execute another migration while this one is running")
chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 100-100,000)")
defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking")
cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout)")
niceRatio := flag.Float64("nice-ratio", 0, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1.5: for every ms spend in a rowcopy chunk, spend 1.5ms sleeping immediately after")
maxLagMillis := flag.Int64("max-lag-millis", 1500, "replication lag at which to throttle operation")
replicationLagQuery := flag.String("replication-lag-query", "", "Query that detects replication lag in seconds. Result can be a floating point (by default gh-ost issues SHOW SLAVE STATUS and reads Seconds_behind_master). If you're using pt-heartbeat, query would be something like: SELECT ROUND(UNIX_TIMESTAMP() - MAX(UNIX_TIMESTAMP(ts))) AS delay FROM my_schema.heartbeat")
throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307")
throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight")
flag.StringVar(&migrationContext.ThrottleFlagFile, "throttle-flag-file", "", "operation pauses when this file exists; hint: use a file that is specific to the table being altered")
flag.StringVar(&migrationContext.ThrottleAdditionalFlagFile, "throttle-additional-flag-file", "/tmp/gh-ost.throttle", "operation pauses when this file exists; hint: keep default, use for throttling multiple gh-ost operations")
flag.StringVar(&migrationContext.PostponeCutOverFlagFile, "postpone-cut-over-flag-file", "", "while this file exists, migration will postpone the final stage of swapping tables, and will keep on syncing the ghost table. Cut-over/swapping would be ready to perform the moment the file is deleted.")
flag.StringVar(&migrationContext.PanicFlagFile, "panic-flag-file", "", "when this file is created, gh-ost will immediately terminate, without cleanup")
flag.BoolVar(&migrationContext.DropServeSocket, "initially-drop-socket-file", false, "Should gh-ost forcibly delete an existing socket file. Be careful: this might drop the socket file of a running migration!")
flag.StringVar(&migrationContext.ServeSocketFile, "serve-socket-file", "", "Unix socket file to serve on. Default: auto-determined and advertised upon startup")
flag.Int64Var(&migrationContext.ServeTCPPort, "serve-tcp-port", 0, "TCP port to serve on. Default: disabled")
maxLoad := flag.String("max-load", "", "Comma delimited status-name=threshold. e.g: 'Threads_running=100,Threads_connected=500'. When status exceeds threshold, app throttles writes")
criticalLoad := flag.String("critical-load", "", "Comma delimited status-name=threshold, same format as `--max-load`. When status exceeds threshold, app panics and quits")
quiet := flag.Bool("quiet", false, "quiet")
verbose := flag.Bool("verbose", false, "verbose")
debug := flag.Bool("debug", false, "debug mode (very verbose)")
stack := flag.Bool("stack", false, "add stack trace upon error")
help := flag.Bool("help", false, "Display usage")
version := flag.Bool("version", false, "Print version & exit")
flag.Parse()
if *help {
fmt.Fprintf(os.Stderr, "Usage of gh-ost:\n")
flag.PrintDefaults()
return
}
if *version {
appVersion := AppVersion
if appVersion == "" {
appVersion = "unversioned"
}
fmt.Println(appVersion)
return
}
log.SetLevel(log.ERROR)
if *verbose {
log.SetLevel(log.INFO)
}
if *debug {
log.SetLevel(log.DEBUG)
}
if *stack {
log.SetPrintStackTrace(*stack)
}
if *quiet {
// Override!!
log.SetLevel(log.ERROR)
}
if migrationContext.DatabaseName == "" {
log.Fatalf("--database must be provided and database name must not be empty")
}
if migrationContext.OriginalTableName == "" {
log.Fatalf("--table must be provided and table name must not be empty")
}
if migrationContext.AlterStatement == "" {
log.Fatalf("--alter must be provided and statement must not be empty")
}
migrationContext.Noop = !(*executeFlag)
if migrationContext.AllowedRunningOnMaster && migrationContext.TestOnReplica {
log.Fatalf("--allow-on-master and --test-on-replica are mutually exclusive")
}
if migrationContext.AllowedRunningOnMaster && migrationContext.MigrateOnReplica {
log.Fatalf("--allow-on-master and --migrate-on-replica are mutually exclusive")
}
if migrationContext.MigrateOnReplica && migrationContext.TestOnReplica {
log.Fatalf("--migrate-on-replica and --test-on-replica are mutually exclusive")
}
switch *cutOver {
case "atomic", "default", "":
migrationContext.CutOverType = base.CutOverAtomic
case "two-step":
migrationContext.CutOverType = base.CutOverTwoStep
default:
log.Fatalf("Unknown cut-over: %s", *cutOver)
}
if err := migrationContext.ReadConfigFile(); err != nil {
log.Fatale(err)
}
if err := migrationContext.ReadThrottleControlReplicaKeys(*throttleControlReplicas); err != nil {
log.Fatale(err)
}
if err := migrationContext.ReadMaxLoad(*maxLoad); err != nil {
log.Fatale(err)
}
if err := migrationContext.ReadCriticalLoad(*criticalLoad); err != nil {
log.Fatale(err)
}
if migrationContext.ServeSocketFile == "" {
migrationContext.ServeSocketFile = fmt.Sprintf("/tmp/gh-ost.%s.%s.sock", migrationContext.DatabaseName, migrationContext.OriginalTableName)
}
migrationContext.SetNiceRatio(*niceRatio)
migrationContext.SetChunkSize(*chunkSize)
migrationContext.SetMaxLagMillisecondsThrottleThreshold(*maxLagMillis)
migrationContext.SetReplicationLagQuery(*replicationLagQuery)
migrationContext.SetThrottleQuery(*throttleQuery)
migrationContext.SetDefaultNumRetries(*defaultRetries)
migrationContext.ApplyCredentials()
if err := migrationContext.SetCutOverLockTimeoutSeconds(*cutOverLockTimeoutSeconds); err != nil {
log.Errore(err)
}
log.Infof("starting gh-ost %+v", AppVersion)
acceptSignals(migrationContext)
migrator := logic.NewMigrator()
err := migrator.Migrate()
if err != nil {
log.Fatale(err)
}
log.Info("Done")
}
// cutOver performs the final step of migration, based on migration
// type (on replica? bumpy? safe?)
func (this *Migrator) cutOver() (err error) {
if this.migrationContext.Noop {
log.Debugf("Noop operation; not really swapping tables")
return nil
}
this.migrationContext.MarkPointOfInterest()
this.throttle(func() {
log.Debugf("throttling before swapping tables")
})
this.migrationContext.MarkPointOfInterest()
this.sleepWhileTrue(
func() (bool, error) {
if this.migrationContext.PostponeCutOverFlagFile == "" {
return false, nil
}
if atomic.LoadInt64(&this.userCommandedUnpostponeFlag) > 0 {
return false, nil
}
if base.FileExists(this.migrationContext.PostponeCutOverFlagFile) {
// Throttle file defined and exists!
atomic.StoreInt64(&this.migrationContext.IsPostponingCutOver, 1)
//log.Debugf("Postponing final table swap as flag file exists: %+v", this.migrationContext.PostponeCutOverFlagFile)
return true, nil
}
return false, nil
},
)
atomic.StoreInt64(&this.migrationContext.IsPostponingCutOver, 0)
this.migrationContext.MarkPointOfInterest()
if this.migrationContext.TestOnReplica {
// With `--test-on-replica` we stop replication thread, and then proceed to use
// the same cut-over phase as the master would use. That means we take locks
// and swap the tables.
// The difference is that we will later swap the tables back.
log.Debugf("testing on replica. Stopping replication IO thread")
if err := this.retryOperation(this.applier.StopReplication); err != nil {
return err
}
// We're merly testing, we don't want to keep this state. Rollback the renames as possible
defer this.applier.RenameTablesRollback()
// We further proceed to do the cutover by normal means; the 'defer' above will rollback the swap
}
if this.migrationContext.CutOverType == base.CutOverAtomic {
// Atomic solution: we use low timeout and multiple attempts. But for
// each failed attempt, we throttle until replication lag is back to normal
err := this.retryOperation(
func() error {
return this.executeAndThrottleOnError(this.atomicCutOver)
},
)
return err
}
if this.migrationContext.CutOverType == base.CutOverTwoStep {
err := this.retryOperation(
func() error {
return this.executeAndThrottleOnError(this.cutOverTwoStep)
},
)
return err
}
return log.Fatalf("Unknown cut-over type: %d; should never get here!", this.migrationContext.CutOverType)
}
// main is the application's entry point. It will either spawn a CLI or HTTP itnerfaces.
func main() {
configFile := flag.String("config", "", "config file name")
command := flag.String("c", "", "command, required. See full list of commands via 'orchestrator -c help'")
strict := flag.Bool("strict", false, "strict mode (more checks, slower)")
instance := flag.String("i", "", "instance, host_fqdn[:port] (e.g. db.company.com:3306, db.company.com)")
sibling := flag.String("s", "", "sibling instance, host_fqdn[:port]")
destination := flag.String("d", "", "destination instance, host_fqdn[:port] (synonym to -s)")
owner := flag.String("owner", "", "operation owner")
reason := flag.String("reason", "", "operation reason")
duration := flag.String("duration", "", "maintenance duration (format: 59s, 59m, 23h, 6d, 4w)")
pattern := flag.String("pattern", "", "regular expression pattern")
clusterAlias := flag.String("alias", "", "cluster alias")
pool := flag.String("pool", "", "Pool logical name (applies for pool-related commands)")
hostnameFlag := flag.String("hostname", "", "Hostname/fqdn/CNAME/VIP (applies for hostname/resolve related commands)")
discovery := flag.Bool("discovery", true, "auto discovery mode")
quiet := flag.Bool("quiet", false, "quiet")
verbose := flag.Bool("verbose", false, "verbose")
debug := flag.Bool("debug", false, "debug mode (very verbose)")
stack := flag.Bool("stack", false, "add stack trace upon error")
config.RuntimeCLIFlags.Databaseless = flag.Bool("databaseless", false, "EXPERIMENTAL! Work without backend database")
config.RuntimeCLIFlags.SkipUnresolveCheck = flag.Bool("skip-unresolve-check", false, "Skip/ignore checking an unresolve mapping (via hostname_unresolve table) resolves back to same hostname")
config.RuntimeCLIFlags.Noop = flag.Bool("noop", false, "Dry run; do not perform destructing operations")
config.RuntimeCLIFlags.BinlogFile = flag.String("binlog", "", "Binary log file name")
flag.Parse()
if *destination != "" && *sibling != "" {
log.Fatalf("-s and -d are synonyms, yet both were specified. You're probably doing the wrong thing.")
}
if *destination == "" {
*destination = *sibling
}
log.SetLevel(log.ERROR)
if *verbose {
log.SetLevel(log.INFO)
}
if *debug {
log.SetLevel(log.DEBUG)
}
if *stack {
log.SetPrintStackTrace(*stack)
}
log.Info("starting")
runtime.GOMAXPROCS(math.MinInt(4, runtime.NumCPU()))
if len(*configFile) > 0 {
config.ForceRead(*configFile)
} else {
config.Read("/etc/orchestrator.conf.json", "conf/orchestrator.conf.json", "orchestrator.conf.json")
}
if *config.RuntimeCLIFlags.Databaseless {
config.Config.DatabaselessMode__experimental = true
}
if config.Config.Debug {
log.SetLevel(log.DEBUG)
}
if *quiet {
// Override!!
log.SetLevel(log.ERROR)
}
if config.Config.EnableSyslog {
log.EnableSyslogWriter("orchestrator")
log.SetSyslogLevel(log.INFO)
}
if config.Config.AuditToSyslog {
inst.EnableAuditSyslog()
}
if len(flag.Args()) == 0 && *command == "" {
// No command, no argument: just prompt
fmt.Println(prompt)
return
}
var executionMode OrchestratorExecutionMode
switch {
case len(flag.Args()) == 0 || flag.Arg(0) == "cli":
executionMode = CliMode
case flag.Arg(0) == "http":
executionMode = HttpMode
default:
fmt.Fprintln(os.Stderr, `Usage:
orchestrator --options... [cli|http]
See complete list of commands:
orchestrator -c help
Full blown documentation:
orchestrator
`)
os.Exit(1)
}
process.ContinuousRegistration(string(executionMode))
switch executionMode {
case CliMode:
app.Cli(*command, *strict, *instance, *destination, *owner, *reason, *duration, *pattern, *clusterAlias, *pool, *hostnameFlag)
case HttpMode:
app.Http(*discovery)
}
}
// Cli initiates a command line interface, executing requested command.
func Cli(command string, strict bool, instance string, destination string, owner string, reason string, duration string, pattern string, clusterAlias string, pool string, hostnameFlag string) {
if instance != "" && !strings.Contains(instance, ":") {
instance = fmt.Sprintf("%s:%d", instance, config.Config.DefaultInstancePort)
}
instanceKey, err := inst.ParseInstanceKey(instance)
if err != nil {
instanceKey = nil
}
rawInstanceKey, err := inst.NewRawInstanceKey(instance)
if err != nil {
rawInstanceKey = nil
}
if destination != "" && !strings.Contains(destination, ":") {
destination = fmt.Sprintf("%s:%d", destination, config.Config.DefaultInstancePort)
}
destinationKey, err := inst.ParseInstanceKey(destination)
if err != nil {
destinationKey = nil
}
if hostname, err := os.Hostname(); err == nil {
thisInstanceKey = &inst.InstanceKey{Hostname: hostname, Port: int(config.Config.DefaultInstancePort)}
}
postponedFunctionsContainer := inst.NewPostponedFunctionsContainer()
if len(owner) == 0 {
// get os username as owner
usr, err := user.Current()
if err != nil {
log.Fatale(err)
}
owner = usr.Username
}
inst.SetMaintenanceOwner(owner)
skipDatabaseCommands := false
switch command {
case "reset-internal-db-deployment":
skipDatabaseCommands = true
case "help":
skipDatabaseCommands = true
}
if !skipDatabaseCommands {
process.ContinuousRegistration(string(process.OrchestratorExecutionCliMode), command)
}
// begin commands
switch command {
// smart mode
case registerCliCommand("relocate", "Smart relocation", `Relocate a slave beneath another instance`), registerCliCommand("relocate-below", "Smart relocation", `Synonym to 'relocate', will be deprecated`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
_, err := inst.RelocateBelow(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case registerCliCommand("relocate-slaves", "Smart relocation", `Relocates all or part of the slaves of a given instance under another instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
slaves, _, err, errs := inst.RelocateSlaves(instanceKey, destinationKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range slaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case registerCliCommand("regroup-slaves", "Smart relocation", `Given an instance, pick one of its slave and make it local master of its siblings`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
lostSlaves, equalSlaves, aheadSlaves, promotedSlave, err := inst.RegroupSlaves(instanceKey, false, func(candidateSlave *inst.Instance) { fmt.Println(candidateSlave.Key.DisplayString()) }, postponedFunctionsContainer)
postponedFunctionsContainer.InvokePostponed()
if promotedSlave == nil {
log.Fatalf("Could not regroup slaves of %+v; error: %+v", *instanceKey, err)
}
fmt.Println(fmt.Sprintf("%s lost: %d, trivial: %d, pseudo-gtid: %d",
promotedSlave.Key.DisplayString(), len(lostSlaves), len(equalSlaves), len(aheadSlaves)))
if err != nil {
log.Fatale(err)
}
}
// General replication commands
// move, binlog file:pos
case registerCliCommand("move-up", "Classic file:pos relocation", `Move a slave one level up the topology`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
instance, err := inst.MoveUp(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case registerCliCommand("move-up-slaves", "Classic file:pos relocation", `Moves slaves of the given instance one level up the topology`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
movedSlaves, _, err, errs := inst.MoveUpSlaves(instanceKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range movedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case registerCliCommand("move-below", "Classic file:pos relocation", `Moves a slave beneath its sibling. Both slaves must be actively replicating from same master.`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if destinationKey == nil {
log.Fatal("Cannot deduce destination/sibling:", destination)
}
_, err := inst.MoveBelow(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case registerCliCommand("move-equivalent", "Classic file:pos relocation", `Moves a slave beneath another server, based on previously recorded "equivalence coordinates"`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
_, err := inst.MoveEquivalent(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case registerCliCommand("repoint", "Classic file:pos relocation", `Make the given instance replicate from another instance without changing the binglog coordinates. Use with care`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
// destinationKey can be null, in which case the instance repoints to its existing master
instance, err := inst.Repoint(instanceKey, destinationKey, inst.GTIDHintNeutral)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case registerCliCommand("repoint-slaves", "Classic file:pos relocation", `Repoint all slaves of given instance to replicate back from the instance. Use with care`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
repointedSlaves, err, errs := inst.RepointSlavesTo(instanceKey, pattern, destinationKey)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range repointedSlaves {
fmt.Println(fmt.Sprintf("%s<%s", slave.Key.DisplayString(), instanceKey.DisplayString()))
}
}
}
case registerCliCommand("enslave-siblings", "Classic file:pos relocation", `Turn all siblings of a slave into its sub-slaves.`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, _, err := inst.EnslaveSiblings(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("enslave-master", "Classic file:pos relocation", `Turn an instance into a master of its own master; essentially switch the two.`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.EnslaveMaster(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("make-co-master", "Classic file:pos relocation", `Create a master-master replication. Given instance is a slave which replicates directly from a master.`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.MakeCoMaster(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("get-candidate-slave", "Classic file:pos relocation", `Information command suggesting the most up-to-date slave of a given instance that is good for promotion`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
instance, _, _, _, err := inst.GetCandidateSlave(instanceKey, false)
if err != nil {
log.Fatale(err)
} else {
fmt.Println(instance.Key.DisplayString())
}
}
case registerCliCommand("regroup-slaves-bls", "Binlog server relocation", `Regroup Binlog Server slaves of a given instance`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, promotedBinlogServer, err := inst.RegroupSlavesBinlogServers(instanceKey, false)
if promotedBinlogServer == nil {
log.Fatalf("Could not regroup binlog server slaves of %+v; error: %+v", *instanceKey, err)
}
fmt.Println(promotedBinlogServer.Key.DisplayString())
if err != nil {
log.Fatale(err)
}
}
// move, GTID
case registerCliCommand("move-gtid", "GTID relocation", `Move a slave beneath another instance.`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
_, err := inst.MoveBelowGTID(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case registerCliCommand("move-slaves-gtid", "GTID relocation", `Moves all slaves of a given instance under another (destination) instance using GTID`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
movedSlaves, _, err, errs := inst.MoveSlavesGTID(instanceKey, destinationKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range movedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case registerCliCommand("regroup-slaves-gtid", "GTID relocation", `Given an instance, pick one of its slave and make it local master of its siblings, using GTID.`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
lostSlaves, movedSlaves, promotedSlave, err := inst.RegroupSlavesGTID(instanceKey, false, func(candidateSlave *inst.Instance) { fmt.Println(candidateSlave.Key.DisplayString()) })
if promotedSlave == nil {
log.Fatalf("Could not regroup slaves of %+v; error: %+v", *instanceKey, err)
}
fmt.Println(fmt.Sprintf("%s lost: %d, moved: %d",
promotedSlave.Key.DisplayString(), len(lostSlaves), len(movedSlaves)))
if err != nil {
log.Fatale(err)
}
}
// Pseudo-GTID
case registerCliCommand("match", "Pseudo-GTID relocation", `Matches a slave beneath another (destination) instance using Pseudo-GTID`),
registerCliCommand("match-below", "Pseudo-GTID relocation", `Synonym to 'match', will be deprecated`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
_, _, err := inst.MatchBelow(instanceKey, destinationKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case registerCliCommand("match-up", "Pseudo-GTID relocation", `Transport the slave one level up the hierarchy, making it child of its grandparent, using Pseudo-GTID`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
instance, _, err := inst.MatchUp(instanceKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case registerCliCommand("rematch", "Pseudo-GTID relocation", `Reconnect a slave onto its master, via PSeudo-GTID.`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
instance, _, err := inst.RematchSlave(instanceKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case registerCliCommand("match-slaves", "Pseudo-GTID relocation", `Matches all slaves of a given instance under another (destination) instance using Pseudo-GTID`),
registerCliCommand("multi-match-slaves", "Pseudo-GTID relocation", `Synonym to 'match-slaves', will be deprecated`):
{
// Move all slaves of "instance" beneath "destination"
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
matchedSlaves, _, err, errs := inst.MultiMatchSlaves(instanceKey, destinationKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range matchedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case registerCliCommand("match-up-slaves", "Pseudo-GTID relocation", `Matches slaves of the given instance one level up the topology, making them siblings of given instance, using Pseudo-GTID`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
matchedSlaves, _, err, errs := inst.MatchUpSlaves(instanceKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range matchedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case registerCliCommand("regroup-slaves-pgtid", "Pseudo-GTID relocation", `Given an instance, pick one of its slave and make it local master of its siblings, using Pseudo-GTID.`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
lostSlaves, equalSlaves, aheadSlaves, promotedSlave, err := inst.RegroupSlavesPseudoGTID(instanceKey, false, func(candidateSlave *inst.Instance) { fmt.Println(candidateSlave.Key.DisplayString()) }, postponedFunctionsContainer)
postponedFunctionsContainer.InvokePostponed()
if promotedSlave == nil {
log.Fatalf("Could not regroup slaves of %+v; error: %+v", *instanceKey, err)
}
fmt.Println(fmt.Sprintf("%s lost: %d, trivial: %d, pseudo-gtid: %d",
promotedSlave.Key.DisplayString(), len(lostSlaves), len(equalSlaves), len(aheadSlaves)))
if err != nil {
log.Fatale(err)
}
}
// General replication commands
case registerCliCommand("enable-gtid", "Replication, general", `If possible, turn on GTID replication`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.EnableGTID(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("disable-gtid", "Replication, general", `Turn off GTID replication, back to file:pos replication`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.DisableGTID(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("reset-master-gtid-remove-own-uuid", "Replication, general", `Reset master on instance, remove GTID entries generated by instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.ResetMasterGTIDOperation(instanceKey, true, "")
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("skip-query", "Replication, general", `Skip a single statement on a slave; either when running with GTID or without`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.SkipQuery(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("stop-slave", "Replication, general", `Issue a STOP SLAVE on an instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.StopSlave(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("start-slave", "Replication, general", `Issue a START SLAVE on an instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.StartSlave(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("restart-slave", "Replication, general", `STOP and START SLAVE on an instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.RestartSlave(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("reset-slave", "Replication, general", `Issues a RESET SLAVE command; use with care`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.ResetSlaveOperation(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("detach-slave", "Replication, general", `Stops replication and modifies binlog position into an impossible, yet reversible, value.`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.DetachSlaveOperation(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("reattach-slave", "Replication, general", `Undo a detach-slave operation`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.ReattachSlaveOperation(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("detach-slave-master-host", "Replication, general", `Stops replication and modifies Master_Host into an impossible, yet reversible, value.`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.DetachSlaveMasterHost(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("reattach-slave-master-host", "Replication, general", `Undo a detach-slave-master-host operation`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.ReattachSlaveMasterHost(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("master-pos-wait", "Replication, general", `Wait until slave reaches given replication coordinates (--binlog=file:pos)`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unresolved instance")
}
instance, err := inst.ReadTopologyInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
var binlogCoordinates *inst.BinlogCoordinates
if binlogCoordinates, err = inst.ParseBinlogCoordinates(*config.RuntimeCLIFlags.BinlogFile); err != nil {
log.Fatalf("Expecing --binlog argument as file:pos")
}
_, err = inst.MasterPosWait(instanceKey, binlogCoordinates)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
// Pool
case registerCliCommand("set-read-only", "Instance", `Turn an instance read-only, via SET GLOBAL read_only := 1`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.SetReadOnly(instanceKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("set-writeable", "Instance", `Turn an instance writeable, via SET GLOBAL read_only := 0`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
_, err := inst.SetReadOnly(instanceKey, false)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
// Binary log operations
case registerCliCommand("flush-binary-logs", "Binary logs", `Flush binary logs on an instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
var err error
if *config.RuntimeCLIFlags.BinlogFile == "" {
_, err = inst.FlushBinaryLogs(instanceKey, 1)
} else {
_, err = inst.FlushBinaryLogsTo(instanceKey, *config.RuntimeCLIFlags.BinlogFile)
}
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("purge-binary-logs", "Binary logs", `Purge binary logs of an instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
var err error
if *config.RuntimeCLIFlags.BinlogFile == "" {
log.Fatal("expecting --binlog value")
}
_, err = inst.PurgeBinaryLogsTo(instanceKey, *config.RuntimeCLIFlags.BinlogFile)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("last-pseudo-gtid", "Binary logs", `Find latest Pseudo-GTID entry in instance's binary logs`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unresolved instance")
}
instance, err := inst.ReadTopologyInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
coordinates, text, err := inst.FindLastPseudoGTIDEntry(instance, instance.RelaylogCoordinates, nil, strict, nil)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%+v:%s", *coordinates, text))
}
case registerCliCommand("find-binlog-entry", "Binary logs", `Get binlog file:pos of entry given by --pattern (exact full match, not a regular expression) in a given instance`):
{
if pattern == "" {
log.Fatal("No pattern given")
}
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unresolved instance")
}
instance, err := inst.ReadTopologyInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
coordinates, err := inst.SearchEntryInInstanceBinlogs(instance, pattern, false)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%+v", *coordinates))
}
case registerCliCommand("correlate-binlog-pos", "Binary logs", `Given an instance (-i) and binlog coordinates (--binlog=file:pos), find the correlated coordinates in another instance (-d)`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unresolved instance")
}
instance, err := inst.ReadTopologyInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
if !instance.LogBinEnabled {
log.Fatalf("Instance does not have binary logs: %+v", *instanceKey)
}
if destinationKey == nil {
log.Fatal("Cannot deduce target instance:", destination)
}
otherInstance, err := inst.ReadTopologyInstance(destinationKey)
if err != nil {
log.Fatale(err)
}
if otherInstance == nil {
log.Fatalf("Instance not found: %+v", *destinationKey)
}
var binlogCoordinates *inst.BinlogCoordinates
if *config.RuntimeCLIFlags.BinlogFile == "" {
binlogCoordinates = &instance.SelfBinlogCoordinates
} else {
if binlogCoordinates, err = inst.ParseBinlogCoordinates(*config.RuntimeCLIFlags.BinlogFile); err != nil {
log.Fatalf("Expecing --binlog argument as file:pos")
}
}
coordinates, _, err := inst.CorrelateBinlogCoordinates(instance, binlogCoordinates, otherInstance)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%+v", *coordinates))
}
// Pool
case registerCliCommand("submit-pool-instances", "Pools", `Submit a pool name with a list of instances in that pool`):
{
if pool == "" {
log.Fatal("Please submit --pool")
}
err := inst.ApplyPoolInstances(pool, instance)
if err != nil {
log.Fatale(err)
}
}
case registerCliCommand("cluster-pool-instances", "Pools", `List all pools and their associated instances`):
{
clusterPoolInstances, err := inst.ReadAllClusterPoolInstances()
if err != nil {
log.Fatale(err)
}
for _, clusterPoolInstance := range clusterPoolInstances {
fmt.Println(fmt.Sprintf("%s\t%s\t%s\t%s:%d", clusterPoolInstance.ClusterName, clusterPoolInstance.ClusterAlias, clusterPoolInstance.Pool, clusterPoolInstance.Hostname, clusterPoolInstance.Port))
}
}
// Information
case registerCliCommand("find", "Information", `Find instances whose hostname matches given regex pattern`):
{
if pattern == "" {
log.Fatal("No pattern given")
}
instances, err := inst.FindInstances(pattern)
if err != nil {
log.Fatale(err)
} else {
for _, instance := range instances {
fmt.Println(instance.Key.DisplayString())
}
}
}
case registerCliCommand("clusters", "Information", `List all clusters known to orchestrator`):
{
clusters, err := inst.ReadClusters()
if err != nil {
log.Fatale(err)
} else {
fmt.Println(strings.Join(clusters, "\n"))
}
}
case registerCliCommand("topology", "Information", `Show an ascii-graph of a replication topology, given a member of that topology`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
output, err := inst.ASCIITopology(instanceKey, pattern)
if err != nil {
log.Fatale(err)
}
fmt.Println(output)
}
case registerCliCommand("which-instance", "Information", `Output the fully-qualified hostname:port representation of the given instance, or error if unknown`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unable to get master: unresolved instance")
}
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
fmt.Println(instance.Key.DisplayString())
}
case registerCliCommand("which-cluster", "Information", `Output the name of the cluster an instance belongs to, or error if unknown to orchestrator`):
{
clusterName := getClusterName(clusterAlias, instanceKey)
fmt.Println(clusterName)
}
case registerCliCommand("which-cluster-instances", "Information", `Output the list of instances participating in same cluster as given instance`):
{
clusterName := getClusterName(clusterAlias, instanceKey)
instances, err := inst.ReadClusterInstances(clusterName)
if err != nil {
log.Fatale(err)
}
for _, clusterInstance := range instances {
fmt.Println(clusterInstance.Key.DisplayString())
}
}
case registerCliCommand("which-cluster-osc-slaves", "Information", `Output a list of slaves in same cluster as given instance, that could serve as a pt-online-schema-change operation control slaves`):
{
clusterName := getClusterName(clusterAlias, instanceKey)
instances, err := inst.GetClusterOSCSlaves(clusterName)
if err != nil {
log.Fatale(err)
}
for _, clusterInstance := range instances {
fmt.Println(clusterInstance.Key.DisplayString())
}
}
case registerCliCommand("which-master", "Information", `Output the fully-qualified hostname:port representation of a given instance's master`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unable to get master: unresolved instance")
}
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
fmt.Println(instance.MasterKey.DisplayString())
}
case registerCliCommand("which-slaves", "Information", `Output the fully-qualified hostname:port list of slaves of a given instance`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unable to get slaves: unresolved instance")
}
slaves, err := inst.ReadSlaveInstances(instanceKey)
if err != nil {
log.Fatale(err)
}
for _, slave := range slaves {
fmt.Println(slave.Key.DisplayString())
}
}
case registerCliCommand("instance-status", "Information", `Output short status on a given instance`):
{
if instanceKey == nil {
instanceKey = assignThisInstanceKey()
}
if instanceKey == nil {
log.Fatalf("Unable to get status: unresolved instance")
}
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
fmt.Println(instance.HumanReadableDescription())
}
case registerCliCommand("get-cluster-heuristic-lag", "Information", `For a given cluster (indicated by an instance or alias), output a heuristic "representative" lag of that cluster`):
{
clusterName := getClusterName(clusterAlias, instanceKey)
lag, err := inst.GetClusterHeuristicLag(clusterName)
if err != nil {
log.Fatale(err)
}
fmt.Println(lag)
}
// Instance management
case registerCliCommand("discover", "Instance management", `Lookup an instance, investigate it`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
instance, err := inst.ReadTopologyInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instance.Key.DisplayString())
}
case registerCliCommand("forget", "Instance management", `Forget about an instance's existence`):
{
if rawInstanceKey == nil {
rawInstanceKey = assignThisInstanceKey()
}
if rawInstanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
err := inst.ForgetInstance(rawInstanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(rawInstanceKey.DisplayString())
}
case registerCliCommand("begin-maintenance", "Instance management", `Request a maintenance lock on an instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if reason == "" {
log.Fatal("--reason option required")
}
var durationSeconds int = 0
if duration != "" {
durationSeconds, err = util.SimpleTimeToSeconds(duration)
if err != nil {
log.Fatale(err)
}
if durationSeconds < 0 {
log.Fatalf("Duration value must be non-negative. Given value: %d", durationSeconds)
}
}
maintenanceKey, err := inst.BeginBoundedMaintenance(instanceKey, inst.GetMaintenanceOwner(), reason, uint(durationSeconds))
if err == nil {
log.Infof("Maintenance key: %+v", maintenanceKey)
log.Infof("Maintenance duration: %d seconds", durationSeconds)
}
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("end-maintenance", "Instance management", `Remove maintenance lock from an instance`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
err := inst.EndMaintenanceByInstanceKey(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("begin-downtime", "Instance management", `Mark an instance as downtimed`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
if reason == "" {
log.Fatal("--reason option required")
}
var durationSeconds int = 0
if duration != "" {
durationSeconds, err = util.SimpleTimeToSeconds(duration)
if err != nil {
log.Fatale(err)
}
if durationSeconds < 0 {
log.Fatalf("Duration value must be non-negative. Given value: %d", durationSeconds)
}
}
err := inst.BeginDowntime(instanceKey, inst.GetMaintenanceOwner(), reason, uint(durationSeconds))
if err == nil {
log.Infof("Downtime duration: %d seconds", durationSeconds)
} else {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("end-downtime", "Instance management", `Indicate an instance is no longer downtimed`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
err := inst.EndDowntime(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
// Recovery & analysis
case registerCliCommand("recover", "Recovery", `Do auto-recovery given a dead instance`), registerCliCommand("recover-lite", "Recovery", `Do auto-recovery given a dead instance. Orchestrator chooses the best course of actionwithout executing external processes`):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
recoveryAttempted, promotedInstanceKey, err := logic.CheckAndRecover(instanceKey, destinationKey, (command == "recover-lite"))
if err != nil {
log.Fatale(err)
}
if recoveryAttempted {
if promotedInstanceKey == nil {
log.Fatalf("Recovery attempted yet no slave promoted")
}
fmt.Println(promotedInstanceKey.DisplayString())
}
}
case registerCliCommand("replication-analysis", "Recovery", `Request an analysis of potential crash incidents in all known topologies`):
{
analysis, err := inst.GetReplicationAnalysis("", false, false)
if err != nil {
log.Fatale(err)
}
for _, entry := range analysis {
fmt.Println(fmt.Sprintf("%s (cluster %s): %s", entry.AnalyzedInstanceKey.DisplayString(), entry.ClusterDetails.ClusterName, entry.Analysis))
}
}
case registerCliCommand("ack-cluster-recoveries", "Recovery", `Acknowledge recoveries for a given cluster; this unblocks pending future recoveries`):
{
if reason == "" {
log.Fatal("--reason option required (comment your ack)")
}
clusterName := getClusterName(clusterAlias, instanceKey)
countRecoveries, err := logic.AcknowledgeClusterRecoveries(clusterName, inst.GetMaintenanceOwner(), reason)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%d recoveries acknowldged", countRecoveries))
}
case registerCliCommand("ack-instance-recoveries", "Recovery", `Acknowledge recoveries for a given instance; this unblocks pending future recoveries`):
{
if reason == "" {
log.Fatal("--reason option required (comment your ack)")
}
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
countRecoveries, err := logic.AcknowledgeInstanceRecoveries(instanceKey, inst.GetMaintenanceOwner(), reason)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%d recoveries acknowldged", countRecoveries))
}
// Instance meta
case registerCliCommand("register-candidate", "Instance, meta", `Indicate that a specific instance is a preferred candidate for master promotion`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
err := inst.RegisterCandidateInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("register-hostname-unresolve", "Instance, meta", `Assigns the given instance a virtual (aka "unresolved") name`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
err := inst.RegisterHostnameUnresolve(instanceKey, hostnameFlag)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case registerCliCommand("deregister-hostname-unresolve", "Instance, meta", `Explicitly deregister/dosassociate a hostname with an "unresolved" name`):
{
instanceKey = deduceInstanceKeyIfNeeded(instance, instanceKey)
err := inst.DeregisterHostnameUnresolve(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
// meta
case registerCliCommand("snapshot-topologies", "Meta", `Take a snapshot of existing topologies.`):
{
err := inst.SnapshotTopologies()
if err != nil {
log.Fatale(err)
}
}
case registerCliCommand("continuous", "Meta", `Enter continuous mode, and actively poll for instances, diagnose problems, do maintenance`):
{
logic.ContinuousDiscovery()
}
case registerCliCommand("resolve", "Meta", `Resolve given hostname`):
{
if rawInstanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if conn, err := net.Dial("tcp", rawInstanceKey.DisplayString()); err == nil {
log.Debugf("tcp test is good; got connection %+v", conn)
conn.Close()
} else {
log.Fatale(err)
}
if cname, err := inst.GetCNAME(rawInstanceKey.Hostname); err == nil {
log.Debugf("GetCNAME() %+v, %+v", cname, err)
rawInstanceKey.Hostname = cname
fmt.Println(rawInstanceKey.DisplayString())
} else {
log.Fatale(err)
}
}
case registerCliCommand("reset-hostname-resolve-cache", "Meta", `Clear the hostname resolve cache`):
{
err := inst.ResetHostnameResolveCache()
if err != nil {
log.Fatale(err)
}
fmt.Println("hostname resolve cache cleared")
}
case registerCliCommand("reset-internal-db-deployment", "Meta, internal", `Clear internal db deployment history, use if somehow corrupted internal deployment history`):
{
config.Config.SkipOrchestratorDatabaseUpdate = true
db.ResetInternalDeployment()
fmt.Println("Internal db deployment history reset. Next orchestrator execution will rebuild internal db structure (no data will be lost)")
}
// Help
case "help":
{
fmt.Fprintf(os.Stderr, availableCommandsUsage())
}
default:
log.Fatalf("Unknown command: \"%s\". %s", command, availableCommandsUsage())
}
}
// Cli initiates a command line interface, executing requested command.
func Cli(command string, strict bool, instance string, destination string, owner string, reason string, duration string, pattern string, clusterAlias string, pool string, hostnameFlag string) {
if instance != "" && !strings.Contains(instance, ":") {
instance = fmt.Sprintf("%s:%d", instance, config.Config.DefaultInstancePort)
}
instanceKey, err := inst.ParseInstanceKey(instance)
if err != nil {
instanceKey = nil
}
rawInstanceKey, err := inst.NewRawInstanceKey(instance)
if err != nil {
rawInstanceKey = nil
}
if destination != "" && !strings.Contains(destination, ":") {
destination = fmt.Sprintf("%s:%d", destination, config.Config.DefaultInstancePort)
}
destinationKey, err := inst.ParseInstanceKey(destination)
if err != nil {
destinationKey = nil
}
if hostname, err := os.Hostname(); err == nil {
thisInstanceKey = &inst.InstanceKey{Hostname: hostname, Port: int(config.Config.DefaultInstancePort)}
}
if len(owner) == 0 {
// get os username as owner
usr, err := user.Current()
if err != nil {
log.Fatale(err)
}
owner = usr.Username
}
inst.SetMaintenanceOwner(owner)
// begin commands
switch command {
// Instance meta
case cliCommand("discover"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
instance, err := inst.ReadTopologyInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instance.Key.DisplayString())
}
case cliCommand("forget"):
{
if rawInstanceKey == nil {
rawInstanceKey = thisInstanceKey
}
if rawInstanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
err := inst.ForgetInstance(rawInstanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(rawInstanceKey.DisplayString())
}
case cliCommand("resolve"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if conn, err := net.Dial("tcp", instanceKey.DisplayString()); err == nil {
conn.Close()
} else {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("register-hostname-unresolve"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
err := inst.RegisterHostnameUnresolve(instanceKey, hostnameFlag)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("deregister-hostname-unresolve"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
err := inst.DeregisterHostnameUnresolve(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("register-candidate"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
err := inst.RegisterCandidateInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
// Instance
case cliCommand("begin-maintenance"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if reason == "" {
log.Fatal("--reason option required")
}
var durationSeconds int = 0
if duration != "" {
durationSeconds, err = util.SimpleTimeToSeconds(duration)
if err != nil {
log.Fatale(err)
}
if durationSeconds < 0 {
log.Fatalf("Duration value must be non-negative. Given value: %d", durationSeconds)
}
}
maintenanceKey, err := inst.BeginBoundedMaintenance(instanceKey, inst.GetMaintenanceOwner(), reason, uint(durationSeconds))
if err == nil {
log.Infof("Maintenance key: %+v", maintenanceKey)
log.Infof("Maintenance duration: %d seconds", durationSeconds)
}
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("end-maintenance"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
err := inst.EndMaintenanceByInstanceKey(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("begin-downtime"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if reason == "" {
log.Fatal("--reason option required")
}
var durationSeconds int = 0
if duration != "" {
durationSeconds, err = util.SimpleTimeToSeconds(duration)
if err != nil {
log.Fatale(err)
}
if durationSeconds < 0 {
log.Fatalf("Duration value must be non-negative. Given value: %d", durationSeconds)
}
}
err := inst.BeginDowntime(instanceKey, inst.GetMaintenanceOwner(), reason, uint(durationSeconds))
if err == nil {
log.Infof("Downtime duration: %d seconds", durationSeconds)
} else {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("end-downtime"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
err := inst.EndDowntime(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("set-read-only"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.SetReadOnly(instanceKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("set-writeable"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.SetReadOnly(instanceKey, false)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("flush-binary-logs"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
var err error
if *config.RuntimeCLIFlags.BinlogFile == "" {
err = inst.FlushBinaryLogs(instanceKey, 1)
} else {
_, err = inst.FlushBinaryLogsTo(instanceKey, *config.RuntimeCLIFlags.BinlogFile)
}
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("last-pseudo-gtid"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatalf("Unresolved instance")
}
instance, err := inst.ReadTopologyInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
coordinates, text, err := inst.FindLastPseudoGTIDEntry(instance, instance.RelaylogCoordinates, strict, nil)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%+v:%s", *coordinates, text))
}
// replication
case cliCommand("stop-slave"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.StopSlave(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("start-slave"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.StartSlave(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("restart-slave"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.RestartSlave(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("reset-slave"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.ResetSlaveOperation(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("detach-slave"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.DetachSlaveOperation(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("reattach-slave"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.ReattachSlaveOperation(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("enable-gtid"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.EnableGTID(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("disable-gtid"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.DisableGTID(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("skip-query"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.SkipQuery(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
// move
case cliCommand("relocate"), cliCommand("relocate-below"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
_, err := inst.RelocateBelow(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case cliCommand("relocate-slaves"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
slaves, _, err, errs := inst.RelocateSlaves(instanceKey, destinationKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range slaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case cliCommand("move-up"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
instance, err := inst.MoveUp(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case cliCommand("move-up-slaves"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
movedSlaves, _, err, errs := inst.MoveUpSlaves(instanceKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range movedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case cliCommand("move-below"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce sibling:", destination)
}
_, err := inst.MoveBelow(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case cliCommand("move-equivalent"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce sibling:", destination)
}
_, err := inst.MoveEquivalent(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case cliCommand("move-gtid"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce sibling:", destination)
}
_, err := inst.MoveBelowGTID(instanceKey, destinationKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case cliCommand("move-slaves-gtid"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
movedSlaves, _, err, errs := inst.MoveSlavesGTID(instanceKey, destinationKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range movedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case cliCommand("repoint"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
// destinationKey can be null, in which case the instance repoints to its existing master
instance, err := inst.Repoint(instanceKey, destinationKey, inst.GTIDHintNeutral)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case cliCommand("repoint-slaves"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
repointedSlaves, err, errs := inst.RepointSlavesTo(instanceKey, pattern, destinationKey)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range repointedSlaves {
fmt.Println(fmt.Sprintf("%s<%s", slave.Key.DisplayString(), instanceKey.DisplayString()))
}
}
}
case cliCommand("enslave-siblings"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, _, err := inst.EnslaveSiblings(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("enslave-master"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.EnslaveMaster(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
case cliCommand("make-co-master"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
_, err := inst.MakeCoMaster(instanceKey)
if err != nil {
log.Fatale(err)
}
fmt.Println(instanceKey.DisplayString())
}
// Pseudo-GTID
case cliCommand("match"), cliCommand("match-below"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
_, _, err := inst.MatchBelow(instanceKey, destinationKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), destinationKey.DisplayString()))
}
case cliCommand("match-up"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
instance, _, err := inst.MatchUp(instanceKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case cliCommand("rematch"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
instance, _, err := inst.RematchSlave(instanceKey, true)
if err != nil {
log.Fatale(err)
}
fmt.Println(fmt.Sprintf("%s<%s", instanceKey.DisplayString(), instance.MasterKey.DisplayString()))
}
case cliCommand("get-candidate-slave"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
instance, _, _, _, err := inst.GetCandidateSlave(instanceKey, false)
if err != nil {
log.Fatale(err)
} else {
fmt.Println(instance.Key.DisplayString())
}
}
case cliCommand("match-slaves"), cliCommand("multi-match-slaves"):
{
// Move all slaves of "instance" beneath "destination"
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
if destinationKey == nil {
log.Fatal("Cannot deduce destination:", destination)
}
matchedSlaves, _, err, errs := inst.MultiMatchSlaves(instanceKey, destinationKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range matchedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case cliCommand("match-up-slaves"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
matchedSlaves, _, err, errs := inst.MatchUpSlaves(instanceKey, pattern)
if err != nil {
log.Fatale(err)
} else {
for _, e := range errs {
log.Errore(e)
}
for _, slave := range matchedSlaves {
fmt.Println(slave.Key.DisplayString())
}
}
}
case cliCommand("regroup-slaves"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
lostSlaves, equalSlaves, aheadSlaves, promotedSlave, err := inst.RegroupSlaves(instanceKey, false, func(candidateSlave *inst.Instance) { fmt.Println(candidateSlave.Key.DisplayString()) })
if promotedSlave == nil {
log.Fatalf("Could not regroup slaves of %+v; error: %+v", *instanceKey, err)
}
fmt.Println(fmt.Sprintf("%s lost: %d, trivial: %d, pseudo-gtid: %d",
promotedSlave.Key.DisplayString(), len(lostSlaves), len(equalSlaves), len(aheadSlaves)))
if err != nil {
log.Fatale(err)
}
}
case cliCommand("regroup-slaves-gtid"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
lostSlaves, movedSlaves, promotedSlave, err := inst.RegroupSlavesGTID(instanceKey, false, func(candidateSlave *inst.Instance) { fmt.Println(candidateSlave.Key.DisplayString()) })
if promotedSlave == nil {
log.Fatalf("Could not regroup slaves of %+v; error: %+v", *instanceKey, err)
}
fmt.Println(fmt.Sprintf("%s lost: %d, moved: %d",
promotedSlave.Key.DisplayString(), len(lostSlaves), len(movedSlaves)))
if err != nil {
log.Fatale(err)
}
}
case cliCommand("regroup-slaves-bls"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
promotedBinlogServer, err := inst.RegroupSlavesBinlogServers(instanceKey, false, nil)
if promotedBinlogServer == nil {
log.Fatalf("Could not regroup binlog server slaves of %+v; error: %+v", *instanceKey, err)
}
fmt.Println(promotedBinlogServer.Key.DisplayString())
if err != nil {
log.Fatale(err)
}
}
// cluster
case cliCommand("clusters"):
{
clusters, err := inst.ReadClusters()
if err != nil {
log.Fatale(err)
} else {
fmt.Println(strings.Join(clusters, "\n"))
}
}
case cliCommand("find"):
{
if pattern == "" {
log.Fatal("No pattern given")
}
instances, err := inst.FindInstances(pattern)
if err != nil {
log.Fatale(err)
} else {
for _, instance := range instances {
fmt.Println(instance.Key.DisplayString())
}
}
}
case cliCommand("topology"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
output, err := inst.ASCIITopology(instanceKey, pattern)
if err != nil {
log.Fatale(err)
}
fmt.Println(output)
}
case cliCommand("which-instance"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatalf("Unable to get master: unresolved instance")
}
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
fmt.Println(instance.Key.DisplayString())
}
case cliCommand("which-master"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatalf("Unable to get master: unresolved instance")
}
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
fmt.Println(instance.MasterKey.DisplayString())
}
case cliCommand("which-cluster"):
{
clusterName := getClusterName(clusterAlias, instanceKey)
fmt.Println(clusterName)
}
case cliCommand("which-cluster-instances"):
{
clusterName := getClusterName(clusterAlias, instanceKey)
instances, err := inst.ReadClusterInstances(clusterName)
if err != nil {
log.Fatale(err)
}
for _, clusterInstance := range instances {
fmt.Println(clusterInstance.Key.DisplayString())
}
}
case cliCommand("which-cluster-osc-slaves"):
{
clusterName := getClusterName(clusterAlias, instanceKey)
instances, err := inst.GetClusterOSCSlaves(clusterName)
if err != nil {
log.Fatale(err)
}
for _, clusterInstance := range instances {
fmt.Println(clusterInstance.Key.DisplayString())
}
}
case cliCommand("which-slaves"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatalf("Unable to get slaves: unresolved instance")
}
slaves, err := inst.ReadSlaveInstances(instanceKey)
if err != nil {
log.Fatale(err)
}
for _, slave := range slaves {
fmt.Println(slave.Key.DisplayString())
}
}
case cliCommand("instance-status"):
{
if instanceKey == nil {
instanceKey = thisInstanceKey
}
if instanceKey == nil {
log.Fatalf("Unable to get status: unresolved instance")
}
instance, _, err := inst.ReadInstance(instanceKey)
if err != nil {
log.Fatale(err)
}
if instance == nil {
log.Fatalf("Instance not found: %+v", *instanceKey)
}
fmt.Println(instance.HumanReadableDescription())
}
case cliCommand("get-cluster-heuristic-lag"):
{
clusterName := getClusterName(clusterAlias, instanceKey)
lag, err := inst.GetClusterHeuristicLag(clusterName)
if err != nil {
log.Fatale(err)
}
fmt.Println(lag)
}
// meta
case cliCommand("snapshot-topologies"):
{
err := inst.SnapshotTopologies()
if err != nil {
log.Fatale(err)
}
}
case cliCommand("continuous"):
{
logic.ContinuousDiscovery()
}
case cliCommand("reset-hostname-resolve-cache"):
{
err := inst.ResetHostnameResolveCache()
if err != nil {
log.Fatale(err)
}
fmt.Println("hostname resolve cache cleared")
}
// Recovery & analysis
case cliCommand("recover"), cliCommand("recover-lite"):
{
if instanceKey == nil {
log.Fatal("Cannot deduce instance:", instance)
}
actionTaken, promotedInstance, err := logic.CheckAndRecover(instanceKey, destinationKey, true, (command == "recover-lite"))
if err != nil {
log.Fatale(err)
}
if actionTaken {
fmt.Println(promotedInstance.Key.DisplayString())
}
}
case cliCommand("replication-analysis"):
{
analysis, err := inst.GetReplicationAnalysis(false)
if err != nil {
log.Fatale(err)
}
for _, entry := range analysis {
fmt.Println(fmt.Sprintf("%s (cluster %s): %s", entry.AnalyzedInstanceKey.DisplayString(), entry.ClusterDetails.ClusterName, entry.Analysis))
}
}
// pool
case cliCommand("submit-pool-instances"):
{
if pool == "" {
log.Fatal("Please submit --pool")
}
err := inst.ApplyPoolInstances(pool, instance)
if err != nil {
log.Fatale(err)
}
}
case cliCommand("cluster-pool-instances"):
{
clusterPoolInstances, err := inst.ReadAllClusterPoolInstances()
if err != nil {
log.Fatale(err)
}
for _, clusterPoolInstance := range clusterPoolInstances {
fmt.Println(fmt.Sprintf("%s\t%s\t%s\t%s:%d", clusterPoolInstance.ClusterName, clusterPoolInstance.ClusterAlias, clusterPoolInstance.Pool, clusterPoolInstance.Hostname, clusterPoolInstance.Port))
}
}
default:
log.Fatalf("Unknown command: \"%s\". Available commands (-c):\n\t%v", command, strings.Join(knownCommands, "\n\t"))
}
}
// main is the application's entry point.
func main() {
servers := flag.String("servers", "", "srv1[:port1][,srv2[:port2]...]")
command := flag.String("c", "", "command, required (exists|get|ls|lsr|create|creater|set|delete|rm|deleter|rmr|getacl|setacl)")
force := flag.Bool("force", false, "force operation")
format := flag.String("format", "txt", "output format (txt|json)")
verbose := flag.Bool("verbose", false, "verbose")
debug := flag.Bool("debug", false, "debug mode (very verbose)")
stack := flag.Bool("stack", false, "add stack trace upon error")
authUser := flag.String("auth_usr", "", "optional, digest scheme, user")
authPwd := flag.String("auth_pwd", "", "optional, digest scheme, pwd")
acls := flag.String("acls", "31", "optional, csv list [1|,2|,4|,8|,16|,31]")
flag.Parse()
log.SetLevel(log.ERROR)
if *verbose {
log.SetLevel(log.INFO)
}
if *debug {
log.SetLevel(log.DEBUG)
}
if *stack {
log.SetPrintStackTrace(*stack)
}
log.Info("starting")
if *servers == "" {
log.Fatal("Expected comma delimited list of servers via --servers")
}
serversArray := strings.Split(*servers, ",")
if len(serversArray) == 0 {
log.Fatal("Expected comma delimited list of servers via --servers")
}
if len(*command) == 0 {
log.Fatal("Expected command (-c) (exists|get|ls|lsr|create|creater|set|delete|rm|deleter|rmr|getacl|setacl)")
}
if len(flag.Args()) < 1 {
log.Fatal("Expected path argument")
}
path := flag.Arg(0)
if *command == "ls" {
} else if strings.HasSuffix(path, "/") {
log.Fatal("Path must not end with '/'")
}
rand.Seed(time.Now().UnixNano())
zk.SetServers(serversArray)
if *authUser != "" && *authPwd != "" {
authExp := fmt.Sprint(*authUser, ":", *authPwd)
zk.SetAuth("digest", []byte(authExp))
}
if *command == "creater" {
*command = "create"
*force = true
}
switch *command {
case "exists":
{
if exists, err := zk.Exists(path); err == nil && exists {
output.PrintString([]byte("true"), *format)
} else {
log.Fatale(err)
}
}
case "get":
{
if result, err := zk.Get(path); err == nil {
output.PrintString(result, *format)
} else {
log.Fatale(err)
}
}
case "getacl":
{
if result, err := zk.GetACL(path); err == nil {
output.PrintStringArray(result, *format)
} else {
log.Fatale(err)
}
}
case "ls":
{
if result, err := zk.Children(path); err == nil {
output.PrintStringArray(result, *format)
} else {
log.Fatale(err)
}
}
case "lsr":
{
if result, err := zk.ChildrenRecursive(path); err == nil {
output.PrintStringArray(result, *format)
} else {
log.Fatale(err)
}
}
case "create":
{
var aclstr string
if len(flag.Args()) < 2 {
log.Fatal("Expected data argument")
}
if len(flag.Args()) >= 3 {
aclstr = flag.Arg(2)
}
if *authUser != "" && *authPwd != "" {
perms, err := zk.BuildACL("digest", *authUser, *authPwd, *acls)
if err != nil {
log.Fatale(err)
}
if result, err := zk.CreateWithACL(path, []byte(flag.Arg(1)), *force, perms); err == nil {
log.Infof("Created %+v", result)
} else {
log.Fatale(err)
}
} else {
if result, err := zk.Create(path, []byte(flag.Arg(1)), aclstr, *force); err == nil {
log.Infof("Created %+v", result)
} else {
log.Fatale(err)
}
}
}
case "set":
{
var info []byte
if len(flag.Args()) > 1 {
info = []byte(flag.Arg(1))
} else {
var err error
info, err = ioutil.ReadAll(os.Stdin)
if err != nil {
log.Fatale(err)
}
}
if result, err := zk.Set(path, info); err == nil {
log.Infof("Set %+v", result)
} else {
log.Fatale(err)
}
}
case "setacl":
{
var aclstr string
if len(flag.Args()) > 1 {
aclstr = flag.Arg(1)
} else {
var err error
data, err := ioutil.ReadAll(os.Stdin)
aclstr = string(data)
if err != nil {
log.Fatale(err)
}
}
if result, err := zk.SetACL(path, aclstr, *force); err == nil {
log.Infof("Set %+v", result)
} else {
log.Fatale(err)
}
}
case "delete", "rm":
{
if err := zk.Delete(path); err != nil {
log.Fatale(err)
}
}
case "deleter", "rmr":
{
if !(*force) {
log.Fatal("deleter (recursive) command requires --force for safety measure")
}
if err := zk.DeleteRecursive(path); err != nil {
log.Fatale(err)
}
}
default:
log.Fatalf("Unknown command: %s", *command)
}
}