// BeginMaintenance will make new maintenance entry for given instanceKey.
func BeginMaintenance(instanceKey *InstanceKey, owner string, reason string) (int64, error) {
db, err := db.OpenOrchestrator()
var maintenanceToken int64 = 0
if err != nil {
return maintenanceToken, log.Errore(err)
}
res, err := sqlutils.Exec(db, `
insert ignore
into database_instance_maintenance (
hostname, port, maintenance_active, begin_timestamp, end_timestamp, owner, reason
) VALUES (
?, ?, 1, NOW(), NULL, ?, ?
)
`,
instanceKey.Hostname,
instanceKey.Port,
owner,
reason,
)
if err != nil {
return maintenanceToken, log.Errore(err)
}
if affected, _ := res.RowsAffected(); affected == 0 {
err = errors.New(fmt.Sprintf("Cannot begin maintenance for instance: %+v", instanceKey))
} else {
// success
maintenanceToken, _ = res.LastInsertId()
AuditOperation("begin-maintenance", instanceKey, fmt.Sprintf("maintenanceToken: %d, owner: %s, reason: %s", maintenanceToken, owner, reason))
}
return maintenanceToken, err
}
// EndMaintenanceByInstanceKey will terminate an active maintenance using given instanceKey as hint
func EndMaintenanceByInstanceKey(instanceKey *InstanceKey) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
res, err := sqlutils.Exec(db, `
update
database_instance_maintenance
set
maintenance_active = NULL,
end_timestamp = NOW()
where
hostname = ?
and port = ?
and maintenance_active = 1
`,
instanceKey.Hostname,
instanceKey.Port,
)
if err != nil {
return log.Errore(err)
}
if affected, _ := res.RowsAffected(); affected == 0 {
err = errors.New(fmt.Sprintf("Instance is not in maintenance mode: %+v", instanceKey))
} else {
// success
AuditOperation("end-maintenance", instanceKey, "")
}
return err
}
// EndMaintenance will terminate an active maintenance via maintenanceToken
func EndMaintenance(maintenanceToken int64) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
res, err := sqlutils.Exec(db, `
update
database_instance_maintenance
set
maintenance_active = NULL,
end_timestamp = NOW()
where
database_instance_maintenance_id = ?
`,
maintenanceToken,
)
if err != nil {
return log.Errore(err)
}
if affected, _ := res.RowsAffected(); affected == 0 {
err = errors.New(fmt.Sprintf("Instance is not in maintenance mode; token = %+v", maintenanceToken))
} else {
// success
instanceKey, _ := ReadMaintenanceInstanceKey(maintenanceToken)
AuditOperation("end-maintenance", instanceKey, fmt.Sprintf("maintenanceToken: %d", maintenanceToken))
}
return err
}
// submitSeedStateEntry submits a seed state: a single step in the overall seed process
func submitSeedStateEntry(seedId int64, action string, errorMessage string) (int64, error) {
db, err := db.OpenOrchestrator()
if err != nil {
return 0, log.Errore(err)
}
res, err := sqlutils.Exec(db, `
insert
into agent_seed_state (
agent_seed_id, state_timestamp, state_action, error_message
) VALUES (
?, NOW(), ?, ?
)
`,
seedId,
action,
errorMessage,
)
if err != nil {
return 0, log.Errore(err)
}
id, err := res.LastInsertId()
return id, err
}
// FailStaleSeeds marks as failed seeds where no progress have been seen recently
func FailStaleSeeds() error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
update
agent_seed
set
is_complete=1,
is_successful=0
where
is_complete=0
and (
select
max(state_timestamp) as last_state_timestamp
from
agent_seed_state
where
agent_seed.agent_seed_id = agent_seed_state.agent_seed_id
) < now() - interval ? minute`,
config.Config.StaleSeedFailMinutes,
)
return err
}
// AuditOperation creates and writes a new audit entry by given params
func AuditOperation(auditType string, instanceKey *InstanceKey, message string) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
if instanceKey == nil {
instanceKey = &InstanceKey{}
}
_, err = sqlutils.Exec(db, `
insert
into audit (
audit_timestamp, audit_type, hostname, port, message
) VALUES (
NOW(), ?, ?, ?, ?
)
`,
auditType,
instanceKey.Hostname,
instanceKey.Port,
message,
)
if err != nil {
return log.Errore(err)
}
return err
}
// UpdateAgentLastChecked updates the last_check timestamp in the orchestrator backed database
// for a given agent
func UpdateAgentInfo(hostname string, agent Agent) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
update
host_agent
set
last_seen = NOW(),
mysql_port = ?,
count_mysql_snapshots = ?
where
hostname = ?`,
agent.MySQLPort,
len(agent.LogicalVolumes),
hostname,
)
if err != nil {
return log.Errore(err)
}
return nil
}
// ExecOrchestrator will execute given query on the orchestrator backend database.
func ExecOrchestrator(query string, args ...interface{}) (sql.Result, error) {
db, err := OpenOrchestrator()
if err != nil {
return nil, err
}
res, err := sqlutils.Exec(db, query, args...)
return res, err
}
// ExecInstance executes a given query on the given MySQL topology instance
func ExecInstance(instanceKey *InstanceKey, query string, args ...interface{}) (sql.Result, error) {
db, err := db.OpenTopology(instanceKey.Hostname, instanceKey.Port)
if err != nil {
return nil, err
}
res, err := sqlutils.Exec(db, query, args...)
return res, err
}
// ForgetLongUnseenInstances will remove entries of all instacnes that have long since been last seen.
func ForgetLongUnseenAgents() error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
delete
from host_agent
where
last_submitted < NOW() - interval ? hour`,
config.Config.UnseenAgentForgetHours,
)
return err
}
// ForgetLongUnseenInstances will remove entries of all instacnes that have long since been last seen.
func ForgetLongUnseenInstances() error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
delete
from database_instance
where
last_seen < NOW() - interval ? hour`,
config.Config.UnseenInstanceForgetHours,
)
AuditOperation("forget-unseen", nil, "")
return err
}
// ForgetInstance removes an instance entry from the orchestrator backed database.
// It may be auto-rediscovered through topology or requested for discovery by multiple means.
func ForgetInstance(instanceKey *InstanceKey) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
delete
from database_instance
where
hostname = ? and port = ?`,
instanceKey.Hostname,
instanceKey.Port,
)
AuditOperation("forget", instanceKey, "")
return err
}
// UpdateAgentLastChecked updates the last_check timestamp in the orchestrator backed database
// for a given agent
func UpdateAgentLastChecked(hostname string) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
update
host_agent
set
last_checked = NOW()
where
hostname = ?`,
hostname,
)
if err != nil {
return log.Errore(err)
}
return nil
}
// updateSeedStateEntry updates seed step state
func updateSeedStateEntry(seedStateId int64, reason error) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
update
agent_seed_state
set error_message = ?
where
agent_seed_state_id = ?
`,
reason.Error(),
seedStateId,
)
if err != nil {
return log.Errore(err)
}
return reason
}
// UpdateInstanceLastChecked updates the last_check timestamp in the orchestrator backed database
// for a given instance
func UpdateInstanceLastChecked(instanceKey *InstanceKey) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
update
database_instance
set
last_checked = NOW()
where
hostname = ?
and port = ?`,
instanceKey.Hostname,
instanceKey.Port,
)
if err != nil {
return log.Errore(err)
}
return nil
}
// SubmitSeedEntry submits a new seed operation entry, returning its unique ID
func SubmitSeedEntry(targetHostname string, sourceHostname string) (int64, error) {
db, err := db.OpenOrchestrator()
if err != nil {
return 0, log.Errore(err)
}
res, err := sqlutils.Exec(db, `
insert
into agent_seed (
target_hostname, source_hostname, start_timestamp
) VALUES (
?, ?, NOW()
)
`,
targetHostname,
sourceHostname,
)
if err != nil {
return 0, log.Errore(err)
}
id, err := res.LastInsertId()
return id, err
}
// SetHostAttributes
func SetHostAttributes(hostname string, attributeName string, attributeValue string) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
replace
into host_attributes (
hostname, attribute_name, attribute_value, submit_timestamp, expire_timestamp
) VALUES (
?, ?, ?, NOW(), NULL
)
`,
hostname,
attributeName,
attributeValue,
)
if err != nil {
return log.Errore(err)
}
return err
}
// SubmitAgent submits a new agent for listing
func SubmitAgent(hostname string, port int, token string) (string, error) {
db, err := db.OpenOrchestrator()
if err != nil {
return "", log.Errore(err)
}
_, err = sqlutils.Exec(db, `
replace
into host_agent (
hostname, port, token, last_submitted
) VALUES (
?, ?, ?, NOW()
)
`,
hostname,
port,
token,
)
if err != nil {
return "", log.Errore(err)
}
return hostname, err
}
// updateSeedComplete updates the seed entry, signing for completion
func updateSeedComplete(seedId int64, seedError error) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
update
agent_seed
set end_timestamp = NOW(),
is_complete = 1,
is_successful = ?
where
agent_seed_id = ?
`,
(seedError == nil),
seedId,
)
if err != nil {
return log.Errore(err)
}
return nil
}
// WriteInstance stores an instance in the orchestrator backend
func WriteInstance(instance *Instance, lastError error) error {
db, err := db.OpenOrchestrator()
if err != nil {
return log.Errore(err)
}
_, err = sqlutils.Exec(db, `
replace into database_instance (
hostname,
port,
last_checked,
server_id,
version,
binlog_format,
log_bin,
log_slave_updates,
binary_log_file,
binary_log_pos,
master_host,
master_port,
slave_sql_running,
slave_io_running,
master_log_file,
read_master_log_pos,
relay_master_log_file,
exec_master_log_pos,
seconds_behind_master,
slave_lag_seconds,
num_slave_hosts,
slave_hosts,
cluster_name
) values (?, ?, NOW(), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
instance.Key.Hostname,
instance.Key.Port,
instance.ServerID,
instance.Version,
instance.Binlog_format,
instance.LogBinEnabled,
instance.LogSlaveUpdatesEnabled,
instance.SelfBinlogCoordinates.LogFile,
instance.SelfBinlogCoordinates.LogPos,
instance.MasterKey.Hostname,
instance.MasterKey.Port,
instance.Slave_SQL_Running,
instance.Slave_IO_Running,
instance.ReadBinlogCoordinates.LogFile,
instance.ReadBinlogCoordinates.LogPos,
instance.ExecBinlogCoordinates.LogFile,
instance.ExecBinlogCoordinates.LogPos,
instance.SecondsBehindMaster,
instance.SlaveLagSeconds,
len(instance.SlaveHosts),
instance.GetSlaveHostsAsJson(),
instance.ClusterName,
)
if err != nil {
return log.Errore(err)
}
if lastError == nil {
sqlutils.Exec(db, `
update database_instance set last_seen = NOW() where hostname=? and port=?
`, instance.Key.Hostname, instance.Key.Port,
)
} else {
log.Debugf("WriteInstance: will not update database_instance due to error: %+v", lastError)
}
return nil
}
