// A wrapper for ImportSqlFile which handles the additional process of
// finding the backup whereever it is stored (local or remote) and putting it in
// the correct place, and then restoring it from that location.
func RestoreInPlace(dbname, basefile string) (err error) {
if strings.Contains(basefile, "/") {
errorMessage := fmt.Sprintf("utils/backup.RestoreInPlace ! '%s' is not a file base name.", basefile, err.Error())
log.Error(errorMessage)
return errors.New(errorMessage)
}
err = StageRestoreInPlace(dbname, basefile)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RestoreInPlace ! utils/backup.StageRestoreInPlace(%s, %s) erred : %s", dbname, basefile, err.Error()))
return err
}
p := pg.NewPG("127.0.0.1", globals.PBPort, "rdpg", "rdpg", globals.PGPass)
if err != nil {
log.Error(fmt.Sprintf(`utils/backup.RestoreInPlace ! pg.NewPG("127.0.0.1", %s, "rdpg", "rdpg", %s) erred : %s`, globals.PBPort, globals.PGPass, err.Error()))
return err
}
exists, err := DatabaseExists(dbname)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RestoreInPlace ! utils/backup.DatabaseExists(%s) erred : %s", dbname, err.Error()))
return err
}
if exists {
err = p.DisableDatabase(dbname)
if err != nil {
log.Error(fmt.Sprintf("utils.backup.RestoreInPlace ! pg.DisableDatabase(%s) erred : %s", dbname, err.Error()))
return err
}
err = p.DropDatabase(dbname)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RestoreInPlace ! pg.DropDatabase(%s) erred : %s", dbname, err.Error()))
return err
}
} else {
errorMessage := fmt.Sprintf("utils/backup.RestoreInPlace ! Restoring database %s doesn't currently exist.", dbname)
log.Warn(errorMessage)
}
username := "******" + strings.TrimPrefix(dbname, "d")
err = p.CreateDatabase(dbname, username)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.StageRestoreInPlace ! pg.CreateDatabase(%s, %s) erred : %s", dbname, username, err.Error()))
return err
}
err = ImportSqlFile(dbname, RestoreLocation(dbname, basefile))
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RestoreInPlace ! utils/backup.ImportSqlFile(%s, %s) erred : %s", dbname, RestoreLocation(dbname, basefile), err.Error()))
return err
}
err = UnstageRestore(dbname, basefile)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RestoreInPlace ! UnstageRestore(%s, %s) erred : %s", dbname, basefile, err.Error()))
return err
}
return nil
}
/* Lists all the backup files at the desired location.
The {where} should be "local" or "remote". The "local" option finds all the backup
files on the local filesystem. The "remote" option will display all of the
backup files on the remote storage, such as S3, but this feature is not yet
implemented. Backups are returned in json format.
The request must be a POST.
Form values:
"fmt", "filename" is the only supported value at present. Defaults to "filename"
if absent or if left blank
"dbname": the name of the database for which to query backups of. Will eventually
allow being left blank to return all database backups, but not yet. */
func BackupListHandler(w http.ResponseWriter, request *http.Request) {
//Default printing format to print pretty timestamps. So pretty.
printFormat := "filename"
if request.Method == "POST" && request.FormValue("fmt") != "" {
printFormat = request.FormValue("fmt")
}
backupList := []databaseBackupList{}
// If the dbname wasn't specified of if the field is blank, then return the backups of
// all databases.
dbname := request.FormValue("dbname")
// Where are we getting the files from?
vars := mux.Vars(request)
var err error
switch vars["where"] {
case "local":
backupList, err = handleLocalListing(dbname)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
}
case "remote":
log.Warn(fmt.Sprintf(`api.BackupListHandler() Remote Backup Listing not yet supported.`))
w.WriteHeader(http.StatusNotImplemented)
w.Write([]byte("Not Yet Implemented"))
return
}
switch printFormat {
case "filename":
outputString := "{ "
var separator string
for i, d := range backupList {
if i == 0 {
separator = ""
} else {
separator = ", "
}
outputString = outputString + fmt.Sprintf("%s\"%s\": [", separator, d.Database)
for j, v := range d.Backups {
if j == 0 {
separator = ""
} else {
separator = ", "
}
outputString = outputString + fmt.Sprintf("%s{ \"Name\": \"%s\", \"Bytes\": \"%s\" }", separator, v.Name, v.Size)
}
outputString = outputString + "]"
}
outputString = outputString + "}"
w.Write([]byte(outputString))
//case "timestamp": TODO
default:
log.Debug(fmt.Sprintf(`api.BackupListHandler() Requested unsupported format.`))
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Unsupported Format Requested"))
}
}
//Sets a retention rule for this cluster for the information specified in the RetentionRuleRow struct.
func (ret *RetentionRuleRow) Put() (err error) {
p := pg.NewPG(`127.0.0.1`, globals.PGPort, `rdpg`, `rdpg`, globals.PGPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RetentionRuleRow.Put ! pg.Connect() : %s", err.Error()))
return err
}
defer db.Close()
exists, err := DatabaseExists(ret.DBName)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RetentionRuleRow.Put ! Error when trying to check if database exists: %s # %s", ret.DBName, err.Error()))
return err
}
if !exists {
errorMessage := fmt.Sprintf("utils/backup.RetentionRuleRow.Put ! Requested retention policy for non-existant database: %s", ret.DBName)
log.Warn(errorMessage)
return errors.New(errorMessage)
}
//Stringify the boolean
rulebool := "false"
if ret.IsRemoteRule {
rulebool = "true"
}
//Check if this row already exists in the database
sql := fmt.Sprintf("SELECT COUNT(*) FROM backups.retention_rules WHERE dbname='%s' AND is_remote_rule=%s", ret.DBName, rulebool)
count := []int{}
err = db.Select(&count, sql)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RetentionRuleRow.Put ! sqlx.db.Select(%s) : %s", sql, err))
return err
}
//If the row already exists, update the row.
if count[0] > 0 {
sql = fmt.Sprintf("UPDATE backups.retention_rules SET hours=%f WHERE dbname='%s' AND is_remote_rule=%s", ret.Hours, ret.DBName, rulebool)
} else { //Otherwise, insert a new row
sql = fmt.Sprintf("INSERT INTO backups.retention_rules(dbname, hours, is_remote_rule) VALUES ('%s', %f, %s)", ret.DBName, ret.Hours, rulebool)
}
//Do the thing with the query.
_, err = db.Exec(sql)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.RetentionRuleRow.Put ! sqlx.db.Select(%s) : %s", sql, err))
return err
}
return
}
func remoteHelper(backupList *[]DatabaseBackupList, input *s3.ListObjectsInput, showGlobals bool) (output *s3.ListObjectsOutput, err error) {
creds := credentials.NewStaticCredentials(rdpgs3.AWSAccessKey, rdpgs3.AWSSecretKey, rdpgs3.Token)
config := &aws.Config{
Region: &rdpgs3.AWSRegion,
Endpoint: &rdpgs3.Endpoint,
S3ForcePathStyle: &rdpgs3.S3ForcePathStyle,
Credentials: creds,
}
s3client := s3.New(config)
output, err = s3client.ListObjects(input)
if err != nil {
log.Error(fmt.Sprintf("admin.backup.handleRemoteListing ! Error when trying to get objects from s3. ! %s", err.Error()))
return
}
var thisDatabase DatabaseBackupList = DatabaseBackupList{Database: "", Backups: []DBBackup{}}
cont := output.Contents
//The Contents list is a list of s3 objects which
// has a Key value which is the full name of the file
// in the storage, including its "path"
sort.Sort(rdpgs3.ByKey(cont))
matchingString := GenerateFiletypeMatcher(showGlobals)
//For each returned content in this page from S3...
for _, c := range cont {
keySplit := strings.Split(*c.Key, "/")
if len(keySplit) < 2 {
log.Warn(fmt.Sprintf("utils/backup.RemoteListing ! S3 key improper structure: %s", *c.Key))
continue
}
filename := keySplit[len(keySplit)-1]
database := keySplit[len(keySplit)-2]
if thisDatabase.Database != database {
//Database set to empty string on first runthrough.
if thisDatabase.Database != "" && len(thisDatabase.Backups) > 0 {
*backupList = append(*backupList, thisDatabase)
}
index := FindDatabaseBackups(database, *backupList)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.remoteHelper ! %s", err))
return output, err
}
//if the database isn't in the list already...
if index < 0 {
thisDatabase = DatabaseBackupList{database, []DBBackup{}}
} else { //Add to the one already in the list
thisDatabase = (*backupList)[index]
}
}
//Make sure it's a backup file by regexping for the proper suffix.
matched, err := regexp.Match(matchingString, []byte(filename))
if err != nil {
log.Error(fmt.Sprintf(`utils/backup.RemoteListing() Error when attempting regexp: %s / %s ! %s`, matchingString, filename, err))
return output, errors.New("A regexp error occurred")
}
//The match matches on "<basename><backupFileSuffix>" e.g "asdf.sql"
if matched {
thisDatabase.Backups = append(thisDatabase.Backups, DBBackup{filename, strconv.FormatInt(*c.Size, 10)})
}
}
if len(cont) > 0 {
*backupList = append(*backupList, thisDatabase)
}
return
}
//BackupDatabase - Perform a schema and database backup of a given database to local disk
func (t *Task) BackupDatabase() (err error) {
b := backupParams{}
//Make sure database actually exists first.
b.databaseName = t.Data
if b.databaseName != "rdpg" {
address := `127.0.0.1`
sq := fmt.Sprintf(`SELECT 1 FROM cfsb.instances WHERE effective_at IS NOT NULL AND decommissioned_at IS NULL AND dbname = '%s';`, b.databaseName)
databasesWithThatName, err := rdpgpg.GetList(address, sq)
if err != nil {
log.Error(fmt.Sprintf("Tasks.BackupDatabase() utils/backup.GetList(%s, %s) Error trying to query for database.", address, b.databaseName))
return err
}
if len(databasesWithThatName) == 0 {
log.Error(fmt.Sprintf("Task.BackupDatabase() Attempt to back up non-existant or non-commissioned database with name: %s", b.databaseName))
return errors.New("Database doesn't exist.")
}
}
lockAcquired, sessID := acquireBackupLock(b.databaseName)
if !lockAcquired {
log.Warn("Aborting Backup: Unable to acquire database lock. Is another backup already in progress?")
return errors.New("Unable to acquire database lock")
}
defer releaseBackupLock(b.databaseName, sessID)
b.pgDumpPath, err = config.GetValue(`pgDumpBinaryLocation`)
if err != nil {
return err
}
b.pgPort, err = config.GetValue(`BackupPort`)
if err != nil {
return err
}
b.basePath, err = config.GetValue(`BackupsPath`)
if err != nil {
return err
}
b.node, err = rdpgconsul.GetNode()
if err != nil {
return err
}
b.baseFileName = getBaseFileName() //Use this to keep schema and data file names the same
err = createTargetFolder(b.basePath + `/` + b.databaseName)
if err != nil {
log.Error(fmt.Sprintf("tasks.BackupDatabase() Could not create target folder %s ! %s", b.basePath, err))
return err
}
schemaDataFileHistory, err := createSchemaAndDataFile(b)
if err != nil {
log.Error(fmt.Sprintf("tasks.BackupDatabase() Could not create schema and data file for database %s ! %s", b.databaseName, err))
schemaDataFileHistory.Status = `error`
}
err = history.InsertBackupHistory(schemaDataFileHistory)
if b.databaseName == `rdpg` {
globalsFileHistory, err := createGlobalsFile(b)
if err != nil {
log.Error(fmt.Sprintf("tasks.BackupDatabase() Could not create globals file for database %s ! %s", b.databaseName, err))
globalsFileHistory.Status = `error`
}
err = history.InsertBackupHistory(globalsFileHistory)
}
return
}
// Copies a backup file into the restore stage and adds it to the register for
// what is prepared to be restored. Any given database can only be staged once
// at any given time - an attempt to stage a database that is already staged
// will delete the previously staged backup and the new file will be registered
// for recovery instead.
// The data section of the task should be valid JSON containing string values
// for the keys "database_name" and "base_file_name"
func StageRestoreInPlace(dbname, filename string) (err error) {
//BDR can't automatically restore, silly!
if !globals.CanAutoRestore {
log.Warn("utils/backup.StageRestoreInPlace ! Attempt made to stage a restore, but this cluster cannot automate restores")
return errors.New("This cluster cannot perform automated restores.")
}
dbExists, err := DatabaseExists(dbname)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.StageRestoreInPlace ! backup.DatabaseExists(%s) erred : %s", dbname, err.Error()))
return err
}
if !dbExists {
errorMessage := fmt.Sprintf("utils/backup.StageRestoreInPlace ! Database doesn't exist: %s", dbname)
log.Error(errorMessage)
return errors.New(errorMessage)
}
if !strings.HasPrefix(dbname, "d") || utf8.RuneCountInString(dbname) != 33 {
log.Warn(fmt.Sprintf("utils/backup.StageRestoreInPlace ! Rejecting attempt made to restore non-user database: %s", dbname))
return errors.New("Cannot auto-restore non-user database")
}
var local, remote []DatabaseBackupList = nil, nil
//I was told to ignore .globals files... change this later if that changes.
local, err = LocalListing(dbname, false)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.StageRestoreInPlace ! backup.LocalListing(%s, false) erred : %s", dbname, err.Error()))
return err
}
if rdpgs3.Configured {
remote, err = RemoteListing(dbname, false)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.StageRestoreInPlace ! backup.RemoteListing(%s, false) erred : %s", dbname, err.Error()))
return err
}
}
all := Combine(local, remote)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.StageRestoreInPlace ! utils/backup.FullListing(%s, false) erred : %s", dbname, err.Error()))
return err
}
//If no backup is specified, use the most recent backup for this database
if dbname == "" {
//Need to check if there are any backups before defaulting to the newest backup.
if len(all) == 0 || len(all[0].Backups) == 0 {
errorMessage := fmt.Sprintf("utils/backup.StageRestoreInPlace ! No backups found for this database.")
log.Warn(errorMessage)
return errors.New(errorMessage)
}
dbname = all[0].Backups[len(all[0].Backups)-1].Name
} else if !ContainsBackup(dbname, filename, all) {
errorMessage := fmt.Sprintf("utils/backup.StageRestoreInPlace ! No backup with name '%s' found for database '%s'", filename, dbname)
log.Error(errorMessage)
return errors.New(errorMessage)
}
//Look for this backup in the local filesystem
isLocal := ContainsBackup(dbname, filename, local)
//At this point, we know that the desired backup is SOMEWHERE. So if it isn't
// local (!isLocal) then it must be in remote storage.
//If we're at this point in the code, the backup actually exists.
//Actually get the file from whereever it is.
if isLocal {
err = stageLocalInPlace(dbname, filename)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.StageRestoreInPlace ! stageLocalInPlace erred : %s", err.Error()))
return err
}
} else {
err = stageRemoteInPlace(dbname, filename)
if err != nil {
log.Error(fmt.Sprintf("utils/backup.StageRestoreInPlace ! stageRemoteInPlace erred : %s", err.Error()))
return err
}
}
return nil
}
func init() {
// Set MyIP variable
client, err := consulapi.NewClient(consulapi.DefaultConfig())
if err != nil {
log.Error(fmt.Sprintf("config.init() consulapi.NewClient()! %s", err))
} else {
agent := client.Agent()
info, err := agent.Self()
if err != nil {
log.Error(fmt.Sprintf("config.init() agent.Self()! %s", err))
} else {
MyIP = info["Config"]["AdvertiseAddr"].(string)
}
}
ClusterService = os.Getenv("RDPGD_CLUSTER_SERVICE")
//Set up the ClusterID
MatrixName := os.Getenv(`RDPGD_MATRIX`)
MatrixNameSplit := strings.SplitAfterN(MatrixName, `-`, -1)
MatrixColumn := os.Getenv(`RDPGD_MATRIX_COLUMN`)
ClusterID = os.Getenv("RDPGD_CLUSTER")
if ClusterID == "" {
for i := 0; i < len(MatrixNameSplit)-1; i++ {
ClusterID = ClusterID + MatrixNameSplit[i]
}
ClusterID = ClusterID + "c" + MatrixColumn
}
//LocalBackupPath has a prerequisite that ClusterID has already been assigned. Be careful if order switching.
LocalBackupPath = fmt.Sprintf("/var/vcap/store/pgbdr/backups/%s/%s", os.Getenv(`RDPGD_ENVIRONMENT_NAME`), ClusterID)
RestoreStagePath = `/var/vcap/store/recover/`
LocalRetentionTime, err = strconv.ParseFloat(os.Getenv(`RDPGD_LOCAL_RETENTION_TIME`), 64)
if err != nil {
log.Error(fmt.Sprintf("globals.init() ! Parsing local retention time: strconv.ParseFloat(%s, 64) : %s", os.Getenv(`RDPGD_LOCAL_RETENTION_TIME`), err))
}
RemoteRetentionTime, err = strconv.ParseFloat(os.Getenv(`RDPGD_REMOTE_RETENTION_TIME`), 64)
if err != nil {
log.Error(fmt.Sprintf("globals.init() ! Parsing remote retention time: strconv.ParseFloat(%s, 64) : %s", os.Getenv(`RDPGD_REMOTE_RETENTION_TIME`), err))
}
//CanAutoRestore requires that ClusterService is already assigned.
CanAutoRestore = (ClusterService != "pgbdr")
PBPort = os.Getenv("RDPGD_PB_PORT")
if PBPort == "" {
PBPort = "6432"
}
PGPass = os.Getenv("RDPGD_PG_PASS")
if PGPass == "" {
PGPass = "******"
}
PGPort = os.Getenv("RDPGD_PG_PORT")
if PGPort == "" {
log.Warn("RDPGD_PG_PORT environment variable was not configured")
}
StuckDuration = os.Getenv("RDPGD_STUCK_DURATION")
if StuckDuration == "" {
StuckDuration = `6 hours`
}
UserExtensions = os.Getenv("RDPGD_PG_EXTENSIONS")
}