/*EnforceFileRetention - Responsible for adding removing files which are no longer
needed on the local file system. For example, backup files which have been created
successfully locally and copied to S3 successfully can be deleted to preserve
local disk storage */
func (t *Task) EnforceFileRetention() (err error) {
/*
If s3 copy is enabled you cannot delete files until they have been copied to s3
otherwise keep the most recent backups, say the last 48 hours worth and delete all others
*/
//Select eligible files
var eligibleBackups []backup.DatabaseBackupList = nil
//Get the list of backups that exist locally...
var localList []backup.DatabaseBackupList
//If S3 backups are enabled, then don't delete a local file unless it has been backed up already
if isS3FileCopyEnabled() {
//Both gets backups that are both in the local filesystem and the remote storage
localList, err = backup.Both("", true)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task<%d>#EnforceFileRetention() ! utils/backup#Both() : %s", t.ID, err.Error()))
return err
}
} else {
localList, err = backup.LocalListing("", true)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task<%d>#EnforceFileRetention() ! utils/backup.LocalListing() : %s", t.ID, err.Error()))
return err
}
}
eligibleBackups, err = findExpiredFiles(localList, false)
if err != nil {
log.Error(fmt.Sprintf("tasks.EnforceFileRetention() ! tasks.findExpiredFiles() : %s", err.Error()))
return err
}
numFilesToDelete := 0
for _, v := range eligibleBackups {
numFilesToDelete += len(v.Backups)
}
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task<%d>#EnforceFileRetention() Failed to load list of files ! %s`, t.ID, err))
return err
}
log.Trace(fmt.Sprintf("tasks.EnforceFileRetention() > Found %d files to delete", numFilesToDelete))
for _, eligibleDatabase := range eligibleBackups {
for _, backupToDelete := range eligibleDatabase.Backups {
fm := S3FileMetadata{
Location: backup.Location(eligibleDatabase.Database, backupToDelete.Name),
DBName: backupToDelete.Name,
Node: globals.MyIP,
ClusterID: globals.ClusterID,
}
byteParams, err := json.Marshal(fm)
if err != nil {
log.Error(fmt.Sprintf("tasks.EnforceFileRetention() > Error attempting to marshal some JSON ! %+v %s", fm, err))
return err
}
fileToDeleteParams := string(byteParams)
log.Trace(fmt.Sprintf("tasks.EnforceFileRetention() > Attempting to add %s", fileToDeleteParams))
newTask := Task{ClusterID: t.ClusterID, Node: t.Node, Role: t.Role, Action: "DeleteFile", Data: fileToDeleteParams, TTL: t.TTL, NodeType: t.NodeType}
err = newTask.Enqueue()
if err != nil {
log.Error(fmt.Sprintf(`tasks.EnforceFileRetention() service task schedules ! %s`, err))
}
}
}
return
}
func BackupDiffHandler(w http.ResponseWriter, request *http.Request) {
//Default printing format to print pretty timestamps. So pretty.
printFormat := "filename"
if request.Method == "POST" && request.FormValue("fmt") != "" {
printFormat = request.FormValue("fmt")
}
showGlobals := false
if request.FormValue("globals") == "true" {
showGlobals = true
}
// If the dbname wasn't specified of if the field is blank, then return the backups of
// all databases.
dbname := request.FormValue("dbname")
// Where are we getting the files from?
vars := mux.Vars(request)
//Do we actually have s3?
rdpgs3.ReinitializeS3Credentials()
if !rdpgs3.Configured {
w.WriteHeader(http.StatusGone)
w.Write([]byte("Remote storage has not been configured"))
}
var localDiff, remoteDiff []backup.DatabaseBackupList
var err error
//The "both" path uses a different function
if vars["where"] != "both" {
localDiff, remoteDiff, err = backup.Diff(dbname, showGlobals)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
}
//====Begin inner function
formatter := func(localList []backup.DatabaseBackupList, remoteList []backup.DatabaseBackupList, printFormat string) (outputString string, err error) {
if localList == nil && remoteList == nil {
errorMessage := fmt.Sprintf("api.BackupDiffHandler ! Can't handle formatting without input")
log.Error(errorMessage)
return "", errors.New(errorMessage)
}
switch printFormat {
case "filename":
if localList == nil {
// only/remote
outputString = formatFilename(remoteList)
} else if remoteList == nil {
//only/local
outputString = formatFilename(localList)
} else {
//only/both
outputString = `{ "local": ` + formatFilename(localList) + `, "remote": ` + formatFilename(remoteList) + ` }`
}
default:
log.Debug(fmt.Sprintf(`api.BackupDiffHandler() Requested unsupported format.`))
return "", errors.New(fmt.Sprintf("Unsupported Printing Format: %s. Valid format is 'filename'", printFormat))
}
return
}
//====End inner function
var output string
switch vars["where"] {
case "local":
output, err = formatter(localDiff, nil, printFormat)
case "remote":
output, err = formatter(nil, remoteDiff, printFormat)
case "diff":
output, err = formatter(localDiff, remoteDiff, printFormat)
case "both":
bothList, err := backup.Both(dbname, false)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
output, err = formatter(bothList, nil, printFormat)
default:
w.WriteHeader(http.StatusNotFound)
}
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(output))
}