//FindFilesToCopyToS3 - Responsible for copying files, such as database backups
//to S3 storage
func (t *Task) FindFilesToCopyToS3() (err error) {
if err != nil {
log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() Could not retrieve S3 Credentials ! %s", err))
return err
}
//If S3 creds/bucket aren't set just exit since they aren't configured
if rdpgs3.Configured == false {
log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() S3 CONFIGURATION MISSING FOR THIS DEPLOYMENT ! S3 Credentials are not configured, skipping attempt to copy until configured "))
return
}
//Select eligible files
//Diff with empty string means get me the diff for ALL THE THINGS
localDiff, _, err := backup.Diff("", true)
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task<%d>#CopyFileToS3() Failed to load list of files ! %s`, t.ID, err))
return
}
numFilesToCopy := 0
for _, dbWithBackupsToCopy := range localDiff {
numFilesToCopy += len(dbWithBackupsToCopy.Backups)
}
log.Trace(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Found %d files to copy over %d unique databases", numFilesToCopy, len(localDiff)))
//Loop and add Tasks CopyFileToS3
for _, dbWithBackupsToCopy := range localDiff {
for _, backupToCopy := range dbWithBackupsToCopy.Backups {
//Gather the info necessary for uploading the file.
fm := S3FileMetadata{}
fm.Location = backup.Location(dbWithBackupsToCopy.Database, backupToCopy.Name)
fm.DBName = dbWithBackupsToCopy.Database
fm.Node = globals.MyIP
fm.ClusterID = globals.ClusterID
//JSONify that info
fileToCopyParams, err := json.Marshal(fm)
if err != nil {
log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Error attempting to marshal some JSON ! %+v %s", fm, err))
return err
}
log.Trace(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Attempting to add %s", fileToCopyParams))
//Insert the task
newTask := Task{ClusterID: t.ClusterID, Node: t.Node, Role: t.Role, Action: "CopyFileToS3", Data: string(fileToCopyParams), TTL: t.TTL, NodeType: t.NodeType}
err = newTask.Enqueue()
if err != nil {
log.Error(fmt.Sprintf(`tasks.FindFilesToCopyToS3() service task schedules ! %s`, err))
}
}
}
return
}
func RemoteCopyHandler(w http.ResponseWriter, request *http.Request) {
dbname := request.FormValue("dbname")
filename := request.FormValue("filename")
//Can't copy to s3 if there's no s3.
if !rdpgs3.Configured {
w.WriteHeader(http.StatusGone)
w.Write([]byte("Remote storage has not been configured"))
return
}
if dbname == "" && filename != "" {
//A backup for no database doesn't make any sense
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Cannot specify filename without database"))
return
}
//Select eligible files
filesToCopy, _, err := backup.Diff(dbname, true)
if err != nil {
log.Error(fmt.Sprintf(`api.CopyFileHelper() ! utils/backup.Diff(\"%s\", true) erred : %s`, dbname, err.Error()))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Error getting file information"))
return
}
//Determine this node type
nType := "read"
if rdpgconsul.IsWriteNode(globals.MyIP) {
nType = "write"
}
numFiles := 0
for _, dbWithBackupsToCopy := range filesToCopy {
for _, backupToCopy := range dbWithBackupsToCopy.Backups {
if filename != "" && backupToCopy.Name != filename {
continue
}
//Gather the info necessary for uploading the file.
fm := tasks.S3FileMetadata{}
fm.Location = backup.Location(dbWithBackupsToCopy.Database, backupToCopy.Name)
fm.DBName = dbWithBackupsToCopy.Database
fm.Node = globals.MyIP
fm.ClusterID = globals.ClusterID
//JSONify that info
fileToCopyParams, err := json.Marshal(fm)
if err != nil {
log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Error attempting t)o marshal some JSON ! %+v %s", fm, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("An error occurred when marshalling JSON"))
return
}
log.Trace(fmt.Sprintf("api.CopyFileHelper > Attempting to copy %s", fileToCopyParams))
//Insert the task
newTask := tasks.Task{
ClusterID: globals.ClusterID,
Node: globals.MyIP,
Role: globals.ServiceRole,
Action: "CopyFileToS3",
Data: string(fileToCopyParams),
TTL: 3600,
NodeType: nType,
}
err = newTask.CopyFileToS3()
if err != nil {
log.Error(fmt.Sprintf(`api.CopyFileHandler ! task.CopyFileToS3 erred : %s`, err.Error()))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("An error occurred when copying files to remote storage"))
return
}
numFiles++
}
}
w.Write([]byte(fmt.Sprintf("%d files were written to S3", numFiles)))
}
func BackupDiffHandler(w http.ResponseWriter, request *http.Request) {
//Default printing format to print pretty timestamps. So pretty.
printFormat := "filename"
if request.Method == "POST" && request.FormValue("fmt") != "" {
printFormat = request.FormValue("fmt")
}
showGlobals := false
if request.FormValue("globals") == "true" {
showGlobals = true
}
// If the dbname wasn't specified of if the field is blank, then return the backups of
// all databases.
dbname := request.FormValue("dbname")
// Where are we getting the files from?
vars := mux.Vars(request)
//Do we actually have s3?
rdpgs3.ReinitializeS3Credentials()
if !rdpgs3.Configured {
w.WriteHeader(http.StatusGone)
w.Write([]byte("Remote storage has not been configured"))
}
var localDiff, remoteDiff []backup.DatabaseBackupList
var err error
//The "both" path uses a different function
if vars["where"] != "both" {
localDiff, remoteDiff, err = backup.Diff(dbname, showGlobals)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
}
//====Begin inner function
formatter := func(localList []backup.DatabaseBackupList, remoteList []backup.DatabaseBackupList, printFormat string) (outputString string, err error) {
if localList == nil && remoteList == nil {
errorMessage := fmt.Sprintf("api.BackupDiffHandler ! Can't handle formatting without input")
log.Error(errorMessage)
return "", errors.New(errorMessage)
}
switch printFormat {
case "filename":
if localList == nil {
// only/remote
outputString = formatFilename(remoteList)
} else if remoteList == nil {
//only/local
outputString = formatFilename(localList)
} else {
//only/both
outputString = `{ "local": ` + formatFilename(localList) + `, "remote": ` + formatFilename(remoteList) + ` }`
}
default:
log.Debug(fmt.Sprintf(`api.BackupDiffHandler() Requested unsupported format.`))
return "", errors.New(fmt.Sprintf("Unsupported Printing Format: %s. Valid format is 'filename'", printFormat))
}
return
}
//====End inner function
var output string
switch vars["where"] {
case "local":
output, err = formatter(localDiff, nil, printFormat)
case "remote":
output, err = formatter(nil, remoteDiff, printFormat)
case "diff":
output, err = formatter(localDiff, remoteDiff, printFormat)
case "both":
bothList, err := backup.Both(dbname, false)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte(err.Error()))
return
}
output, err = formatter(bothList, nil, printFormat)
default:
w.WriteHeader(http.StatusNotFound)
}
if err != nil {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
w.Header().Set("Content-Type", "application/json")
w.Write([]byte(output))
}