//FindFilesToCopyToS3 - Responsible for copying files, such as database backups //to S3 storage func (t *Task) FindFilesToCopyToS3() (err error) { if err != nil { log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() Could not retrieve S3 Credentials ! %s", err)) return err } //If S3 creds/bucket aren't set just exit since they aren't configured if rdpgs3.Configured == false { log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() S3 CONFIGURATION MISSING FOR THIS DEPLOYMENT ! S3 Credentials are not configured, skipping attempt to copy until configured ")) return } //Select eligible files //Diff with empty string means get me the diff for ALL THE THINGS localDiff, _, err := backup.Diff("", true) if err != nil { log.Error(fmt.Sprintf(`tasks.Task<%d>#CopyFileToS3() Failed to load list of files ! %s`, t.ID, err)) return } numFilesToCopy := 0 for _, dbWithBackupsToCopy := range localDiff { numFilesToCopy += len(dbWithBackupsToCopy.Backups) } log.Trace(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Found %d files to copy over %d unique databases", numFilesToCopy, len(localDiff))) //Loop and add Tasks CopyFileToS3 for _, dbWithBackupsToCopy := range localDiff { for _, backupToCopy := range dbWithBackupsToCopy.Backups { //Gather the info necessary for uploading the file. fm := S3FileMetadata{} fm.Location = backup.Location(dbWithBackupsToCopy.Database, backupToCopy.Name) fm.DBName = dbWithBackupsToCopy.Database fm.Node = globals.MyIP fm.ClusterID = globals.ClusterID //JSONify that info fileToCopyParams, err := json.Marshal(fm) if err != nil { log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Error attempting to marshal some JSON ! %+v %s", fm, err)) return err } log.Trace(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Attempting to add %s", fileToCopyParams)) //Insert the task newTask := Task{ClusterID: t.ClusterID, Node: t.Node, Role: t.Role, Action: "CopyFileToS3", Data: string(fileToCopyParams), TTL: t.TTL, NodeType: t.NodeType} err = newTask.Enqueue() if err != nil { log.Error(fmt.Sprintf(`tasks.FindFilesToCopyToS3() service task schedules ! %s`, err)) } } } return }
func RemoteCopyHandler(w http.ResponseWriter, request *http.Request) { dbname := request.FormValue("dbname") filename := request.FormValue("filename") //Can't copy to s3 if there's no s3. if !rdpgs3.Configured { w.WriteHeader(http.StatusGone) w.Write([]byte("Remote storage has not been configured")) return } if dbname == "" && filename != "" { //A backup for no database doesn't make any sense w.WriteHeader(http.StatusBadRequest) w.Write([]byte("Cannot specify filename without database")) return } //Select eligible files filesToCopy, _, err := backup.Diff(dbname, true) if err != nil { log.Error(fmt.Sprintf(`api.CopyFileHelper() ! utils/backup.Diff(\"%s\", true) erred : %s`, dbname, err.Error())) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("Error getting file information")) return } //Determine this node type nType := "read" if rdpgconsul.IsWriteNode(globals.MyIP) { nType = "write" } numFiles := 0 for _, dbWithBackupsToCopy := range filesToCopy { for _, backupToCopy := range dbWithBackupsToCopy.Backups { if filename != "" && backupToCopy.Name != filename { continue } //Gather the info necessary for uploading the file. fm := tasks.S3FileMetadata{} fm.Location = backup.Location(dbWithBackupsToCopy.Database, backupToCopy.Name) fm.DBName = dbWithBackupsToCopy.Database fm.Node = globals.MyIP fm.ClusterID = globals.ClusterID //JSONify that info fileToCopyParams, err := json.Marshal(fm) if err != nil { log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Error attempting t)o marshal some JSON ! %+v %s", fm, err)) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("An error occurred when marshalling JSON")) return } log.Trace(fmt.Sprintf("api.CopyFileHelper > Attempting to copy %s", fileToCopyParams)) //Insert the task newTask := tasks.Task{ ClusterID: globals.ClusterID, Node: globals.MyIP, Role: globals.ServiceRole, Action: "CopyFileToS3", Data: string(fileToCopyParams), TTL: 3600, NodeType: nType, } err = newTask.CopyFileToS3() if err != nil { log.Error(fmt.Sprintf(`api.CopyFileHandler ! task.CopyFileToS3 erred : %s`, err.Error())) w.WriteHeader(http.StatusInternalServerError) w.Write([]byte("An error occurred when copying files to remote storage")) return } numFiles++ } } w.Write([]byte(fmt.Sprintf("%d files were written to S3", numFiles))) }
func BackupDiffHandler(w http.ResponseWriter, request *http.Request) { //Default printing format to print pretty timestamps. So pretty. printFormat := "filename" if request.Method == "POST" && request.FormValue("fmt") != "" { printFormat = request.FormValue("fmt") } showGlobals := false if request.FormValue("globals") == "true" { showGlobals = true } // If the dbname wasn't specified of if the field is blank, then return the backups of // all databases. dbname := request.FormValue("dbname") // Where are we getting the files from? vars := mux.Vars(request) //Do we actually have s3? rdpgs3.ReinitializeS3Credentials() if !rdpgs3.Configured { w.WriteHeader(http.StatusGone) w.Write([]byte("Remote storage has not been configured")) } var localDiff, remoteDiff []backup.DatabaseBackupList var err error //The "both" path uses a different function if vars["where"] != "both" { localDiff, remoteDiff, err = backup.Diff(dbname, showGlobals) if err != nil { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(err.Error())) return } } //====Begin inner function formatter := func(localList []backup.DatabaseBackupList, remoteList []backup.DatabaseBackupList, printFormat string) (outputString string, err error) { if localList == nil && remoteList == nil { errorMessage := fmt.Sprintf("api.BackupDiffHandler ! Can't handle formatting without input") log.Error(errorMessage) return "", errors.New(errorMessage) } switch printFormat { case "filename": if localList == nil { // only/remote outputString = formatFilename(remoteList) } else if remoteList == nil { //only/local outputString = formatFilename(localList) } else { //only/both outputString = `{ "local": ` + formatFilename(localList) + `, "remote": ` + formatFilename(remoteList) + ` }` } default: log.Debug(fmt.Sprintf(`api.BackupDiffHandler() Requested unsupported format.`)) return "", errors.New(fmt.Sprintf("Unsupported Printing Format: %s. Valid format is 'filename'", printFormat)) } return } //====End inner function var output string switch vars["where"] { case "local": output, err = formatter(localDiff, nil, printFormat) case "remote": output, err = formatter(nil, remoteDiff, printFormat) case "diff": output, err = formatter(localDiff, remoteDiff, printFormat) case "both": bothList, err := backup.Both(dbname, false) if err != nil { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(err.Error())) return } output, err = formatter(bothList, nil, printFormat) default: w.WriteHeader(http.StatusNotFound) } if err != nil { w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } w.Header().Set("Content-Type", "application/json") w.Write([]byte(output)) }