예제 #1
0
//FindFilesToCopyToS3 - Responsible for copying files, such as database backups
//to S3 storage
func (t *Task) FindFilesToCopyToS3() (err error) {
	if err != nil {
		log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() Could not retrieve S3 Credentials ! %s", err))
		return err
	}

	//If S3 creds/bucket aren't set just exit since they aren't configured
	if rdpgs3.Configured == false {
		log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() S3 CONFIGURATION MISSING FOR THIS DEPLOYMENT ! S3 Credentials are not configured, skipping attempt to copy until configured "))
		return
	}

	//Select eligible files
	//Diff with empty string means get me the diff for ALL THE THINGS
	localDiff, _, err := backup.Diff("", true)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.Task<%d>#CopyFileToS3() Failed to load list of files ! %s`, t.ID, err))
		return
	}
	numFilesToCopy := 0
	for _, dbWithBackupsToCopy := range localDiff {
		numFilesToCopy += len(dbWithBackupsToCopy.Backups)
	}

	log.Trace(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Found %d files to copy over %d unique databases", numFilesToCopy, len(localDiff)))

	//Loop and add Tasks CopyFileToS3
	for _, dbWithBackupsToCopy := range localDiff {
		for _, backupToCopy := range dbWithBackupsToCopy.Backups {
			//Gather the info necessary for uploading the file.
			fm := S3FileMetadata{}
			fm.Location = backup.Location(dbWithBackupsToCopy.Database, backupToCopy.Name)
			fm.DBName = dbWithBackupsToCopy.Database
			fm.Node = globals.MyIP
			fm.ClusterID = globals.ClusterID
			//JSONify that info
			fileToCopyParams, err := json.Marshal(fm)
			if err != nil {
				log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Error attempting to marshal some JSON ! %+v %s", fm, err))
				return err
			}
			log.Trace(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Attempting to add %s", fileToCopyParams))
			//Insert the task
			newTask := Task{ClusterID: t.ClusterID, Node: t.Node, Role: t.Role, Action: "CopyFileToS3", Data: string(fileToCopyParams), TTL: t.TTL, NodeType: t.NodeType}
			err = newTask.Enqueue()
			if err != nil {
				log.Error(fmt.Sprintf(`tasks.FindFilesToCopyToS3() service task schedules ! %s`, err))
			}
		}

	}
	return

}
예제 #2
0
func RemoteCopyHandler(w http.ResponseWriter, request *http.Request) {
	dbname := request.FormValue("dbname")
	filename := request.FormValue("filename")
	//Can't copy to s3 if there's no s3.
	if !rdpgs3.Configured {
		w.WriteHeader(http.StatusGone)
		w.Write([]byte("Remote storage has not been configured"))
		return
	}

	if dbname == "" && filename != "" {
		//A backup for no database doesn't make any sense
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Cannot specify filename without database"))
		return
	}

	//Select eligible files
	filesToCopy, _, err := backup.Diff(dbname, true)
	if err != nil {
		log.Error(fmt.Sprintf(`api.CopyFileHelper() ! utils/backup.Diff(\"%s\", true) erred : %s`, dbname, err.Error()))
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("Error getting file information"))
		return
	}
	//Determine this node type
	nType := "read"
	if rdpgconsul.IsWriteNode(globals.MyIP) {
		nType = "write"
	}

	numFiles := 0
	for _, dbWithBackupsToCopy := range filesToCopy {
		for _, backupToCopy := range dbWithBackupsToCopy.Backups {
			if filename != "" && backupToCopy.Name != filename {
				continue
			}
			//Gather the info necessary for uploading the file.
			fm := tasks.S3FileMetadata{}
			fm.Location = backup.Location(dbWithBackupsToCopy.Database, backupToCopy.Name)
			fm.DBName = dbWithBackupsToCopy.Database
			fm.Node = globals.MyIP
			fm.ClusterID = globals.ClusterID
			//JSONify that info
			fileToCopyParams, err := json.Marshal(fm)
			if err != nil {
				log.Error(fmt.Sprintf("tasks.FindFilesToCopyToS3() > Error attempting t)o marshal some JSON ! %+v %s", fm, err))
				w.WriteHeader(http.StatusInternalServerError)
				w.Write([]byte("An error occurred when marshalling JSON"))
				return
			}
			log.Trace(fmt.Sprintf("api.CopyFileHelper > Attempting to copy %s", fileToCopyParams))
			//Insert the task
			newTask := tasks.Task{
				ClusterID: globals.ClusterID,
				Node:      globals.MyIP,
				Role:      globals.ServiceRole,
				Action:    "CopyFileToS3",
				Data:      string(fileToCopyParams),
				TTL:       3600,
				NodeType:  nType,
			}
			err = newTask.CopyFileToS3()
			if err != nil {
				log.Error(fmt.Sprintf(`api.CopyFileHandler ! task.CopyFileToS3 erred : %s`, err.Error()))
				w.WriteHeader(http.StatusInternalServerError)
				w.Write([]byte("An error occurred when copying files to remote storage"))
				return
			}
			numFiles++
		}
	}

	w.Write([]byte(fmt.Sprintf("%d files were written to S3", numFiles)))

}
예제 #3
0
func BackupDiffHandler(w http.ResponseWriter, request *http.Request) {
	//Default printing format to print pretty timestamps. So pretty.
	printFormat := "filename"
	if request.Method == "POST" && request.FormValue("fmt") != "" {
		printFormat = request.FormValue("fmt")
	}
	showGlobals := false
	if request.FormValue("globals") == "true" {
		showGlobals = true
	}

	// If the dbname wasn't specified of if the field is blank, then return the backups of
	// all databases.
	dbname := request.FormValue("dbname")
	// Where are we getting the files from?
	vars := mux.Vars(request)

	//Do we actually have s3?

	rdpgs3.ReinitializeS3Credentials()
	if !rdpgs3.Configured {
		w.WriteHeader(http.StatusGone)
		w.Write([]byte("Remote storage has not been configured"))
	}

	var localDiff, remoteDiff []backup.DatabaseBackupList
	var err error
	//The "both" path uses a different function
	if vars["where"] != "both" {
		localDiff, remoteDiff, err = backup.Diff(dbname, showGlobals)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}
	}

	//====Begin inner function
	formatter := func(localList []backup.DatabaseBackupList, remoteList []backup.DatabaseBackupList, printFormat string) (outputString string, err error) {
		if localList == nil && remoteList == nil {
			errorMessage := fmt.Sprintf("api.BackupDiffHandler ! Can't handle formatting without input")
			log.Error(errorMessage)
			return "", errors.New(errorMessage)
		}
		switch printFormat {
		case "filename":
			if localList == nil {
				// only/remote
				outputString = formatFilename(remoteList)
			} else if remoteList == nil {
				//only/local
				outputString = formatFilename(localList)
			} else {
				//only/both
				outputString = `{ "local": ` + formatFilename(localList) + `, "remote": ` + formatFilename(remoteList) + ` }`
			}
		default:
			log.Debug(fmt.Sprintf(`api.BackupDiffHandler() Requested unsupported format.`))
			return "", errors.New(fmt.Sprintf("Unsupported Printing Format: %s. Valid format is 'filename'", printFormat))
		}
		return
	}
	//====End inner function

	var output string
	switch vars["where"] {
	case "local":
		output, err = formatter(localDiff, nil, printFormat)
	case "remote":
		output, err = formatter(nil, remoteDiff, printFormat)
	case "diff":
		output, err = formatter(localDiff, remoteDiff, printFormat)
	case "both":
		bothList, err := backup.Both(dbname, false)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte(err.Error()))
			return
		}
		output, err = formatter(bothList, nil, printFormat)
	default:
		w.WriteHeader(http.StatusNotFound)
	}
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(err.Error()))
		return
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write([]byte(output))
}