// AdminStoppg stops a postgres database on a given container
func AdminStoppg(w rest.ResponseWriter, r *rest.Request) {
	dbConn, err := util.GetConnection(CLUSTERADMIN_DB)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), 400)
		return

	}
	defer dbConn.Close()

	err = secimpl.Authorize(dbConn, r.PathParam("Token"), "perm-cluster")
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusUnauthorized)
		return
	}

	//logit.Info.Println("AdminStoppg:called")
	ID := r.PathParam("ID")
	if ID == "" {
		logit.Error.Println("AdminStoppg:ID not found error")
		rest.Error(w, "node ID required", http.StatusBadRequest)
		return
	}

	var dbNode types.Container
	dbNode, err = admindb.GetContainer(dbConn, ID)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	//logit.Info.Println("AdminStoppg: in stop with dbnode")

	if dbNode.Role == "pgpool" {
		var stoppoolResp cpmcontainerapi.StopPgpoolResponse
		stoppoolResp, err = cpmcontainerapi.StopPgpoolClient(dbNode.Name)
		logit.Info.Println("AdminStoppg:" + stoppoolResp.Output)
	} else {
		var stoppgResp cpmcontainerapi.StopPGResponse
		stoppgResp, err = cpmcontainerapi.StopPGClient(dbNode.Name)
		logit.Info.Println("AdminStoppg:" + stoppgResp.Output)
	}
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.WriteHeader(http.StatusOK)
	status := types.SimpleStatus{}
	status.Status = "OK"
	w.WriteJson(&status)
}
// AdminFailover causes a cluster failorver to be performed for a given cluster
func AdminFailover(w rest.ResponseWriter, r *rest.Request) {
	dbConn, err := util.GetConnection(CLUSTERADMIN_DB)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), 400)
		return

	}
	defer dbConn.Close()
	err = secimpl.Authorize(dbConn, r.PathParam("Token"), "perm-cluster")
	if err != nil {
		logit.Error.Println("authorize error " + err.Error())
		rest.Error(w, err.Error(), http.StatusUnauthorized)
		return
	}
	ID := r.PathParam("ID")
	if ID == "" {
		logit.Error.Println("node ID required error")
		rest.Error(w, "node ID required", http.StatusBadRequest)
		return
	}

	//dbNode is the standby node we are going to fail over and
	//make the new master in the cluster
	var dbNode types.Container
	dbNode, err = admindb.GetContainer(dbConn, ID)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	cluster, err := admindb.GetCluster(dbConn, dbNode.ClusterID)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	var failoverResp cpmcontainerapi.FailoverResponse
	failoverResp, err = cpmcontainerapi.FailoverClient(dbNode.Name)
	if err != nil {
		logit.Error.Println("fail-over error " + err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	logit.Info.Println("AdminFailover: fail-over output " + failoverResp.Output)

	//update the old master to standalone role
	oldMaster := types.Container{}
	oldMaster, err = admindb.GetContainerMaster(dbConn, dbNode.ClusterID)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	oldMaster.Role = "standalone"
	oldMaster.ClusterID = "-1"
	err = admindb.UpdateContainer(dbConn, oldMaster)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	//update the failover node to master role
	dbNode.Role = "master"
	err = admindb.UpdateContainer(dbConn, dbNode)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	//stop pg on the old master
	//params.IPAddress1 = oldMaster.IPAddress
	var stopPGResp cpmcontainerapi.StopPGResponse
	stopPGResp, err = cpmcontainerapi.StopPGClient(oldMaster.Name)
	if err != nil {
		logit.Error.Println(err.Error() + stopPGResp.Output)
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	err = configureCluster("SM", dbConn, cluster, false)
	if err != nil {
		logit.Error.Println(err.Error())
		rest.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	w.WriteHeader(http.StatusOK)
	status := types.SimpleStatus{}
	status.Status = "OK"
	w.WriteJson(&status)

	return
}
func configureCluster(profile string, dbConn *sql.DB, cluster types.Cluster, autocluster bool) error {
	logit.Info.Println("configureCluster:GetCluster")

	//get master node for this cluster
	master, err := admindb.GetContainerMaster(dbConn, cluster.ID)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	var pgport types.Setting
	pgport, err = admindb.GetSetting(dbConn, "PG-PORT")
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	var sleepSetting types.Setting
	sleepSetting, err = admindb.GetSetting(dbConn, "SLEEP-PROV")
	if err != nil {
		logit.Error.Println("configureCluster:" + err.Error())
		return err
	}

	//logit.Info.Println("configureCluster:GetContainerMaster")

	//configure master postgresql.conf file
	var data string
	info := new(template.PostgresqlParameters)
	info.PG_PORT = pgport.Value
	err = template.GetTuningParms(dbConn, profile, info)
	if err != nil {
		logit.Error.Println("configureCluster:" + err.Error())
		return err
	}

	if cluster.ClusterType == "synchronous" {
		info.CLUSTER_TYPE = "*"
		data, err = template.Postgresql("master", info)
	} else {
		//TODO verify these next 2 lines look erroneous
		info.CLUSTER_TYPE = "*"
		data, err = template.Postgresql("master", info)

		//info.CLUSTER_TYPE = ""
		//data, err = template.Postgresql("master", info)
	}
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master postgresql.conf generated")

	//write master postgresql.conf file remotely
	_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master postgresql.conf copied to remote")

	//get domain name
	var domainname types.Setting
	domainname, err = admindb.GetSetting(dbConn, "DOMAIN-NAME")
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	//configure master pg_hba.conf file
	rules := make([]template.Rule, 0)
	data, err = template.Hba(dbConn, "master", master.Name, pgport.Value, cluster.ID, domainname.Value, rules)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master pg_hba.conf generated")

	//write master pg_hba.conf file remotely
	_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master pg_hba.conf copied remotely")

	//restart postgres after the config file changes
	var stopResp cpmcontainerapi.StopPGResponse
	stopResp, err = cpmcontainerapi.StopPGClient(master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster: master stoppg output was" + stopResp.Output)

	var startResp cpmcontainerapi.StartPGResponse
	startResp, err = cpmcontainerapi.StartPGClient(master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:master startpg output was" + startResp.Output)

	//sleep loop until the master's PG can respond
	var sleepTime time.Duration
	sleepTime, err = time.ParseDuration(sleepSetting.Value)
	var found = false
	var currentStatus string
	var masterhost = master.Name
	for i := 0; i < 20; i++ {
		//currentStatus, err = GetPGStatus2(dbConn, master.Name, masterhost)
		currentStatus, err = util.FastPing(pgport.Value, master.Name)
		if currentStatus == "RUNNING" {
			logit.Info.Println("master is running...continuing")
			found = true
			break
		} else {
			logit.Info.Println("sleeping waiting on master..")
			time.Sleep(sleepTime)
		}
	}
	if !found {
		logit.Info.Println("configureCluster: timed out waiting on master pg to start")
		return errors.New("timeout waiting for master pg to respond")
	}

	standbynodes, err2 := admindb.GetAllStandbyContainers(dbConn, cluster.ID)
	if err2 != nil {
		logit.Error.Println(err.Error())
		return err
	}
	//configure all standby nodes
	var stopPGResp cpmcontainerapi.StopPGResponse
	i := 0
	for i = range standbynodes {
		if standbynodes[i].Role == STANDBY {

			//stop standby
			if !autocluster {
				stopPGResp, err = cpmcontainerapi.StopPGClient(standbynodes[i].Name)
				if err != nil {
					logit.Error.Println(err.Error())
					logit.Error.Println("configureCluster:stop output was" + stopPGResp.Output)
					return err
				}
				//logit.Info.Println("configureCluster:stop output was" + stopPGResp.Output)
			}

			//
			var username = "******"
			var password = "******"

			//create base backup from master
			var backupresp cpmcontainerapi.BasebackupResponse
			backupresp, err = cpmcontainerapi.BasebackupClient(masterhost+"."+domainname.Value, standbynodes[i].Name, username, password)
			if err != nil {
				logit.Error.Println(err.Error())
				logit.Error.Println("configureCluster:basebackup output was" + backupresp.Output)
				return err
			}
			//logit.Info.Println("configureCluster:basebackup output was" + backupresp.Output)

			data, err = template.Recovery(masterhost, pgport.Value, "postgres")
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby recovery.conf generated")

			//write standby recovery.conf file remotely
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/recovery.conf", data, standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby recovery.conf copied remotely")

			info := new(template.PostgresqlParameters)
			info.PG_PORT = pgport.Value
			info.CLUSTER_TYPE = ""
			err = template.GetTuningParms(dbConn, profile, info)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}

			data, err = template.Postgresql(STANDBY, info)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}

			//write standby postgresql.conf file remotely
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby postgresql.conf copied remotely")

			//configure standby pg_hba.conf file
			data, err = template.Hba(dbConn, STANDBY, standbynodes[i].Name, pgport.Value, cluster.ID, domainname.Value, rules)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}

			logit.Info.Println("configureCluster:standby pg_hba.conf generated")

			//write standby pg_hba.conf file remotely
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby pg_hba.conf copied remotely")

			//start standby

			var stResp cpmcontainerapi.StartPGOnStandbyResponse
			stResp, err = cpmcontainerapi.StartPGOnStandbyClient(standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby startpg output was" + stResp.Output)
		}
		i++
	}

	logit.Info.Println("configureCluster: sleeping 5 seconds before configuring pgpool...")
	clustersleepTime, _ := time.ParseDuration("5s")
	time.Sleep(clustersleepTime)

	pgpoolNode, err4 := admindb.GetContainerPgpool(dbConn, cluster.ID)
	//logit.Info.Println("configureCluster: lookup pgpool node")
	if err4 != nil {
		logit.Error.Println(err.Error())
		return err
	}
	//logit.Info.Println("configureCluster:" + pgpoolNode.Name)

	//configure the pgpool includes all standby nodes AND the master node
	poolnames := make([]string, len(standbynodes)+1)

	i = 0
	for i = range standbynodes {
		poolnames[i] = standbynodes[i].Name + "." + domainname.Value
		i++
	}
	poolnames[i] = master.Name + "." + domainname.Value

	//generate pgpool.conf HOST_LIST
	data, err = template.Poolconf(poolnames)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:pgpool pgpool.conf generated")

	//write pgpool.conf to remote pool node
	_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pgpool.conf", data, pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:pgpool pgpool.conf copied remotely")

	//generate pool_passwd
	data, err = template.Poolpasswd()
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:pgpool pool_passwd generated")

	//write pgpool.conf to remote pool node
	_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_passwd", data, pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:pgpool pool_passwd copied remotely")

	//g enerate pool_hba.conf
	cars := make([]template.Rule, 0)
	data, err = template.Poolhba(cars)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:pgpool pool_hba generated")

	//write pgpool.conf to remote pool node
	_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_hba.conf", data, pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:pgpool pool_hba copied remotely")

	//start pgpool
	var startPoolResp cpmcontainerapi.StartPgpoolResponse
	startPoolResp, err = cpmcontainerapi.StartPgpoolClient(pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster: pgpool startpgpool output was" + startPoolResp.Output)

	//finally, update the cluster to show that it is
	//initialized!
	cluster.Status = "initialized"
	err = admindb.UpdateCluster(dbConn, cluster)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	return nil

}
Exemplo n.º 4
0
func performConfigUpdate(dbConn *sql.DB, ContainerID string) error {
	logit.Info.Println("performConfigUpdate....")

	cars, err := GetAllContainerAccessRule(dbConn, ContainerID)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	container, err := admindb.GetContainer(dbConn, ContainerID)
	if err != nil {
		logit.Error.Println("GetNode: " + err.Error())
		return err
	}

	var currentStatus string
	//currentStatus, err = GetPGStatus2(dbConn, container.Name, container.Name)
	var pgport types.Setting
	pgport, err = admindb.GetSetting(dbConn, "PG-PORT")
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	currentStatus, err = util.FastPing(pgport.Value, container.Name)
	if err != nil {
		logit.Error.Println("GetNode:" + err.Error())
		return err
	}

	if currentStatus != "RUNNING" {
		logit.Info.Println("performConfigUpdate....starting postgres")
		if container.Role == "pgpool" {
			var spgresp cpmcontainerapi.StartPgpoolResponse
			spgresp, err = cpmcontainerapi.StartPgpoolClient(container.Name)
			logit.Info.Println("AdminStartpg:" + spgresp.Output)
		} else {
			var srep cpmcontainerapi.StartPGResponse
			srep, err = cpmcontainerapi.StartPGClient(container.Name)
			logit.Info.Println("AdminStartpg:" + srep.Output)
		}

		if err != nil {
			logit.Error.Println("AdminStartpg:" + err.Error())
			return err
		}

	}

	//make template changes here
	logit.Info.Println("performConfigUpdate....making template changes")
	templateChange(dbConn, container.Name, cars, container.Role)

	//restart postgres

	if container.Role == "pgpool" {
		logit.Info.Println("performConfigUpdate....stopping pgpool")
		var stoppoolResp cpmcontainerapi.StopPgpoolResponse
		stoppoolResp, err = cpmcontainerapi.StopPgpoolClient(container.Name)
		logit.Info.Println("AdminStoppg:" + stoppoolResp.Output)
	} else {
		logit.Info.Println("performConfigUpdate....stopping postgres")
		var stoppgResp cpmcontainerapi.StopPGResponse
		stoppgResp, err = cpmcontainerapi.StopPGClient(container.Name)
		logit.Info.Println("AdminStoppg:" + stoppgResp.Output)
	}
	if err != nil {
		logit.Error.Println("AdminStoppg:" + err.Error())
		return err
	}

	if container.Role == "pgpool" {
		logit.Info.Println("performConfigUpdate....starting pgpool")
		var spgresp cpmcontainerapi.StartPgpoolResponse
		spgresp, err = cpmcontainerapi.StartPgpoolClient(container.Name)
		logit.Info.Println("AdminStartpg:" + spgresp.Output)
	} else {
		logit.Info.Println("performConfigUpdate....starting postgres")
		var srep cpmcontainerapi.StartPGResponse
		srep, err = cpmcontainerapi.StartPGClient(container.Name)
		logit.Info.Println("AdminStartpg:" + srep.Output)
	}

	if err != nil {
		logit.Error.Println("AdminStartpg:" + err.Error())
		return err
	}

	return err

}
func main() {

	startTime = time.Now()
	startTimeString = startTime.String()
	logit.Info.Println("backrestrestore running....")

	err := getEnvVars()
	if err != nil {
		logit.Error.Println(err.Error())
		return
	}

	s := task.TaskStatus{}
	eDuration := time.Since(startTime)
	s.StartTime = startTimeString
	s.ElapsedTime = eDuration.String()
	s.Status = "initializing"
	s.TaskSize = "n/a"
	sendStats(&s)

	logit.Info.Println("giving DNS time to register the backup job....sleeping for 7 secs")
	sleepTime, _ := time.ParseDuration("7s")
	time.Sleep(sleepTime)

	stats("restore job starting")

	stats("stopping postgres...")
	var stopResponse cpmcontainerapi.StopPGResponse
	stopResponse, err = cpmcontainerapi.StopPGClient(restoreContainerName)
	if err != nil {
		logit.Error.Println(err.Error())
		s.Status = "error in stopPG"
		sendStats(&s)
		os.Exit(1)
	}
	logit.Info.Println("StopPG....")
	logit.Info.Println(stopResponse.Output)
	logit.Info.Println(stopResponse.Status)
	logit.Info.Println("End of StopPG....")

	//wait for postgres to quit
	time.Sleep(sleepTime)

	stats("performing the restore...")
	//perform the restore
	restoreRequest := cpmcontainerapi.RestoreRequest{}
	restoreRequest.RestoreRemotePath = restoreRemotePath
	restoreRequest.RestoreRemoteHost = restoreRemoteHost
	restoreRequest.RestoreRemoteUser = restoreRemoteUser
	restoreRequest.RestoreDbUser = restoreDbUser
	restoreRequest.RestoreDbPass = restoreDbPass
	restoreRequest.RestoreSet = restoreSet

	var restoreResponse cpmcontainerapi.RestoreResponse
	restoreResponse, err = cpmcontainerapi.RestoreClient(restoreContainerName, &restoreRequest)
	if err != nil {
		logit.Error.Println(err.Error())
		s.Status = "error in restore"
		sendStats(&s)
		os.Exit(1)
	}
	logit.Info.Println("Restore....")
	logit.Info.Println(restoreResponse.Output)
	logit.Info.Println(restoreResponse.Status)
	logit.Info.Println("End of Restore....")

	stats("starting postgres after the restore...")

	var startResponse cpmcontainerapi.StartPGResponse
	startResponse, err = cpmcontainerapi.StartPGClient(restoreContainerName)
	if err != nil {
		logit.Error.Println(err.Error())
		s.Status = "error in startPG"
		sendStats(&s)
		os.Exit(1)
	}
	logit.Info.Println("StartPG....")
	logit.Info.Println(startResponse.Output)
	logit.Info.Println(startResponse.Status)
	logit.Info.Println("End of StartPG....")

	stats("seeding the database...")
	var seedResponse cpmcontainerapi.SeedResponse
	seedResponse, err = cpmcontainerapi.SeedClient(restoreContainerName)
	if err != nil {
		logit.Error.Println(err.Error())
		s.Status = "error in Seed"
		sendStats(&s)
		os.Exit(1)
	}
	logit.Info.Println("Seed....")
	logit.Info.Println(seedResponse.Output)
	logit.Info.Println(seedResponse.Status)
	logit.Info.Println("End of Seed....")

	//send final stats to backup
	finalstats("restore completed")

}
func main() {

	startTime = time.Now()
	startTimeString = startTime.String()
	logit.Info.Println("backrestrestore running....")

	err := getEnvVars()
	if err != nil {
		logit.Error.Println(err.Error())
		return
	}

	s := task.TaskStatus{}
	eDuration := time.Since(startTime)
	s.StartTime = startTimeString
	//s.ElapsedTime = eDuration.String()
	s.ElapsedTime = fmt.Sprintf("%.3fs", eDuration.Seconds())
	s.Status = "initializing"
	s.TaskSize = "n/a"
	sendStats(&s)

	//	logit.Info.Println("giving DNS time to register the backup job....sleeping for 7 secs")
	sleepTime, _ := time.ParseDuration("7s")
	//	time.Sleep(sleepTime)

	stats("restore job starting")

	stats("stopping postgres...")
	var stopResponse cpmcontainerapi.StopPGResponse
	stopResponse, err = cpmcontainerapi.StopPGClient(restoreContainerName)
	if err != nil {
		logit.Error.Println(err.Error())
		s.Status = "error in stopPG"
		sendStats(&s)
		os.Exit(1)
	}
	logit.Info.Println("StopPG....")
	logit.Info.Println(stopResponse.Output)
	logit.Info.Println(stopResponse.Status)
	logit.Info.Println("End of StopPG....")

	//wait for postgres to quit
	time.Sleep(sleepTime)

	stats("performing the restore...")
	//perform the restore
	//remove anything left in the /pgdata on the receiving container
	logit.Info.Println("removing any existing pgdata files")
	var frompath string
	frompath = "/pgdata/" + restorePath + "/*"
	logit.Info.Println("/bin/rm -rf " + frompath)
	var cmd *exec.Cmd
	cmd = exec.Command("/bin/rm", "-rf", frompath)
	var out bytes.Buffer
	var stderr bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &stderr
	err = cmd.Run()
	if err != nil {
		logit.Error.Println(err.Error())
		logit.Error.Println("rm stdout=" + out.String())
		logit.Error.Println("rm stderr=" + stderr.String())
		s.Status = "error in removing old files"
		sendStats(&s)
		os.Exit(1)
	}
	logit.Info.Println("remove was successful")

	//I'm choosing to do the remove here this way since this restore
	//might be a HUGE amount of data and the copy command could run a LONG
	//time, longer than an http timeout might allow for, since restorecommand
	//is running inside a container itself, this copy can run any amount of time

	//copy from the backup path all files to the /pgdata on the receiving container
	frompath = " /pgdata" + restoreBackupPath
	topath := "  /pgdata/" + restorePath
	logit.Info.Println("/var/cpm/bin/copyfiles.sh" + frompath + topath)
	cmd = exec.Command("/var/cpm/bin/copyfiles.sh", frompath, topath)
	cmd.Stdout = &out
	cmd.Stderr = &stderr
	err = cmd.Run()
	if err != nil {
		logit.Error.Println(err.Error())
		s.Status = "error in copying backup files"
		logit.Error.Println("cp stdout=" + out.String())
		logit.Error.Println("cp stderr=" + stderr.String())
		sendStats(&s)
		os.Exit(1)
	}

	logit.Info.Println("restore - copy of files was successful...")

	stats("starting postgres after the restore...")

	var startResponse cpmcontainerapi.StartPGResponse
	startResponse, err = cpmcontainerapi.StartPGClient(restoreContainerName)
	if err != nil {
		logit.Error.Println(err.Error())
		s.Status = "error in startPG"
		sendStats(&s)
		os.Exit(1)
	}
	logit.Info.Println("StartPG....")
	logit.Info.Println(startResponse.Output)
	logit.Info.Println(startResponse.Status)
	logit.Info.Println("End of StartPG....")

	//send final stats to backup
	finalstats("restore completed")

}