// AdminStoppg stops a postgres database on a given container
func AdminStoppg(w rest.ResponseWriter, r *rest.Request) {
dbConn, err := util.GetConnection(CLUSTERADMIN_DB)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), 400)
return
}
defer dbConn.Close()
err = secimpl.Authorize(dbConn, r.PathParam("Token"), "perm-cluster")
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusUnauthorized)
return
}
//logit.Info.Println("AdminStoppg:called")
ID := r.PathParam("ID")
if ID == "" {
logit.Error.Println("AdminStoppg:ID not found error")
rest.Error(w, "node ID required", http.StatusBadRequest)
return
}
var dbNode types.Container
dbNode, err = admindb.GetContainer(dbConn, ID)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
//logit.Info.Println("AdminStoppg: in stop with dbnode")
if dbNode.Role == "pgpool" {
var stoppoolResp cpmcontainerapi.StopPgpoolResponse
stoppoolResp, err = cpmcontainerapi.StopPgpoolClient(dbNode.Name)
logit.Info.Println("AdminStoppg:" + stoppoolResp.Output)
} else {
var stoppgResp cpmcontainerapi.StopPGResponse
stoppgResp, err = cpmcontainerapi.StopPGClient(dbNode.Name)
logit.Info.Println("AdminStoppg:" + stoppgResp.Output)
}
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusOK)
status := types.SimpleStatus{}
status.Status = "OK"
w.WriteJson(&status)
}
// AdminFailover causes a cluster failorver to be performed for a given cluster
func AdminFailover(w rest.ResponseWriter, r *rest.Request) {
dbConn, err := util.GetConnection(CLUSTERADMIN_DB)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), 400)
return
}
defer dbConn.Close()
err = secimpl.Authorize(dbConn, r.PathParam("Token"), "perm-cluster")
if err != nil {
logit.Error.Println("authorize error " + err.Error())
rest.Error(w, err.Error(), http.StatusUnauthorized)
return
}
ID := r.PathParam("ID")
if ID == "" {
logit.Error.Println("node ID required error")
rest.Error(w, "node ID required", http.StatusBadRequest)
return
}
//dbNode is the standby node we are going to fail over and
//make the new master in the cluster
var dbNode types.Container
dbNode, err = admindb.GetContainer(dbConn, ID)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
cluster, err := admindb.GetCluster(dbConn, dbNode.ClusterID)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
var failoverResp cpmcontainerapi.FailoverResponse
failoverResp, err = cpmcontainerapi.FailoverClient(dbNode.Name)
if err != nil {
logit.Error.Println("fail-over error " + err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
logit.Info.Println("AdminFailover: fail-over output " + failoverResp.Output)
//update the old master to standalone role
oldMaster := types.Container{}
oldMaster, err = admindb.GetContainerMaster(dbConn, dbNode.ClusterID)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
oldMaster.Role = "standalone"
oldMaster.ClusterID = "-1"
err = admindb.UpdateContainer(dbConn, oldMaster)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
//update the failover node to master role
dbNode.Role = "master"
err = admindb.UpdateContainer(dbConn, dbNode)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
//stop pg on the old master
//params.IPAddress1 = oldMaster.IPAddress
var stopPGResp cpmcontainerapi.StopPGResponse
stopPGResp, err = cpmcontainerapi.StopPGClient(oldMaster.Name)
if err != nil {
logit.Error.Println(err.Error() + stopPGResp.Output)
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
err = configureCluster("SM", dbConn, cluster, false)
if err != nil {
logit.Error.Println(err.Error())
rest.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.WriteHeader(http.StatusOK)
status := types.SimpleStatus{}
status.Status = "OK"
w.WriteJson(&status)
return
}
func configureCluster(profile string, dbConn *sql.DB, cluster types.Cluster, autocluster bool) error {
logit.Info.Println("configureCluster:GetCluster")
//get master node for this cluster
master, err := admindb.GetContainerMaster(dbConn, cluster.ID)
if err != nil {
logit.Error.Println(err.Error())
return err
}
var pgport types.Setting
pgport, err = admindb.GetSetting(dbConn, "PG-PORT")
if err != nil {
logit.Error.Println(err.Error())
return err
}
var sleepSetting types.Setting
sleepSetting, err = admindb.GetSetting(dbConn, "SLEEP-PROV")
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
//logit.Info.Println("configureCluster:GetContainerMaster")
//configure master postgresql.conf file
var data string
info := new(template.PostgresqlParameters)
info.PG_PORT = pgport.Value
err = template.GetTuningParms(dbConn, profile, info)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
if cluster.ClusterType == "synchronous" {
info.CLUSTER_TYPE = "*"
data, err = template.Postgresql("master", info)
} else {
//TODO verify these next 2 lines look erroneous
info.CLUSTER_TYPE = "*"
data, err = template.Postgresql("master", info)
//info.CLUSTER_TYPE = ""
//data, err = template.Postgresql("master", info)
}
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:master postgresql.conf generated")
//write master postgresql.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, master.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:master postgresql.conf copied to remote")
//get domain name
var domainname types.Setting
domainname, err = admindb.GetSetting(dbConn, "DOMAIN-NAME")
if err != nil {
logit.Error.Println(err.Error())
return err
}
//configure master pg_hba.conf file
rules := make([]template.Rule, 0)
data, err = template.Hba(dbConn, "master", master.Name, pgport.Value, cluster.ID, domainname.Value, rules)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:master pg_hba.conf generated")
//write master pg_hba.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, master.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:master pg_hba.conf copied remotely")
//restart postgres after the config file changes
var stopResp cpmcontainerapi.StopPGResponse
stopResp, err = cpmcontainerapi.StopPGClient(master.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster: master stoppg output was" + stopResp.Output)
var startResp cpmcontainerapi.StartPGResponse
startResp, err = cpmcontainerapi.StartPGClient(master.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:master startpg output was" + startResp.Output)
//sleep loop until the master's PG can respond
var sleepTime time.Duration
sleepTime, err = time.ParseDuration(sleepSetting.Value)
var found = false
var currentStatus string
var masterhost = master.Name
for i := 0; i < 20; i++ {
//currentStatus, err = GetPGStatus2(dbConn, master.Name, masterhost)
currentStatus, err = util.FastPing(pgport.Value, master.Name)
if currentStatus == "RUNNING" {
logit.Info.Println("master is running...continuing")
found = true
break
} else {
logit.Info.Println("sleeping waiting on master..")
time.Sleep(sleepTime)
}
}
if !found {
logit.Info.Println("configureCluster: timed out waiting on master pg to start")
return errors.New("timeout waiting for master pg to respond")
}
standbynodes, err2 := admindb.GetAllStandbyContainers(dbConn, cluster.ID)
if err2 != nil {
logit.Error.Println(err.Error())
return err
}
//configure all standby nodes
var stopPGResp cpmcontainerapi.StopPGResponse
i := 0
for i = range standbynodes {
if standbynodes[i].Role == STANDBY {
//stop standby
if !autocluster {
stopPGResp, err = cpmcontainerapi.StopPGClient(standbynodes[i].Name)
if err != nil {
logit.Error.Println(err.Error())
logit.Error.Println("configureCluster:stop output was" + stopPGResp.Output)
return err
}
//logit.Info.Println("configureCluster:stop output was" + stopPGResp.Output)
}
//
var username = "******"
var password = "******"
//create base backup from master
var backupresp cpmcontainerapi.BasebackupResponse
backupresp, err = cpmcontainerapi.BasebackupClient(masterhost+"."+domainname.Value, standbynodes[i].Name, username, password)
if err != nil {
logit.Error.Println(err.Error())
logit.Error.Println("configureCluster:basebackup output was" + backupresp.Output)
return err
}
//logit.Info.Println("configureCluster:basebackup output was" + backupresp.Output)
data, err = template.Recovery(masterhost, pgport.Value, "postgres")
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:standby recovery.conf generated")
//write standby recovery.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/recovery.conf", data, standbynodes[i].Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:standby recovery.conf copied remotely")
info := new(template.PostgresqlParameters)
info.PG_PORT = pgport.Value
info.CLUSTER_TYPE = ""
err = template.GetTuningParms(dbConn, profile, info)
if err != nil {
logit.Error.Println(err.Error())
return err
}
data, err = template.Postgresql(STANDBY, info)
if err != nil {
logit.Error.Println(err.Error())
return err
}
//write standby postgresql.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, standbynodes[i].Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:standby postgresql.conf copied remotely")
//configure standby pg_hba.conf file
data, err = template.Hba(dbConn, STANDBY, standbynodes[i].Name, pgport.Value, cluster.ID, domainname.Value, rules)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:standby pg_hba.conf generated")
//write standby pg_hba.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, standbynodes[i].Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:standby pg_hba.conf copied remotely")
//start standby
var stResp cpmcontainerapi.StartPGOnStandbyResponse
stResp, err = cpmcontainerapi.StartPGOnStandbyClient(standbynodes[i].Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:standby startpg output was" + stResp.Output)
}
i++
}
logit.Info.Println("configureCluster: sleeping 5 seconds before configuring pgpool...")
clustersleepTime, _ := time.ParseDuration("5s")
time.Sleep(clustersleepTime)
pgpoolNode, err4 := admindb.GetContainerPgpool(dbConn, cluster.ID)
//logit.Info.Println("configureCluster: lookup pgpool node")
if err4 != nil {
logit.Error.Println(err.Error())
return err
}
//logit.Info.Println("configureCluster:" + pgpoolNode.Name)
//configure the pgpool includes all standby nodes AND the master node
poolnames := make([]string, len(standbynodes)+1)
i = 0
for i = range standbynodes {
poolnames[i] = standbynodes[i].Name + "." + domainname.Value
i++
}
poolnames[i] = master.Name + "." + domainname.Value
//generate pgpool.conf HOST_LIST
data, err = template.Poolconf(poolnames)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pgpool.conf generated")
//write pgpool.conf to remote pool node
_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pgpool.conf", data, pgpoolNode.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pgpool.conf copied remotely")
//generate pool_passwd
data, err = template.Poolpasswd()
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_passwd generated")
//write pgpool.conf to remote pool node
_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_passwd", data, pgpoolNode.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_passwd copied remotely")
//g enerate pool_hba.conf
cars := make([]template.Rule, 0)
data, err = template.Poolhba(cars)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_hba generated")
//write pgpool.conf to remote pool node
_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_hba.conf", data, pgpoolNode.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_hba copied remotely")
//start pgpool
var startPoolResp cpmcontainerapi.StartPgpoolResponse
startPoolResp, err = cpmcontainerapi.StartPgpoolClient(pgpoolNode.Name)
if err != nil {
logit.Error.Println(err.Error())
return err
}
logit.Info.Println("configureCluster: pgpool startpgpool output was" + startPoolResp.Output)
//finally, update the cluster to show that it is
//initialized!
cluster.Status = "initialized"
err = admindb.UpdateCluster(dbConn, cluster)
if err != nil {
logit.Error.Println(err.Error())
return err
}
return nil
}
func performConfigUpdate(dbConn *sql.DB, ContainerID string) error {
logit.Info.Println("performConfigUpdate....")
cars, err := GetAllContainerAccessRule(dbConn, ContainerID)
if err != nil {
logit.Error.Println(err.Error())
return err
}
container, err := admindb.GetContainer(dbConn, ContainerID)
if err != nil {
logit.Error.Println("GetNode: " + err.Error())
return err
}
var currentStatus string
//currentStatus, err = GetPGStatus2(dbConn, container.Name, container.Name)
var pgport types.Setting
pgport, err = admindb.GetSetting(dbConn, "PG-PORT")
if err != nil {
logit.Error.Println(err.Error())
return err
}
currentStatus, err = util.FastPing(pgport.Value, container.Name)
if err != nil {
logit.Error.Println("GetNode:" + err.Error())
return err
}
if currentStatus != "RUNNING" {
logit.Info.Println("performConfigUpdate....starting postgres")
if container.Role == "pgpool" {
var spgresp cpmcontainerapi.StartPgpoolResponse
spgresp, err = cpmcontainerapi.StartPgpoolClient(container.Name)
logit.Info.Println("AdminStartpg:" + spgresp.Output)
} else {
var srep cpmcontainerapi.StartPGResponse
srep, err = cpmcontainerapi.StartPGClient(container.Name)
logit.Info.Println("AdminStartpg:" + srep.Output)
}
if err != nil {
logit.Error.Println("AdminStartpg:" + err.Error())
return err
}
}
//make template changes here
logit.Info.Println("performConfigUpdate....making template changes")
templateChange(dbConn, container.Name, cars, container.Role)
//restart postgres
if container.Role == "pgpool" {
logit.Info.Println("performConfigUpdate....stopping pgpool")
var stoppoolResp cpmcontainerapi.StopPgpoolResponse
stoppoolResp, err = cpmcontainerapi.StopPgpoolClient(container.Name)
logit.Info.Println("AdminStoppg:" + stoppoolResp.Output)
} else {
logit.Info.Println("performConfigUpdate....stopping postgres")
var stoppgResp cpmcontainerapi.StopPGResponse
stoppgResp, err = cpmcontainerapi.StopPGClient(container.Name)
logit.Info.Println("AdminStoppg:" + stoppgResp.Output)
}
if err != nil {
logit.Error.Println("AdminStoppg:" + err.Error())
return err
}
if container.Role == "pgpool" {
logit.Info.Println("performConfigUpdate....starting pgpool")
var spgresp cpmcontainerapi.StartPgpoolResponse
spgresp, err = cpmcontainerapi.StartPgpoolClient(container.Name)
logit.Info.Println("AdminStartpg:" + spgresp.Output)
} else {
logit.Info.Println("performConfigUpdate....starting postgres")
var srep cpmcontainerapi.StartPGResponse
srep, err = cpmcontainerapi.StartPGClient(container.Name)
logit.Info.Println("AdminStartpg:" + srep.Output)
}
if err != nil {
logit.Error.Println("AdminStartpg:" + err.Error())
return err
}
return err
}
func main() {
startTime = time.Now()
startTimeString = startTime.String()
logit.Info.Println("backrestrestore running....")
err := getEnvVars()
if err != nil {
logit.Error.Println(err.Error())
return
}
s := task.TaskStatus{}
eDuration := time.Since(startTime)
s.StartTime = startTimeString
s.ElapsedTime = eDuration.String()
s.Status = "initializing"
s.TaskSize = "n/a"
sendStats(&s)
logit.Info.Println("giving DNS time to register the backup job....sleeping for 7 secs")
sleepTime, _ := time.ParseDuration("7s")
time.Sleep(sleepTime)
stats("restore job starting")
stats("stopping postgres...")
var stopResponse cpmcontainerapi.StopPGResponse
stopResponse, err = cpmcontainerapi.StopPGClient(restoreContainerName)
if err != nil {
logit.Error.Println(err.Error())
s.Status = "error in stopPG"
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("StopPG....")
logit.Info.Println(stopResponse.Output)
logit.Info.Println(stopResponse.Status)
logit.Info.Println("End of StopPG....")
//wait for postgres to quit
time.Sleep(sleepTime)
stats("performing the restore...")
//perform the restore
restoreRequest := cpmcontainerapi.RestoreRequest{}
restoreRequest.RestoreRemotePath = restoreRemotePath
restoreRequest.RestoreRemoteHost = restoreRemoteHost
restoreRequest.RestoreRemoteUser = restoreRemoteUser
restoreRequest.RestoreDbUser = restoreDbUser
restoreRequest.RestoreDbPass = restoreDbPass
restoreRequest.RestoreSet = restoreSet
var restoreResponse cpmcontainerapi.RestoreResponse
restoreResponse, err = cpmcontainerapi.RestoreClient(restoreContainerName, &restoreRequest)
if err != nil {
logit.Error.Println(err.Error())
s.Status = "error in restore"
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("Restore....")
logit.Info.Println(restoreResponse.Output)
logit.Info.Println(restoreResponse.Status)
logit.Info.Println("End of Restore....")
stats("starting postgres after the restore...")
var startResponse cpmcontainerapi.StartPGResponse
startResponse, err = cpmcontainerapi.StartPGClient(restoreContainerName)
if err != nil {
logit.Error.Println(err.Error())
s.Status = "error in startPG"
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("StartPG....")
logit.Info.Println(startResponse.Output)
logit.Info.Println(startResponse.Status)
logit.Info.Println("End of StartPG....")
stats("seeding the database...")
var seedResponse cpmcontainerapi.SeedResponse
seedResponse, err = cpmcontainerapi.SeedClient(restoreContainerName)
if err != nil {
logit.Error.Println(err.Error())
s.Status = "error in Seed"
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("Seed....")
logit.Info.Println(seedResponse.Output)
logit.Info.Println(seedResponse.Status)
logit.Info.Println("End of Seed....")
//send final stats to backup
finalstats("restore completed")
}
func main() {
startTime = time.Now()
startTimeString = startTime.String()
logit.Info.Println("backrestrestore running....")
err := getEnvVars()
if err != nil {
logit.Error.Println(err.Error())
return
}
s := task.TaskStatus{}
eDuration := time.Since(startTime)
s.StartTime = startTimeString
//s.ElapsedTime = eDuration.String()
s.ElapsedTime = fmt.Sprintf("%.3fs", eDuration.Seconds())
s.Status = "initializing"
s.TaskSize = "n/a"
sendStats(&s)
// logit.Info.Println("giving DNS time to register the backup job....sleeping for 7 secs")
sleepTime, _ := time.ParseDuration("7s")
// time.Sleep(sleepTime)
stats("restore job starting")
stats("stopping postgres...")
var stopResponse cpmcontainerapi.StopPGResponse
stopResponse, err = cpmcontainerapi.StopPGClient(restoreContainerName)
if err != nil {
logit.Error.Println(err.Error())
s.Status = "error in stopPG"
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("StopPG....")
logit.Info.Println(stopResponse.Output)
logit.Info.Println(stopResponse.Status)
logit.Info.Println("End of StopPG....")
//wait for postgres to quit
time.Sleep(sleepTime)
stats("performing the restore...")
//perform the restore
//remove anything left in the /pgdata on the receiving container
logit.Info.Println("removing any existing pgdata files")
var frompath string
frompath = "/pgdata/" + restorePath + "/*"
logit.Info.Println("/bin/rm -rf " + frompath)
var cmd *exec.Cmd
cmd = exec.Command("/bin/rm", "-rf", frompath)
var out bytes.Buffer
var stderr bytes.Buffer
cmd.Stdout = &out
cmd.Stderr = &stderr
err = cmd.Run()
if err != nil {
logit.Error.Println(err.Error())
logit.Error.Println("rm stdout=" + out.String())
logit.Error.Println("rm stderr=" + stderr.String())
s.Status = "error in removing old files"
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("remove was successful")
//I'm choosing to do the remove here this way since this restore
//might be a HUGE amount of data and the copy command could run a LONG
//time, longer than an http timeout might allow for, since restorecommand
//is running inside a container itself, this copy can run any amount of time
//copy from the backup path all files to the /pgdata on the receiving container
frompath = " /pgdata" + restoreBackupPath
topath := " /pgdata/" + restorePath
logit.Info.Println("/var/cpm/bin/copyfiles.sh" + frompath + topath)
cmd = exec.Command("/var/cpm/bin/copyfiles.sh", frompath, topath)
cmd.Stdout = &out
cmd.Stderr = &stderr
err = cmd.Run()
if err != nil {
logit.Error.Println(err.Error())
s.Status = "error in copying backup files"
logit.Error.Println("cp stdout=" + out.String())
logit.Error.Println("cp stderr=" + stderr.String())
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("restore - copy of files was successful...")
stats("starting postgres after the restore...")
var startResponse cpmcontainerapi.StartPGResponse
startResponse, err = cpmcontainerapi.StartPGClient(restoreContainerName)
if err != nil {
logit.Error.Println(err.Error())
s.Status = "error in startPG"
sendStats(&s)
os.Exit(1)
}
logit.Info.Println("StartPG....")
logit.Info.Println(startResponse.Output)
logit.Info.Println(startResponse.Status)
logit.Info.Println("End of StartPG....")
//send final stats to backup
finalstats("restore completed")
}