func AutoCluster(w rest.ResponseWriter, r *rest.Request) {
dbConn, err := util.GetConnection(CLUSTERADMIN_DB)
if err != nil {
logit.Error.Println("BackupNow: error " + err.Error())
rest.Error(w, err.Error(), 400)
return
}
defer dbConn.Close()
logit.Info.Println("AUTO CLUSTER PROFILE starts")
logit.Info.Println("AutoCluster: start AutoCluster")
params := AutoClusterInfo{}
err = r.DecodeJsonPayload(¶ms)
if err != nil {
logit.Error.Println("AutoCluster: error in decode" + err.Error())
rest.Error(w, err.Error(), http.StatusInternalServerError)
return
}
err = secimpl.Authorize(dbConn, params.Token, "perm-cluster")
if err != nil {
logit.Error.Println("AutoCluster: authorize error " + err.Error())
rest.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if params.Name == "" {
logit.Error.Println("AutoCluster: error in Name")
rest.Error(w, "cluster name required", http.StatusBadRequest)
return
}
if params.ClusterType == "" {
logit.Error.Println("AutoCluster: error in ClusterType")
rest.Error(w, "ClusterType name required", http.StatusBadRequest)
return
}
if params.ProjectID == "" {
logit.Error.Println("AutoCluster: error in ProjectID")
rest.Error(w, "ProjectID name required", http.StatusBadRequest)
return
}
if params.ClusterProfile == "" {
logit.Error.Println("AutoCluster: error in ClusterProfile")
rest.Error(w, "ClusterProfile name required", http.StatusBadRequest)
return
}
logit.Info.Println("AutoCluster: Name=" + params.Name + " ClusterType=" + params.ClusterType + " Profile=" + params.ClusterProfile + " ProjectID=" + params.ProjectID)
//create cluster definition
dbcluster := admindb.Cluster{"", params.ProjectID, params.Name, params.ClusterType, "uninitialized", "", make(map[string]string)}
var ival int
ival, err = admindb.InsertCluster(dbConn, dbcluster)
clusterID := strconv.Itoa(ival)
dbcluster.ID = clusterID
logit.Info.Println(clusterID)
if err != nil {
logit.Error.Println("AutoCluster:" + err.Error())
rest.Error(w, "Insert Cluster error:"+err.Error(), http.StatusBadRequest)
return
}
//lookup profile
profile, err2 := getClusterProfileInfo(dbConn, params.ClusterProfile)
if err2 != nil {
logit.Error.Println("AutoCluster: error-" + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
var masterServer admindb.Server
var chosenServers []admindb.Server
if profile.Algo == "round-robin" {
masterServer, chosenServers, err2 = roundRobin(dbConn, profile)
} else {
logit.Error.Println("AutoCluster: error-unsupported algorithm request")
rest.Error(w, "AutoCluster error: unsupported algorithm", http.StatusBadRequest)
return
}
//create master container
dockermaster := cpmserverapi.DockerRunRequest{}
dockermaster.Image = CPM_NODE_IMAGE
dockermaster.ContainerName = params.Name + "-master"
dockermaster.ServerID = masterServer.ID
dockermaster.ProjectID = params.ProjectID
dockermaster.Standalone = "false"
if err != nil {
logit.Error.Println("AutoCluster: error-create master node " + err.Error())
rest.Error(w, "AutoCluster error"+err.Error(), http.StatusBadRequest)
return
}
// provision the master
err2 = provisionImpl(dbConn, &dockermaster, profile.MasterProfile, false)
if err2 != nil {
logit.Error.Println("AutoCluster: error-provision master " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
logit.Info.Println("AUTO CLUSTER PROFILE master container created")
var node admindb.Container
//update node with cluster iD
node, err2 = admindb.GetContainerByName(dbConn, dockermaster.ContainerName)
if err2 != nil {
logit.Error.Println("AutoCluster: error-get node by name " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
node.ClusterID = clusterID
node.Role = "master"
err2 = admindb.UpdateContainer(dbConn, node)
if err2 != nil {
logit.Error.Println("AutoCluster: error-update standby node " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
//create standby containers
var count int
count, err2 = strconv.Atoi(profile.Count)
if err2 != nil {
logit.Error.Println(err2.Error())
rest.Error(w, err2.Error(), http.StatusBadRequest)
return
}
dockerstandby := make([]cpmserverapi.DockerRunRequest, count)
for i := 0; i < count; i++ {
logit.Info.Println("working on standby ....")
// loop - provision standby
dockerstandby[i].ServerID = chosenServers[i].ID
dockerstandby[i].ProjectID = params.ProjectID
dockerstandby[i].Image = CPM_NODE_IMAGE
dockerstandby[i].ContainerName = params.Name + "-" + STANDBY + "-" + strconv.Itoa(i)
dockerstandby[i].Standalone = "false"
err2 = provisionImpl(dbConn, &dockerstandby[i], profile.StandbyProfile, true)
if err2 != nil {
logit.Error.Println("AutoCluster: error-provision master " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
//update node with cluster iD
node, err2 = admindb.GetContainerByName(dbConn, dockerstandby[i].ContainerName)
if err2 != nil {
logit.Error.Println("AutoCluster: error-get node by name " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
node.ClusterID = clusterID
node.Role = STANDBY
err2 = admindb.UpdateContainer(dbConn, node)
if err2 != nil {
logit.Error.Println("AutoCluster: error-update standby node " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
}
logit.Info.Println("AUTO CLUSTER PROFILE standbys created")
//create pgpool container
// provision
dockerpgpool := cpmserverapi.DockerRunRequest{}
dockerpgpool.ContainerName = params.Name + "-pgpool"
dockerpgpool.Image = CPM_PGPOOL_IMAGE
dockerpgpool.ServerID = chosenServers[count].ID
dockerpgpool.ProjectID = params.ProjectID
dockerpgpool.Standalone = "false"
err2 = provisionImpl(dbConn, &dockerpgpool, profile.StandbyProfile, true)
if err2 != nil {
logit.Error.Println("AutoCluster: error-provision pgpool " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
logit.Info.Println("AUTO CLUSTER PROFILE pgpool created")
//update node with cluster ID
node, err2 = admindb.GetContainerByName(dbConn, dockerpgpool.ContainerName)
if err2 != nil {
logit.Error.Println("AutoCluster: error-get pgpool node by name " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
node.ClusterID = clusterID
node.Role = "pgpool"
err2 = admindb.UpdateContainer(dbConn, node)
if err2 != nil {
logit.Error.Println("AutoCluster: error-update pgpool node " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
//init the master DB
// provision the master
err2 = provisionImplInit(dbConn, &dockermaster, profile.MasterProfile, false)
if err2 != nil {
logit.Error.Println("AutoCluster: error-provisionInit master " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
//make sure every node is ready
err2 = waitTillAllReady(dockermaster, dockerpgpool, dockerstandby)
if err2 != nil {
logit.Error.Println("cluster members not responding in time")
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
//configure cluster
// ConfigureCluster
logit.Info.Println("AUTO CLUSTER PROFILE configure cluster ")
err2 = configureCluster(dbConn, dbcluster, true)
if err2 != nil {
logit.Error.Println("AutoCluster: error-configure cluster " + err2.Error())
rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest)
return
}
logit.Info.Println("AUTO CLUSTER PROFILE done")
w.WriteHeader(http.StatusOK)
status := SimpleStatus{}
status.Status = "OK"
w.WriteJson(&status)
}
func configureCluster(dbConn *sql.DB, cluster admindb.Cluster, autocluster bool) error {
logit.Info.Println("configureCluster:GetCluster")
//get master node for this cluster
master, err := admindb.GetContainerMaster(dbConn, cluster.ID)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
var pgport admindb.Setting
pgport, err = admindb.GetSetting(dbConn, "PG-PORT")
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:GetContainerMaster")
//configure master postgresql.conf file
var data string
if cluster.ClusterType == "synchronous" {
data, err = template.Postgresql("master", pgport.Value, "*")
} else {
data, err = template.Postgresql("master", pgport.Value, "")
}
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:master postgresql.conf generated")
//write master postgresql.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, master.Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:master postgresql.conf copied to remote")
//get domain name
var domainname admindb.Setting
domainname, err = admindb.GetSetting(dbConn, "DOMAIN-NAME")
if err != nil {
logit.Error.Println("configureCluster: DOMAIN-NAME err " + err.Error())
return err
}
//configure master pg_hba.conf file
rules := make([]template.Rule, 0)
data, err = template.Hba(dbConn, "master", master.Name, pgport.Value, cluster.ID, domainname.Value, rules)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:master pg_hba.conf generated")
//write master pg_hba.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, master.Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:master pg_hba.conf copied remotely")
//restart postgres after the config file changes
var stopResp cpmcontainerapi.StopPGResponse
stopResp, err = cpmcontainerapi.StopPGClient(master.Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster: master stoppg output was" + stopResp.Output)
var startResp cpmcontainerapi.StartPGResponse
startResp, err = cpmcontainerapi.StartPGClient(master.Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:master startpg output was" + startResp.Output)
//sleep loop until the master's PG can respond
var found = false
var currentStatus string
var masterhost = master.Name
for i := 0; i < 20; i++ {
currentStatus, err = GetPGStatus2(dbConn, master.Name, masterhost)
if currentStatus == "RUNNING" {
logit.Info.Println("master is running...continuing")
found = true
break
} else {
logit.Info.Println("sleeping 1 sec waiting on master..")
time.Sleep(1000 * time.Millisecond)
}
}
if !found {
logit.Info.Println("configureCluster: timed out waiting on master pg to start")
return errors.New("timeout waiting for master pg to respond")
}
standbynodes, err2 := admindb.GetAllStandbyContainers(dbConn, cluster.ID)
if err2 != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
//configure all standby nodes
var stopPGResp cpmcontainerapi.StopPGResponse
i := 0
for i = range standbynodes {
if standbynodes[i].Role == STANDBY {
//stop standby
if !autocluster {
stopPGResp, err = cpmcontainerapi.StopPGClient(standbynodes[i].Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:stop output was" + stopPGResp.Output)
}
//create base backup from master
var backupresp cpmcontainerapi.BasebackupResponse
backupresp, err = cpmcontainerapi.BasebackupClient(masterhost+"."+domainname.Value, standbynodes[i].Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:basebackup output was" + backupresp.Output)
data, err = template.Recovery(masterhost, pgport.Value, "postgres")
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:standby recovery.conf generated")
//write standby recovery.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/recovery.conf", data, standbynodes[i].Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:standby recovery.conf copied remotely")
data, err = template.Postgresql(STANDBY, pgport.Value, "")
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
//write standby postgresql.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, standbynodes[i].Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:standby postgresql.conf copied remotely")
//configure standby pg_hba.conf file
data, err = template.Hba(dbConn, STANDBY, standbynodes[i].Name, pgport.Value, cluster.ID, domainname.Value, rules)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:standby pg_hba.conf generated")
//write standby pg_hba.conf file remotely
_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, standbynodes[i].Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:standby pg_hba.conf copied remotely")
//start standby
var stResp cpmcontainerapi.StartPGOnStandbyResponse
stResp, err = cpmcontainerapi.StartPGOnStandbyClient(standbynodes[i].Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:standby startpg output was" + stResp.Output)
}
i++
}
logit.Info.Println("configureCluster: sleeping 5 seconds before configuring pgpool...")
time.Sleep(5000 * time.Millisecond)
pgpoolNode, err4 := admindb.GetContainerPgpool(dbConn, cluster.ID)
logit.Info.Println("configureCluster: lookup pgpool node")
if err4 != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:" + pgpoolNode.Name)
//configure the pgpool includes all standby nodes AND the master node
poolnames := make([]string, len(standbynodes)+1)
i = 0
for i = range standbynodes {
poolnames[i] = standbynodes[i].Name + "." + domainname.Value
i++
}
poolnames[i] = master.Name + "." + domainname.Value
//generate pgpool.conf HOST_LIST
data, err = template.Poolconf(poolnames)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pgpool.conf generated")
//write pgpool.conf to remote pool node
_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pgpool.conf", data, pgpoolNode.Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pgpool.conf copied remotely")
//generate pool_passwd
data, err = template.Poolpasswd()
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_passwd generated")
//write pgpool.conf to remote pool node
_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_passwd", data, pgpoolNode.Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_passwd copied remotely")
//generate pool_hba.conf
data, err = template.Poolhba()
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_hba generated")
//write pgpool.conf to remote pool node
_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_hba.conf", data, pgpoolNode.Name)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
logit.Info.Println("configureCluster:pgpool pool_hba copied remotely")
//start pgpool
var startPoolResp cpmcontainerapi.StartPgpoolResponse
startPoolResp, err = cpmcontainerapi.StartPgpoolClient(pgpoolNode.Name)
if err != nil {
logit.Error.Println("configureCluster: " + err.Error())
return err
}
logit.Info.Println("configureCluster: pgpool startpgpool output was" + startPoolResp.Output)
//finally, update the cluster to show that it is
//initialized!
cluster.Status = "initialized"
err = admindb.UpdateCluster(dbConn, cluster)
if err != nil {
logit.Error.Println("configureCluster:" + err.Error())
return err
}
return nil
}