// GetCluster returns a cluster object from the database for a given cluster ID func GetCluster(dbConn *sql.DB, id string) (types.Cluster, error) { //logit.Info.Println("admindb:GetCluster: called") cluster := types.Cluster{} err := dbConn.QueryRow(fmt.Sprintf("select projectid, id, name, clustertype, status, to_char(createdt, 'MM-DD-YYYY HH24:MI:SS') from cluster where id=%s", id)).Scan(&cluster.ProjectID, &cluster.ID, &cluster.Name, &cluster.ClusterType, &cluster.Status, &cluster.CreateDate) switch { case err == sql.ErrNoRows: logit.Info.Println("admindb:GetCluster:no cluster with that id") return cluster, err case err != nil: logit.Info.Println("admindb:GetCluster:" + err.Error()) return cluster, err default: logit.Info.Println("admindb:GetCluster: cluster name returned is " + cluster.Name) } var containers []types.Container cluster.Containers = make(map[string]string) containers, err = GetAllContainersForCluster(dbConn, cluster.ID) if err != nil { logit.Info.Println("admindb:GetCluster:" + err.Error()) return cluster, err } for i := range containers { cluster.Containers[containers[i].ID] = containers[i].Name } return cluster, nil }
// PostCluster updates or inserts a new cluster definition func PostCluster(w rest.ResponseWriter, r *rest.Request) { dbConn, err := util.GetConnection(CLUSTERADMIN_DB) if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), 400) return } defer dbConn.Close() //logit.Info.Println("PostCluster: in PostCluster") cluster := types.Cluster{} err = r.DecodeJsonPayload(&cluster) if err != nil { logit.Error.Println("error in decode" + err.Error()) rest.Error(w, err.Error(), http.StatusInternalServerError) return } err = secimpl.Authorize(dbConn, cluster.Token, "perm-cluster") if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), http.StatusUnauthorized) return } if cluster.Name == "" { logit.Error.Println("PostCluster: error in Name") rest.Error(w, "cluster name required", http.StatusBadRequest) return } //logit.Info.Println("PostCluster: have ID=" + cluster.ID + " Name=" + cluster.Name + " type=" + cluster.ClusterType + " status=" + cluster.Status) dbcluster := types.Cluster{} dbcluster.ID = cluster.ID dbcluster.ProjectID = cluster.ProjectID dbcluster.Name = cluster.Name dbcluster.ClusterType = cluster.ClusterType dbcluster.Status = cluster.Status dbcluster.Containers = cluster.Containers if cluster.ID == "" { strid, err := admindb.InsertCluster(dbConn, dbcluster) newid := strconv.Itoa(strid) if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), http.StatusBadRequest) return } cluster.ID = newid } else { //logit.Info.Println("PostCluster: about to call UpdateCluster") err2 := admindb.UpdateCluster(dbConn, dbcluster) if err2 != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), http.StatusBadRequest) return } } w.WriteJson(&cluster) }
// GetCluster returns a given cluster definition func GetCluster(w rest.ResponseWriter, r *rest.Request) { dbConn, err := util.GetConnection(CLUSTERADMIN_DB) if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), 400) return } defer dbConn.Close() err = secimpl.Authorize(dbConn, r.PathParam("Token"), "perm-read") if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), http.StatusUnauthorized) return } ID := r.PathParam("ID") results, err := admindb.GetCluster(dbConn, ID) if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), http.StatusBadRequest) } cluster := types.Cluster{} cluster.ID = results.ID cluster.ProjectID = results.ProjectID cluster.Name = results.Name cluster.ClusterType = results.ClusterType cluster.Status = results.Status cluster.CreateDate = results.CreateDate cluster.Containers = results.Containers //logit.Info.Println("GetCluser:db call results=" + results.ID) w.WriteJson(&cluster) }
// GetAllClustersForProject returns a list of cluster objects from the database for a given project func GetAllClustersForProject(dbConn *sql.DB, projectId string) ([]types.Cluster, error) { //logit.Info.Println("admindb:GetAllClusters: called") var rows *sql.Rows var err error rows, err = dbConn.Query("select id, projectid, name, clustertype, status, to_char(createdt, 'MM-DD-YYYY HH24:MI:SS') from cluster where projectid = " + projectId + " order by name") if err != nil { return nil, err } defer rows.Close() var containers []types.Container clusters := make([]types.Cluster, 0) for rows.Next() { cluster := types.Cluster{} if err = rows.Scan( &cluster.ID, &cluster.ProjectID, &cluster.Name, &cluster.ClusterType, &cluster.Status, &cluster.CreateDate); err != nil { return nil, err } cluster.Containers = make(map[string]string) containers, err = GetAllContainersForCluster(dbConn, cluster.ID) if err != nil { logit.Info.Println("admindb:GetCluster:" + err.Error()) } for i := range containers { cluster.Containers[containers[i].ID] = containers[i].Name logit.Info.Println("admindb:GetCluster: add to map " + cluster.Containers[containers[i].ID]) } clusters = append(clusters, cluster) } if err = rows.Err(); err != nil { return nil, err } return clusters, nil }
// AutoCluster creates a new cluster func AutoCluster(w rest.ResponseWriter, r *rest.Request) { dbConn, err := util.GetConnection(CLUSTERADMIN_DB) if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), 400) return } defer dbConn.Close() logit.Info.Println("AUTO CLUSTER PROFILE starts") params := AutoClusterInfo{} err = r.DecodeJsonPayload(¶ms) if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), http.StatusInternalServerError) return } err = secimpl.Authorize(dbConn, params.Token, "perm-cluster") if err != nil { logit.Error.Println(err.Error()) rest.Error(w, err.Error(), http.StatusUnauthorized) return } if params.Name == "" { logit.Error.Println("AutoCluster: error in Name") rest.Error(w, "cluster name required", http.StatusBadRequest) return } if params.ClusterType == "" { logit.Error.Println("AutoCluster: error in ClusterType") rest.Error(w, "ClusterType name required", http.StatusBadRequest) return } if params.ProjectID == "" { logit.Error.Println("AutoCluster: error in ProjectID") rest.Error(w, "ProjectID name required", http.StatusBadRequest) return } if params.ClusterProfile == "" { logit.Error.Println("AutoCluster: error in ClusterProfile") rest.Error(w, "ClusterProfile name required", http.StatusBadRequest) return } logit.Info.Println("AutoCluster: Name=" + params.Name + " ClusterType=" + params.ClusterType + " Profile=" + params.ClusterProfile + " ProjectID=" + params.ProjectID) //create cluster definition dbcluster := types.Cluster{} dbcluster.ID = "" dbcluster.ProjectID = params.ProjectID dbcluster.Name = util.CleanName(params.Name) dbcluster.ClusterType = params.ClusterType dbcluster.Status = "uninitialized" dbcluster.Containers = make(map[string]string) var ival int ival, err = admindb.InsertCluster(dbConn, dbcluster) clusterID := strconv.Itoa(ival) dbcluster.ID = clusterID //logit.Info.Println(clusterID) if err != nil { logit.Error.Println(err.Error()) rest.Error(w, "Insert Cluster error:"+err.Error(), http.StatusBadRequest) return } //lookup profile profile, err2 := getClusterProfileInfo(dbConn, params.ClusterProfile) if err2 != nil { logit.Error.Println(err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } //var masterServer types.Server //var chosenServers []types.Server if profile.Algo == "round-robin" { //masterServer, chosenServers, err2 = roundRobin(dbConn, profile) } else { logit.Error.Println("AutoCluster: error-unsupported algorithm request") rest.Error(w, "AutoCluster error: unsupported algorithm", http.StatusBadRequest) return } //create master container dockermaster := swarmapi.DockerRunRequest{} dockermaster.Image = "cpm-node" dockermaster.ContainerName = params.Name + "-master" dockermaster.ProjectID = params.ProjectID dockermaster.Standalone = "false" dockermaster.Profile = profile.MasterProfile if err != nil { logit.Error.Println("AutoCluster: error-create master node " + err.Error()) rest.Error(w, "AutoCluster error"+err.Error(), http.StatusBadRequest) return } // provision the master logit.Info.Println("dockermaster profile is " + dockermaster.Profile) _, err2 = provisionImpl(dbConn, &dockermaster, false) if err2 != nil { logit.Error.Println("AutoCluster: error-provision master " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } logit.Info.Println("AUTO CLUSTER PROFILE master container created") var node types.Container //update node with cluster iD node, err2 = admindb.GetContainerByName(dbConn, dockermaster.ContainerName) if err2 != nil { logit.Error.Println("AutoCluster: error-get node by name " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } node.ClusterID = clusterID node.Role = "master" err2 = admindb.UpdateContainer(dbConn, node) if err2 != nil { logit.Error.Println("AutoCluster: error-update standby node " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } var sleepSetting types.Setting sleepSetting, err2 = admindb.GetSetting(dbConn, "SLEEP-PROV") if err2 != nil { logit.Error.Println("SLEEP-PROV setting error " + err2.Error()) rest.Error(w, err2.Error(), http.StatusInternalServerError) return } var sleepTime time.Duration sleepTime, err2 = time.ParseDuration(sleepSetting.Value) if err2 != nil { logit.Error.Println(err2.Error()) rest.Error(w, err2.Error(), http.StatusInternalServerError) return } //create standby containers var count int count, err2 = strconv.Atoi(profile.Count) if err2 != nil { logit.Error.Println(err2.Error()) rest.Error(w, err2.Error(), http.StatusBadRequest) return } dockerstandby := make([]swarmapi.DockerRunRequest, count) for i := 0; i < count; i++ { logit.Info.Println("working on standby ....") // loop - provision standby dockerstandby[i].ProjectID = params.ProjectID dockerstandby[i].Image = "cpm-node" dockerstandby[i].ContainerName = params.Name + "-" + STANDBY + "-" + strconv.Itoa(i) dockerstandby[i].Standalone = "false" dockerstandby[i].Profile = profile.StandbyProfile _, err2 = provisionImpl(dbConn, &dockerstandby[i], true) if err2 != nil { logit.Error.Println("AutoCluster: error-provision master " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } //update node with cluster iD node, err2 = admindb.GetContainerByName(dbConn, dockerstandby[i].ContainerName) if err2 != nil { logit.Error.Println("AutoCluster: error-get node by name " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } node.ClusterID = clusterID node.Role = STANDBY err2 = admindb.UpdateContainer(dbConn, node) if err2 != nil { logit.Error.Println("AutoCluster: error-update standby node " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } } logit.Info.Println("AUTO CLUSTER PROFILE standbys created") //create pgpool container // provision dockerpgpool := swarmapi.DockerRunRequest{} dockerpgpool.ContainerName = params.Name + "-pgpool" dockerpgpool.Image = "cpm-pgpool" dockerpgpool.ProjectID = params.ProjectID dockerpgpool.Standalone = "false" dockerpgpool.Profile = profile.StandbyProfile _, err2 = provisionImpl(dbConn, &dockerpgpool, true) if err2 != nil { logit.Error.Println("AutoCluster: error-provision pgpool " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } logit.Info.Println("AUTO CLUSTER PROFILE pgpool created") //update node with cluster ID node, err2 = admindb.GetContainerByName(dbConn, dockerpgpool.ContainerName) if err2 != nil { logit.Error.Println("AutoCluster: error-get pgpool node by name " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } node.ClusterID = clusterID node.Role = "pgpool" err2 = admindb.UpdateContainer(dbConn, node) if err2 != nil { logit.Error.Println("AutoCluster: error-update pgpool node " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } //init the master DB // provision the master dockermaster.Profile = profile.MasterProfile err2 = provisionImplInit(dbConn, &dockermaster, false) if err2 != nil { logit.Error.Println("AutoCluster: error-provisionInit master " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } //make sure every node is ready err2 = waitTillAllReady(dockermaster, dockerpgpool, dockerstandby, sleepTime) if err2 != nil { logit.Error.Println("cluster members not responding in time") rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } //configure cluster // ConfigureCluster logit.Info.Println("AUTO CLUSTER PROFILE configure cluster ") err2 = configureCluster(profile.MasterProfile, dbConn, dbcluster, true) if err2 != nil { logit.Error.Println("AutoCluster: error-configure cluster " + err2.Error()) rest.Error(w, "AutoCluster error"+err2.Error(), http.StatusBadRequest) return } logit.Info.Println("AUTO CLUSTER PROFILE done") w.WriteHeader(http.StatusOK) status := types.SimpleStatus{} status.Status = "OK" w.WriteJson(&status) }
func configureCluster(profile string, dbConn *sql.DB, cluster types.Cluster, autocluster bool) error { logit.Info.Println("configureCluster:GetCluster") //get master node for this cluster master, err := admindb.GetContainerMaster(dbConn, cluster.ID) if err != nil { logit.Error.Println(err.Error()) return err } var pgport types.Setting pgport, err = admindb.GetSetting(dbConn, "PG-PORT") if err != nil { logit.Error.Println(err.Error()) return err } var sleepSetting types.Setting sleepSetting, err = admindb.GetSetting(dbConn, "SLEEP-PROV") if err != nil { logit.Error.Println("configureCluster:" + err.Error()) return err } //logit.Info.Println("configureCluster:GetContainerMaster") //configure master postgresql.conf file var data string info := new(template.PostgresqlParameters) info.PG_PORT = pgport.Value err = template.GetTuningParms(dbConn, profile, info) if err != nil { logit.Error.Println("configureCluster:" + err.Error()) return err } if cluster.ClusterType == "synchronous" { info.CLUSTER_TYPE = "*" data, err = template.Postgresql("master", info) } else { //TODO verify these next 2 lines look erroneous info.CLUSTER_TYPE = "*" data, err = template.Postgresql("master", info) //info.CLUSTER_TYPE = "" //data, err = template.Postgresql("master", info) } if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:master postgresql.conf generated") //write master postgresql.conf file remotely _, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, master.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:master postgresql.conf copied to remote") //get domain name var domainname types.Setting domainname, err = admindb.GetSetting(dbConn, "DOMAIN-NAME") if err != nil { logit.Error.Println(err.Error()) return err } //configure master pg_hba.conf file rules := make([]template.Rule, 0) data, err = template.Hba(dbConn, "master", master.Name, pgport.Value, cluster.ID, domainname.Value, rules) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:master pg_hba.conf generated") //write master pg_hba.conf file remotely _, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, master.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:master pg_hba.conf copied remotely") //restart postgres after the config file changes var stopResp cpmcontainerapi.StopPGResponse stopResp, err = cpmcontainerapi.StopPGClient(master.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster: master stoppg output was" + stopResp.Output) var startResp cpmcontainerapi.StartPGResponse startResp, err = cpmcontainerapi.StartPGClient(master.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:master startpg output was" + startResp.Output) //sleep loop until the master's PG can respond var sleepTime time.Duration sleepTime, err = time.ParseDuration(sleepSetting.Value) var found = false var currentStatus string var masterhost = master.Name for i := 0; i < 20; i++ { //currentStatus, err = GetPGStatus2(dbConn, master.Name, masterhost) currentStatus, err = util.FastPing(pgport.Value, master.Name) if currentStatus == "RUNNING" { logit.Info.Println("master is running...continuing") found = true break } else { logit.Info.Println("sleeping waiting on master..") time.Sleep(sleepTime) } } if !found { logit.Info.Println("configureCluster: timed out waiting on master pg to start") return errors.New("timeout waiting for master pg to respond") } standbynodes, err2 := admindb.GetAllStandbyContainers(dbConn, cluster.ID) if err2 != nil { logit.Error.Println(err.Error()) return err } //configure all standby nodes var stopPGResp cpmcontainerapi.StopPGResponse i := 0 for i = range standbynodes { if standbynodes[i].Role == STANDBY { //stop standby if !autocluster { stopPGResp, err = cpmcontainerapi.StopPGClient(standbynodes[i].Name) if err != nil { logit.Error.Println(err.Error()) logit.Error.Println("configureCluster:stop output was" + stopPGResp.Output) return err } //logit.Info.Println("configureCluster:stop output was" + stopPGResp.Output) } // var username = "******" var password = "******" //create base backup from master var backupresp cpmcontainerapi.BasebackupResponse backupresp, err = cpmcontainerapi.BasebackupClient(masterhost+"."+domainname.Value, standbynodes[i].Name, username, password) if err != nil { logit.Error.Println(err.Error()) logit.Error.Println("configureCluster:basebackup output was" + backupresp.Output) return err } //logit.Info.Println("configureCluster:basebackup output was" + backupresp.Output) data, err = template.Recovery(masterhost, pgport.Value, "postgres") if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:standby recovery.conf generated") //write standby recovery.conf file remotely _, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/recovery.conf", data, standbynodes[i].Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:standby recovery.conf copied remotely") info := new(template.PostgresqlParameters) info.PG_PORT = pgport.Value info.CLUSTER_TYPE = "" err = template.GetTuningParms(dbConn, profile, info) if err != nil { logit.Error.Println(err.Error()) return err } data, err = template.Postgresql(STANDBY, info) if err != nil { logit.Error.Println(err.Error()) return err } //write standby postgresql.conf file remotely _, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, standbynodes[i].Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:standby postgresql.conf copied remotely") //configure standby pg_hba.conf file data, err = template.Hba(dbConn, STANDBY, standbynodes[i].Name, pgport.Value, cluster.ID, domainname.Value, rules) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:standby pg_hba.conf generated") //write standby pg_hba.conf file remotely _, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, standbynodes[i].Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:standby pg_hba.conf copied remotely") //start standby var stResp cpmcontainerapi.StartPGOnStandbyResponse stResp, err = cpmcontainerapi.StartPGOnStandbyClient(standbynodes[i].Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:standby startpg output was" + stResp.Output) } i++ } logit.Info.Println("configureCluster: sleeping 5 seconds before configuring pgpool...") clustersleepTime, _ := time.ParseDuration("5s") time.Sleep(clustersleepTime) pgpoolNode, err4 := admindb.GetContainerPgpool(dbConn, cluster.ID) //logit.Info.Println("configureCluster: lookup pgpool node") if err4 != nil { logit.Error.Println(err.Error()) return err } //logit.Info.Println("configureCluster:" + pgpoolNode.Name) //configure the pgpool includes all standby nodes AND the master node poolnames := make([]string, len(standbynodes)+1) i = 0 for i = range standbynodes { poolnames[i] = standbynodes[i].Name + "." + domainname.Value i++ } poolnames[i] = master.Name + "." + domainname.Value //generate pgpool.conf HOST_LIST data, err = template.Poolconf(poolnames) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:pgpool pgpool.conf generated") //write pgpool.conf to remote pool node _, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pgpool.conf", data, pgpoolNode.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:pgpool pgpool.conf copied remotely") //generate pool_passwd data, err = template.Poolpasswd() if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:pgpool pool_passwd generated") //write pgpool.conf to remote pool node _, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_passwd", data, pgpoolNode.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:pgpool pool_passwd copied remotely") //g enerate pool_hba.conf cars := make([]template.Rule, 0) data, err = template.Poolhba(cars) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:pgpool pool_hba generated") //write pgpool.conf to remote pool node _, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_hba.conf", data, pgpoolNode.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster:pgpool pool_hba copied remotely") //start pgpool var startPoolResp cpmcontainerapi.StartPgpoolResponse startPoolResp, err = cpmcontainerapi.StartPgpoolClient(pgpoolNode.Name) if err != nil { logit.Error.Println(err.Error()) return err } logit.Info.Println("configureCluster: pgpool startpgpool output was" + startPoolResp.Output) //finally, update the cluster to show that it is //initialized! cluster.Status = "initialized" err = admindb.UpdateCluster(dbConn, cluster) if err != nil { logit.Error.Println(err.Error()) return err } return nil }