func waitTillReady(container string, sleepTime time.Duration) error {
	for i := 0; i < 40; i++ {
		_, err := cpmcontainerapi.RemoteWritefileClient("/tmp/waitTest", "waitTillReady was here", container)
		if err != nil {
			logit.Error.Println("waitTillReady:waited for cpmcontainerapi on " + container)
			time.Sleep(sleepTime)
		} else {
			logit.Info.Println("waitTillReady:connected to cpmcontainerapi on " + container)
			return nil
		}
	}
	logit.Info.Println("waitTillReady: could not connect to cpmcontainerapi on " + container)
	return errors.New("could not connect to cpmcontainerapi on " + container)

}
func configureCluster(profile string, dbConn *sql.DB, cluster types.Cluster, autocluster bool) error {
	logit.Info.Println("configureCluster:GetCluster")

	//get master node for this cluster
	master, err := admindb.GetContainerMaster(dbConn, cluster.ID)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	var pgport types.Setting
	pgport, err = admindb.GetSetting(dbConn, "PG-PORT")
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	var sleepSetting types.Setting
	sleepSetting, err = admindb.GetSetting(dbConn, "SLEEP-PROV")
	if err != nil {
		logit.Error.Println("configureCluster:" + err.Error())
		return err
	}

	//logit.Info.Println("configureCluster:GetContainerMaster")

	//configure master postgresql.conf file
	var data string
	info := new(template.PostgresqlParameters)
	info.PG_PORT = pgport.Value
	err = template.GetTuningParms(dbConn, profile, info)
	if err != nil {
		logit.Error.Println("configureCluster:" + err.Error())
		return err
	}

	if cluster.ClusterType == "synchronous" {
		info.CLUSTER_TYPE = "*"
		data, err = template.Postgresql("master", info)
	} else {
		//TODO verify these next 2 lines look erroneous
		info.CLUSTER_TYPE = "*"
		data, err = template.Postgresql("master", info)

		//info.CLUSTER_TYPE = ""
		//data, err = template.Postgresql("master", info)
	}
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master postgresql.conf generated")

	//write master postgresql.conf file remotely
	_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master postgresql.conf copied to remote")

	//get domain name
	var domainname types.Setting
	domainname, err = admindb.GetSetting(dbConn, "DOMAIN-NAME")
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	//configure master pg_hba.conf file
	rules := make([]template.Rule, 0)
	data, err = template.Hba(dbConn, "master", master.Name, pgport.Value, cluster.ID, domainname.Value, rules)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master pg_hba.conf generated")

	//write master pg_hba.conf file remotely
	_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:master pg_hba.conf copied remotely")

	//restart postgres after the config file changes
	var stopResp cpmcontainerapi.StopPGResponse
	stopResp, err = cpmcontainerapi.StopPGClient(master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster: master stoppg output was" + stopResp.Output)

	var startResp cpmcontainerapi.StartPGResponse
	startResp, err = cpmcontainerapi.StartPGClient(master.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:master startpg output was" + startResp.Output)

	//sleep loop until the master's PG can respond
	var sleepTime time.Duration
	sleepTime, err = time.ParseDuration(sleepSetting.Value)
	var found = false
	var currentStatus string
	var masterhost = master.Name
	for i := 0; i < 20; i++ {
		//currentStatus, err = GetPGStatus2(dbConn, master.Name, masterhost)
		currentStatus, err = util.FastPing(pgport.Value, master.Name)
		if currentStatus == "RUNNING" {
			logit.Info.Println("master is running...continuing")
			found = true
			break
		} else {
			logit.Info.Println("sleeping waiting on master..")
			time.Sleep(sleepTime)
		}
	}
	if !found {
		logit.Info.Println("configureCluster: timed out waiting on master pg to start")
		return errors.New("timeout waiting for master pg to respond")
	}

	standbynodes, err2 := admindb.GetAllStandbyContainers(dbConn, cluster.ID)
	if err2 != nil {
		logit.Error.Println(err.Error())
		return err
	}
	//configure all standby nodes
	var stopPGResp cpmcontainerapi.StopPGResponse
	i := 0
	for i = range standbynodes {
		if standbynodes[i].Role == STANDBY {

			//stop standby
			if !autocluster {
				stopPGResp, err = cpmcontainerapi.StopPGClient(standbynodes[i].Name)
				if err != nil {
					logit.Error.Println(err.Error())
					logit.Error.Println("configureCluster:stop output was" + stopPGResp.Output)
					return err
				}
				//logit.Info.Println("configureCluster:stop output was" + stopPGResp.Output)
			}

			//
			var username = "******"
			var password = "******"

			//create base backup from master
			var backupresp cpmcontainerapi.BasebackupResponse
			backupresp, err = cpmcontainerapi.BasebackupClient(masterhost+"."+domainname.Value, standbynodes[i].Name, username, password)
			if err != nil {
				logit.Error.Println(err.Error())
				logit.Error.Println("configureCluster:basebackup output was" + backupresp.Output)
				return err
			}
			//logit.Info.Println("configureCluster:basebackup output was" + backupresp.Output)

			data, err = template.Recovery(masterhost, pgport.Value, "postgres")
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby recovery.conf generated")

			//write standby recovery.conf file remotely
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/recovery.conf", data, standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby recovery.conf copied remotely")

			info := new(template.PostgresqlParameters)
			info.PG_PORT = pgport.Value
			info.CLUSTER_TYPE = ""
			err = template.GetTuningParms(dbConn, profile, info)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}

			data, err = template.Postgresql(STANDBY, info)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}

			//write standby postgresql.conf file remotely
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby postgresql.conf copied remotely")

			//configure standby pg_hba.conf file
			data, err = template.Hba(dbConn, STANDBY, standbynodes[i].Name, pgport.Value, cluster.ID, domainname.Value, rules)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}

			logit.Info.Println("configureCluster:standby pg_hba.conf generated")

			//write standby pg_hba.conf file remotely
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby pg_hba.conf copied remotely")

			//start standby

			var stResp cpmcontainerapi.StartPGOnStandbyResponse
			stResp, err = cpmcontainerapi.StartPGOnStandbyClient(standbynodes[i].Name)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("configureCluster:standby startpg output was" + stResp.Output)
		}
		i++
	}

	logit.Info.Println("configureCluster: sleeping 5 seconds before configuring pgpool...")
	clustersleepTime, _ := time.ParseDuration("5s")
	time.Sleep(clustersleepTime)

	pgpoolNode, err4 := admindb.GetContainerPgpool(dbConn, cluster.ID)
	//logit.Info.Println("configureCluster: lookup pgpool node")
	if err4 != nil {
		logit.Error.Println(err.Error())
		return err
	}
	//logit.Info.Println("configureCluster:" + pgpoolNode.Name)

	//configure the pgpool includes all standby nodes AND the master node
	poolnames := make([]string, len(standbynodes)+1)

	i = 0
	for i = range standbynodes {
		poolnames[i] = standbynodes[i].Name + "." + domainname.Value
		i++
	}
	poolnames[i] = master.Name + "." + domainname.Value

	//generate pgpool.conf HOST_LIST
	data, err = template.Poolconf(poolnames)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:pgpool pgpool.conf generated")

	//write pgpool.conf to remote pool node
	_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pgpool.conf", data, pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:pgpool pgpool.conf copied remotely")

	//generate pool_passwd
	data, err = template.Poolpasswd()
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:pgpool pool_passwd generated")

	//write pgpool.conf to remote pool node
	_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_passwd", data, pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:pgpool pool_passwd copied remotely")

	//g enerate pool_hba.conf
	cars := make([]template.Rule, 0)
	data, err = template.Poolhba(cars)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	logit.Info.Println("configureCluster:pgpool pool_hba generated")

	//write pgpool.conf to remote pool node
	_, err = cpmcontainerapi.RemoteWritefileClient(util.GetBase()+"/bin/"+"pool_hba.conf", data, pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster:pgpool pool_hba copied remotely")

	//start pgpool
	var startPoolResp cpmcontainerapi.StartPgpoolResponse
	startPoolResp, err = cpmcontainerapi.StartPgpoolClient(pgpoolNode.Name)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("configureCluster: pgpool startpgpool output was" + startPoolResp.Output)

	//finally, update the cluster to show that it is
	//initialized!
	cluster.Status = "initialized"
	err = admindb.UpdateCluster(dbConn, cluster)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	return nil

}
func provisionImplInit(dbConn *sql.DB, params *swarmapi.DockerRunRequest, standby bool) error {
	//go get the domain name from the settings
	var domainname types.Setting
	var pgport types.Setting
	var sleepSetting types.Setting
	var err error

	domainname, err = admindb.GetSetting(dbConn, "DOMAIN-NAME")
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	pgport, err = admindb.GetSetting(dbConn, "PG-PORT")
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}

	sleepSetting, err = admindb.GetSetting(dbConn, "SLEEP-PROV")
	if err != nil {
		logit.Error.Println("Provision:SLEEP-PROV setting error " + err.Error())
		return err
	}
	var sleepTime time.Duration
	sleepTime, err = time.ParseDuration(sleepSetting.Value)

	fqdn := params.ContainerName + "." + domainname.Value

	//we are depending on a DNS entry being created shortly after
	//creating the node in Docker
	//you might need to wait here until you can reach the new node's agent
	logit.Info.Println("PROFILE waiting till DNS ready")
	err = waitTillReady(fqdn, sleepTime)
	if err != nil {
		logit.Error.Println(err.Error())
		return err
	}
	logit.Info.Println("checkpt 1")

	if standby {
		logit.Info.Println("standby node being created, will not initdb")
	} else {
		if params.RestoreJob != "" {
			logit.Info.Println("RestoreJob found, not doing initdb...")
		} else {
			//initdb on the new node
			logit.Info.Println("PROFILE running initdb on the node")
			var resp cpmcontainerapi.InitdbResponse

			logit.Info.Println("checkpt 2")
			resp, err = cpmcontainerapi.InitdbClient(fqdn)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("checkpt 3")
			logit.Info.Println("initdb output was" + resp.Output)
			//create postgresql.conf
			var data string
			var mode = "standalone"

			info := new(template.PostgresqlParameters)
			info.PG_PORT = pgport.Value
			info.CLUSTER_TYPE = ""
			err = template.GetTuningParms(dbConn, params.Profile, info)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("after GetTuning call with MWM = " + info.TUNE_MWM)
			data, err = template.Postgresql(mode, info)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("provision chkpt 4")

			//place postgresql.conf on new node
			logit.Info.Println("fqdn is " + fqdn)
			logit.Info.Println("postgresql.conf file is " + data)
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/postgresql.conf", data, fqdn)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("provision chkpt 5")
			//create pg_hba.conf
			rules := make([]template.Rule, 0)
			data, err = template.Hba(dbConn, mode, params.ContainerName, pgport.Value, "", domainname.Value, rules)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("provision chkpt 6")
			//place pg_hba.conf on new node
			_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, fqdn)
			if err != nil {
				logit.Error.Println(err.Error())
				return err
			}
			logit.Info.Println("PROFILE templates all built and copied to node")
		}

		//start pg on new node
		var startResp cpmcontainerapi.StartPGResponse
		startResp, err = cpmcontainerapi.StartPGClient(fqdn)
		if err != nil {
			logit.Error.Println(err.Error())
			return err
		}
		logit.Info.Println("startpg output was" + startResp.Output)

		//seed database with initial objects
		var seedResp cpmcontainerapi.SeedResponse
		seedResp, err = cpmcontainerapi.SeedClient(fqdn)
		if err != nil {
			logit.Error.Println(err.Error())
			return err
		}
		logit.Info.Println("seed output was" + seedResp.Output)
	}
	logit.Info.Println("PROFILE node provisioning completed")

	return nil
}
func templateChange(dbConn *sql.DB, containerName string, cars []ContainerAccessRule, containerRole string) error {
	var err error

	logit.Info.Println("templateChange called")
	//create pg_hba.conf
	var mode = containerRole

	domainname, err := admindb.GetSetting(dbConn, "DOMAIN-NAME")
	if err != nil {
		logit.Error.Println("templateChange:DOMAIN-NAME error " + err.Error())
		return err
	}

	rules := make([]template.Rule, 0)
	var ar Rule
	for i := range cars {
		logit.Info.Println("templateChange cars found")
		if cars[i].Selected == "true" {
			logit.Info.Println("templateChange cars found to be true")
			rule := template.Rule{}
			ar, err = GetAccessRule(dbConn, cars[i].AccessRuleID)
			if err != nil {
				logit.Error.Println("templateChange:get access rule error " + err.Error())
				return err
			}
			rule.Type = ar.Type
			rule.Database = ar.Database
			rule.User = ar.User
			rule.Address = ar.Address
			rule.Method = ar.Method
			rules = append(rules, rule)
		}
	}

	logit.Info.Printf("templateChange rules going to template %d\n", len(rules))
	var data string

	fqdn := containerName + "." + domainname.Value

	//place pg_hba.conf on node
	if containerRole == "pgpool" {
		//generate pool_hba.conf
		data, err = template.Poolhba(rules)
		if err != nil {
			logit.Error.Println(err.Error())
			return err
		}

		logit.Info.Println("configureCluster:pgpool pool_hba generated")

		//write pgpool.conf to remote pool node
		var dest = util.GetBase() + "/bin/pool_hba.conf"
		_, err = cpmcontainerapi.RemoteWritefileClient(dest, data, fqdn)
		if err != nil {
			logit.Error.Println(err.Error())
			return err
		}
		logit.Info.Println("configureCluster:pgpool pool_hba copied remotely")

	} else {
		data, err = template.Hba(dbConn, mode, containerName, "", "", domainname.Value, rules)

		if err != nil {
			logit.Error.Println("templateChange:" + err.Error())
			return err
		}

		_, err = cpmcontainerapi.RemoteWritefileClient("/pgdata/pg_hba.conf", data, fqdn)
		if err != nil {
			logit.Error.Println("templateChange:" + err.Error())
			return err
		}
	}

	return err
}