func TestRDPGSystemIntegration(t *testing.T) {
return
config := consulapi.DefaultConfig()
config.Address = `10.244.2.2:8500`
consulClient, _ := consulapi.NewClient(config)
kv := consulClient.KV()
key := fmt.Sprintf(`rdpg/sc-pgbdr-m0-c0/capacity/instances/allowed`)
kvp, _, _ := kv.Get(key, nil)
capacityAllowedString := string(kvp.Value)
capacityAllowed, _ := strconv.Atoi(capacityAllowedString)
SkipConvey(`RDPG System, given four service clusters`, t, func() {
Convey(`When (Capacity Size -1 (rdpg)) * 4 + 1 databases are assigned`, func() {
pSize := os.Getenv(`RDPGD_POOL_SIZE`)
fmt.Printf("pool size is :%s", pSize)
poolSize, err := strconv.Atoi(pSize)
So(err, ShouldBeNil)
So(poolSize, ShouldEqual, 10)
So(capacityAllowed, ShouldEqual, 12)
numberInstance := 0
organizationID := uuid.NewUUID().String()
spaceID := uuid.NewUUID().String()
req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
So(err, ShouldBeNil)
req.SetBasicAuth("cfadmin", "cfadmin")
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
fmt.Printf(`Getting Catalog`)
decoder := json.NewDecoder(resp.Body)
var c cfsb.Catalog
err = decoder.Decode(&c)
So(err, ShouldBeNil)
serviceID := c.Services[0].ServiceID
planID := c.Services[0].Plans[0].PlanID
type InstanceBody struct {
ServiceID string `json:"service_id"`
PlanID string `json:"plan_id"`
OrganizationID string `json:"organization_guid"`
SpaceID string `json:"space_guid"`
}
ins := &InstanceBody{
ServiceID: serviceID,
PlanID: planID,
OrganizationID: organizationID,
SpaceID: spaceID,
}
scpgbdrm0c2Count := 0
scpgbdrm0c3Count := 0
scpgbdrm0c0Count := 0
scpgbdrm0c1Count := 0
time.Sleep(10 * time.Second) // Wait for precreated databases
Convey("assigning Capacity Size * 4 +1 database, oldest available selected at each iteration", func() {
for ps := 0; ps < capacityAllowed*4; ps++ {
p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
sq := fmt.Sprintf(`SELECT id,cluster_id,dbname FROM cfsb.instances WHERE instance_id IS NULL ORDER BY created_at ASC LIMIT 1`)
iOldestAvailable := instances.Instance{}
if ps < capacityAllowed*4 {
for {
err = db.Get(&iOldestAvailable, sq)
if err == sql.ErrNoRows {
time.Sleep(1 * time.Second) // Wait for a pre-created database to be ready.
continue
} else {
break
}
}
} else {
err = db.Get(&iOldestAvailable, sq)
}
instanceID := uuid.NewUUID().String()
insbody, err := json.Marshal(ins)
So(err, ShouldBeNil)
url := cfsbAPIURL("/v2/service_instances/" + instanceID)
req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody))
req.SetBasicAuth("cfadmin", "cfadmin")
fmt.Printf(`Assigning database # %d`, ps)
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
if ps == capacityAllowed*4 {
fmt.Printf(`Assigning one more database than capacity, should says no reach capacity`)
Convey("Trying to asign one more database than total capacity", func() {
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
So(resp.Body, ShouldEqual, "status: 500,description: Provisioning failed, out of capacity. Need to increase capacity setting for service cluster through admin API or redeploy to scale out, please notify the operations team.")
})
Convey(`The number of instances should be equal with the instance capacity.`, func() {
sq := fmt.Sprintf(`SELECT count(id) FROM cfsb.instances`)
err = db.Get(numberInstance, sq)
So(numberInstance, ShouldEqual, capacityAllowed)
})
}
if ps < capacityAllowed*4 {
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
fmt.Printf(`Assigning database!!!! # %d`, ps)
time.Sleep(500 * time.Millisecond) // Wait a second for the transaction commit...
i := instances.Instance{}
sq := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
err = db.Get(&i, sq)
So(i.ID, ShouldEqual, iOldestAvailable.ID)
So(i.ClusterID, ShouldEqual, iOldestAvailable.ClusterID)
So(i.Database, ShouldEqual, iOldestAvailable.Database)
if i.ClusterID == "sc-pgbdr-m0-c2" {
scpgbdrm0c2Count += 1
}
if i.ClusterID == "sc-pgbdr-m0-c3" {
scpgbdrm0c3Count += 1
}
if i.ClusterID == "sc-pgbdr-m0-c0" {
scpgbdrm0c0Count += 1
}
if i.ClusterID == "sc-pgbdr-m0-c1" {
scpgbdrm0c1Count += 1
}
if ps == poolSize*3 {
Convey(`Poolsize*3 +1 Databases should be assigned to all the service cluster.`, func() {
// TODO: count the # assigned on each service cluster and
// Both should be > 0
So(scpgbdrm0c2Count, ShouldBeGreaterThan, 0)
So(scpgbdrm0c3Count, ShouldBeGreaterThan, 0)
So(scpgbdrm0c0Count, ShouldBeGreaterThan, 0)
So(scpgbdrm0c1Count, ShouldBeGreaterThan, 0)
})
}
}
} // capacityAllowed for looooop
})
})
})
}
func (t *Task) bdrPrecreateDatabase(client *consulapi.Client) (err error) {
/*
key := fmt.Sprintf(`rdpg/%s/cluster/service`, ClusterID)
kvp, _, err := kv.Get(key, nil)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrGroupCreate() kv.Get() ! %s`, ClusterID, err))
return
}
v := ``
if kvp != nil {
v = string(kvp.Value)
}
if len(v) > 0 {
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrPrecreateDatabase()`, ClusterID))
return
}
*/
b := bdr.NewBDR(t.ClusterID, client)
re := regexp.MustCompile("[^A-Za-z0-9_]")
u1 := uuid.NewUUID().String()
u2 := uuid.NewUUID().String()
identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte(""))))
dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte(""))))
i := &instances.Instance{
ClusterID: ClusterID,
Database: "d" + identifier,
User: "******" + identifier,
Pass: dbpass,
ClusterService: t.ClusterService,
}
// TODO: Keep the databases under rdpg schema, link to them in the
// cfsb.instances table so that we separate the concerns of CF and databases.
err = b.CreateUser(i.User, i.Pass)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err))
return err
}
err = b.CreateDatabase(i.Database, i.User)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err))
return err
}
defer db.Close()
sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass, cluster_service) VALUES ('%s','%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass, t.ClusterService)
log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err))
return err
}
err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`, `pg_stat_statements`, `uuid-ossp`, `hstore`, `pg_trgm`, `pgcrypto`})
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
//Loop through and add any additional extensions specified in the rdpgd_service properties of the deployment manifest
if len(globals.UserExtensions) > 1 {
userExtensions := strings.Split(globals.UserExtensions, " ")
err = b.CreateExtensions(i.Database, userExtensions)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) Creating Extra User Extensions ! %s", i.Database, i.Database, i.User, err))
return err
}
}
err = b.CreateReplicationGroup(i.Database)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err))
return err
}
sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err))
return err
}
// Tell the management cluster about the newly available database.
// TODO: This can be in a function.
catalog := client.Catalog()
svcs, _, err := catalog.Service("rdpgmc", "", nil)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) catalog.Service() ! %s", i.Database, err))
return err
}
if len(svcs) == 0 {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) ! No services found, no known nodes?!", i.Database))
return err
}
body, err := json.Marshal(i)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err))
return err
}
url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`)
req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body)))
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) http.NewRequest() POST %s ! %s`, i.Database, url, err))
return err
}
log.Trace(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) POST %s`, i.Database, url))
req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS"))
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err))
return err
}
resp.Body.Close()
return
}
func TestRDPGSystemIntegration(t *testing.T) {
Convey(`RDPG System, given two service clusters`, t, func() {
Convey(`When Pool Size + 1 databases are assigned`, func() {
// TODO: Provision Pool Size + 1 databases here...
pSize := os.Getenv(`RDPGD_POOL_SIZE`)
poolSize, err := strconv.Atoi(pSize)
So(err, ShouldBeNil)
So(poolSize, ShouldEqual, 10)
organizationID := uuid.NewUUID().String()
spaceID := uuid.NewUUID().String()
req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
So(err, ShouldBeNil)
req.SetBasicAuth("cfadmin", "cfadmin")
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
decoder := json.NewDecoder(resp.Body)
var c cfsb.Catalog
err = decoder.Decode(&c)
So(err, ShouldBeNil)
serviceID := c.Services[0].ServiceID
planID := c.Services[0].Plans[0].PlanID
type InstanceBody struct {
ServiceID string `json:"service_id"`
PlanID string `json:"plan_id"`
OrganizationID string `json:"organization_guid"`
SpaceID string `json:"space_guid"`
}
ins := &InstanceBody{
ServiceID: serviceID,
PlanID: planID,
OrganizationID: organizationID,
SpaceID: spaceID,
}
sc1Count := 0
sc2Count := 0
time.Sleep(10 * time.Second) // Wait for precreated databases
Convey("assigning pool size +1, oldest available selected at each iteration", func() {
for ps := 0; ps < poolSize; ps++ {
p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
sq := fmt.Sprintf(`SELECT id,cluster_id,dbname FROM cfsb.instances WHERE instance_id IS NULL ORDER BY created_at ASC LIMIT 1`)
iOldestAvailable := instances.Instance{}
err = db.Get(&iOldestAvailable, sq)
instanceID := uuid.NewUUID().String()
insbody, err := json.Marshal(ins)
So(err, ShouldBeNil)
url := cfsbAPIURL("/v2/service_instances/" + instanceID)
req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody))
req.SetBasicAuth("cfadmin", "cfadmin")
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
time.Sleep(500 * time.Millisecond) // Wait a second for the transaction commit...
i := instances.Instance{}
sq = fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
for {
err = db.Get(&i, sq)
if err == sql.ErrNoRows {
time.Sleep(1 * time.Second) // Wait for a pre-created database to be ready.
continue
} else {
break
}
}
So(i.ID, ShouldEqual, iOldestAvailable.ID)
So(i.ClusterID, ShouldEqual, iOldestAvailable.ClusterID)
So(i.Database, ShouldEqual, iOldestAvailable.Database)
if i.ClusterID == "rdpgsc1" {
sc1Count += 1
}
if i.ClusterID == "rdpgsc2" {
sc2Count += 1
}
} // pollSize for looooop :)
Convey(`Databases should be assigned to more than one service cluster.`, func() {
// TODO: count the # assigned on each service cluster and
// Both should be > 0
So(sc1Count, ShouldBeGreaterThan, 0)
So(sc2Count, ShouldBeGreaterThan, 0)
})
})
})
})
}
// Precreate database functionality, note that the database creation lock is
// expected to be held when this is called as this must be done in sequence.
func (t *Task) precreateDatabase(workRole string, client *consulapi.Client) (err error) {
log.Trace(fmt.Sprintf("tasks.Task#precreateDatabase()..."))
b := bdr.NewBDR(t.ClusterID, client)
re := regexp.MustCompile("[^A-Za-z0-9_]")
u1 := uuid.NewUUID().String()
u2 := uuid.NewUUID().String()
identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte(""))))
dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte(""))))
i := &instances.Instance{
ClusterID: ClusterID,
Database: "d" + identifier,
User: "******" + identifier,
Pass: dbpass,
}
// TODO: Keep the databases under rdpg schema, link to them in the
// cfsb.instances table so that we separate the concerns of CF and databases.
err = b.CreateUser(i.User, i.Pass)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err))
return err
}
err = b.CreateDatabase(i.Database, i.User)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err))
return err
}
defer db.Close()
sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass) VALUES ('%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass)
log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err))
return err
}
err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`})
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
err = b.CreateReplicationGroup(i.Database)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err))
return err
}
sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err))
return err
}
// Tell the management cluster about the newly available database.
// TODO: This can be in a function.
catalog := client.Catalog()
svcs, _, err := catalog.Service("rdpgmc", "", nil)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) catalog.Service() ! %s", i.Database, err))
return err
}
if len(svcs) == 0 {
log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) ! No services found, no known nodes?!", i.Database))
return err
}
body, err := json.Marshal(i)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err))
return err
}
url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`)
req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body)))
log.Trace(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) POST %s`, i.Database, url))
// req.Header.Set("Content-Type", "application/json")
// TODO: Retrieve from configuration in database.
req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS"))
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err))
return err
}
resp.Body.Close()
return
}
func TestCFSBIntegration(t *testing.T) {
Convey("CFSB API Authorization", t, func() {
// API Version Header is set test
// basic_auth test, need username and password (Authentication :header) to do broker registrations
// return 401 Unauthorized if credentials are not valid test, auth only tested here
// test when reject a request, response a 412 Precondition Failed message
var getBasicAuthTests = []struct {
username, password string
status int
}{
{"cfadmin", "cfadmin", 200},
{"Aladdin", "open:sesame", 401},
{"", "", 401},
{"cf", "bala", 401},
{"", "cf", 401},
}
for _, authTest := range getBasicAuthTests {
req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
So(err, ShouldBeNil)
req.SetBasicAuth(authTest.username, authTest.password)
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
}
})
// Complete integration test,
// get catalg
// use results to provision instance
// user results to bind
// user results to unbind
// user results to deprovision
// - check for ineffective_at timestamp set
Convey("CFSB API", t, func() {
config := consulapi.DefaultConfig()
config.Address = `10.244.2.2:8500`
consulClient, err := consulapi.NewClient(config)
So(err, ShouldBeNil)
organizationID := uuid.NewUUID().String()
spaceID := uuid.NewUUID().String()
Convey("Get Catalog", func() {
req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
So(err, ShouldBeNil)
req.SetBasicAuth("cfadmin", "cfadmin")
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
decoder := json.NewDecoder(resp.Body)
var c cfsb.Catalog
err = decoder.Decode(&c)
So(err, ShouldBeNil)
So(len(c.Services), ShouldNotEqual, 0)
Convey("Services in Catalog Response", func() {
firstService := c.Services[0]
So(firstService.ServiceID, ShouldNotBeBlank)
So(firstService.Name, ShouldNotBeBlank)
So(firstService.Description, ShouldNotBeBlank)
So(len(firstService.Plans), ShouldNotEqual, 0)
Convey("Plans in Services", func() {
firstPlan := firstService.Plans[0]
So(firstPlan.PlanID, ShouldNotBeBlank)
So(firstPlan.Name, ShouldNotBeBlank)
So(firstPlan.Description, ShouldNotBeBlank)
})
})
serviceID := c.Services[0].ServiceID
planID := c.Services[0].Plans[0].PlanID
Convey("Provision Instance", func() {
type InstanceBody struct {
ServiceID string `json:"service_id"`
PlanID string `json:"plan_id"`
OrganizationID string `json:"organization_guid"`
SpaceID string `json:"space_guid"`
}
ins := &InstanceBody{
ServiceID: serviceID,
PlanID: planID,
OrganizationID: organizationID,
SpaceID: spaceID,
}
instanceID := uuid.NewUUID().String()
insbody, err := json.Marshal(ins)
So(err, ShouldBeNil)
// TODO: Perform this test after checking/waiting for databases to be
// pre-created otherwise we get a false-fail.
url := cfsbAPIURL("/v2/service_instances/" + instanceID)
req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody))
req.SetBasicAuth("cfadmin", "cfadmin")
So(err, ShouldBeNil)
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
time.Sleep(1 * time.Second) // Allow replication to catch up.
// At this point the instance within the database should have the clusterID
// Note that this will be fetching from the management cluster database
i := instances.Instance{}
p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
q := fmt.Sprintf(`SELECT id, cluster_id, instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
db.Get(&i, q)
db.Close()
Convey("The fields should be set in management cluster write master instance table", func() {
So(err, ShouldBeNil)
So(i.ClusterID, ShouldNotBeBlank)
So(i.ServiceID, ShouldEqual, serviceID)
So(i.PlanID, ShouldEqual, planID)
So(i.OrganizationID, ShouldEqual, organizationID)
So(i.SpaceID, ShouldEqual, spaceID)
})
managementCluster, err := rdpg.NewCluster(`rdpgmc`, consulClient)
So(err, ShouldBeNil)
Convey("Each management cluster node should have the correct fields set in the cfsb.instances table", func() {
for _, node := range managementCluster.Nodes { // Loop over management cluster nodes.
p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
in := instances.Instance{}
q := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
db.Get(&in, q)
So(in.ClusterID, ShouldNotBeBlank)
So(in.ServiceID, ShouldEqual, serviceID)
So(in.PlanID, ShouldEqual, planID)
So(in.OrganizationID, ShouldEqual, organizationID)
So(in.SpaceID, ShouldEqual, spaceID)
db.Close()
}
})
serviceCluster, err := rdpg.NewCluster(i.ClusterID, consulClient)
So(err, ShouldBeNil)
// TODO: Find out which SC the instance is on and for that cluster's nodes, do the below.
Convey("Each service cluster node should have a correct record of the instance in rdpg.cfsb.instances.", func() {
for _, node := range serviceCluster.Nodes {
sp := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := sp.Connect()
So(err, ShouldBeNil)
in := instances.Instance{}
q := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
db.Get(&in, q)
So(in.ClusterID, ShouldEqual, i.ClusterID)
So(in.ServiceID, ShouldEqual, serviceID)
So(in.PlanID, ShouldEqual, planID)
So(in.OrganizationID, ShouldEqual, organizationID)
So(in.SpaceID, ShouldEqual, spaceID)
db.Close()
}
})
Convey("Each service cluster node should have the instance's user and database created on it", func() {
for _, node := range serviceCluster.Nodes { // Loop over service cluster nodes.
sp := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := sp.Connect()
So(err, ShouldBeNil)
// user should be created on each service cluster node
r := ""
q := fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname='%s'`, i.User)
db.Get(&r, q)
So(r, ShouldEqual, i.User)
// database should be created on each service cluster node
d := ""
q = fmt.Sprintf(`SELECT datname FROM pg_catalog.pg_database WHERE datname='%s'`, i.Database)
db.Get(&d, q)
So(d, ShouldEqual, i.Database)
db.Close()
}
})
Convey("Each service cluster node for the instance's database should have bdr and btree_gist extension, and have the same count of bdr.bdr_nodes.", func() {
var count int
for _, node := range serviceCluster.Nodes {
p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, i.Database, i.Pass)
db, err := p.Connect()
So(err, ShouldBeNil)
var s string
// extensions on all cluster nodes should have bdr, btree_gist
exts := []string{"bdr", "btree_gist"}
for _, ext := range exts {
q := fmt.Sprintf(`SELECT extname FROM pg_extension WHERE extname ='%s'`, ext)
db.Get(&s, q)
So(s, ShouldEqual, ext)
}
// replication group should have the same count as rdpg bdr.bdr_nodes;
db.Get(&count, "SELECT count(node_status) FROM bdr.bdr_nodes WHERE node_status='r'")
So(count, ShouldEqual, len(serviceCluster.Nodes))
db.Close()
}
})
Convey("Bind", func() {
bindingID := uuid.NewUUID().String()
appGuid := uuid.NewUUID().String()
type BindingBody struct {
PlanID string `json:"plan_id"`
ServiceID string `json:"service_id"`
AppGuid string `json:app_guid"`
}
bind := &BindingBody{
PlanID: planID,
ServiceID: serviceID,
AppGuid: appGuid,
}
bindbody, err := json.Marshal(bind)
So(err, ShouldBeNil)
url := cfsbAPIURL("/v2/service_instances/" + instanceID + "/service_bindings/" + bindingID)
req, err := http.NewRequest("PUT", url, bytes.NewBuffer(bindbody))
req.SetBasicAuth("cfadmin", "cfadmin")
So(err, ShouldBeNil)
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
time.Sleep(1 * time.Second) // Allow replication to replicate before proceeding
// Binding that it returns should have values for it's fields
decoder := json.NewDecoder(resp.Body)
var b cfsb.Binding
err = decoder.Decode(&b)
So(err, ShouldBeNil)
Convey("Bind response body should be consistant with management cluster node bind info", func() {
So(b.BindingID, ShouldEqual, bindingID)
So(b.InstanceID, ShouldEqual, instanceID)
So(b.Creds, ShouldNotBeNil)
creds := b.Creds
catalog := consulClient.Catalog()
services, _, err := catalog.Service(fmt.Sprintf(`%s-master`, i.ClusterID), "", nil)
So(err, ShouldBeNil)
So(len(services), ShouldEqual, 1)
dns := fmt.Sprintf(`%s:5432`, services[0].Address)
s_dns := strings.Split(dns, ":")
uri := fmt.Sprintf(`postgres://%s:%s@%s/%s?sslmode=%s`, i.User, i.Pass, dns, i.Database, `disable`)
dsn := fmt.Sprintf(`host=%s port=%s user=%s password=%s dbname=%s connect_timeout=%s sslmode=%s`, s_dns[0], s_dns[1], i.User, i.Pass, i.Database, `5`, `disable`)
jdbc := fmt.Sprintf(`jdbc:postgres://%s:%s@%s/%s?sslmode=%s`, i.User, i.Pass, dns, i.Database, `disable`)
So(creds.URI, ShouldEqual, uri)
So(creds.DSN, ShouldEqual, dsn)
So(creds.JDBCURI, ShouldEqual, jdbc)
So(creds.Host, ShouldEqual, s_dns[0])
So(creds.Port, ShouldEqual, s_dns[1])
So(creds.UserName, ShouldEqual, i.User)
So(creds.Password, ShouldEqual, i.Pass)
So(creds.Database, ShouldEqual, i.Database)
})
Convey("Each management cluster node should have correct cfsb.binding record for the binding id.", func() {
for _, node := range managementCluster.Nodes {
p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
var instanceIdTest string
q := fmt.Sprintf(`SELECT instance_id FROM cfsb.bindings WHERE binding_id = '%s'`, bindingID)
db.Get(&instanceIdTest, q)
So(instanceIdTest, ShouldEqual, instanceID)
db.Close()
}
})
// TODO: Come back to this and get this test passing.
SkipConvey("Each management cluster node should have correct cfsb.credentials record for the binding id.", func() {
for _, node := range managementCluster.Nodes {
p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
credsTest := cfsb.Credentials{}
q = fmt.Sprintf(`SELECT instance_id,dbuser,dbname,dbpass FROM cfsb.credentials WHERE binding_id = '%s'`, bindingID)
db.Get(&credsTest, q)
So(credsTest.InstanceID, ShouldEqual, instanceID)
So(credsTest.UserName, ShouldEqual, i.User)
So(credsTest.Database, ShouldEqual, i.Database)
So(credsTest.Password, ShouldEqual, i.Pass)
db.Close()
}
})
Convey("Un Bind", func() {
url := cfsbAPIURL("/v2/service_instances/" + instanceID + "/service_bindings/" + bindingID + "?service_id=" + serviceID + "&plan_id=" + planID)
req, err = http.NewRequest("DELETE", url, nil)
req.SetBasicAuth("cfadmin", "cfadmin")
So(err, ShouldBeNil)
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
Convey("cfsb.binding record for each management node should become inefective for the binding id .", func() {
for _, node := range managementCluster.Nodes {
p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
var s string
q := fmt.Sprintf(`SELECT ineffective_at::text FROM cfsb.bindings WHERE binding_id = '%s'`, bindingID)
db.Get(&s, q)
So(s, ShouldNotBeBlank)
db.Close()
}
})
}) //unbind
}) // bind
// Skipping for now, deprovision is not completed yet.
Convey("Deprovision", func() {
url := cfsbAPIURL("/v2/service_instances/" + instanceID + "?service_id=" + serviceID + "&plan_id=" + planID)
req, err = http.NewRequest("DELETE", url, nil)
req.SetBasicAuth("cfadmin", "cfadmin")
So(err, ShouldBeNil)
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
So(err, ShouldBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
time.Sleep(1 * time.Second) // Wait for MC to tell SC about the udpate
Convey("cfsb.instances for each management node should become inefective for the instanceID .", func() {
for _, node := range managementCluster.Nodes {
p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
var ia pq.NullTime
q := fmt.Sprintf(`SELECT ineffective_at FROM cfsb.instances WHERE instance_id='%s' LIMIT 1`, instanceID)
db.Get(&ia, q)
So(ia.Valid, ShouldEqual, true)
So(ia.Time, ShouldNotBeNil)
db.Close()
}
})
Convey("cfsb.instances for each service node should become inefective for the instanceID.", func() {
for _, node := range serviceCluster.Nodes {
p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
var ia pq.NullTime
q := fmt.Sprintf(`SELECT ineffective_at FROM cfsb.instances WHERE instance_id='%s'`, instanceID)
db.Get(&ia, q)
So(ia.Time, ShouldNotBeNil)
db.Close()
}
})
SkipConvey("Each node should NOT have the user and database anymore", func() {
for _, node := range serviceCluster.Nodes {
p := pg.NewPG(node.PG.IP, `7432`, `postgres`, `rdpg`, `admin`)
db, err := p.Connect()
So(err, ShouldBeNil)
// user should have been deleted on all service cluster nodes
var s string
q := fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname='%s'`, i.User)
db.Get(&s, q)
So(s, ShouldBeBlank)
// database should be deleted all service cluster node
q = fmt.Sprintf(`SELECT datname FROM pg_catalog.pg_database WHERE datname=?`, i.Database)
db.Get(&s, q)
So(s, ShouldBeBlank)
db.Close()
}
})
}) // Deprovision
}) // Provision
}) // catalog
}) // integratetion workflow
}