func TestRDPGSystemIntegration(t *testing.T) { return config := consulapi.DefaultConfig() config.Address = `10.244.2.2:8500` consulClient, _ := consulapi.NewClient(config) kv := consulClient.KV() key := fmt.Sprintf(`rdpg/sc-pgbdr-m0-c0/capacity/instances/allowed`) kvp, _, _ := kv.Get(key, nil) capacityAllowedString := string(kvp.Value) capacityAllowed, _ := strconv.Atoi(capacityAllowedString) SkipConvey(`RDPG System, given four service clusters`, t, func() { Convey(`When (Capacity Size -1 (rdpg)) * 4 + 1 databases are assigned`, func() { pSize := os.Getenv(`RDPGD_POOL_SIZE`) fmt.Printf("pool size is :%s", pSize) poolSize, err := strconv.Atoi(pSize) So(err, ShouldBeNil) So(poolSize, ShouldEqual, 10) So(capacityAllowed, ShouldEqual, 12) numberInstance := 0 organizationID := uuid.NewUUID().String() spaceID := uuid.NewUUID().String() req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil) So(err, ShouldBeNil) req.SetBasicAuth("cfadmin", "cfadmin") httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) fmt.Printf(`Getting Catalog`) decoder := json.NewDecoder(resp.Body) var c cfsb.Catalog err = decoder.Decode(&c) So(err, ShouldBeNil) serviceID := c.Services[0].ServiceID planID := c.Services[0].Plans[0].PlanID type InstanceBody struct { ServiceID string `json:"service_id"` PlanID string `json:"plan_id"` OrganizationID string `json:"organization_guid"` SpaceID string `json:"space_guid"` } ins := &InstanceBody{ ServiceID: serviceID, PlanID: planID, OrganizationID: organizationID, SpaceID: spaceID, } scpgbdrm0c2Count := 0 scpgbdrm0c3Count := 0 scpgbdrm0c0Count := 0 scpgbdrm0c1Count := 0 time.Sleep(10 * time.Second) // Wait for precreated databases Convey("assigning Capacity Size * 4 +1 database, oldest available selected at each iteration", func() { for ps := 0; ps < capacityAllowed*4; ps++ { p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() sq := fmt.Sprintf(`SELECT id,cluster_id,dbname FROM cfsb.instances WHERE instance_id IS NULL ORDER BY created_at ASC LIMIT 1`) iOldestAvailable := instances.Instance{} if ps < capacityAllowed*4 { for { err = db.Get(&iOldestAvailable, sq) if err == sql.ErrNoRows { time.Sleep(1 * time.Second) // Wait for a pre-created database to be ready. continue } else { break } } } else { err = db.Get(&iOldestAvailable, sq) } instanceID := uuid.NewUUID().String() insbody, err := json.Marshal(ins) So(err, ShouldBeNil) url := cfsbAPIURL("/v2/service_instances/" + instanceID) req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody)) req.SetBasicAuth("cfadmin", "cfadmin") fmt.Printf(`Assigning database # %d`, ps) httpClient := &http.Client{} resp, err := httpClient.Do(req) if ps == capacityAllowed*4 { fmt.Printf(`Assigning one more database than capacity, should says no reach capacity`) Convey("Trying to asign one more database than total capacity", func() { So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError) So(resp.Body, ShouldEqual, "status: 500,description: Provisioning failed, out of capacity. Need to increase capacity setting for service cluster through admin API or redeploy to scale out, please notify the operations team.") }) Convey(`The number of instances should be equal with the instance capacity.`, func() { sq := fmt.Sprintf(`SELECT count(id) FROM cfsb.instances`) err = db.Get(numberInstance, sq) So(numberInstance, ShouldEqual, capacityAllowed) }) } if ps < capacityAllowed*4 { So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) fmt.Printf(`Assigning database!!!! # %d`, ps) time.Sleep(500 * time.Millisecond) // Wait a second for the transaction commit... i := instances.Instance{} sq := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID) err = db.Get(&i, sq) So(i.ID, ShouldEqual, iOldestAvailable.ID) So(i.ClusterID, ShouldEqual, iOldestAvailable.ClusterID) So(i.Database, ShouldEqual, iOldestAvailable.Database) if i.ClusterID == "sc-pgbdr-m0-c2" { scpgbdrm0c2Count += 1 } if i.ClusterID == "sc-pgbdr-m0-c3" { scpgbdrm0c3Count += 1 } if i.ClusterID == "sc-pgbdr-m0-c0" { scpgbdrm0c0Count += 1 } if i.ClusterID == "sc-pgbdr-m0-c1" { scpgbdrm0c1Count += 1 } if ps == poolSize*3 { Convey(`Poolsize*3 +1 Databases should be assigned to all the service cluster.`, func() { // TODO: count the # assigned on each service cluster and // Both should be > 0 So(scpgbdrm0c2Count, ShouldBeGreaterThan, 0) So(scpgbdrm0c3Count, ShouldBeGreaterThan, 0) So(scpgbdrm0c0Count, ShouldBeGreaterThan, 0) So(scpgbdrm0c1Count, ShouldBeGreaterThan, 0) }) } } } // capacityAllowed for looooop }) }) }) }
func (t *Task) bdrPrecreateDatabase(client *consulapi.Client) (err error) { /* key := fmt.Sprintf(`rdpg/%s/cluster/service`, ClusterID) kvp, _, err := kv.Get(key, nil) if err != nil { log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrGroupCreate() kv.Get() ! %s`, ClusterID, err)) return } v := `` if kvp != nil { v = string(kvp.Value) } if len(v) > 0 { log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrPrecreateDatabase()`, ClusterID)) return } */ b := bdr.NewBDR(t.ClusterID, client) re := regexp.MustCompile("[^A-Za-z0-9_]") u1 := uuid.NewUUID().String() u2 := uuid.NewUUID().String() identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte("")))) dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte("")))) i := &instances.Instance{ ClusterID: ClusterID, Database: "d" + identifier, User: "******" + identifier, Pass: dbpass, ClusterService: t.ClusterService, } // TODO: Keep the databases under rdpg schema, link to them in the // cfsb.instances table so that we separate the concerns of CF and databases. err = b.CreateUser(i.User, i.Pass) if err != nil { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err)) return err } err = b.CreateDatabase(i.Database, i.User) if err != nil { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err)) return err } p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass) db, err := p.Connect() if err != nil { log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err)) return err } defer db.Close() sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass, cluster_service) VALUES ('%s','%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass, t.ClusterService) log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq)) _, err = db.Query(sq) if err != nil { log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err)) return err } err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`, `pg_stat_statements`, `uuid-ossp`, `hstore`, `pg_trgm`, `pgcrypto`}) if err != nil { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err)) return err } //Loop through and add any additional extensions specified in the rdpgd_service properties of the deployment manifest if len(globals.UserExtensions) > 1 { userExtensions := strings.Split(globals.UserExtensions, " ") err = b.CreateExtensions(i.Database, userExtensions) if err != nil { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) Creating Extra User Extensions ! %s", i.Database, i.Database, i.User, err)) return err } } err = b.CreateReplicationGroup(i.Database) if err != nil { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err)) return err } sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database) log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq)) _, err = db.Query(sq) if err != nil { log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err)) return err } // Tell the management cluster about the newly available database. // TODO: This can be in a function. catalog := client.Catalog() svcs, _, err := catalog.Service("rdpgmc", "", nil) if err != nil { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) catalog.Service() ! %s", i.Database, err)) return err } if len(svcs) == 0 { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) ! No services found, no known nodes?!", i.Database)) return err } body, err := json.Marshal(i) if err != nil { log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err)) return err } url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`) req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body))) if err != nil { log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) http.NewRequest() POST %s ! %s`, i.Database, url, err)) return err } log.Trace(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) POST %s`, i.Database, url)) req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS")) httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err)) return err } resp.Body.Close() return }
func TestRDPGSystemIntegration(t *testing.T) { Convey(`RDPG System, given two service clusters`, t, func() { Convey(`When Pool Size + 1 databases are assigned`, func() { // TODO: Provision Pool Size + 1 databases here... pSize := os.Getenv(`RDPGD_POOL_SIZE`) poolSize, err := strconv.Atoi(pSize) So(err, ShouldBeNil) So(poolSize, ShouldEqual, 10) organizationID := uuid.NewUUID().String() spaceID := uuid.NewUUID().String() req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil) So(err, ShouldBeNil) req.SetBasicAuth("cfadmin", "cfadmin") httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) decoder := json.NewDecoder(resp.Body) var c cfsb.Catalog err = decoder.Decode(&c) So(err, ShouldBeNil) serviceID := c.Services[0].ServiceID planID := c.Services[0].Plans[0].PlanID type InstanceBody struct { ServiceID string `json:"service_id"` PlanID string `json:"plan_id"` OrganizationID string `json:"organization_guid"` SpaceID string `json:"space_guid"` } ins := &InstanceBody{ ServiceID: serviceID, PlanID: planID, OrganizationID: organizationID, SpaceID: spaceID, } sc1Count := 0 sc2Count := 0 time.Sleep(10 * time.Second) // Wait for precreated databases Convey("assigning pool size +1, oldest available selected at each iteration", func() { for ps := 0; ps < poolSize; ps++ { p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() sq := fmt.Sprintf(`SELECT id,cluster_id,dbname FROM cfsb.instances WHERE instance_id IS NULL ORDER BY created_at ASC LIMIT 1`) iOldestAvailable := instances.Instance{} err = db.Get(&iOldestAvailable, sq) instanceID := uuid.NewUUID().String() insbody, err := json.Marshal(ins) So(err, ShouldBeNil) url := cfsbAPIURL("/v2/service_instances/" + instanceID) req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody)) req.SetBasicAuth("cfadmin", "cfadmin") httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) time.Sleep(500 * time.Millisecond) // Wait a second for the transaction commit... i := instances.Instance{} sq = fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID) for { err = db.Get(&i, sq) if err == sql.ErrNoRows { time.Sleep(1 * time.Second) // Wait for a pre-created database to be ready. continue } else { break } } So(i.ID, ShouldEqual, iOldestAvailable.ID) So(i.ClusterID, ShouldEqual, iOldestAvailable.ClusterID) So(i.Database, ShouldEqual, iOldestAvailable.Database) if i.ClusterID == "rdpgsc1" { sc1Count += 1 } if i.ClusterID == "rdpgsc2" { sc2Count += 1 } } // pollSize for looooop :) Convey(`Databases should be assigned to more than one service cluster.`, func() { // TODO: count the # assigned on each service cluster and // Both should be > 0 So(sc1Count, ShouldBeGreaterThan, 0) So(sc2Count, ShouldBeGreaterThan, 0) }) }) }) }) }
// Precreate database functionality, note that the database creation lock is // expected to be held when this is called as this must be done in sequence. func (t *Task) precreateDatabase(workRole string, client *consulapi.Client) (err error) { log.Trace(fmt.Sprintf("tasks.Task#precreateDatabase()...")) b := bdr.NewBDR(t.ClusterID, client) re := regexp.MustCompile("[^A-Za-z0-9_]") u1 := uuid.NewUUID().String() u2 := uuid.NewUUID().String() identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte("")))) dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte("")))) i := &instances.Instance{ ClusterID: ClusterID, Database: "d" + identifier, User: "******" + identifier, Pass: dbpass, } // TODO: Keep the databases under rdpg schema, link to them in the // cfsb.instances table so that we separate the concerns of CF and databases. err = b.CreateUser(i.User, i.Pass) if err != nil { log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err)) return err } err = b.CreateDatabase(i.Database, i.User) if err != nil { log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err)) return err } p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass) db, err := p.Connect() if err != nil { log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err)) return err } defer db.Close() sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass) VALUES ('%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass) log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq)) _, err = db.Query(sq) if err != nil { log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err)) return err } err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`}) if err != nil { log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err)) return err } err = b.CreateReplicationGroup(i.Database) if err != nil { log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err)) return err } sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database) log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq)) _, err = db.Query(sq) if err != nil { log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err)) return err } // Tell the management cluster about the newly available database. // TODO: This can be in a function. catalog := client.Catalog() svcs, _, err := catalog.Service("rdpgmc", "", nil) if err != nil { log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) catalog.Service() ! %s", i.Database, err)) return err } if len(svcs) == 0 { log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) ! No services found, no known nodes?!", i.Database)) return err } body, err := json.Marshal(i) if err != nil { log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err)) return err } url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`) req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body))) log.Trace(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) POST %s`, i.Database, url)) // req.Header.Set("Content-Type", "application/json") // TODO: Retrieve from configuration in database. req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS")) httpClient := &http.Client{} resp, err := httpClient.Do(req) if err != nil { log.Error(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err)) return err } resp.Body.Close() return }
func TestCFSBIntegration(t *testing.T) { Convey("CFSB API Authorization", t, func() { // API Version Header is set test // basic_auth test, need username and password (Authentication :header) to do broker registrations // return 401 Unauthorized if credentials are not valid test, auth only tested here // test when reject a request, response a 412 Precondition Failed message var getBasicAuthTests = []struct { username, password string status int }{ {"cfadmin", "cfadmin", 200}, {"Aladdin", "open:sesame", 401}, {"", "", 401}, {"cf", "bala", 401}, {"", "cf", 401}, } for _, authTest := range getBasicAuthTests { req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil) So(err, ShouldBeNil) req.SetBasicAuth(authTest.username, authTest.password) httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) } }) // Complete integration test, // get catalg // use results to provision instance // user results to bind // user results to unbind // user results to deprovision // - check for ineffective_at timestamp set Convey("CFSB API", t, func() { config := consulapi.DefaultConfig() config.Address = `10.244.2.2:8500` consulClient, err := consulapi.NewClient(config) So(err, ShouldBeNil) organizationID := uuid.NewUUID().String() spaceID := uuid.NewUUID().String() Convey("Get Catalog", func() { req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil) So(err, ShouldBeNil) req.SetBasicAuth("cfadmin", "cfadmin") httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) decoder := json.NewDecoder(resp.Body) var c cfsb.Catalog err = decoder.Decode(&c) So(err, ShouldBeNil) So(len(c.Services), ShouldNotEqual, 0) Convey("Services in Catalog Response", func() { firstService := c.Services[0] So(firstService.ServiceID, ShouldNotBeBlank) So(firstService.Name, ShouldNotBeBlank) So(firstService.Description, ShouldNotBeBlank) So(len(firstService.Plans), ShouldNotEqual, 0) Convey("Plans in Services", func() { firstPlan := firstService.Plans[0] So(firstPlan.PlanID, ShouldNotBeBlank) So(firstPlan.Name, ShouldNotBeBlank) So(firstPlan.Description, ShouldNotBeBlank) }) }) serviceID := c.Services[0].ServiceID planID := c.Services[0].Plans[0].PlanID Convey("Provision Instance", func() { type InstanceBody struct { ServiceID string `json:"service_id"` PlanID string `json:"plan_id"` OrganizationID string `json:"organization_guid"` SpaceID string `json:"space_guid"` } ins := &InstanceBody{ ServiceID: serviceID, PlanID: planID, OrganizationID: organizationID, SpaceID: spaceID, } instanceID := uuid.NewUUID().String() insbody, err := json.Marshal(ins) So(err, ShouldBeNil) // TODO: Perform this test after checking/waiting for databases to be // pre-created otherwise we get a false-fail. url := cfsbAPIURL("/v2/service_instances/" + instanceID) req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody)) req.SetBasicAuth("cfadmin", "cfadmin") So(err, ShouldBeNil) httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) time.Sleep(1 * time.Second) // Allow replication to catch up. // At this point the instance within the database should have the clusterID // Note that this will be fetching from the management cluster database i := instances.Instance{} p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) q := fmt.Sprintf(`SELECT id, cluster_id, instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID) db.Get(&i, q) db.Close() Convey("The fields should be set in management cluster write master instance table", func() { So(err, ShouldBeNil) So(i.ClusterID, ShouldNotBeBlank) So(i.ServiceID, ShouldEqual, serviceID) So(i.PlanID, ShouldEqual, planID) So(i.OrganizationID, ShouldEqual, organizationID) So(i.SpaceID, ShouldEqual, spaceID) }) managementCluster, err := rdpg.NewCluster(`rdpgmc`, consulClient) So(err, ShouldBeNil) Convey("Each management cluster node should have the correct fields set in the cfsb.instances table", func() { for _, node := range managementCluster.Nodes { // Loop over management cluster nodes. p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) in := instances.Instance{} q := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID) db.Get(&in, q) So(in.ClusterID, ShouldNotBeBlank) So(in.ServiceID, ShouldEqual, serviceID) So(in.PlanID, ShouldEqual, planID) So(in.OrganizationID, ShouldEqual, organizationID) So(in.SpaceID, ShouldEqual, spaceID) db.Close() } }) serviceCluster, err := rdpg.NewCluster(i.ClusterID, consulClient) So(err, ShouldBeNil) // TODO: Find out which SC the instance is on and for that cluster's nodes, do the below. Convey("Each service cluster node should have a correct record of the instance in rdpg.cfsb.instances.", func() { for _, node := range serviceCluster.Nodes { sp := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := sp.Connect() So(err, ShouldBeNil) in := instances.Instance{} q := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID) db.Get(&in, q) So(in.ClusterID, ShouldEqual, i.ClusterID) So(in.ServiceID, ShouldEqual, serviceID) So(in.PlanID, ShouldEqual, planID) So(in.OrganizationID, ShouldEqual, organizationID) So(in.SpaceID, ShouldEqual, spaceID) db.Close() } }) Convey("Each service cluster node should have the instance's user and database created on it", func() { for _, node := range serviceCluster.Nodes { // Loop over service cluster nodes. sp := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := sp.Connect() So(err, ShouldBeNil) // user should be created on each service cluster node r := "" q := fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname='%s'`, i.User) db.Get(&r, q) So(r, ShouldEqual, i.User) // database should be created on each service cluster node d := "" q = fmt.Sprintf(`SELECT datname FROM pg_catalog.pg_database WHERE datname='%s'`, i.Database) db.Get(&d, q) So(d, ShouldEqual, i.Database) db.Close() } }) Convey("Each service cluster node for the instance's database should have bdr and btree_gist extension, and have the same count of bdr.bdr_nodes.", func() { var count int for _, node := range serviceCluster.Nodes { p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, i.Database, i.Pass) db, err := p.Connect() So(err, ShouldBeNil) var s string // extensions on all cluster nodes should have bdr, btree_gist exts := []string{"bdr", "btree_gist"} for _, ext := range exts { q := fmt.Sprintf(`SELECT extname FROM pg_extension WHERE extname ='%s'`, ext) db.Get(&s, q) So(s, ShouldEqual, ext) } // replication group should have the same count as rdpg bdr.bdr_nodes; db.Get(&count, "SELECT count(node_status) FROM bdr.bdr_nodes WHERE node_status='r'") So(count, ShouldEqual, len(serviceCluster.Nodes)) db.Close() } }) Convey("Bind", func() { bindingID := uuid.NewUUID().String() appGuid := uuid.NewUUID().String() type BindingBody struct { PlanID string `json:"plan_id"` ServiceID string `json:"service_id"` AppGuid string `json:app_guid"` } bind := &BindingBody{ PlanID: planID, ServiceID: serviceID, AppGuid: appGuid, } bindbody, err := json.Marshal(bind) So(err, ShouldBeNil) url := cfsbAPIURL("/v2/service_instances/" + instanceID + "/service_bindings/" + bindingID) req, err := http.NewRequest("PUT", url, bytes.NewBuffer(bindbody)) req.SetBasicAuth("cfadmin", "cfadmin") So(err, ShouldBeNil) httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) time.Sleep(1 * time.Second) // Allow replication to replicate before proceeding // Binding that it returns should have values for it's fields decoder := json.NewDecoder(resp.Body) var b cfsb.Binding err = decoder.Decode(&b) So(err, ShouldBeNil) Convey("Bind response body should be consistant with management cluster node bind info", func() { So(b.BindingID, ShouldEqual, bindingID) So(b.InstanceID, ShouldEqual, instanceID) So(b.Creds, ShouldNotBeNil) creds := b.Creds catalog := consulClient.Catalog() services, _, err := catalog.Service(fmt.Sprintf(`%s-master`, i.ClusterID), "", nil) So(err, ShouldBeNil) So(len(services), ShouldEqual, 1) dns := fmt.Sprintf(`%s:5432`, services[0].Address) s_dns := strings.Split(dns, ":") uri := fmt.Sprintf(`postgres://%s:%s@%s/%s?sslmode=%s`, i.User, i.Pass, dns, i.Database, `disable`) dsn := fmt.Sprintf(`host=%s port=%s user=%s password=%s dbname=%s connect_timeout=%s sslmode=%s`, s_dns[0], s_dns[1], i.User, i.Pass, i.Database, `5`, `disable`) jdbc := fmt.Sprintf(`jdbc:postgres://%s:%s@%s/%s?sslmode=%s`, i.User, i.Pass, dns, i.Database, `disable`) So(creds.URI, ShouldEqual, uri) So(creds.DSN, ShouldEqual, dsn) So(creds.JDBCURI, ShouldEqual, jdbc) So(creds.Host, ShouldEqual, s_dns[0]) So(creds.Port, ShouldEqual, s_dns[1]) So(creds.UserName, ShouldEqual, i.User) So(creds.Password, ShouldEqual, i.Pass) So(creds.Database, ShouldEqual, i.Database) }) Convey("Each management cluster node should have correct cfsb.binding record for the binding id.", func() { for _, node := range managementCluster.Nodes { p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) var instanceIdTest string q := fmt.Sprintf(`SELECT instance_id FROM cfsb.bindings WHERE binding_id = '%s'`, bindingID) db.Get(&instanceIdTest, q) So(instanceIdTest, ShouldEqual, instanceID) db.Close() } }) // TODO: Come back to this and get this test passing. SkipConvey("Each management cluster node should have correct cfsb.credentials record for the binding id.", func() { for _, node := range managementCluster.Nodes { p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) credsTest := cfsb.Credentials{} q = fmt.Sprintf(`SELECT instance_id,dbuser,dbname,dbpass FROM cfsb.credentials WHERE binding_id = '%s'`, bindingID) db.Get(&credsTest, q) So(credsTest.InstanceID, ShouldEqual, instanceID) So(credsTest.UserName, ShouldEqual, i.User) So(credsTest.Database, ShouldEqual, i.Database) So(credsTest.Password, ShouldEqual, i.Pass) db.Close() } }) Convey("Un Bind", func() { url := cfsbAPIURL("/v2/service_instances/" + instanceID + "/service_bindings/" + bindingID + "?service_id=" + serviceID + "&plan_id=" + planID) req, err = http.NewRequest("DELETE", url, nil) req.SetBasicAuth("cfadmin", "cfadmin") So(err, ShouldBeNil) httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) Convey("cfsb.binding record for each management node should become inefective for the binding id .", func() { for _, node := range managementCluster.Nodes { p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) var s string q := fmt.Sprintf(`SELECT ineffective_at::text FROM cfsb.bindings WHERE binding_id = '%s'`, bindingID) db.Get(&s, q) So(s, ShouldNotBeBlank) db.Close() } }) }) //unbind }) // bind // Skipping for now, deprovision is not completed yet. Convey("Deprovision", func() { url := cfsbAPIURL("/v2/service_instances/" + instanceID + "?service_id=" + serviceID + "&plan_id=" + planID) req, err = http.NewRequest("DELETE", url, nil) req.SetBasicAuth("cfadmin", "cfadmin") So(err, ShouldBeNil) httpClient := &http.Client{} resp, err := httpClient.Do(req) So(err, ShouldBeNil) So(resp.StatusCode, ShouldEqual, http.StatusOK) time.Sleep(1 * time.Second) // Wait for MC to tell SC about the udpate Convey("cfsb.instances for each management node should become inefective for the instanceID .", func() { for _, node := range managementCluster.Nodes { p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) var ia pq.NullTime q := fmt.Sprintf(`SELECT ineffective_at FROM cfsb.instances WHERE instance_id='%s' LIMIT 1`, instanceID) db.Get(&ia, q) So(ia.Valid, ShouldEqual, true) So(ia.Time, ShouldNotBeNil) db.Close() } }) Convey("cfsb.instances for each service node should become inefective for the instanceID.", func() { for _, node := range serviceCluster.Nodes { p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) var ia pq.NullTime q := fmt.Sprintf(`SELECT ineffective_at FROM cfsb.instances WHERE instance_id='%s'`, instanceID) db.Get(&ia, q) So(ia.Time, ShouldNotBeNil) db.Close() } }) SkipConvey("Each node should NOT have the user and database anymore", func() { for _, node := range serviceCluster.Nodes { p := pg.NewPG(node.PG.IP, `7432`, `postgres`, `rdpg`, `admin`) db, err := p.Connect() So(err, ShouldBeNil) // user should have been deleted on all service cluster nodes var s string q := fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname='%s'`, i.User) db.Get(&s, q) So(s, ShouldBeBlank) // database should be deleted all service cluster node q = fmt.Sprintf(`SELECT datname FROM pg_catalog.pg_database WHERE datname=?`, i.Database) db.Get(&s, q) So(s, ShouldBeBlank) db.Close() } }) }) // Deprovision }) // Provision }) // catalog }) // integratetion workflow }