Example #1
0
func TestRDPGSystemIntegration(t *testing.T) {
	return
	config := consulapi.DefaultConfig()
	config.Address = `10.244.2.2:8500`
	consulClient, _ := consulapi.NewClient(config)
	kv := consulClient.KV()
	key := fmt.Sprintf(`rdpg/sc-pgbdr-m0-c0/capacity/instances/allowed`)
	kvp, _, _ := kv.Get(key, nil)
	capacityAllowedString := string(kvp.Value)
	capacityAllowed, _ := strconv.Atoi(capacityAllowedString)

	SkipConvey(`RDPG System, given four service clusters`, t, func() {
		Convey(`When (Capacity Size -1 (rdpg)) * 4 + 1 databases are assigned`, func() {

			pSize := os.Getenv(`RDPGD_POOL_SIZE`)
			fmt.Printf("pool size is :%s", pSize)
			poolSize, err := strconv.Atoi(pSize)
			So(err, ShouldBeNil)
			So(poolSize, ShouldEqual, 10)

			So(capacityAllowed, ShouldEqual, 12)

			numberInstance := 0

			organizationID := uuid.NewUUID().String()
			spaceID := uuid.NewUUID().String()

			req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
			So(err, ShouldBeNil)
			req.SetBasicAuth("cfadmin", "cfadmin")
			httpClient := &http.Client{}
			resp, err := httpClient.Do(req)
			So(err, ShouldBeNil)
			So(resp.StatusCode, ShouldEqual, http.StatusOK)
			fmt.Printf(`Getting Catalog`)

			decoder := json.NewDecoder(resp.Body)
			var c cfsb.Catalog
			err = decoder.Decode(&c)
			So(err, ShouldBeNil)

			serviceID := c.Services[0].ServiceID
			planID := c.Services[0].Plans[0].PlanID

			type InstanceBody struct {
				ServiceID      string `json:"service_id"`
				PlanID         string `json:"plan_id"`
				OrganizationID string `json:"organization_guid"`
				SpaceID        string `json:"space_guid"`
			}

			ins := &InstanceBody{
				ServiceID:      serviceID,
				PlanID:         planID,
				OrganizationID: organizationID,
				SpaceID:        spaceID,
			}

			scpgbdrm0c2Count := 0
			scpgbdrm0c3Count := 0
			scpgbdrm0c0Count := 0
			scpgbdrm0c1Count := 0

			time.Sleep(10 * time.Second) // Wait for precreated databases
			Convey("assigning Capacity Size * 4 +1  database, oldest available selected at each iteration", func() {
				for ps := 0; ps < capacityAllowed*4; ps++ {

					p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`)
					db, err := p.Connect()
					sq := fmt.Sprintf(`SELECT id,cluster_id,dbname FROM cfsb.instances WHERE instance_id IS NULL ORDER BY created_at ASC LIMIT 1`)
					iOldestAvailable := instances.Instance{}
					if ps < capacityAllowed*4 {
						for {
							err = db.Get(&iOldestAvailable, sq)
							if err == sql.ErrNoRows {
								time.Sleep(1 * time.Second) // Wait for a pre-created database to be ready.
								continue
							} else {
								break
							}
						}
					} else {
						err = db.Get(&iOldestAvailable, sq)
					}

					instanceID := uuid.NewUUID().String()
					insbody, err := json.Marshal(ins)
					So(err, ShouldBeNil)

					url := cfsbAPIURL("/v2/service_instances/" + instanceID)
					req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody))
					req.SetBasicAuth("cfadmin", "cfadmin")

					fmt.Printf(`Assigning database # %d`, ps)
					httpClient := &http.Client{}
					resp, err := httpClient.Do(req)

					if ps == capacityAllowed*4 {
						fmt.Printf(`Assigning one more database than capacity, should says no reach capacity`)

						Convey("Trying to asign one more database than total capacity", func() {
							So(err, ShouldBeNil)
							So(resp.StatusCode, ShouldEqual, http.StatusInternalServerError)
							So(resp.Body, ShouldEqual, "status: 500,description: Provisioning failed, out of capacity. Need to increase capacity setting for service cluster through admin API or redeploy to scale out, please notify the operations team.")
						})
						Convey(`The number of instances should be equal with the instance capacity.`, func() {
							sq := fmt.Sprintf(`SELECT count(id) FROM cfsb.instances`)
							err = db.Get(numberInstance, sq)
							So(numberInstance, ShouldEqual, capacityAllowed)
						})
					}

					if ps < capacityAllowed*4 {
						So(err, ShouldBeNil)
						So(resp.StatusCode, ShouldEqual, http.StatusOK)
						fmt.Printf(`Assigning database!!!! # %d`, ps)

						time.Sleep(500 * time.Millisecond) // Wait a second for the transaction commit...

						i := instances.Instance{}
						sq := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
						err = db.Get(&i, sq)

						So(i.ID, ShouldEqual, iOldestAvailable.ID)
						So(i.ClusterID, ShouldEqual, iOldestAvailable.ClusterID)
						So(i.Database, ShouldEqual, iOldestAvailable.Database)
						if i.ClusterID == "sc-pgbdr-m0-c2" {
							scpgbdrm0c2Count += 1
						}
						if i.ClusterID == "sc-pgbdr-m0-c3" {
							scpgbdrm0c3Count += 1
						}
						if i.ClusterID == "sc-pgbdr-m0-c0" {
							scpgbdrm0c0Count += 1
						}
						if i.ClusterID == "sc-pgbdr-m0-c1" {
							scpgbdrm0c1Count += 1
						}

						if ps == poolSize*3 {
							Convey(`Poolsize*3 +1 Databases should be assigned to all the service cluster.`, func() {
								// TODO: count the # assigned on each service cluster and
								// Both should be > 0
								So(scpgbdrm0c2Count, ShouldBeGreaterThan, 0)
								So(scpgbdrm0c3Count, ShouldBeGreaterThan, 0)
								So(scpgbdrm0c0Count, ShouldBeGreaterThan, 0)
								So(scpgbdrm0c1Count, ShouldBeGreaterThan, 0)
							})
						}

					}
				} // capacityAllowed for looooop
			})
		})
	})

}
Example #2
0
func (t *Task) bdrPrecreateDatabase(client *consulapi.Client) (err error) {
	/*
	   key := fmt.Sprintf(`rdpg/%s/cluster/service`, ClusterID)
	   kvp, _, err := kv.Get(key, nil)
	   if err != nil {
	   	log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrGroupCreate() kv.Get() ! %s`, ClusterID, err))
	   	return
	   }
	   v := ``
	   if kvp != nil {
	   	v = string(kvp.Value)
	   }
	   if len(v) > 0 {
	   	log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrPrecreateDatabase()`, ClusterID))
	   	return
	   }
	*/
	b := bdr.NewBDR(t.ClusterID, client)
	re := regexp.MustCompile("[^A-Za-z0-9_]")
	u1 := uuid.NewUUID().String()
	u2 := uuid.NewUUID().String()
	identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte(""))))
	dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte(""))))

	i := &instances.Instance{
		ClusterID:      ClusterID,
		Database:       "d" + identifier,
		User:           "u" + identifier,
		Pass:           dbpass,
		ClusterService: t.ClusterService,
	}
	// TODO: Keep the databases under rdpg schema, link to them in the
	// cfsb.instances table so that we separate the concerns of CF and databases.
	err = b.CreateUser(i.User, i.Pass)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err))
		return err
	}

	err = b.CreateDatabase(i.Database, i.User)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err))
		return err
	}

	p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err))
		return err
	}
	defer db.Close()

	sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass, cluster_service) VALUES ('%s','%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass, t.ClusterService)
	log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq))
	_, err = db.Query(sq)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err))
		return err
	}

	err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`, `pg_stat_statements`, `uuid-ossp`, `hstore`, `pg_trgm`, `pgcrypto`})
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err))
		return err
	}

	//Loop through and add any additional extensions specified in the rdpgd_service properties of the deployment manifest
	if len(globals.UserExtensions) > 1 {
		userExtensions := strings.Split(globals.UserExtensions, " ")
		err = b.CreateExtensions(i.Database, userExtensions)
		if err != nil {
			log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) Creating Extra User Extensions ! %s", i.Database, i.Database, i.User, err))
			return err
		}
	}

	err = b.CreateReplicationGroup(i.Database)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err))
		return err
	}

	sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
	log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq))
	_, err = db.Query(sq)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err))
		return err
	}

	// Tell the management cluster about the newly available database.
	// TODO: This can be in a function.
	catalog := client.Catalog()
	svcs, _, err := catalog.Service("rdpgmc", "", nil)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) catalog.Service() ! %s", i.Database, err))
		return err
	}
	if len(svcs) == 0 {
		log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) ! No services found, no known nodes?!", i.Database))
		return err
	}
	body, err := json.Marshal(i)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err))
		return err
	}
	url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body)))
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) http.NewRequest() POST %s ! %s`, i.Database, url, err))
		return err
	}
	log.Trace(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) POST %s`, i.Database, url))
	req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS"))
	httpClient := &http.Client{}
	resp, err := httpClient.Do(req)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err))
		return err
	}
	resp.Body.Close()
	return
}
Example #3
0
func TestRDPGSystemIntegration(t *testing.T) {
	Convey(`RDPG System, given two service clusters`, t, func() {
		Convey(`When Pool Size + 1 databases are assigned`, func() {
			// TODO: Provision Pool Size + 1 databases here...
			pSize := os.Getenv(`RDPGD_POOL_SIZE`)
			poolSize, err := strconv.Atoi(pSize)
			So(err, ShouldBeNil)
			So(poolSize, ShouldEqual, 10)

			organizationID := uuid.NewUUID().String()
			spaceID := uuid.NewUUID().String()

			req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
			So(err, ShouldBeNil)
			req.SetBasicAuth("cfadmin", "cfadmin")
			httpClient := &http.Client{}
			resp, err := httpClient.Do(req)
			So(err, ShouldBeNil)
			So(resp.StatusCode, ShouldEqual, http.StatusOK)

			decoder := json.NewDecoder(resp.Body)
			var c cfsb.Catalog
			err = decoder.Decode(&c)
			So(err, ShouldBeNil)

			serviceID := c.Services[0].ServiceID
			planID := c.Services[0].Plans[0].PlanID

			type InstanceBody struct {
				ServiceID      string `json:"service_id"`
				PlanID         string `json:"plan_id"`
				OrganizationID string `json:"organization_guid"`
				SpaceID        string `json:"space_guid"`
			}

			ins := &InstanceBody{
				ServiceID:      serviceID,
				PlanID:         planID,
				OrganizationID: organizationID,
				SpaceID:        spaceID,
			}

			sc1Count := 0
			sc2Count := 0

			time.Sleep(10 * time.Second) // Wait for precreated databases
			Convey("assigning pool size +1, oldest available selected at each iteration", func() {
				for ps := 0; ps < poolSize; ps++ {
					p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`)
					db, err := p.Connect()
					sq := fmt.Sprintf(`SELECT id,cluster_id,dbname FROM cfsb.instances WHERE instance_id IS NULL ORDER BY created_at ASC LIMIT 1`)
					iOldestAvailable := instances.Instance{}
					err = db.Get(&iOldestAvailable, sq)

					instanceID := uuid.NewUUID().String()
					insbody, err := json.Marshal(ins)
					So(err, ShouldBeNil)
					url := cfsbAPIURL("/v2/service_instances/" + instanceID)
					req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody))
					req.SetBasicAuth("cfadmin", "cfadmin")

					httpClient := &http.Client{}
					resp, err := httpClient.Do(req)
					So(err, ShouldBeNil)
					So(resp.StatusCode, ShouldEqual, http.StatusOK)

					time.Sleep(500 * time.Millisecond) // Wait a second for the transaction commit...

					i := instances.Instance{}
					sq = fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
					for {
						err = db.Get(&i, sq)
						if err == sql.ErrNoRows {
							time.Sleep(1 * time.Second) // Wait for a pre-created database to be ready.
							continue
						} else {
							break
						}
					}
					So(i.ID, ShouldEqual, iOldestAvailable.ID)
					So(i.ClusterID, ShouldEqual, iOldestAvailable.ClusterID)
					So(i.Database, ShouldEqual, iOldestAvailable.Database)
					if i.ClusterID == "rdpgsc1" {
						sc1Count += 1
					}
					if i.ClusterID == "rdpgsc2" {
						sc2Count += 1
					}
				} // pollSize for looooop :)

				Convey(`Databases should be assigned to more than one service cluster.`, func() {
					// TODO: count the # assigned on each service cluster and
					// Both should be > 0
					So(sc1Count, ShouldBeGreaterThan, 0)
					So(sc2Count, ShouldBeGreaterThan, 0)
				})
			})
		})
	})

}
Example #4
0
// Precreate database functionality, note that the database creation lock is
// expected to be held when this is called as this must be done in sequence.
func (t *Task) precreateDatabase(workRole string, client *consulapi.Client) (err error) {
	log.Trace(fmt.Sprintf("tasks.Task#precreateDatabase()..."))

	b := bdr.NewBDR(t.ClusterID, client)
	re := regexp.MustCompile("[^A-Za-z0-9_]")
	u1 := uuid.NewUUID().String()
	u2 := uuid.NewUUID().String()
	identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte(""))))
	dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte(""))))

	i := &instances.Instance{
		ClusterID: ClusterID,
		Database:  "d" + identifier,
		User:      "u" + identifier,
		Pass:      dbpass,
	}
	// TODO: Keep the databases under rdpg schema, link to them in the
	// cfsb.instances table so that we separate the concerns of CF and databases.
	err = b.CreateUser(i.User, i.Pass)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err))
		return err
	}

	err = b.CreateDatabase(i.Database, i.User)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err))
		return err
	}

	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err))
		return err
	}
	defer db.Close()

	sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass) VALUES ('%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass)
	log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq))
	_, err = db.Query(sq)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err))
		return err
	}

	err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`})
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err))
		return err
	}

	err = b.CreateReplicationGroup(i.Database)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err))
		return err
	}

	sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
	log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq))
	_, err = db.Query(sq)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err))
		return err
	}

	// Tell the management cluster about the newly available database.
	// TODO: This can be in a function.
	catalog := client.Catalog()
	svcs, _, err := catalog.Service("rdpgmc", "", nil)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) catalog.Service() ! %s", i.Database, err))
		return err
	}
	if len(svcs) == 0 {
		log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) ! No services found, no known nodes?!", i.Database))
		return err
	}
	body, err := json.Marshal(i)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err))
		return err
	}
	url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`)
	req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body)))
	log.Trace(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) POST %s`, i.Database, url))
	// req.Header.Set("Content-Type", "application/json")
	// TODO: Retrieve from configuration in database.
	req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS"))
	httpClient := &http.Client{}
	resp, err := httpClient.Do(req)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err))
		return err
	}
	resp.Body.Close()
	return
}
Example #5
0
func TestCFSBIntegration(t *testing.T) {
	Convey("CFSB API Authorization", t, func() {
		// API Version Header is set test
		// basic_auth test, need username and password (Authentication :header) to do broker registrations
		// return 401 Unauthorized if credentials are not valid  test, auth only tested here
		// test when reject a request, response a 412 Precondition Failed message
		var getBasicAuthTests = []struct {
			username, password string
			status             int
		}{
			{"cfadmin", "cfadmin", 200},
			{"Aladdin", "open:sesame", 401},
			{"", "", 401},
			{"cf", "bala", 401},
			{"", "cf", 401},
		}

		for _, authTest := range getBasicAuthTests {
			req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
			So(err, ShouldBeNil)
			req.SetBasicAuth(authTest.username, authTest.password)
			httpClient := &http.Client{}
			resp, err := httpClient.Do(req)
			So(err, ShouldBeNil)
			So(resp.StatusCode, ShouldEqual, http.StatusOK)
		}
	})
	// Complete integration test,
	// get catalg
	// use results to provision instance
	// user results to bind
	// user results to unbind
	// user results to deprovision
	// - check for ineffective_at timestamp set
	Convey("CFSB API", t, func() {
		config := consulapi.DefaultConfig()
		config.Address = `10.244.2.2:8500`
		consulClient, err := consulapi.NewClient(config)
		So(err, ShouldBeNil)

		organizationID := uuid.NewUUID().String()
		spaceID := uuid.NewUUID().String()

		Convey("Get Catalog", func() {
			req, err := http.NewRequest("GET", cfsbAPIURL(`/v2/catalog`), nil)
			So(err, ShouldBeNil)
			req.SetBasicAuth("cfadmin", "cfadmin")

			httpClient := &http.Client{}
			resp, err := httpClient.Do(req)
			So(err, ShouldBeNil)
			So(resp.StatusCode, ShouldEqual, http.StatusOK)

			decoder := json.NewDecoder(resp.Body)
			var c cfsb.Catalog
			err = decoder.Decode(&c)
			So(err, ShouldBeNil)
			So(len(c.Services), ShouldNotEqual, 0)

			Convey("Services in Catalog Response", func() {
				firstService := c.Services[0]
				So(firstService.ServiceID, ShouldNotBeBlank)
				So(firstService.Name, ShouldNotBeBlank)
				So(firstService.Description, ShouldNotBeBlank)
				So(len(firstService.Plans), ShouldNotEqual, 0)
				Convey("Plans in Services", func() {
					firstPlan := firstService.Plans[0]
					So(firstPlan.PlanID, ShouldNotBeBlank)
					So(firstPlan.Name, ShouldNotBeBlank)
					So(firstPlan.Description, ShouldNotBeBlank)
				})
			})

			serviceID := c.Services[0].ServiceID
			planID := c.Services[0].Plans[0].PlanID

			Convey("Provision Instance", func() {
				type InstanceBody struct {
					ServiceID      string `json:"service_id"`
					PlanID         string `json:"plan_id"`
					OrganizationID string `json:"organization_guid"`
					SpaceID        string `json:"space_guid"`
				}

				ins := &InstanceBody{
					ServiceID:      serviceID,
					PlanID:         planID,
					OrganizationID: organizationID,
					SpaceID:        spaceID,
				}

				instanceID := uuid.NewUUID().String()
				insbody, err := json.Marshal(ins)
				So(err, ShouldBeNil)

				// TODO: Perform this test after checking/waiting for databases to be
				// pre-created otherwise we get a false-fail.
				url := cfsbAPIURL("/v2/service_instances/" + instanceID)
				req, err := http.NewRequest("PUT", url, bytes.NewBuffer(insbody))
				req.SetBasicAuth("cfadmin", "cfadmin")
				So(err, ShouldBeNil)
				httpClient := &http.Client{}
				resp, err := httpClient.Do(req)
				So(err, ShouldBeNil)
				So(resp.StatusCode, ShouldEqual, http.StatusOK)

				time.Sleep(1 * time.Second) // Allow replication to catch up.

				// At this point the instance within the database should have the clusterID
				// Note that this will be fetching from the management cluster database
				i := instances.Instance{}
				p := pg.NewPG(`10.244.2.2`, `7432`, `rdpg`, `rdpg`, `admin`)
				db, err := p.Connect()
				So(err, ShouldBeNil)
				q := fmt.Sprintf(`SELECT id, cluster_id, instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
				db.Get(&i, q)
				db.Close()

				Convey("The fields should be set in management cluster write master instance table", func() {
					So(err, ShouldBeNil)
					So(i.ClusterID, ShouldNotBeBlank)
					So(i.ServiceID, ShouldEqual, serviceID)
					So(i.PlanID, ShouldEqual, planID)
					So(i.OrganizationID, ShouldEqual, organizationID)
					So(i.SpaceID, ShouldEqual, spaceID)
				})

				managementCluster, err := rdpg.NewCluster(`rdpgmc`, consulClient)
				So(err, ShouldBeNil)

				Convey("Each management cluster node should have the correct fields set in the cfsb.instances table", func() {
					for _, node := range managementCluster.Nodes { // Loop over management cluster nodes.
						p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
						db, err := p.Connect()
						So(err, ShouldBeNil)

						in := instances.Instance{}
						q := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
						db.Get(&in, q)
						So(in.ClusterID, ShouldNotBeBlank)
						So(in.ServiceID, ShouldEqual, serviceID)
						So(in.PlanID, ShouldEqual, planID)
						So(in.OrganizationID, ShouldEqual, organizationID)
						So(in.SpaceID, ShouldEqual, spaceID)
						db.Close()
					}
				})

				serviceCluster, err := rdpg.NewCluster(i.ClusterID, consulClient)
				So(err, ShouldBeNil)
				// TODO: Find out which SC the instance is on and for that cluster's nodes, do the below.
				Convey("Each service cluster node should have a correct record of the instance in rdpg.cfsb.instances.", func() {
					for _, node := range serviceCluster.Nodes {
						sp := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
						db, err := sp.Connect()
						So(err, ShouldBeNil)
						in := instances.Instance{}
						q := fmt.Sprintf(`SELECT id, cluster_id,instance_id, service_id, plan_id, organization_id, space_id, dbname, dbuser, dbpass FROM cfsb.instances WHERE instance_id=lower('%s') LIMIT 1`, instanceID)
						db.Get(&in, q)
						So(in.ClusterID, ShouldEqual, i.ClusterID)
						So(in.ServiceID, ShouldEqual, serviceID)
						So(in.PlanID, ShouldEqual, planID)
						So(in.OrganizationID, ShouldEqual, organizationID)
						So(in.SpaceID, ShouldEqual, spaceID)
						db.Close()
					}
				})

				Convey("Each service cluster node should have the instance's user and database created on it", func() {
					for _, node := range serviceCluster.Nodes { // Loop over service cluster nodes.
						sp := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
						db, err := sp.Connect()
						So(err, ShouldBeNil)
						// user should be created on each service cluster node
						r := ""
						q := fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname='%s'`, i.User)
						db.Get(&r, q)
						So(r, ShouldEqual, i.User)

						// database should be created on each service cluster node
						d := ""
						q = fmt.Sprintf(`SELECT datname FROM pg_catalog.pg_database WHERE datname='%s'`, i.Database)
						db.Get(&d, q)
						So(d, ShouldEqual, i.Database)
						db.Close()
					}
				})

				Convey("Each service cluster node for the instance's database should have bdr and btree_gist extension, and have the same count of bdr.bdr_nodes.", func() {
					var count int
					for _, node := range serviceCluster.Nodes {
						p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, i.Database, i.Pass)
						db, err := p.Connect()
						So(err, ShouldBeNil)
						var s string
						// extensions on all cluster nodes should have bdr, btree_gist
						exts := []string{"bdr", "btree_gist"}
						for _, ext := range exts {
							q := fmt.Sprintf(`SELECT extname FROM pg_extension WHERE extname ='%s'`, ext)
							db.Get(&s, q)
							So(s, ShouldEqual, ext)
						}
						// replication group should have the same count as rdpg bdr.bdr_nodes;
						db.Get(&count, "SELECT count(node_status) FROM bdr.bdr_nodes WHERE node_status='r'")
						So(count, ShouldEqual, len(serviceCluster.Nodes))
						db.Close()
					}
				})

				Convey("Bind", func() {
					bindingID := uuid.NewUUID().String()
					appGuid := uuid.NewUUID().String()

					type BindingBody struct {
						PlanID    string `json:"plan_id"`
						ServiceID string `json:"service_id"`
						AppGuid   string `json:app_guid"`
					}

					bind := &BindingBody{
						PlanID:    planID,
						ServiceID: serviceID,
						AppGuid:   appGuid,
					}

					bindbody, err := json.Marshal(bind)
					So(err, ShouldBeNil)

					url := cfsbAPIURL("/v2/service_instances/" + instanceID + "/service_bindings/" + bindingID)
					req, err := http.NewRequest("PUT", url, bytes.NewBuffer(bindbody))
					req.SetBasicAuth("cfadmin", "cfadmin")
					So(err, ShouldBeNil)

					httpClient := &http.Client{}
					resp, err := httpClient.Do(req)
					So(err, ShouldBeNil)
					So(resp.StatusCode, ShouldEqual, http.StatusOK)
					time.Sleep(1 * time.Second) // Allow replication to replicate before proceeding

					// Binding that it returns should have values for it's fields
					decoder := json.NewDecoder(resp.Body)
					var b cfsb.Binding
					err = decoder.Decode(&b)
					So(err, ShouldBeNil)
					Convey("Bind response body should be consistant with management cluster node bind info", func() {
						So(b.BindingID, ShouldEqual, bindingID)
						So(b.InstanceID, ShouldEqual, instanceID)
						So(b.Creds, ShouldNotBeNil)
						creds := b.Creds

						catalog := consulClient.Catalog()
						services, _, err := catalog.Service(fmt.Sprintf(`%s-master`, i.ClusterID), "", nil)
						So(err, ShouldBeNil)
						So(len(services), ShouldEqual, 1)
						dns := fmt.Sprintf(`%s:5432`, services[0].Address)
						s_dns := strings.Split(dns, ":")
						uri := fmt.Sprintf(`postgres://%s:%[email protected]%s/%s?sslmode=%s`, i.User, i.Pass, dns, i.Database, `disable`)
						dsn := fmt.Sprintf(`host=%s port=%s user=%s password=%s dbname=%s connect_timeout=%s sslmode=%s`, s_dns[0], s_dns[1], i.User, i.Pass, i.Database, `5`, `disable`)
						jdbc := fmt.Sprintf(`jdbc:postgres://%s:%[email protected]%s/%s?sslmode=%s`, i.User, i.Pass, dns, i.Database, `disable`)

						So(creds.URI, ShouldEqual, uri)
						So(creds.DSN, ShouldEqual, dsn)
						So(creds.JDBCURI, ShouldEqual, jdbc)
						So(creds.Host, ShouldEqual, s_dns[0])
						So(creds.Port, ShouldEqual, s_dns[1])
						So(creds.UserName, ShouldEqual, i.User)
						So(creds.Password, ShouldEqual, i.Pass)
						So(creds.Database, ShouldEqual, i.Database)
					})

					Convey("Each management cluster node should have correct cfsb.binding record for the binding id.", func() {
						for _, node := range managementCluster.Nodes {
							p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
							db, err := p.Connect()
							So(err, ShouldBeNil)

							var instanceIdTest string
							q := fmt.Sprintf(`SELECT instance_id FROM cfsb.bindings WHERE binding_id = '%s'`, bindingID)
							db.Get(&instanceIdTest, q)
							So(instanceIdTest, ShouldEqual, instanceID)
							db.Close()
						}
					})

					// TODO: Come back to this and get this test passing.
					SkipConvey("Each management cluster node should have correct cfsb.credentials record for the binding id.", func() {
						for _, node := range managementCluster.Nodes {
							p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
							db, err := p.Connect()
							So(err, ShouldBeNil)

							credsTest := cfsb.Credentials{}
							q = fmt.Sprintf(`SELECT instance_id,dbuser,dbname,dbpass FROM cfsb.credentials WHERE binding_id = '%s'`, bindingID)
							db.Get(&credsTest, q)
							So(credsTest.InstanceID, ShouldEqual, instanceID)
							So(credsTest.UserName, ShouldEqual, i.User)
							So(credsTest.Database, ShouldEqual, i.Database)
							So(credsTest.Password, ShouldEqual, i.Pass)
							db.Close()
						}
					})

					Convey("Un Bind", func() {
						url := cfsbAPIURL("/v2/service_instances/" + instanceID + "/service_bindings/" + bindingID + "?service_id=" + serviceID + "&plan_id=" + planID)
						req, err = http.NewRequest("DELETE", url, nil)
						req.SetBasicAuth("cfadmin", "cfadmin")
						So(err, ShouldBeNil)

						httpClient := &http.Client{}
						resp, err := httpClient.Do(req)
						So(err, ShouldBeNil)
						So(resp.StatusCode, ShouldEqual, http.StatusOK)

						Convey("cfsb.binding record for each management node should become inefective for the binding id .", func() {
							for _, node := range managementCluster.Nodes {
								p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
								db, err := p.Connect()
								So(err, ShouldBeNil)
								var s string
								q := fmt.Sprintf(`SELECT ineffective_at::text FROM cfsb.bindings WHERE binding_id = '%s'`, bindingID)
								db.Get(&s, q)
								So(s, ShouldNotBeBlank)
								db.Close()
							}
						})
					}) //unbind
				}) // bind

				// Skipping for now, deprovision is not completed yet.
				Convey("Deprovision", func() {
					url := cfsbAPIURL("/v2/service_instances/" + instanceID + "?service_id=" + serviceID + "&plan_id=" + planID)
					req, err = http.NewRequest("DELETE", url, nil)
					req.SetBasicAuth("cfadmin", "cfadmin")
					So(err, ShouldBeNil)

					httpClient := &http.Client{}
					resp, err := httpClient.Do(req)
					So(err, ShouldBeNil)
					So(resp.StatusCode, ShouldEqual, http.StatusOK)

					time.Sleep(1 * time.Second) // Wait for MC to tell SC about the udpate

					Convey("cfsb.instances for each management node should become inefective for the instanceID .", func() {
						for _, node := range managementCluster.Nodes {
							p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
							db, err := p.Connect()
							So(err, ShouldBeNil)

							var ia pq.NullTime
							q := fmt.Sprintf(`SELECT ineffective_at FROM cfsb.instances WHERE instance_id='%s' LIMIT 1`, instanceID)
							db.Get(&ia, q)
							So(ia.Valid, ShouldEqual, true)
							So(ia.Time, ShouldNotBeNil)
							db.Close()
						}
					})

					Convey("cfsb.instances for each service node should become inefective for the instanceID.", func() {
						for _, node := range serviceCluster.Nodes {
							p := pg.NewPG(node.PG.IP, `7432`, `rdpg`, `rdpg`, `admin`)
							db, err := p.Connect()
							So(err, ShouldBeNil)
							var ia pq.NullTime
							q := fmt.Sprintf(`SELECT ineffective_at FROM cfsb.instances WHERE instance_id='%s'`, instanceID)
							db.Get(&ia, q)
							So(ia.Time, ShouldNotBeNil)
							db.Close()
						}
					})

					SkipConvey("Each node should NOT have the user and database anymore", func() {
						for _, node := range serviceCluster.Nodes {
							p := pg.NewPG(node.PG.IP, `7432`, `postgres`, `rdpg`, `admin`)
							db, err := p.Connect()
							So(err, ShouldBeNil)

							// user  should have been deleted on all service cluster nodes
							var s string
							q := fmt.Sprintf(`SELECT rolname FROM pg_roles WHERE rolname='%s'`, i.User)
							db.Get(&s, q)
							So(s, ShouldBeBlank)

							// database should be deleted all service cluster node
							q = fmt.Sprintf(`SELECT datname FROM pg_catalog.pg_database WHERE datname=?`, i.Database)
							db.Get(&s, q)
							So(s, ShouldBeBlank)
							db.Close()
						}
					})
				}) // Deprovision

			}) // Provision
		}) // catalog

	}) // integratetion workflow
}