// serviceConfig constructs the config for all good instances of a single service.
func serviceConfig(client *api.Client, name string, passing map[string]bool, tagPrefix string) (config []string) {
if name == "" || len(passing) == 0 {
return nil
}
q := &api.QueryOptions{RequireConsistent: true}
svcs, _, err := client.Catalog().Service(name, "", q)
if err != nil {
log.Printf("[WARN] consul: Error getting catalog service %s. %v", name, err)
return nil
}
for _, svc := range svcs {
// check if the instance is in the list of instances
// which passed the health check
if _, ok := passing[svc.ServiceID]; !ok {
continue
}
for _, tag := range svc.ServiceTags {
if host, path, ok := parseURLPrefixTag(tag, tagPrefix); ok {
name, addr, port := svc.ServiceName, svc.ServiceAddress, svc.ServicePort
if runtime.GOOS == "darwin" && !strings.Contains(addr, ".") {
addr += ".local"
}
config = append(config, fmt.Sprintf("route add %s %s%s http://%s:%d/ tags %q", name, host, path, addr, port, strings.Join(svc.ServiceTags, ",")))
}
}
}
return config
}
func (q *QueryCommand) Query(client *api.Client, tag string) []*api.CatalogService {
services, _, err := client.Catalog().Service(q.service, tag, &api.QueryOptions{AllowStale: true, RequireConsistent: false})
if err != nil {
panic(err)
return nil
}
return services
}
// Ping confirms that Consul can be reached and it has a leader. If the return
// is nil, then consul should be ready to accept requests.
//
// If the return is non-nil, this typically indicates that either Consul is
// unreachable (eg the agent is not listening on the target port) or has not
// found a leader (in which case Consul returns a 500 to all endpoints, except
// the status types).
//
// If a cluster is starting for the first time, it may report a leader just
// before beginning raft replication, thus rejecting requests made at that
// exact moment.
func Ping(client *api.Client) error {
_, qm, err := client.Catalog().Nodes(&api.QueryOptions{RequireConsistent: true})
if err != nil {
return consulutil.NewKVError("ping", "/catalog/nodes", err)
}
if qm == nil || !qm.KnownLeader {
return util.Errorf("No known leader")
}
return nil
}
// serviceConfig constructs the config for all good instances of a single service.
func serviceConfig(client *api.Client, name string, passing map[string]bool, tagPrefix string) (config []string) {
if name == "" || len(passing) == 0 {
return nil
}
dc, err := datacenter(client)
if err != nil {
log.Printf("[WARN] consul: Error getting datacenter. %s", err)
return nil
}
q := &api.QueryOptions{RequireConsistent: true}
svcs, _, err := client.Catalog().Service(name, "", q)
if err != nil {
log.Printf("[WARN] consul: Error getting catalog service %s. %v", name, err)
return nil
}
env := map[string]string{
"DC": dc,
}
for _, svc := range svcs {
// check if the instance is in the list of instances
// which passed the health check
if _, ok := passing[svc.ServiceID]; !ok {
continue
}
for _, tag := range svc.ServiceTags {
if route, opts, ok := parseURLPrefixTag(tag, tagPrefix, env); ok {
name, addr, port := svc.ServiceName, svc.ServiceAddress, svc.ServicePort
// use consul node address if service address is not set
if addr == "" {
addr = svc.Address
}
// add .local suffix on OSX for simple host names w/o domain
if runtime.GOOS == "darwin" && !strings.Contains(addr, ".") && !strings.HasSuffix(addr, ".local") {
addr += ".local"
}
addr = net.JoinHostPort(addr, strconv.Itoa(port))
cfg := fmt.Sprintf("route add %s %s http://%s/ tags %q", name, route, addr, strings.Join(svc.ServiceTags, ","))
if opts != "" {
cfg += ` opts "` + opts + `"`
}
config = append(config, cfg)
}
}
}
return config
}
func discover(client *consul.Client, r *discoveryRecord) {
services, _, err := client.Catalog().Service(r.name, r.tag, nil)
if err != nil {
log.Warnf("Couldn't get %s services from consul: %v", r.name, err)
}
var hosts []string
for _, svc := range services {
host := svc.Node
port := svc.ServicePort
location := fmt.Sprintf("%s:%d", host, port)
hosts = append(hosts, location)
}
r.discoveredHosts = hosts
log.Debugf("Discovered %s service hosts: %v", r.name, r.discoveredHosts)
}
// Create a new RDPG Cluster.
func NewCluster(clusterID string, client *consulapi.Client) (c *Cluster, err error) {
c = &Cluster{ID: clusterID, ConsulClient: client}
catalog := client.Catalog()
services, _, err := catalog.Service(clusterID, "", nil)
if err != nil {
log.Error(fmt.Sprintf("rdpg.NewCluster(%s) ! %s", clusterID, err))
return
}
if len(services) == 0 {
log.Error(fmt.Sprintf("rdpg.NewCluster(%s) ! No services found, no known nodes?!", clusterID))
return
}
for index, _ := range services {
p := pg.NewPG(services[index].Address, pgPort, `postgres`, `postgres`, ``)
c.Nodes = append(c.Nodes, Node{PG: p})
}
return
}
func consulQuery(service string, tag string, client *api.Client, options *api.QueryOptions, channel chan []*api.CatalogService) {
catalog := client.Catalog()
failures := 0
for {
nodes, qm, err := catalog.Service(service, tag, options)
if err != nil {
failures++
retry := retryInterval * time.Duration(failures*failures)
if retry > maxBackoffTime {
retry = maxBackoffTime
}
log.Printf("Consul monitor errored: %s, retry in %s", err, retry)
<-time.After(retry)
continue
}
failures = 0
if options.WaitIndex == qm.LastIndex {
continue
}
options.WaitIndex = qm.LastIndex
channel <- nodes
}
}
func discover(client *consul.Client, r *discoveryRecord) {
services, _, err := client.Catalog().Service(r.name, r.tag, nil)
if err != nil {
log.Warnf("Couldn't get %s services from consul: %v", r.name, err)
}
if len(services) > 0 {
serviceName := services[0].ServiceName
r.port = services[0].ServicePort
location := fmt.Sprintf("%s.service.%s:%d", serviceName, r.domain, r.port)
if r.tag != "" {
location = fmt.Sprintf("%s.%s", r.tag, location)
}
if r.scheme != "" {
r.discoveredUrl = fmt.Sprintf("%s://%s", r.scheme, location)
} else {
r.discoveredUrl = location
}
} else {
r.discoveredUrl = r.defaultUrl
}
log.Debugf("Discovered %s service url: %s", r.name, r.discoveredUrl)
}
func getConsulServiceCatalog(consulClient *consul.Client, name, tag string) ([]*consul.CatalogService, error) {
catalog, _, err := consulClient.Catalog().Service(name, tag, nil)
return catalog, err
}
var _ = BeforeSuite(func() {
logging.SetLevel(logging.CRITICAL)
consulPath, err := gexec.Build("github.com/hashicorp/consul")
Expect(err).ToNot(HaveOccurred())
tmpDir, err = ioutil.TempDir("", "consul")
Expect(err).ToNot(HaveOccurred())
consulCmd := exec.Command(consulPath, "agent", "-server", "-bootstrap-expect", "1", "-data-dir", tmpDir, "-bind", "127.0.0.1")
consulSession, err = gexec.Start(consulCmd, nil, nil)
Expect(err).ToNot(HaveOccurred())
Consistently(consulSession).ShouldNot(gexec.Exit())
consulClient, err = api.NewClient(api.DefaultConfig())
Expect(err).ToNot(HaveOccurred())
f := func() error {
_, _, err := consulClient.Catalog().Nodes(nil)
return err
}
Eventually(f, 10).Should(BeNil())
})
var _ = AfterSuite(func() {
consulSession.Kill()
consulSession.Wait("60s", "200ms")
Expect(os.RemoveAll(tmpDir)).To(Succeed())
gexec.CleanupBuildArtifacts()
})
func (t *Task) bdrPrecreateDatabase(client *consulapi.Client) (err error) {
/*
key := fmt.Sprintf(`rdpg/%s/cluster/service`, ClusterID)
kvp, _, err := kv.Get(key, nil)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrGroupCreate() kv.Get() ! %s`, ClusterID, err))
return
}
v := ``
if kvp != nil {
v = string(kvp.Value)
}
if len(v) > 0 {
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrPrecreateDatabase()`, ClusterID))
return
}
*/
b := bdr.NewBDR(t.ClusterID, client)
re := regexp.MustCompile("[^A-Za-z0-9_]")
u1 := uuid.NewUUID().String()
u2 := uuid.NewUUID().String()
identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte(""))))
dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte(""))))
i := &instances.Instance{
ClusterID: ClusterID,
Database: "d" + identifier,
User: "******" + identifier,
Pass: dbpass,
ClusterService: t.ClusterService,
}
// TODO: Keep the databases under rdpg schema, link to them in the
// cfsb.instances table so that we separate the concerns of CF and databases.
err = b.CreateUser(i.User, i.Pass)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err))
return err
}
err = b.CreateDatabase(i.Database, i.User)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err))
return err
}
defer db.Close()
sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass, cluster_service) VALUES ('%s','%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass, t.ClusterService)
log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err))
return err
}
err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`, `pg_stat_statements`, `uuid-ossp`, `hstore`, `pg_trgm`, `pgcrypto`})
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
//Loop through and add any additional extensions specified in the rdpgd_service properties of the deployment manifest
if len(globals.UserExtensions) > 1 {
userExtensions := strings.Split(globals.UserExtensions, " ")
err = b.CreateExtensions(i.Database, userExtensions)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateExtensions(%s,%s) Creating Extra User Extensions ! %s", i.Database, i.Database, i.User, err))
return err
}
}
err = b.CreateReplicationGroup(i.Database)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err))
return err
}
sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
log.Trace(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.bdrPrecreateDatabase(%s) ! %s`, i.Database, err))
return err
}
// Tell the management cluster about the newly available database.
// TODO: This can be in a function.
catalog := client.Catalog()
svcs, _, err := catalog.Service("rdpgmc", "", nil)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) catalog.Service() ! %s", i.Database, err))
return err
}
if len(svcs) == 0 {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) ! No services found, no known nodes?!", i.Database))
return err
}
body, err := json.Marshal(i)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#bdrPrecreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err))
return err
}
url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`)
req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body)))
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) http.NewRequest() POST %s ! %s`, i.Database, url, err))
return err
}
log.Trace(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) POST %s`, i.Database, url))
req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS"))
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task#bdrPrecreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err))
return err
}
resp.Body.Close()
return
}
// Precreate database functionality, note that the database creation lock is
// expected to be held when this is called as this must be done in sequence.
func (t *Task) precreateDatabase(workRole string, client *consulapi.Client) (err error) {
log.Trace(fmt.Sprintf("tasks.Task#precreateDatabase()..."))
b := bdr.NewBDR(t.ClusterID, client)
re := regexp.MustCompile("[^A-Za-z0-9_]")
u1 := uuid.NewUUID().String()
u2 := uuid.NewUUID().String()
identifier := strings.ToLower(string(re.ReplaceAll([]byte(u1), []byte(""))))
dbpass := strings.ToLower(string(re.ReplaceAll([]byte(u2), []byte(""))))
i := &instances.Instance{
ClusterID: ClusterID,
Database: "d" + identifier,
User: "******" + identifier,
Pass: dbpass,
}
// TODO: Keep the databases under rdpg schema, link to them in the
// cfsb.instances table so that we separate the concerns of CF and databases.
err = b.CreateUser(i.User, i.Pass)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateUser(%s) ! %s", i.Database, i.User, err))
return err
}
err = b.CreateDatabase(i.Database, i.User)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateDatabase(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("tasks.Work() Failed connecting to %s err: %s", p.URI, err))
return err
}
defer db.Close()
sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass) VALUES ('%s','%s','%s','%s')`, ClusterID, i.Database, i.User, i.Pass)
log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err))
return err
}
err = b.CreateExtensions(i.Database, []string{`btree_gist`, `bdr`})
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateExtensions(%s,%s) ! %s", i.Database, i.Database, i.User, err))
return err
}
err = b.CreateReplicationGroup(i.Database)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#PrecreateDatabases(%s) CreateReplicationGroup() ! %s", i.Database, err))
return err
}
sq = fmt.Sprintf(`UPDATE cfsb.instances SET effective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
log.Trace(fmt.Sprintf(`tasks.precreateDatabase(%s) > %s`, i.Database, sq))
_, err = db.Query(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.precreateDatabase(%s) ! %s`, i.Database, err))
return err
}
// Tell the management cluster about the newly available database.
// TODO: This can be in a function.
catalog := client.Catalog()
svcs, _, err := catalog.Service("rdpgmc", "", nil)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) catalog.Service() ! %s", i.Database, err))
return err
}
if len(svcs) == 0 {
log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) ! No services found, no known nodes?!", i.Database))
return err
}
body, err := json.Marshal(i)
if err != nil {
log.Error(fmt.Sprintf("tasks.Task#precreateDatabase(%s) json.Marchal(i) ! %s", i.Database, err))
return err
}
url := fmt.Sprintf("http://%s:%s/%s", svcs[0].Address, os.Getenv("RDPGD_ADMIN_PORT"), `databases/register`)
req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(body)))
log.Trace(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) POST %s`, i.Database, url))
// req.Header.Set("Content-Type", "application/json")
// TODO: Retrieve from configuration in database.
req.SetBasicAuth(os.Getenv("RDPGD_ADMIN_USER"), os.Getenv("RDPGD_ADMIN_PASS"))
httpClient := &http.Client{}
resp, err := httpClient.Do(req)
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task#precreateDatabase(%s) httpClient.Do() %s ! %s`, i.Database, url, err))
return err
}
resp.Body.Close()
return
}
