/* Should contain a form value dbname which equals the database name
e.g. curl www.hostname.com/backup/now -X POST -d "dbname=nameofdatabase"
The {how} should be either "now" or "enqueue" */
func BackupHandler(w http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
dbname := request.FormValue("dbname")
t := tasks.NewTask()
t.Action = "BackupDatabase"
t.Data = dbname
t.Node = globals.MyIP
t.Role = globals.ServiceRole
t.TTL = 3600
t.ClusterService = globals.ClusterService
t.NodeType = "read"
if rdpgconsul.IsWriteNode(globals.MyIP) {
t.NodeType = "write"
}
var err error
if dbname != "rdpg" {
//Using FindByDatabase to determine if the database actually exists to be backed up.
inst, err := instances.FindByDatabase(dbname)
if err != nil {
log.Error(fmt.Sprintf("admin.BackupHandler() instances.FindByDatabase(%s) Error occurred when searching for database.", dbname))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Error encountered while searching for database"))
return
}
if inst == nil {
//...then the database doesn't exist on this cluster.
log.Debug(fmt.Sprintf("admin.BackupHandler() Attempt to initiate backup on non-existant database with name: %s", dbname))
w.WriteHeader(http.StatusNotFound)
w.Write([]byte("Database not found"))
return
}
}
switch vars[`how`] {
//Immediately calls Backup() and performs the backup
case "now":
err = t.BackupDatabase()
if err != nil {
log.Error(fmt.Sprintf(`api.BackupHandler() Task.BackupDatabase() %+v ! %s`, t, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Error encountered while trying to perform backup"))
return
}
w.Write([]byte("Backup completed."))
case "enqueue":
// Queues up a backup to be done with a worker thread gets around to it.
// This call returns after the queuing process is done; not after the backup is done.
err = t.Enqueue()
if err != nil {
log.Error(fmt.Sprintf(`api.BackupHandler() Task.Enqueue() %+v ! %s`, t, err))
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Error while trying to queue"))
return
}
w.Write([]byte("Backup successfully queued."))
default:
w.WriteHeader(http.StatusNotFound)
}
}
func getMasterIP(clusterName string) (masterIp string, err error) {
log.Trace(fmt.Sprintf("gpb#consul.getMasterIP() Calling out to Consul at address %s", mcConsulIP))
consulConfig := consulapi.DefaultConfig()
consulConfig.Address = mcConsulIP
consulClient, err := consulapi.NewClient(consulConfig)
if err != nil {
log.Error(fmt.Sprintf(`gpb#consul.getMasterIP() Consul IP: %s ! %s`, mcConsulIP, err))
return
}
masterNode, _, err := consulClient.Catalog().Service(fmt.Sprintf(`%s-master`, clusterName), "", nil)
if err != nil {
log.Error(fmt.Sprintf("gpb#consul.getMasterIP() Cluster Name: %s ! %s", clusterName, err))
return
}
if len(masterNode) == 0 {
masterIp = "0.0.0.0"
return masterIp, errors.New("Could not find the consul master ip")
}
masterIp = masterNode[0].Address
log.Trace(fmt.Sprintf("gpb#consul.getMasterIP() Found master ip for %s = %s", clusterName, masterIp))
return masterIp, err
}
func httpAuth(h http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, request *http.Request) {
if len(request.Header["Authorization"]) == 0 {
log.Trace(fmt.Sprintf("httpAuth(): 'Authorization' Header Required"))
http.Error(w, "Authorization Required", http.StatusUnauthorized)
return
}
auth := strings.SplitN(request.Header["Authorization"][0], " ", 2)
if len(auth) != 2 || auth[0] != "Basic" {
log.Error(fmt.Sprintf("httpAuth(): Unhandled Authorization Type, Expected Basic"))
http.Error(w, "Unhandled Authorization Type, Expected Basic\n", http.StatusBadRequest)
return
}
payload, err := base64.StdEncoding.DecodeString(auth[1])
if err != nil {
log.Error(fmt.Sprintf("httpAuth(): Authorization base64.StdEncoding.DecodeString() Failed"))
http.Error(w, "Authorization Failed\n", http.StatusUnauthorized)
return
}
nv := strings.SplitN(string(payload), ":", 2)
if (len(nv) != 2) || !isAuthorized(nv[0], nv[1]) {
log.Error(fmt.Sprintf("httpAuth(): Authorization Failed: !isAuthorized() nv: %+v", nv))
http.Error(w, "Authorization Failed\n", http.StatusUnauthorized)
return
}
h(w, request)
}
}
// This is called on the management cluster when the service cluster has created
// a new database and is registering it's avialabilityh with the management cluster.
func (i *Instance) Register() (err error) {
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("instances.Register() p.Connect(%s) ! %s", p.URI, err))
return err
}
defer db.Close()
err = i.Lock()
if err != nil {
log.Error(fmt.Sprintf("instances.Instance#Register(%s) Failed Locking instance %s ! %s", i.Database, i.Database, err))
return
}
sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass,effective_at) VALUES ('%s','%s','%s','%s',CURRENT_TIMESTAMP)`, i.ClusterID, i.Database, i.User, i.Pass)
log.Trace(fmt.Sprintf(`instances.Instance#Register(%s) > %s`, i.Database, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf("instances.Instance#Register(%s) ! %s", i.Database, err))
}
err = i.Unlock()
if err != nil {
log.Error(fmt.Sprintf(`instances.Instance#Register(%s) Unlocking ! %s`, i.InstanceID, err))
}
return
}
func MoveRemoteBackupFiles(oldPath, newPath string) (err error) {
log.Trace("tasks.MoveRemoteBackupFiles ! Beginning to move remote backup files")
if !rdpgs3.Configured {
errorMessage := "tasks.MoveRemoteBackupFiles ! s3 storage is not configured; exiting"
log.Error(errorMessage)
return errors.New(errorMessage)
}
//Helper functions rely on pointing to a specific place for globals.LocalBackupLocation...
// so, I'm going to hack that for the duration of this function
storedPath := globals.LocalBackupPath
globals.LocalBackupPath = oldPath
defer func() { globals.LocalBackupPath = storedPath }() //mmmm lambdas
//Get all of the old files...
backupList, err := RemoteListing("", true)
if err != nil {
log.Error(fmt.Sprintf("tasks.MoveRemoteBackupFiles ! backup.RemoteListing erred : %s", err.Error()))
return err
}
//Move them over to the new location...
err = moveS3(backupList, oldPath, newPath)
if err != nil {
log.Error(fmt.Sprintf("tasks.MoveRemoteBackupFiles ! tasks.moveS3 erred : %s", err.Error()))
return err
}
return nil
}
func (r *RDPG) bdrBootstrap() (err error) {
_, err = r.bootstrapLock()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() r.bootstrapLock() ! %s`, ClusterID, err))
return
}
leader := false
key := fmt.Sprintf(`rdpg/%s/bdr/join/ip`, ClusterID)
bdrJoinIP, err = r.getValue(key)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() kv.getValue(%s) ! %s ...`, ClusterID, key, err))
return err
}
if len(bdrJoinIP) == 0 || bdrJoinIP == globals.MyIP {
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() kv.getValue(%s) BDR Join IP has not been set`, ClusterID, key))
leader = true
} else {
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() kv.getValue(%s) BDR Join Node IP has been set to %s`, ClusterID, key, bdrJoinIP))
leader = false
}
if leader {
err = r.bdrLeaderBootstrap()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() r.bdrLeaderBootstrap() ! %s`, ClusterID, err))
}
} else {
err = r.bdrNonLeaderBootstrap()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() r.bdrNonLeaderBootstrap() ! %s`, ClusterID, err))
}
}
return
}
/*
GetWriteMasterIP returns the write master IP for the service cluster.
*/
func (s *Service) GetWriteMasterIP() (ip string, err error) {
log.Trace(fmt.Sprintf(`services.Service<%s>#GetWriteMaster()`, s.Name))
client, err := consulapi.NewClient(consulapi.DefaultConfig())
if err != nil {
log.Error(fmt.Sprintf("services.Service<%s>#GetWriteMaster() ! %s", s.Name, err))
return
}
catalog := client.Catalog()
clusterID := os.Getenv("RDPGD_CLUSTER")
if clusterID == "" {
matrixName := os.Getenv(`RDPGD_MATRIX`)
matrixNameSplit := strings.SplitAfterN(matrixName, `-`, -1)
matrixColumn := os.Getenv(`RDPGD_MATRIX_COLUMN`)
for i := 0; i < len(matrixNameSplit)-1; i++ {
clusterID = clusterID + matrixNameSplit[i]
}
clusterID = clusterID + "c" + matrixColumn
}
svcs, _, err := catalog.Service(fmt.Sprintf(`%s-master`, clusterID), "", nil)
if err != nil {
log.Error(fmt.Sprintf(`services.Service<%s>#GetWriteMaster() ! %s`, s.Name, err))
return
}
if len(svcs) == 0 {
return "", nil
}
ip = svcs[0].Address
return
}
func (p *PG) DropDatabase(dbname string) (err error) {
log.Trace(fmt.Sprintf(`pg.PG<%s>#DropDatabase(%s) Dropping postgres database...`, p.IP, dbname))
p.Set(`database`, `postgres`)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbname, err))
return
}
defer db.Close()
exists, err := p.DatabaseExists(dbname)
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbname, err))
return
}
if !exists {
log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) Database %s already does not exist.", p.IP, dbname, err))
return
}
// TODO: How do we drop a database in bdr properly?
sq := fmt.Sprintf(`DROP DATABASE IF EXISTS %s`, dbname)
log.Trace(fmt.Sprintf(`pg.PG<%s>#DropDatabase(%s) > %s`, p.IP, dbname, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbname, err))
return
}
return
}
func (p *PG) DropUser(dbuser string) (err error) {
log.Trace(fmt.Sprintf(`pg.PG<%s>#DropUser(%s) Dropping postgres user...`, p.IP, dbuser))
p.Set(`database`, `postgres`)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#DropUser(%s) %s ! %s", p.IP, dbuser, p.URI, err))
return
}
defer db.Close()
exists, err := p.UserExists(dbuser)
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#DropUser(%s) ! %s", p.IP, dbuser, err))
return
}
if !exists {
log.Error(fmt.Sprintf("pg.PG<%s>#DropUser(%s) User already does not exist, skipping.", p.IP, dbuser))
return
}
// TODO: How do we drop a database in bdr properly?
sq := fmt.Sprintf(`DROP USER %s`, dbuser)
log.Trace(fmt.Sprintf(`pg.PG<%s>#DropDatabase(%s) > %s`, p.IP, dbuser, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbuser, err))
return
}
return
}
// Create given extensions on a single target host.
func (p *PG) CreateExtensions(dbname string, exts []string) (err error) {
log.Trace(fmt.Sprintf(`pg.PG<%s>#CreateExtensions(%s) Creating postgres extensions %+v on database...`, p.IP, dbname, exts))
p.Set(`database`, dbname)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#CreateExtensions(%s) %s ! %s", p.IP, dbname, p.URI, err))
return
}
ddlLockRE := regexp.MustCompile(`cannot acquire DDL lock|Database is locked against DDL operations`)
// TODO: Only create extension if it doesn't already exist.
for _, ext := range exts {
sq := fmt.Sprintf(`CREATE EXTENSION IF NOT EXISTS "%s";`, ext)
log.Trace(fmt.Sprintf(`pg.PG<%s>#CreateExtensions() > %s`, p.IP, sq))
for { // Retry loop for acquiring DDL schema lock.
_, err = db.Exec(sq)
if err != nil {
if ddlLockRE.MatchString(err.Error()) {
log.Trace("pg.PG#CreateExtensions() DDL Lock not available, waiting...")
time.Sleep(1 * time.Second)
continue
}
db.Close()
log.Error(fmt.Sprintf("pg.PG<%s>#CreateExtensions() %s ! %s", p.IP, ext, err))
return
}
break
}
}
db.Close()
return
}
// First break replication within the specified database
// Then disables the specific database from the postgres database
func (p *PG) BDRDisableDatabase(dbname string) (err error) {
log.Trace(fmt.Sprintf(`pg.PG<%s>#BDRDisableDatabase(%s) Disabling postgres database...`, p.IP, dbname))
p.Set(`database`, dbname)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#BDRDisableDatabase(%s) %s ! %s", p.IP, dbname, p.URI, err))
return
}
nodes := []string{}
sq := `SELECT node_name FROM bdr.bdr_nodes;`
err = db.Select(&nodes, sq)
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#BDRDisableDatabase(%s) %s ! %s", p.IP, dbname, sq, err))
}
for index, _ := range nodes {
sq := fmt.Sprintf(`SELECT bdr.bdr_part_by_node_names(ARRAY['%s']);`, nodes[index])
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#BDRDisableDatabase(%s) %s ! %s", p.IP, dbname, sq, err))
}
}
db.Close()
p.DisableDatabase(dbname) // Call the non-BDR disabling function.
return
}
// Create a given user on a single target host.
func (p *PG) GrantUserPrivileges(dbuser string, priviliges []string) (err error) {
log.Trace(fmt.Sprintf(`pg.PG<%s>#GrantUserPrivileges(%s) Granting postgres user priviliges %+v...`, p.IP, dbuser, priviliges))
p.Set(`database`, `postgres`)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#UserGrantPrivileges(%s) %s ! %s", p.IP, dbuser, p.URI, err))
return
}
defer db.Close()
for _, priv := range priviliges {
sq := fmt.Sprintf(`ALTER USER %s WITH %s;`, dbuser, priv)
log.Trace(fmt.Sprintf(`pg.PG<%s>#UserGrantPrivileges(%s) > %s`, p.IP, dbuser, sq))
result, err := db.Exec(sq)
rows, _ := result.RowsAffected()
if rows > 0 {
log.Trace(fmt.Sprintf(`pg.PG<%s>#CreateUser(%s) Successfully Created.`, p.IP, dbuser))
}
if err != nil {
log.Error(fmt.Sprintf(`pg.PG<%s>#CreateUser(%s) ! %s`, p.IP, dbuser, err))
return err
}
}
return nil
}
func (b *Binding) Find() (err error) {
log.Trace(fmt.Sprintf(`cfsb.Binding#Find(%s) ... `, b.BindingID))
if b.BindingID == "" {
return errors.New("Binding ID is empty, can not Binding#Find()")
}
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("cfsb.Binding#Find(%s) ! %s", b.BindingID, err))
return
}
defer db.Close()
sq := fmt.Sprintf(`SELECT id,instance_id FROM cfsb.bindings WHERE binding_id=lower('%s') LIMIT 1`, b.BindingID)
log.Trace(fmt.Sprintf(`cfsb.Binding#Find(%s) > %s`, b.BindingID, sq))
err = db.Get(b, sq)
if err != nil {
if err == sql.ErrNoRows {
log.Error(fmt.Sprintf("cfsb.Binding#Find(%s) ! Could not find binding with given Binding ID", b.BindingID))
} else {
log.Error(fmt.Sprintf("cfsb.Binding#Find(%s) ! %s", b.BindingID, err))
}
} else {
// TODO: Load creds: b.Creds := Credentials{} ... b.Creds.Find()
}
return
}
func (b *Binding) Remove() (err error) {
log.Trace(fmt.Sprintf(`cfsb.Binding#Remove(%s) ... `, b.BindingID))
err = b.Find()
if err != nil {
log.Error(fmt.Sprintf(`cfsb.Binding#Remove(%s) ! %s`, b.BindingID, err))
return
}
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("cfsb.Binding#Remove(%s) ! %s", b.BindingID, err))
return
}
defer db.Close()
// TODO: Scheduled background task that does any cleanup necessary for an
// unbinding (remove credentials?)
sq := fmt.Sprintf(`UPDATE cfsb.bindings SET ineffective_at=CURRENT_TIMESTAMP WHERE binding_id=lower('%s')`, b.BindingID)
log.Trace(fmt.Sprintf(`cfsb.Binding#Remove(%s) SQL > %s`, b.BindingID, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf(`cfsb.Binding#Remove(%s) ! %s`, b.BindingID, err))
}
b.Creds = &Credentials{
InstanceID: b.InstanceID,
BindingID: b.BindingID,
}
err = b.Creds.Remove()
if err != nil {
log.Error(fmt.Sprintf(`cfsb.Binding#Remove(%s) b.Creds.Remove() ! %s`, b.BindingID, err))
}
return
}
func (r *RDPG) serviceClusterCapacityStore() (err error) {
if ClusterID == "rdpgmc" {
return nil
}
instanceAllowed := os.Getenv(`RDPGD_INSTANCE_ALLOWED`)
if instanceAllowed == "" {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceAllowed is not configured or failed to export as environment variable! %s`, ClusterID))
}
instanceLimit := os.Getenv(`RDPGD_INSTANCE_LIMIT`)
if instanceLimit == "" {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceLimit is not configured or failed to export as environment variable! %s`, ClusterID))
}
kv := r.ConsulClient.KV()
key := fmt.Sprintf(`rdpg/%s/capacity/instances/limit`, ClusterID)
kvp := &consulapi.KVPair{Key: key, Value: []byte(instanceLimit)}
_, err = kv.Put(kvp, &consulapi.WriteOptions{})
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceLimit ! %s`, ClusterID, err))
}
key = fmt.Sprintf(`rdpg/%s/capacity/instances/allowed`, ClusterID)
kvp.Key = key
kvp.Value = []byte(instanceAllowed)
_, err = kv.Put(kvp, &consulapi.WriteOptions{})
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceAloowed ! %s`, ClusterID, err))
}
return
}
// Check if the given PostgreSQL Database Exists on the host.
func (p *PG) DatabaseExists(dbname string) (exists bool, err error) {
log.Trace(fmt.Sprintf(`pg.PG<%s>#DatabaseExists(%s) Checking if postgres database exists...`, p.IP, dbname))
p.Set(`database`, `postgres`)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("pg.PG<%s>#DatabaseExists(%s) %s ! %s", p.IP, dbname, p.URI, err))
return
}
defer db.Close()
type name struct {
Name string `db:"name"`
}
var n name
sq := fmt.Sprintf(`SELECT datname AS name FROM pg_database WHERE datname='%s' LIMIT 1;`, dbname)
err = db.Get(&n, sq)
if err != nil {
if err == sql.ErrNoRows {
exists = false
err = nil
} else {
log.Error(fmt.Sprintf(`pg.PG<%s>#DatabaseExists(%s) ! %s`, p.IP, dbname, err))
return
}
}
if n.Name != "" {
exists = true
} else {
exists = false
}
return
}
func (r *RDPG) waitForClusterNodes() (err error) {
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#waitForClusterNodes() waiting for all nodes to be registered as Consul services...`, ClusterID))
cluster, err := NewCluster(ClusterID, r.ConsulClient)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#waitForClusterNodes() NewCluster() ! %s`, ClusterID, err))
return err
}
for {
ips, err := cluster.ClusterIPs()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#waitForClusterNodes() cluster.ClusterIPs() ! %s`, ClusterID, err))
return err
}
switch ClusterID {
case `rdpgmc`:
if len(ips) > 2 {
return nil
}
default: // rdpgsc*
if len(ips) > 1 {
return nil
}
}
time.Sleep(2 * time.Second)
}
}
// createGlobalsFile - Create a pg backup file which contains only globals (roles/logins)
func createGlobalsFile(b backupParams) (f backupFileHistory, err error) {
start := time.Now()
f.duration = 0
f.status = `ok`
f.backupFile = b.baseFileName + ".globals"
f.backupPathAndFile = b.basePath + "/" + b.databaseName + "/" + f.backupFile
f.dbname = b.databaseName
f.node = b.node
pgDumpallPath := b.pgDumpPath + `all`
out, err := exec.Command(pgDumpallPath, "-p", b.pgPort, "-U", "vcap", "--globals-only").CombinedOutput()
if err != nil {
log.Error(fmt.Sprintf(`tasks.createGlobalsFile() Error running pg_dumpall command for: %s out: %s ! %s`, b.databaseName, out, err))
return
}
err = ioutil.WriteFile(f.backupPathAndFile, out, 0644)
if err != nil {
log.Error(fmt.Sprintf(`tasks.createGlobalsFile() Error running output to file: %s ! %s`, f.backupPathAndFile, err))
return
}
f.duration = int(time.Since(start).Seconds())
return
}
func EnvHandler(w http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
envKey := vars["key"]
envValue := os.Getenv(envKey)
log.Trace(fmt.Sprintf(`%s /env/%s => %s`, request.Method, envKey, envValue))
switch request.Method {
case `GET`:
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(http.StatusOK)
ev := EnvVar{Key: envKey, Value: envValue}
msg, err := json.Marshal(ev)
if err != nil {
log.Error(fmt.Sprintf(`admin.EnvHandler() %s`, err))
}
w.Write(msg)
default:
resp := ResponseObject{Status: http.StatusMethodNotAllowed, Description: request.Method}
msg, err := json.Marshal(resp)
if err != nil {
log.Error(fmt.Sprintf(`admin.EnvHandler() %s`, err))
}
log.Error(fmt.Sprintf(`admin.EnvHandler() %+v %s`, vars, string(msg)))
http.Error(w, string(msg), http.StatusMethodNotAllowed)
}
}
func (b *BDR) CreateReplicationGroup(dbname string) (err error) {
nodes, err := b.PGNodes()
if err != nil {
log.Error(fmt.Sprintf(`bdr.BDR#CreateReplicationGroup(%s) b.PGNodes() ! %s`, dbname, err))
}
// TODO: Drop Database on all nodes if err != nil for any operation below
for index, pg := range nodes {
localNodeName := fmt.Sprintf(`%s_%d`, dbname, index)
if index == 0 {
err = pg.BDRGroupCreate(localNodeName, dbname)
if err != nil {
log.Error(fmt.Sprintf(`bdr.BDR<%s>#CreateReplicationGroup(%s) pg.BDRGroupCreate() ! %s`, pg.IP, dbname, err))
break
}
} else {
err = pg.BDRGroupJoin(localNodeName, dbname, nodes[0])
if err != nil {
log.Error(fmt.Sprintf(`bdr.BDR<%s>#CreateReplicationGroup(%s) pg.BDRGroupJoin() ! %s`, pg.IP, dbname, err))
break
}
}
}
if err != nil {
// Cleanup in BDR currently requires droping the database and trying again...
err = b.DropDatabase(dbname)
if err != nil {
log.Error(fmt.Sprintf(`bdr.BDR#CreateReplicationGroup(%s) Dropping Database due to Create Error ! %s`, dbname, err))
}
}
return err
}
/*
Dequeue dequeue's a given task from the database's rdpg.tasks table.
*/
func (t *Task) Dequeue() (err error) {
tasks := []Task{}
sq := fmt.Sprintf(`SELECT id,node,cluster_id,role,action,data,ttl,node_type,cluster_service FROM tasks.tasks WHERE id=%d LIMIT 1`, t.ID)
log.Trace(fmt.Sprintf(`tasks.Task<%d>#Dequeue() > %s`, t.ID, sq))
OpenWorkDB()
err = workDB.Select(&tasks, sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task<%d>#Dequeue() Selecting Task %+v ! %s`, t.ID, t, err.Error()))
return
}
if len(tasks) == 0 {
log.Error(fmt.Sprintf(`tasks.Task<%d>#Dequeue() No rows returned for task %+v`, t.ID, t))
return
}
t = &tasks[0]
// TODO: Add the information for who has this task locked using IP
sq = fmt.Sprintf(`UPDATE tasks.tasks SET locked_by='%s', processing_at=CURRENT_TIMESTAMP WHERE id=%d`, myIP, t.ID)
log.Trace(fmt.Sprintf(`tasks.Task<%d>#Dequeue() > %s`, t.ID, sq))
_, err = workDB.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf(`tasks.Task<%d>#Dequeue() Updating Task processing_at ! %s`, t.ID, err))
return
}
log.Trace(fmt.Sprintf(`tasks.Task<%d>#Dequeue() Task Dequeued > %+v`, t.ID, t))
return
}
// Attempt to obtain a lock on the boostrap leader and bootstrap the cluster if we get the lock..
func (r *RDPG) bootstrapLock() (locked bool, err error) {
const MAX_ACQUIRE_ATTEMPTS int = 10
const TIME_BETWEEN_ATTEMPTS time.Duration = 5 * time.Second
var numAcquireAttempts = 0
key := fmt.Sprintf(`rdpg/%s/bootstrap/lock`, ClusterID)
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() Attempting to acquire boostrap leader lock /%s...`, ClusterID, key))
locked = false
bootstrapLock, err = r.ConsulClient.LockKey(key)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() LockKey() Error Locking Bootstrap Key %s ! %s`, ClusterID, key, err))
return
}
tryLockingBootstrap:
bootstrapLockCh, err := bootstrapLock.Lock(nil)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() Lock() Attempt: %d, Error Acquiring Bootstrap Key lock %s ! %s`, ClusterID, numAcquireAttempts, key, err))
if numAcquireAttempts < MAX_ACQUIRE_ATTEMPTS {
numAcquireAttempts++
time.Sleep(TIME_BETWEEN_ATTEMPTS)
goto tryLockingBootstrap
}
return
}
if bootstrapLockCh == nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() Bootstrap Lock not aquired, halting bootstrap.`, ClusterID))
return
}
locked = true
return
}
func (c *Catalog) Fetch() (err error) {
log.Trace(`cfsb.Catalog#Fetch()...`)
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("cfsb.Catalog#Fetch() ! %s", err))
return
}
defer db.Close()
sq := `SELECT service_id,name,description,bindable FROM cfsb.services;`
log.Trace(fmt.Sprintf(`cfsb.Catalog#Fetch() > %s`, sq))
err = db.Select(&c.Services, sq)
if err != nil {
log.Error(fmt.Sprintf("cfsb.Catalog#Fetch() db.Select() ! %s", err.Error()))
return
}
// TODO: Account for plans being associated with a service.
for i, _ := range c.Services {
service := &c.Services[i]
sq := `SELECT plan_id,name,description FROM cfsb.plans;`
log.Trace(fmt.Sprintf(`cfsb.Catalog#Fetch() > %s`, sq))
err = db.Select(&service.Plans, sq)
if err != nil {
log.Error(fmt.Sprintf("cfsb.Catalog#Fetch() db.Select() ! %s", err.Error()))
return
}
c.Services[i].Tags = []string{"rdpg", "postgresql"}
// c.Services[i].Dashboard = DashboardClient{}
}
return
}
func (r *RDPG) bootstrapSystem() (err error) {
p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass)
exts := []string{`pgcrypto`, `pg_stat_statements`, `uuid-ossp`, `hstore`, `pg_trgm`}
err = p.CreateExtensions(`rdpg`, exts)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#initialBootstrap() CreateExtensions() ! %s`, ClusterID, err))
return
}
err = r.InitSchema()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem() r.InitSchema(%s) ! %s`, ClusterID, globals.ServiceRole, err))
return
}
cluster, err := NewCluster(ClusterID, r.ConsulClient)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem(%s) NewCluster() ! %s`, ClusterID, globals.ServiceRole, err))
return err
}
err = cluster.SetWriteMaster(globals.MyIP)
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem() SetWriteMaster() ! %s`, ClusterID, err))
return
}
return
}
func (i *Instance) Decommission() (err error) {
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("instances.Decommission() p.Connect(%s) ! %s", p.URI, err))
return
}
defer db.Close()
// TODO: i.SetIneffective()
sq := fmt.Sprintf(`UPDATE cfsb.instances SET ineffective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
log.Trace(fmt.Sprintf(`instances.Instance<%s>#Decommission() SQL > %s`, i.InstanceID, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf("Instance#Decommission(%s) setting inefective_at ! %s", i.InstanceID, err))
return
}
// TODO: tasks.Task{ClusterID: ,Node: ,Role: ,Action:, Data: }.Enqueue()
// Question is how to do this without an import cycle? Some tasks require instances.
sq = fmt.Sprintf(`INSERT INTO tasks.tasks (cluster_id,role,action,data, cluster_service) VALUES ('%s','all','DecommissionDatabase','%s', '%s')`, i.ClusterID, i.Database, i.ClusterService)
log.Trace(fmt.Sprintf(`instances.Instance#Decommission(%s) Scheduling Instance Removal > %s`, i.InstanceID, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf(`instances.Instance#Decommission(%s) ! %s`, i.InstanceID, err))
}
return
}
// Leader specific bootstrapping.
func (r *RDPG) bdrLeaderBootstrap() (err error) {
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() bootstrapping leader for cluster...`, ClusterID))
err = r.serviceClusterCapacityStore()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem() Store Service CLuster Instance Capacity in Consul KeyValue! %s`, ClusterID, err))
return
}
err = r.bdrGroupCreate()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() Error Creating BDR Group ! %s`, ClusterID, err))
}
r.bootstrapUnlock()
err = r.waitForBDRNodes()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() Waiting for BDR Nodes ! %s`, ClusterID, err))
return
}
err = r.bootstrapSystem()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() r.bootstrapSystem(%s,%s) ! %s`, ClusterID, globals.ServiceRole, globals.ClusterService, err))
return
}
// TODO: Write a Consul Key /rdpg/%s/schema/initialized indicating completion.
// Wait for this value on the non-leader nodes before they start.
return
}
// WorkLock - Acquire consul for cluster to aquire right to schedule tasks.
func WorkLock() (err error) {
clusterID := os.Getenv("RDPGD_CLUSTER")
if clusterID == "" {
matrixName := os.Getenv(`RDPGD_MATRIX`)
matrixNameSplit := strings.SplitAfterN(matrixName, `-`, -1)
matrixColumn := os.Getenv(`RDPGD_MATRIX_COLUMN`)
for i := 0; i < len(matrixNameSplit)-1; i++ {
clusterID = clusterID + matrixNameSplit[i]
}
clusterID = clusterID + "c" + matrixColumn
}
key := fmt.Sprintf("rdpg/%s/tasks/work/lock", clusterID)
client, _ := consulapi.NewClient(consulapi.DefaultConfig())
workLock, err = client.LockKey(key)
if err != nil {
log.Error(fmt.Sprintf("tasks.WorkLock() Error Locking Work Key %s ! %s", key, err))
return
}
workLockCh, err = workLock.Lock(nil) // Acquire Consul K/V Lock
if err != nil {
log.Error(fmt.Sprintf("tasks.WorkLock() Error Acquiring Work Key lock %s ! %s", key, err))
return
}
if workLockCh == nil {
err = fmt.Errorf(`tasks.WorkLock() Work Lock not acquired`)
}
return
}
// Non-Leader specifc bootstrapping.
func (r *RDPG) bdrNonLeaderBootstrap() (err error) {
log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() bootstrapping non-leader...`, ClusterID))
err = r.bdrGroupJoin()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() bdrGroupJoin() ! %s`, ClusterID, err))
r.bootstrapUnlock()
return err // BDR join during bootstrap is critical path, unlock and exit.
}
r.bootstrapUnlock()
err = r.waitForBDRNodes()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() r.waitForBDRNodes() ! %s`, ClusterID, err))
}
p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("rdpg.RDPG<%s>#bdrNonLeaderBootstrap() ! %s", ClusterID, err))
return
}
defer db.Close()
err = p.WaitForRegClass("cfsb.instances")
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() p.WaitForRegClass() ! %s`, ClusterID, err))
}
err = r.waitForWriteMasterIP()
if err != nil {
log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() p.waitForWriteMasterIP() ! %s`, ClusterID, err))
}
return
}
// This is called on the management cluster when it is running the scheduled task
// which reconciles the databases comparing against the service clusters lists.
func (i *Instance) Reconcile() (err error) {
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("instances.Register() p.Connect(%s) ! %s", p.URI, err))
return err
}
defer db.Close()
err = i.Lock()
if err != nil {
log.Error(fmt.Sprintf("instances.Instance#Reconcile(%s) Failed Locking instance %s ! %s", i.Database, i.Database, err))
return
}
ei, err := FindByDatabase(i.Database)
if err != nil {
log.Error(fmt.Sprintf("instances.Instance#Reconcile() ! %s", err))
} else if ei == nil {
log.Trace(fmt.Sprintf(`instances.Instance#Reconcile() Reconciling database %s for cluster %s`, i.Database, i.ClusterID))
sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,service_id ,plan_id ,instance_id ,organization_id ,space_id,dbname, dbuser, dbpass,effective_at) VALUES ('%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s',CURRENT_TIMESTAMP)`, i.ClusterID, i.ServiceID, i.PlanID, i.InstanceID, i.OrganizationID, i.SpaceID, i.Database, i.User, i.Pass)
log.Trace(fmt.Sprintf(`instances.Instance#Reconcile(%s) > %s`, i.Database, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf("instances.Instance#Reconcile(%s) ! %s", i.Database, err))
}
}
err = i.Unlock()
if err != nil {
log.Error(fmt.Sprintf(`instances.Instance#Reconcile(%s) Unlocking ! %s`, i.InstanceID, err))
}
return
}
// Create Credentials in the data store
func (c *Credentials) Create() (err error) {
log.Trace(fmt.Sprintf(`cfsb.Credentials#Create(%s,%s) ... `, c.InstanceID, c.BindingID))
p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
db, err := p.Connect()
if err != nil {
log.Error(fmt.Sprintf("cfsb.Credentials#Create(%s) ! %s", c.BindingID, err))
return
}
defer db.Close()
err = c.Find()
if err != nil { // Does not yet exist, insert the credentials.
if err == sql.ErrNoRows { // Does not yet exist, insert the credentials.
sq := fmt.Sprintf(`INSERT INTO cfsb.credentials (instance_id,binding_id,host,port,dbuser,dbpass,dbname) VALUES (lower('%s'),lower('%s'),'%s','%s','%s','%s','%s');`, c.InstanceID, c.BindingID, c.Host, c.Port, c.UserName, c.Password, c.Database)
log.Trace(fmt.Sprintf(`cfsb.Credentials#Create() > %s`, sq))
_, err = db.Exec(sq)
if err != nil {
log.Error(fmt.Sprintf(`cfsb.Credentials#Create() %s ! %s`, sq, err))
}
} else {
log.Error(fmt.Sprintf(`cfsb.Credentials#Create() c.Find() binding %s ! %s`, c.BindingID, err))
}
return
} else { // Credentials already exists, return.
log.Trace(fmt.Sprintf(`cfsb.Credentials#Create() Credentials already exist for binding %s, returning`, c.BindingID))
return
}
}