Пример #1
0
/* Should contain a form value dbname which equals the database name
   e.g. curl www.hostname.com/backup/now -X POST -d "dbname=nameofdatabase"
   The {how} should be either "now" or "enqueue" */
func BackupHandler(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	dbname := request.FormValue("dbname")
	t := tasks.NewTask()
	t.Action = "BackupDatabase"
	t.Data = dbname
	t.Node = globals.MyIP
	t.Role = globals.ServiceRole
	t.TTL = 3600
	t.ClusterService = globals.ClusterService
	t.NodeType = "read"
	if rdpgconsul.IsWriteNode(globals.MyIP) {
		t.NodeType = "write"
	}

	var err error
	if dbname != "rdpg" {
		//Using FindByDatabase to determine if the database actually exists to be backed up.
		inst, err := instances.FindByDatabase(dbname)
		if err != nil {
			log.Error(fmt.Sprintf("admin.BackupHandler() instances.FindByDatabase(%s) Error occurred when searching for database.", dbname))
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte("Error encountered while searching for database"))
			return
		}
		if inst == nil {
			//...then the database doesn't exist on this cluster.
			log.Debug(fmt.Sprintf("admin.BackupHandler() Attempt to initiate backup on non-existant database with name: %s", dbname))
			w.WriteHeader(http.StatusNotFound)
			w.Write([]byte("Database not found"))
			return
		}
	}

	switch vars[`how`] {
	//Immediately calls Backup() and performs the backup
	case "now":
		err = t.BackupDatabase()
		if err != nil {
			log.Error(fmt.Sprintf(`api.BackupHandler() Task.BackupDatabase() %+v ! %s`, t, err))
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte("Error encountered while trying to perform backup"))
			return
		}
		w.Write([]byte("Backup completed."))
	case "enqueue":
		// Queues up a backup to be done with a worker thread gets around to it.
		// This call returns after the queuing process is done; not after the backup is done.
		err = t.Enqueue()
		if err != nil {
			log.Error(fmt.Sprintf(`api.BackupHandler() Task.Enqueue() %+v ! %s`, t, err))
			w.WriteHeader(http.StatusInternalServerError)
			w.Write([]byte("Error while trying to queue"))
			return
		}
		w.Write([]byte("Backup successfully queued."))
	default:
		w.WriteHeader(http.StatusNotFound)
	}
}
Пример #2
0
func getMasterIP(clusterName string) (masterIp string, err error) {

	log.Trace(fmt.Sprintf("gpb#consul.getMasterIP() Calling out to Consul at address %s", mcConsulIP))

	consulConfig := consulapi.DefaultConfig()
	consulConfig.Address = mcConsulIP
	consulClient, err := consulapi.NewClient(consulConfig)
	if err != nil {
		log.Error(fmt.Sprintf(`gpb#consul.getMasterIP() Consul IP: %s ! %s`, mcConsulIP, err))
		return
	}

	masterNode, _, err := consulClient.Catalog().Service(fmt.Sprintf(`%s-master`, clusterName), "", nil)
	if err != nil {
		log.Error(fmt.Sprintf("gpb#consul.getMasterIP() Cluster Name: %s ! %s", clusterName, err))
		return
	}

	if len(masterNode) == 0 {
		masterIp = "0.0.0.0"
		return masterIp, errors.New("Could not find the consul master ip")
	}

	masterIp = masterNode[0].Address
	log.Trace(fmt.Sprintf("gpb#consul.getMasterIP() Found master ip for %s = %s", clusterName, masterIp))
	return masterIp, err

}
Пример #3
0
func httpAuth(h http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, request *http.Request) {
		if len(request.Header["Authorization"]) == 0 {
			log.Trace(fmt.Sprintf("httpAuth(): 'Authorization' Header Required"))
			http.Error(w, "Authorization Required", http.StatusUnauthorized)
			return
		}

		auth := strings.SplitN(request.Header["Authorization"][0], " ", 2)
		if len(auth) != 2 || auth[0] != "Basic" {
			log.Error(fmt.Sprintf("httpAuth(): Unhandled Authorization Type, Expected Basic"))
			http.Error(w, "Unhandled Authorization Type, Expected Basic\n", http.StatusBadRequest)
			return
		}
		payload, err := base64.StdEncoding.DecodeString(auth[1])
		if err != nil {
			log.Error(fmt.Sprintf("httpAuth(): Authorization base64.StdEncoding.DecodeString() Failed"))
			http.Error(w, "Authorization Failed\n", http.StatusUnauthorized)
			return
		}
		nv := strings.SplitN(string(payload), ":", 2)
		if (len(nv) != 2) || !isAuthorized(nv[0], nv[1]) {
			log.Error(fmt.Sprintf("httpAuth(): Authorization Failed: !isAuthorized() nv: %+v", nv))
			http.Error(w, "Authorization Failed\n", http.StatusUnauthorized)
			return
		}
		h(w, request)
	}
}
Пример #4
0
// This is called on the management cluster when the service cluster has created
// a new database and is registering it's avialabilityh with the management cluster.
func (i *Instance) Register() (err error) {
	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("instances.Register() p.Connect(%s) ! %s", p.URI, err))
		return err
	}
	defer db.Close()

	err = i.Lock()
	if err != nil {
		log.Error(fmt.Sprintf("instances.Instance#Register(%s) Failed Locking instance %s ! %s", i.Database, i.Database, err))
		return
	}
	sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,dbname, dbuser, dbpass,effective_at) VALUES ('%s','%s','%s','%s',CURRENT_TIMESTAMP)`, i.ClusterID, i.Database, i.User, i.Pass)
	log.Trace(fmt.Sprintf(`instances.Instance#Register(%s) > %s`, i.Database, sq))
	_, err = db.Exec(sq)
	if err != nil {
		log.Error(fmt.Sprintf("instances.Instance#Register(%s) ! %s", i.Database, err))
	}
	err = i.Unlock()
	if err != nil {
		log.Error(fmt.Sprintf(`instances.Instance#Register(%s) Unlocking ! %s`, i.InstanceID, err))
	}

	return
}
Пример #5
0
func MoveRemoteBackupFiles(oldPath, newPath string) (err error) {
	log.Trace("tasks.MoveRemoteBackupFiles ! Beginning to move remote backup files")
	if !rdpgs3.Configured {
		errorMessage := "tasks.MoveRemoteBackupFiles !  s3 storage is not configured; exiting"
		log.Error(errorMessage)
		return errors.New(errorMessage)
	}
	//Helper functions rely on pointing to a specific place for globals.LocalBackupLocation...
	// so, I'm going to hack that for the duration of this function
	storedPath := globals.LocalBackupPath
	globals.LocalBackupPath = oldPath
	defer func() { globals.LocalBackupPath = storedPath }() //mmmm lambdas
	//Get all of the old files...
	backupList, err := RemoteListing("", true)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.MoveRemoteBackupFiles ! backup.RemoteListing erred : %s", err.Error()))
		return err
	}
	//Move them over to the new location...
	err = moveS3(backupList, oldPath, newPath)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.MoveRemoteBackupFiles ! tasks.moveS3 erred : %s", err.Error()))
		return err
	}
	return nil
}
Пример #6
0
func (r *RDPG) bdrBootstrap() (err error) {
	_, err = r.bootstrapLock()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() r.bootstrapLock() ! %s`, ClusterID, err))
		return
	}

	leader := false
	key := fmt.Sprintf(`rdpg/%s/bdr/join/ip`, ClusterID)
	bdrJoinIP, err = r.getValue(key)
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() kv.getValue(%s) ! %s ...`, ClusterID, key, err))
		return err
	}
	if len(bdrJoinIP) == 0 || bdrJoinIP == globals.MyIP {
		log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() kv.getValue(%s) BDR Join IP has not been set`, ClusterID, key))
		leader = true
	} else {
		log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() kv.getValue(%s) BDR Join Node IP has been set to %s`, ClusterID, key, bdrJoinIP))
		leader = false
	}

	if leader {
		err = r.bdrLeaderBootstrap()
		if err != nil {
			log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() r.bdrLeaderBootstrap() ! %s`, ClusterID, err))
		}
	} else {
		err = r.bdrNonLeaderBootstrap()
		if err != nil {
			log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#Bootstrap() r.bdrNonLeaderBootstrap() ! %s`, ClusterID, err))
		}
	}
	return
}
Пример #7
0
/*
GetWriteMasterIP returns the write master IP for the service cluster.
*/
func (s *Service) GetWriteMasterIP() (ip string, err error) {
	log.Trace(fmt.Sprintf(`services.Service<%s>#GetWriteMaster()`, s.Name))
	client, err := consulapi.NewClient(consulapi.DefaultConfig())
	if err != nil {
		log.Error(fmt.Sprintf("services.Service<%s>#GetWriteMaster() ! %s", s.Name, err))
		return
	}
	catalog := client.Catalog()

	clusterID := os.Getenv("RDPGD_CLUSTER")
	if clusterID == "" {
		matrixName := os.Getenv(`RDPGD_MATRIX`)
		matrixNameSplit := strings.SplitAfterN(matrixName, `-`, -1)
		matrixColumn := os.Getenv(`RDPGD_MATRIX_COLUMN`)
		for i := 0; i < len(matrixNameSplit)-1; i++ {
			clusterID = clusterID + matrixNameSplit[i]
		}
		clusterID = clusterID + "c" + matrixColumn
	}
	svcs, _, err := catalog.Service(fmt.Sprintf(`%s-master`, clusterID), "", nil)

	if err != nil {
		log.Error(fmt.Sprintf(`services.Service<%s>#GetWriteMaster() ! %s`, s.Name, err))
		return
	}

	if len(svcs) == 0 {
		return "", nil
	}
	ip = svcs[0].Address
	return
}
Пример #8
0
func (p *PG) DropDatabase(dbname string) (err error) {
	log.Trace(fmt.Sprintf(`pg.PG<%s>#DropDatabase(%s) Dropping postgres database...`, p.IP, dbname))
	p.Set(`database`, `postgres`)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbname, err))
		return
	}
	defer db.Close()

	exists, err := p.DatabaseExists(dbname)
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbname, err))
		return
	}
	if !exists {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) Database %s already does not exist.", p.IP, dbname, err))
		return
	}

	// TODO: How do we drop a database in bdr properly?
	sq := fmt.Sprintf(`DROP DATABASE IF EXISTS %s`, dbname)
	log.Trace(fmt.Sprintf(`pg.PG<%s>#DropDatabase(%s) > %s`, p.IP, dbname, sq))
	_, err = db.Exec(sq)
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbname, err))
		return
	}
	return
}
Пример #9
0
func (p *PG) DropUser(dbuser string) (err error) {
	log.Trace(fmt.Sprintf(`pg.PG<%s>#DropUser(%s) Dropping postgres user...`, p.IP, dbuser))
	p.Set(`database`, `postgres`)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropUser(%s) %s ! %s", p.IP, dbuser, p.URI, err))
		return
	}
	defer db.Close()

	exists, err := p.UserExists(dbuser)
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropUser(%s) ! %s", p.IP, dbuser, err))
		return
	}
	if !exists {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropUser(%s) User already does not exist, skipping.", p.IP, dbuser))
		return
	}
	// TODO: How do we drop a database in bdr properly?
	sq := fmt.Sprintf(`DROP USER %s`, dbuser)
	log.Trace(fmt.Sprintf(`pg.PG<%s>#DropDatabase(%s) > %s`, p.IP, dbuser, sq))
	_, err = db.Exec(sq)
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#DropDatabase(%s) ! %s", p.IP, dbuser, err))
		return
	}
	return
}
Пример #10
0
// Create given extensions on a single target host.
func (p *PG) CreateExtensions(dbname string, exts []string) (err error) {
	log.Trace(fmt.Sprintf(`pg.PG<%s>#CreateExtensions(%s) Creating postgres extensions %+v on database...`, p.IP, dbname, exts))
	p.Set(`database`, dbname)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#CreateExtensions(%s) %s ! %s", p.IP, dbname, p.URI, err))
		return
	}

	ddlLockRE := regexp.MustCompile(`cannot acquire DDL lock|Database is locked against DDL operations`)
	// TODO: Only create extension if it doesn't already exist.
	for _, ext := range exts {
		sq := fmt.Sprintf(`CREATE EXTENSION IF NOT EXISTS "%s";`, ext)
		log.Trace(fmt.Sprintf(`pg.PG<%s>#CreateExtensions() > %s`, p.IP, sq))
		for { // Retry loop for acquiring DDL schema lock.
			_, err = db.Exec(sq)
			if err != nil {
				if ddlLockRE.MatchString(err.Error()) {
					log.Trace("pg.PG#CreateExtensions() DDL Lock not available, waiting...")
					time.Sleep(1 * time.Second)
					continue
				}
				db.Close()
				log.Error(fmt.Sprintf("pg.PG<%s>#CreateExtensions() %s ! %s", p.IP, ext, err))
				return
			}
			break
		}
	}
	db.Close()
	return
}
Пример #11
0
// First break replication within the specified database
// Then disables the specific database from the postgres database
func (p *PG) BDRDisableDatabase(dbname string) (err error) {
	log.Trace(fmt.Sprintf(`pg.PG<%s>#BDRDisableDatabase(%s) Disabling postgres database...`, p.IP, dbname))
	p.Set(`database`, dbname)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#BDRDisableDatabase(%s) %s ! %s", p.IP, dbname, p.URI, err))
		return
	}

	nodes := []string{}
	sq := `SELECT node_name FROM bdr.bdr_nodes;`
	err = db.Select(&nodes, sq)
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#BDRDisableDatabase(%s) %s ! %s", p.IP, dbname, sq, err))
	}
	for index, _ := range nodes {
		sq := fmt.Sprintf(`SELECT bdr.bdr_part_by_node_names(ARRAY['%s']);`, nodes[index])
		_, err = db.Exec(sq)
		if err != nil {
			log.Error(fmt.Sprintf("pg.PG<%s>#BDRDisableDatabase(%s) %s ! %s", p.IP, dbname, sq, err))
		}
	}
	db.Close()

	p.DisableDatabase(dbname) // Call the non-BDR disabling function.
	return
}
Пример #12
0
// Create a given user on a single target host.
func (p *PG) GrantUserPrivileges(dbuser string, priviliges []string) (err error) {
	log.Trace(fmt.Sprintf(`pg.PG<%s>#GrantUserPrivileges(%s) Granting postgres user priviliges %+v...`, p.IP, dbuser, priviliges))
	p.Set(`database`, `postgres`)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#UserGrantPrivileges(%s) %s ! %s", p.IP, dbuser, p.URI, err))
		return
	}
	defer db.Close()

	for _, priv := range priviliges {
		sq := fmt.Sprintf(`ALTER USER %s WITH %s;`, dbuser, priv)
		log.Trace(fmt.Sprintf(`pg.PG<%s>#UserGrantPrivileges(%s) > %s`, p.IP, dbuser, sq))
		result, err := db.Exec(sq)
		rows, _ := result.RowsAffected()
		if rows > 0 {
			log.Trace(fmt.Sprintf(`pg.PG<%s>#CreateUser(%s) Successfully Created.`, p.IP, dbuser))
		}
		if err != nil {
			log.Error(fmt.Sprintf(`pg.PG<%s>#CreateUser(%s) ! %s`, p.IP, dbuser, err))
			return err
		}
	}
	return nil
}
Пример #13
0
func (b *Binding) Find() (err error) {
	log.Trace(fmt.Sprintf(`cfsb.Binding#Find(%s) ... `, b.BindingID))

	if b.BindingID == "" {
		return errors.New("Binding ID is empty, can not Binding#Find()")
	}
	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("cfsb.Binding#Find(%s) ! %s", b.BindingID, err))
		return
	}
	defer db.Close()

	sq := fmt.Sprintf(`SELECT id,instance_id FROM cfsb.bindings WHERE binding_id=lower('%s') LIMIT 1`, b.BindingID)
	log.Trace(fmt.Sprintf(`cfsb.Binding#Find(%s) > %s`, b.BindingID, sq))
	err = db.Get(b, sq)
	if err != nil {
		if err == sql.ErrNoRows {
			log.Error(fmt.Sprintf("cfsb.Binding#Find(%s) ! Could not find binding with given Binding ID", b.BindingID))
		} else {
			log.Error(fmt.Sprintf("cfsb.Binding#Find(%s) ! %s", b.BindingID, err))
		}
	} else {
		// TODO: Load creds: b.Creds := Credentials{} ... b.Creds.Find()
	}
	return
}
Пример #14
0
func (b *Binding) Remove() (err error) {
	log.Trace(fmt.Sprintf(`cfsb.Binding#Remove(%s) ... `, b.BindingID))
	err = b.Find()
	if err != nil {
		log.Error(fmt.Sprintf(`cfsb.Binding#Remove(%s) ! %s`, b.BindingID, err))
		return
	}
	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("cfsb.Binding#Remove(%s) ! %s", b.BindingID, err))
		return
	}
	defer db.Close()

	// TODO: Scheduled background task that does any cleanup necessary for an
	// unbinding (remove credentials?)
	sq := fmt.Sprintf(`UPDATE cfsb.bindings SET ineffective_at=CURRENT_TIMESTAMP WHERE binding_id=lower('%s')`, b.BindingID)
	log.Trace(fmt.Sprintf(`cfsb.Binding#Remove(%s) SQL > %s`, b.BindingID, sq))
	_, err = db.Exec(sq)
	if err != nil {
		log.Error(fmt.Sprintf(`cfsb.Binding#Remove(%s) ! %s`, b.BindingID, err))
	}

	b.Creds = &Credentials{
		InstanceID: b.InstanceID,
		BindingID:  b.BindingID,
	}

	err = b.Creds.Remove()
	if err != nil {
		log.Error(fmt.Sprintf(`cfsb.Binding#Remove(%s) b.Creds.Remove() ! %s`, b.BindingID, err))
	}
	return
}
Пример #15
0
func (r *RDPG) serviceClusterCapacityStore() (err error) {
	if ClusterID == "rdpgmc" {
		return nil
	}
	instanceAllowed := os.Getenv(`RDPGD_INSTANCE_ALLOWED`)
	if instanceAllowed == "" {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceAllowed is not configured or failed to export as environment variable! %s`, ClusterID))
	}
	instanceLimit := os.Getenv(`RDPGD_INSTANCE_LIMIT`)
	if instanceLimit == "" {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceLimit is not configured or failed to export as environment variable! %s`, ClusterID))
	}
	kv := r.ConsulClient.KV()

	key := fmt.Sprintf(`rdpg/%s/capacity/instances/limit`, ClusterID)
	kvp := &consulapi.KVPair{Key: key, Value: []byte(instanceLimit)}
	_, err = kv.Put(kvp, &consulapi.WriteOptions{})
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceLimit ! %s`, ClusterID, err))
	}

	key = fmt.Sprintf(`rdpg/%s/capacity/instances/allowed`, ClusterID)
	kvp.Key = key
	kvp.Value = []byte(instanceAllowed)
	_, err = kv.Put(kvp, &consulapi.WriteOptions{})
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#serviceClusterCapacityStore(): instanceAloowed ! %s`, ClusterID, err))
	}

	return
}
Пример #16
0
// Check if the given PostgreSQL Database Exists on the host.
func (p *PG) DatabaseExists(dbname string) (exists bool, err error) {
	log.Trace(fmt.Sprintf(`pg.PG<%s>#DatabaseExists(%s) Checking if postgres database exists...`, p.IP, dbname))
	p.Set(`database`, `postgres`)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("pg.PG<%s>#DatabaseExists(%s) %s ! %s", p.IP, dbname, p.URI, err))
		return
	}
	defer db.Close()

	type name struct {
		Name string `db:"name"`
	}
	var n name
	sq := fmt.Sprintf(`SELECT datname AS name FROM pg_database WHERE datname='%s' LIMIT 1;`, dbname)
	err = db.Get(&n, sq)
	if err != nil {
		if err == sql.ErrNoRows {
			exists = false
			err = nil
		} else {
			log.Error(fmt.Sprintf(`pg.PG<%s>#DatabaseExists(%s) ! %s`, p.IP, dbname, err))
			return
		}
	}
	if n.Name != "" {
		exists = true
	} else {
		exists = false
	}
	return
}
Пример #17
0
func (r *RDPG) waitForClusterNodes() (err error) {
	log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#waitForClusterNodes() waiting for all nodes to be registered as Consul services...`, ClusterID))
	cluster, err := NewCluster(ClusterID, r.ConsulClient)
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#waitForClusterNodes() NewCluster() ! %s`, ClusterID, err))
		return err
	}
	for {
		ips, err := cluster.ClusterIPs()
		if err != nil {
			log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#waitForClusterNodes() cluster.ClusterIPs() ! %s`, ClusterID, err))
			return err
		}
		switch ClusterID {
		case `rdpgmc`:
			if len(ips) > 2 {
				return nil
			}
		default: // rdpgsc*
			if len(ips) > 1 {
				return nil
			}
		}
		time.Sleep(2 * time.Second)
	}
}
Пример #18
0
// createGlobalsFile - Create a pg backup file which contains only globals (roles/logins)
func createGlobalsFile(b backupParams) (f backupFileHistory, err error) {

	start := time.Now()
	f.duration = 0
	f.status = `ok`
	f.backupFile = b.baseFileName + ".globals"
	f.backupPathAndFile = b.basePath + "/" + b.databaseName + "/" + f.backupFile
	f.dbname = b.databaseName
	f.node = b.node

	pgDumpallPath := b.pgDumpPath + `all`

	out, err := exec.Command(pgDumpallPath, "-p", b.pgPort, "-U", "vcap", "--globals-only").CombinedOutput()
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.createGlobalsFile() Error running pg_dumpall command for: %s out: %s ! %s`, b.databaseName, out, err))
		return
	}

	err = ioutil.WriteFile(f.backupPathAndFile, out, 0644)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.createGlobalsFile() Error running output to file: %s ! %s`, f.backupPathAndFile, err))
		return
	}

	f.duration = int(time.Since(start).Seconds())
	return

}
Пример #19
0
func EnvHandler(w http.ResponseWriter, request *http.Request) {
	vars := mux.Vars(request)
	envKey := vars["key"]
	envValue := os.Getenv(envKey)
	log.Trace(fmt.Sprintf(`%s /env/%s => %s`, request.Method, envKey, envValue))
	switch request.Method {
	case `GET`:
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(http.StatusOK)
		ev := EnvVar{Key: envKey, Value: envValue}
		msg, err := json.Marshal(ev)
		if err != nil {
			log.Error(fmt.Sprintf(`admin.EnvHandler() %s`, err))
		}
		w.Write(msg)
	default:
		resp := ResponseObject{Status: http.StatusMethodNotAllowed, Description: request.Method}
		msg, err := json.Marshal(resp)
		if err != nil {
			log.Error(fmt.Sprintf(`admin.EnvHandler() %s`, err))
		}
		log.Error(fmt.Sprintf(`admin.EnvHandler() %+v %s`, vars, string(msg)))
		http.Error(w, string(msg), http.StatusMethodNotAllowed)
	}
}
Пример #20
0
func (b *BDR) CreateReplicationGroup(dbname string) (err error) {
	nodes, err := b.PGNodes()
	if err != nil {
		log.Error(fmt.Sprintf(`bdr.BDR#CreateReplicationGroup(%s) b.PGNodes() ! %s`, dbname, err))
	}

	// TODO: Drop Database on all nodes if err != nil for any operation below
	for index, pg := range nodes {
		localNodeName := fmt.Sprintf(`%s_%d`, dbname, index)
		if index == 0 {
			err = pg.BDRGroupCreate(localNodeName, dbname)
			if err != nil {
				log.Error(fmt.Sprintf(`bdr.BDR<%s>#CreateReplicationGroup(%s) pg.BDRGroupCreate() ! %s`, pg.IP, dbname, err))
				break
			}
		} else {
			err = pg.BDRGroupJoin(localNodeName, dbname, nodes[0])
			if err != nil {
				log.Error(fmt.Sprintf(`bdr.BDR<%s>#CreateReplicationGroup(%s) pg.BDRGroupJoin() ! %s`, pg.IP, dbname, err))
				break
			}
		}
	}
	if err != nil {
		// Cleanup in BDR currently requires droping the database and trying again...
		err = b.DropDatabase(dbname)
		if err != nil {
			log.Error(fmt.Sprintf(`bdr.BDR#CreateReplicationGroup(%s) Dropping Database due to Create Error ! %s`, dbname, err))
		}
	}
	return err
}
Пример #21
0
/*
Dequeue dequeue's a given task from the database's rdpg.tasks table.
*/
func (t *Task) Dequeue() (err error) {
	tasks := []Task{}
	sq := fmt.Sprintf(`SELECT id,node,cluster_id,role,action,data,ttl,node_type,cluster_service FROM tasks.tasks WHERE id=%d LIMIT 1`, t.ID)
	log.Trace(fmt.Sprintf(`tasks.Task<%d>#Dequeue() > %s`, t.ID, sq))
	OpenWorkDB()
	err = workDB.Select(&tasks, sq)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.Task<%d>#Dequeue() Selecting Task %+v ! %s`, t.ID, t, err.Error()))
		return
	}
	if len(tasks) == 0 {
		log.Error(fmt.Sprintf(`tasks.Task<%d>#Dequeue() No rows returned for task %+v`, t.ID, t))
		return
	}
	t = &tasks[0]
	// TODO: Add the information for who has this task locked using IP
	sq = fmt.Sprintf(`UPDATE tasks.tasks SET locked_by='%s', processing_at=CURRENT_TIMESTAMP WHERE id=%d`, myIP, t.ID)
	log.Trace(fmt.Sprintf(`tasks.Task<%d>#Dequeue() > %s`, t.ID, sq))
	_, err = workDB.Exec(sq)
	if err != nil {
		log.Error(fmt.Sprintf(`tasks.Task<%d>#Dequeue() Updating Task processing_at ! %s`, t.ID, err))
		return
	}
	log.Trace(fmt.Sprintf(`tasks.Task<%d>#Dequeue() Task Dequeued > %+v`, t.ID, t))
	return
}
Пример #22
0
// Attempt to obtain a lock on the boostrap leader and bootstrap the cluster if we get the lock..
func (r *RDPG) bootstrapLock() (locked bool, err error) {
	const MAX_ACQUIRE_ATTEMPTS int = 10
	const TIME_BETWEEN_ATTEMPTS time.Duration = 5 * time.Second
	var numAcquireAttempts = 0
	key := fmt.Sprintf(`rdpg/%s/bootstrap/lock`, ClusterID)
	log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() Attempting to acquire boostrap leader lock /%s...`, ClusterID, key))
	locked = false
	bootstrapLock, err = r.ConsulClient.LockKey(key)
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() LockKey() Error Locking Bootstrap Key %s ! %s`, ClusterID, key, err))
		return
	}
tryLockingBootstrap:
	bootstrapLockCh, err := bootstrapLock.Lock(nil)
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() Lock() Attempt: %d, Error Acquiring Bootstrap Key lock %s ! %s`, ClusterID, numAcquireAttempts, key, err))
		if numAcquireAttempts < MAX_ACQUIRE_ATTEMPTS {
			numAcquireAttempts++
			time.Sleep(TIME_BETWEEN_ATTEMPTS)
			goto tryLockingBootstrap
		}
		return
	}
	if bootstrapLockCh == nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapLock() Bootstrap Lock not aquired, halting bootstrap.`, ClusterID))
		return
	}
	locked = true
	return
}
Пример #23
0
func (c *Catalog) Fetch() (err error) {
	log.Trace(`cfsb.Catalog#Fetch()...`)
	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("cfsb.Catalog#Fetch() ! %s", err))
		return
	}
	defer db.Close()

	sq := `SELECT service_id,name,description,bindable FROM cfsb.services;`
	log.Trace(fmt.Sprintf(`cfsb.Catalog#Fetch() > %s`, sq))
	err = db.Select(&c.Services, sq)
	if err != nil {
		log.Error(fmt.Sprintf("cfsb.Catalog#Fetch() db.Select() ! %s", err.Error()))
		return
	}

	// TODO: Account for plans being associated with a service.
	for i, _ := range c.Services {
		service := &c.Services[i]
		sq := `SELECT plan_id,name,description FROM cfsb.plans;`
		log.Trace(fmt.Sprintf(`cfsb.Catalog#Fetch() > %s`, sq))
		err = db.Select(&service.Plans, sq)
		if err != nil {
			log.Error(fmt.Sprintf("cfsb.Catalog#Fetch() db.Select() ! %s", err.Error()))
			return
		}
		c.Services[i].Tags = []string{"rdpg", "postgresql"}
		// c.Services[i].Dashboard = DashboardClient{}
	}
	return
}
Пример #24
0
func (r *RDPG) bootstrapSystem() (err error) {
	p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass)
	exts := []string{`pgcrypto`, `pg_stat_statements`, `uuid-ossp`, `hstore`, `pg_trgm`}
	err = p.CreateExtensions(`rdpg`, exts)
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#initialBootstrap() CreateExtensions() ! %s`, ClusterID, err))
		return
	}

	err = r.InitSchema()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem() r.InitSchema(%s) ! %s`, ClusterID, globals.ServiceRole, err))
		return
	}

	cluster, err := NewCluster(ClusterID, r.ConsulClient)
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem(%s) NewCluster() ! %s`, ClusterID, globals.ServiceRole, err))
		return err
	}
	err = cluster.SetWriteMaster(globals.MyIP)
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem() SetWriteMaster() ! %s`, ClusterID, err))
		return
	}

	return
}
Пример #25
0
func (i *Instance) Decommission() (err error) {
	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("instances.Decommission() p.Connect(%s) ! %s", p.URI, err))
		return
	}
	defer db.Close()

	// TODO: i.SetIneffective()
	sq := fmt.Sprintf(`UPDATE cfsb.instances SET ineffective_at=CURRENT_TIMESTAMP WHERE dbname='%s'`, i.Database)
	log.Trace(fmt.Sprintf(`instances.Instance<%s>#Decommission() SQL > %s`, i.InstanceID, sq))
	_, err = db.Exec(sq)
	if err != nil {
		log.Error(fmt.Sprintf("Instance#Decommission(%s) setting inefective_at ! %s", i.InstanceID, err))
		return
	}

	// TODO: tasks.Task{ClusterID: ,Node: ,Role: ,Action:, Data: }.Enqueue()
	// Question is how to do this without an import cycle? Some tasks require instances.
	sq = fmt.Sprintf(`INSERT INTO tasks.tasks (cluster_id,role,action,data, cluster_service) VALUES ('%s','all','DecommissionDatabase','%s', '%s')`, i.ClusterID, i.Database, i.ClusterService)
	log.Trace(fmt.Sprintf(`instances.Instance#Decommission(%s) Scheduling Instance Removal > %s`, i.InstanceID, sq))
	_, err = db.Exec(sq)
	if err != nil {
		log.Error(fmt.Sprintf(`instances.Instance#Decommission(%s) ! %s`, i.InstanceID, err))
	}
	return
}
Пример #26
0
// Leader specific bootstrapping.
func (r *RDPG) bdrLeaderBootstrap() (err error) {
	log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() bootstrapping leader for cluster...`, ClusterID))

	err = r.serviceClusterCapacityStore()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bootstrapSystem() Store Service CLuster Instance Capacity in Consul KeyValue! %s`, ClusterID, err))
		return
	}

	err = r.bdrGroupCreate()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() Error Creating BDR Group ! %s`, ClusterID, err))
	}

	r.bootstrapUnlock()

	err = r.waitForBDRNodes()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() Waiting for BDR Nodes ! %s`, ClusterID, err))
		return
	}

	err = r.bootstrapSystem()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrLeaderBootstrap() r.bootstrapSystem(%s,%s) ! %s`, ClusterID, globals.ServiceRole, globals.ClusterService, err))
		return
	}

	// TODO: Write a Consul Key /rdpg/%s/schema/initialized indicating completion.
	// Wait for this value on the non-leader nodes before they start.

	return
}
Пример #27
0
// WorkLock - Acquire consul for cluster to aquire right to schedule tasks.
func WorkLock() (err error) {
	clusterID := os.Getenv("RDPGD_CLUSTER")
	if clusterID == "" {
		matrixName := os.Getenv(`RDPGD_MATRIX`)
		matrixNameSplit := strings.SplitAfterN(matrixName, `-`, -1)
		matrixColumn := os.Getenv(`RDPGD_MATRIX_COLUMN`)
		for i := 0; i < len(matrixNameSplit)-1; i++ {
			clusterID = clusterID + matrixNameSplit[i]
		}
		clusterID = clusterID + "c" + matrixColumn
	}

	key := fmt.Sprintf("rdpg/%s/tasks/work/lock", clusterID)
	client, _ := consulapi.NewClient(consulapi.DefaultConfig())
	workLock, err = client.LockKey(key)
	if err != nil {
		log.Error(fmt.Sprintf("tasks.WorkLock() Error Locking Work Key %s ! %s", key, err))
		return
	}

	workLockCh, err = workLock.Lock(nil) // Acquire Consul K/V Lock
	if err != nil {
		log.Error(fmt.Sprintf("tasks.WorkLock() Error Acquiring Work Key lock %s ! %s", key, err))
		return
	}

	if workLockCh == nil {
		err = fmt.Errorf(`tasks.WorkLock() Work Lock not acquired`)
	}

	return
}
Пример #28
0
// Non-Leader specifc bootstrapping.
func (r *RDPG) bdrNonLeaderBootstrap() (err error) {
	log.Trace(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() bootstrapping non-leader...`, ClusterID))
	err = r.bdrGroupJoin()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() bdrGroupJoin() ! %s`, ClusterID, err))
		r.bootstrapUnlock()
		return err // BDR join during bootstrap is critical path, unlock and exit.
	}
	r.bootstrapUnlock()

	err = r.waitForBDRNodes()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() r.waitForBDRNodes() ! %s`, ClusterID, err))
	}

	p := pg.NewPG(`127.0.0.1`, pgPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("rdpg.RDPG<%s>#bdrNonLeaderBootstrap() ! %s", ClusterID, err))
		return
	}
	defer db.Close()

	err = p.WaitForRegClass("cfsb.instances")
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() p.WaitForRegClass() ! %s`, ClusterID, err))
	}

	err = r.waitForWriteMasterIP()
	if err != nil {
		log.Error(fmt.Sprintf(`rdpg.RDPG<%s>#bdrNonLeaderBootstrap() p.waitForWriteMasterIP() ! %s`, ClusterID, err))
	}
	return
}
Пример #29
0
// This is called on the management cluster when it is running the scheduled task
// which reconciles the databases comparing against the service clusters lists.
func (i *Instance) Reconcile() (err error) {
	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("instances.Register() p.Connect(%s) ! %s", p.URI, err))
		return err
	}
	defer db.Close()
	err = i.Lock()
	if err != nil {
		log.Error(fmt.Sprintf("instances.Instance#Reconcile(%s) Failed Locking instance %s ! %s", i.Database, i.Database, err))
		return
	}
	ei, err := FindByDatabase(i.Database)
	if err != nil {
		log.Error(fmt.Sprintf("instances.Instance#Reconcile() ! %s", err))
	} else if ei == nil {
		log.Trace(fmt.Sprintf(`instances.Instance#Reconcile() Reconciling database %s for cluster %s`, i.Database, i.ClusterID))
		sq := fmt.Sprintf(`INSERT INTO cfsb.instances (cluster_id,service_id ,plan_id ,instance_id ,organization_id ,space_id,dbname, dbuser, dbpass,effective_at) VALUES ('%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s',CURRENT_TIMESTAMP)`, i.ClusterID, i.ServiceID, i.PlanID, i.InstanceID, i.OrganizationID, i.SpaceID, i.Database, i.User, i.Pass)
		log.Trace(fmt.Sprintf(`instances.Instance#Reconcile(%s) > %s`, i.Database, sq))
		_, err = db.Exec(sq)
		if err != nil {
			log.Error(fmt.Sprintf("instances.Instance#Reconcile(%s) ! %s", i.Database, err))
		}
	}
	err = i.Unlock()
	if err != nil {
		log.Error(fmt.Sprintf(`instances.Instance#Reconcile(%s) Unlocking ! %s`, i.InstanceID, err))
	}
	return
}
Пример #30
0
// Create Credentials in the data store
func (c *Credentials) Create() (err error) {
	log.Trace(fmt.Sprintf(`cfsb.Credentials#Create(%s,%s) ... `, c.InstanceID, c.BindingID))

	p := pg.NewPG(`127.0.0.1`, pbPort, `rdpg`, `rdpg`, pgPass)
	db, err := p.Connect()
	if err != nil {
		log.Error(fmt.Sprintf("cfsb.Credentials#Create(%s) ! %s", c.BindingID, err))
		return
	}
	defer db.Close()

	err = c.Find()
	if err != nil { // Does not yet exist, insert the credentials.
		if err == sql.ErrNoRows { // Does not yet exist, insert the credentials.
			sq := fmt.Sprintf(`INSERT INTO cfsb.credentials (instance_id,binding_id,host,port,dbuser,dbpass,dbname) VALUES (lower('%s'),lower('%s'),'%s','%s','%s','%s','%s');`, c.InstanceID, c.BindingID, c.Host, c.Port, c.UserName, c.Password, c.Database)
			log.Trace(fmt.Sprintf(`cfsb.Credentials#Create() > %s`, sq))
			_, err = db.Exec(sq)
			if err != nil {
				log.Error(fmt.Sprintf(`cfsb.Credentials#Create()  %s ! %s`, sq, err))
			}
		} else {
			log.Error(fmt.Sprintf(`cfsb.Credentials#Create() c.Find() binding %s ! %s`, c.BindingID, err))
		}
		return
	} else { // Credentials already exists, return.
		log.Trace(fmt.Sprintf(`cfsb.Credentials#Create() Credentials already exist for binding %s, returning`, c.BindingID))
		return
	}
}