func NewRealDB(url string) (DB, error) {
pgUrl, err := pq.ParseURL(url)
if err != nil {
return nil, err
}
db, err := sql.Open("postgres", pgUrl)
if err != nil {
return nil, err
}
if err = db.Ping(); err != nil {
return nil, err
}
jobIdFind, err := db.Prepare("SELECT id FROM logs WHERE job_id=$1")
if err != nil {
return nil, err
}
logPartsCreate, err := db.Prepare("INSERT INTO log_parts (log_id, number, content, final, created_at) VALUES ($1, $2, $3, $4, $5) RETURNING id")
if err != nil {
return nil, err
}
return &RealDB{db, jobIdFind, logPartsCreate}, nil
}
func Connect() (*sqlx.DB, error) {
open := os.Getenv("DATABASE_URL")
fmt.Println("connecting to", open)
if parsedURL, err := pq.ParseURL(open); err == nil && parsedURL != "" {
open = parsedURL
}
con, err := sqlx.Connect("postgres", open)
if err != nil {
fmt.Println(err)
return nil, err
}
err = con.Ping()
if err != nil {
fmt.Println(err)
return nil, err
}
return con, err
}
// Connect to the database.
func NewPostgresStorage() *PostgresStorage {
postgresUrlString := os.Getenv("STORAGE_URL")
if glog.V(2) {
glog.Infoln("postgresUrlString: ", postgresUrlString)
}
if postgresUrlString == "" {
glog.Fatal("STORAGE_URL cannot be empty.\nexport STORAGE_URL=postgres://user:password@host:port/db_name")
}
dataSource, err := pq.ParseURL(postgresUrlString)
if err != nil {
glog.Fatal("Could not read database string", err)
}
db, err := sql.Open("postgres", dataSource+" sslmode=disable fallback_application_name=bot")
if err != nil {
glog.Fatal("Could not connect to database.", err)
}
// The following 2 lines mitigate the leak of postgresql connection leak
// explicitly setting a maximum number of postgresql connections
db.SetMaxOpenConns(10)
// explicitly setting a maximum number of Idle postgresql connections
db.SetMaxIdleConns(2)
return &PostgresStorage{db}
}
func Connect() gorm.DB {
url := os.Getenv("DATABASE_URL")
werckerDbUrl := os.Getenv("WERCKER_POSTGRESQL_URL")
if werckerDbUrl != "" {
url = werckerDbUrl
}
connection, err := pq.ParseURL(url)
if err != nil {
log.Fatalf("Failed to parse database URL: %s", err)
}
connection += " sslmode=disable" // TODO: Use require in production
db, err := gorm.Open("postgres", connection)
if err != nil {
log.Fatalf("Failed to connecto to database: %s", err)
}
db.AutoMigrate(models.Project{})
if martini.Env == martini.Dev {
db.LogMode(true)
}
return db
}
// NewDatabase creates a (postgres) *Database from a database URL string
func NewDatabase(url string, log *logrus.Logger) (*Database, error) {
db := &Database{
url: url,
st: &dbStatements{},
log: log,
migrations: defaultMigrations,
unpreparedStatements: defaultStatements,
}
if db.log == nil {
db.log = logrus.New()
}
parsedURL, err := pq.ParseURL(url)
if err != nil {
return nil, err
}
conn, err := sql.Open("postgres", parsedURL)
if err != nil {
return nil, err
}
db.conn = conn
return db, nil
}
// Open creates a new connection to the Arco database.
// URL is a Postgres connection string in the form:
// "postgres://*****:*****@1.2.3.4:5432/mydb?option=value"
func Open(url string) (*DB, error) {
dsn, err := pq.ParseURL(url)
if err != nil {
return nil, err
}
db, err := sql.Open("postgres", dsn)
return &DB{db}, err
}
func SetupDb() (*sql.DB, *gorp.DbMap) {
parsedUrl, _ := pq.ParseURL(dbUrl)
db, err := sql.Open("postgres", parsedUrl)
if err != nil {
panic(err.Error())
}
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
return db, dbmap
}
func NewDBConf(conf, p, env string) (*DBConf, error) {
if !filepath.IsAbs(conf) {
dir, file := filepath.Split(conf)
if dir == "" {
// Path is neither relative nor absolute (just filename)
conf = filepath.Join(p, file)
}
}
f, err := yaml.ReadFile(conf)
if err != nil {
return nil, err
}
drv, err := f.Get(fmt.Sprintf("%s.driver", env))
if err != nil {
return nil, err
}
open, err := f.Get(fmt.Sprintf("%s.open", env))
if err != nil {
return nil, err
}
open = os.ExpandEnv(open)
// Automatically parse postgres urls
if drv == "postgres" {
// Assumption: If we can parse the URL, we should
if parsedURL, err := pq.ParseURL(open); err == nil && parsedURL != "" {
open = parsedURL
}
}
d := newDBDriver(drv, open)
// allow the configuration to override the Import for this driver
if imprt, err := f.Get(fmt.Sprintf("%s.import", env)); err == nil {
d.Import = imprt
}
// allow the configuration to override the Dialect for this driver
if dialect, err := f.Get(fmt.Sprintf("%s.dialect", env)); err == nil {
d.Dialect = dialectByName(dialect)
}
if !d.IsValid() {
return nil, errors.New(fmt.Sprintf("Invalid DBConf: %v", d))
}
return &DBConf{
MigrationsDir: filepath.Join(p, "migrations"),
Env: env,
Driver: d,
}, nil
}
func testAccStepDropTable(t *testing.T, b logical.Backend, s logical.Storage, name string, connURL string) logicaltest.TestStep {
return logicaltest.TestStep{
Operation: logical.ReadOperation,
Path: path.Join("creds", name),
Check: func(resp *logical.Response) error {
var d struct {
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
}
if err := mapstructure.Decode(resp.Data, &d); err != nil {
return err
}
log.Printf("[TRACE] Generated credentials: %v", d)
conn, err := pq.ParseURL(connURL)
if err != nil {
t.Fatal(err)
}
conn += " timezone=utc"
db, err := sql.Open("postgres", conn)
if err != nil {
t.Fatal(err)
}
_, err = db.Exec("DROP TABLE test;")
if err != nil {
t.Fatal(err)
}
resp, err = b.HandleRequest(&logical.Request{
Operation: logical.RevokeOperation,
Storage: s,
Secret: &logical.Secret{
InternalData: map[string]interface{}{
"secret_type": "creds",
"username": d.Username,
},
},
})
if err != nil {
return err
}
if resp != nil {
if resp.IsError() {
return fmt.Errorf("Error on resp: %#v", *resp)
}
}
return nil
},
}
}
func (me *postgresTest) SetupSuite() {
uri, err := pq.ParseURL(db_uri)
if err != nil {
panic(err)
}
db, err := sql.Open("postgres", uri)
if err != nil {
panic(err)
}
me.db = goqu.New("postgres", db)
}
// extract configuration details from the given file
func NewDBConf(p, env string, pgschema string) (*DBConf, error) {
cfgFile := filepath.Join(p, "dbconf.yml")
f, err := yaml.ReadFile(cfgFile)
if err != nil {
return nil, err
}
drv, err := f.Get(fmt.Sprintf("%s.driver", env))
if err != nil {
return nil, err
}
drv = os.ExpandEnv(drv)
open, err := f.Get(fmt.Sprintf("%s.open", env))
if err != nil {
return nil, err
}
open = os.ExpandEnv(open)
// Automatically parse postgres urls
if drv == "postgres" {
// Assumption: If we can parse the URL, we should
if parsedURL, err := pq.ParseURL(open); err == nil && parsedURL != "" {
open = parsedURL
}
}
d := newDBDriver(drv, open)
// allow the configuration to override the Import for this driver
if imprt, err := f.Get(fmt.Sprintf("%s.import", env)); err == nil {
d.Import = imprt
}
// allow the configuration to override the Dialect for this driver
if dialect, err := f.Get(fmt.Sprintf("%s.dialect", env)); err == nil {
d.Dialect = dialectByName(dialect)
}
if !d.IsValid() {
return nil, errors.New(fmt.Sprintf("Invalid DBConf: %v", d))
}
return &DBConf{
MigrationsDir: filepath.Join(p, "migrations"),
Env: env,
Driver: d,
PgSchema: pgschema,
}, nil
}
func SetupDB() *sql.DB {
url := os.Getenv("DATABASE_URL")
connection, _ := pq.ParseURL(url)
connection += " sslmode=require"
db, err := sql.Open("postgres", connection)
if err != nil {
log.Println(err)
}
return db
}
// DB returns the database connection.
func (b *backend) DB(s logical.Storage) (*sql.DB, error) {
b.lock.Lock()
defer b.lock.Unlock()
// If we already have a DB, we got it!
if b.db != nil {
return b.db, nil
}
// Otherwise, attempt to make connection
entry, err := s.Get("config/connection")
if err != nil {
return nil, err
}
if entry == nil {
return nil,
fmt.Errorf("configure the DB connection with config/connection first")
}
var connConfig connectionConfig
if err := entry.DecodeJSON(&connConfig); err != nil {
return nil, err
}
conn := connConfig.ConnectionString
if len(conn) == 0 {
conn = connConfig.ConnectionURL
}
// Ensure timezone is set to UTC for all the conenctions
if strings.HasPrefix(conn, "postgres://") || strings.HasPrefix(conn, "postgresql://") {
var err error
conn, err = pq.ParseURL(conn)
if err != nil {
return nil, err
}
}
conn += " timezone=utc"
b.db, err = sql.Open("postgres", conn)
if err != nil {
return nil, err
}
// Set some connection pool settings. We don't need much of this,
// since the request rate shouldn't be high.
b.db.SetMaxOpenConns(connConfig.MaxOpenConnections)
b.db.SetMaxIdleConns(connConfig.MaxIdleConnections)
return b.db, nil
}
func (p *Postgresql) SanitizedAddress() (_ string, err error) {
var canonicalizedAddress string
if strings.HasPrefix(p.Address, "postgres://") || strings.HasPrefix(p.Address, "postgresql://") {
canonicalizedAddress, err = pq.ParseURL(p.Address)
if err != nil {
return p.sanitizedAddress, err
}
} else {
canonicalizedAddress = p.Address
}
p.sanitizedAddress = passwordKVMatcher.ReplaceAllString(canonicalizedAddress, "")
return p.sanitizedAddress, err
}
func SetupPostgres() error {
dbUrl := env.StringDefault("DATABASE_URL", "postgres://")
dsn, err := pq.ParseURL(dbUrl)
if err != nil {
return err
}
db, err := sql.Open("postgres", dsn)
if err != nil {
return err
}
dbmap.Db = db
dbmap.Dialect = gorp.PostgresDialect{}
return nil
}
func NewPostgres() gorm.DB {
var err error
databaseUrl, _ := pq.ParseURL(os.Getenv("DATABASE_URL"))
Postgres, err = gorm.Open("postgres", databaseUrl)
if err != nil {
log.Fatalln(err.Error())
}
Postgres.LogMode(true)
Postgres.AutoMigrate(models.App{})
Postgres.AutoMigrate(models.User{})
return Postgres
}
func initDb() error {
parsed_url, err := pq.ParseURL(*db_url)
if err != nil {
return err
}
db, err = sql.Open("postgres", parsed_url)
if err != nil {
return err
}
dbmap = &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
err = logs.InitTables(dbmap)
if err != nil {
return err
}
return nil
}
func ParseURL(mod mb.Module, rawURL string) (mb.HostData, error) {
c := struct {
Username string `config:"username"`
Password string `config:"password"`
}{}
if err := mod.UnpackConfig(&c); err != nil {
return mb.HostData{}, err
}
if parts := strings.SplitN(rawURL, "://", 2); len(parts) != 2 {
// Add scheme.
rawURL = fmt.Sprintf("postgres://%s", rawURL)
}
u, err := url.Parse(rawURL)
if err != nil {
return mb.HostData{}, fmt.Errorf("error parsing URL: %v", err)
}
parse.SetURLUser(u, c.Username, c.Password)
if timeout := mod.Config().Timeout; timeout > 0 {
q := u.Query()
q.Set("connect_timeout", strconv.Itoa(int(timeout.Seconds())))
u.RawQuery = q.Encode()
}
// https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
connString, err := pq.ParseURL(u.String())
if err != nil {
return mb.HostData{}, err
}
h := parse.NewHostDataFromURL(u)
// Store the connection string instead of URL to avoid the cost of sql.Open
// parsing the URL on each call.
h.URI = connString
// Postgres URLs can use a host query param to specify the host. This is
// used for unix domain sockets (postgres:///dbname?host=/var/lib/postgres).
if host := u.Query().Get("host"); u.Host == "" && host != "" {
h.Host = host
}
return h, nil
}
func DbConnect() {
var err error
DATABASE_URL := os.Getenv("DATABASE_URL")
if DATABASE_URL == "" {
DATABASE_URL = "postgres://*****:*****@localhost:5432/quiet?sslmode=disable"
}
connStr, err := pq.ParseURL(DATABASE_URL)
if err != nil {
log.Fatal(err)
}
Db, err = sql.Open("postgres", connStr)
if err != nil {
log.Fatal(err)
}
}
func GetConnection() *sql.DB {
// XXX
//config, err := toml.LoadFile("config.toml")
//checkError(err)
//configTree := config.Get("postgres").(*toml.TomlTree)
//user := configTree.Get("user").(string)
//password := configTree.Get("password").(string)
//host := configTree.Get("host").(string)
//database := configTree.Get("database").(string)
//port := configTree.Get("port").(int64)
rawUrl := "postgres://postgres_arena@localhost:5432/arena?sslmode=disable"
url, err := pq.ParseURL(rawUrl)
checkError(err)
db, err := sql.Open("postgres", url)
checkError(err)
return db
}
func New(conf database.Config, m *mapping.Mapping) (database.DB, error) {
db := &PostGIS{}
db.Tables = make(map[string]*TableSpec)
db.GeneralizedTables = make(map[string]*GeneralizedTableSpec)
db.Config = conf
if strings.HasPrefix(db.Config.ConnectionParams, "postgis://") {
db.Config.ConnectionParams = strings.Replace(
db.Config.ConnectionParams,
"postgis", "postgres", 1,
)
}
params, err := pq.ParseURL(db.Config.ConnectionParams)
if err != nil {
return nil, err
}
params = disableDefaultSslOnLocalhost(params)
db.Prefix = prefixFromConnectionParams(params)
for name, table := range m.Tables {
db.Tables[name] = NewTableSpec(db, table)
}
for name, table := range m.GeneralizedTables {
db.GeneralizedTables[name] = NewGeneralizedTableSpec(db, table)
}
db.prepareGeneralizedTableSources()
db.prepareGeneralizations()
db.pointTagMatcher = m.PointMatcher()
db.lineStringTagMatcher = m.LineStringMatcher()
db.polygonTagMatcher = m.PolygonMatcher()
db.Params = params
err = db.Open()
if err != nil {
return nil, err
}
return db, nil
}
//NewPostgresMigrator returns a new migrator capable of performing a sequence
//of migrations. Note that this typically will fail (returning the
//error when the DB connection cannot be established). There is no
//connection pooling or other optimizations done in Migrator because it
//is designed to be used primarily in one-off migration situations.
func NewPostgresMigrator(url string) (Migrator, error) {
//fmt.Printf("url to parse: %s\n", url)
opts, err := pq.ParseURL(url)
if err != nil {
return nil, err
}
//fmt.Printf("opts that resulted: %s\n", opts)
db, err := sql.Open("postgres", opts)
if err != nil {
return nil, err
}
result := &postgresMigrator{
db: db,
}
if err := result.createTableIfNotExists(); err != nil {
return nil, err
}
return result, nil
}
func NewHandler(c Config) (http.Handler, error) {
p := &podslingHandler{
files: static.FileSystemHandler(),
}
if c.DatabaseURL == "" {
return nil, fmt.Errorf("Missing Database URL")
} else if info, err := pq.ParseURL(c.DatabaseURL); err != nil {
return nil, fmt.Errorf("Invalid Database URL: %s", err)
} else if db, err := sql.Open("postgres", info); err != nil {
return nil, fmt.Errorf("Failed to open Database: %s", err)
} else if err := p.initDB(db); err != nil {
return nil, fmt.Errorf("Failed to setup Database: %s", err)
}
h := http.Handler(p)
if c.Log {
h = requestlog.Wrap(h)
}
return h, nil
}
// ParseURL parses s into a ConnectionURL struct.
func ParseURL(s string) (u ConnectionURL, err error) {
o := make(values)
if strings.HasPrefix(s, "postgres://") {
s, err = pq.ParseURL(s)
if err != nil {
return u, err
}
}
if err := parseOpts(s, o); err != nil {
return u, err
}
u.User = o.Get("user")
u.Password = o.Get("password")
h := o.Get("host")
if strings.HasPrefix(h, "/") {
u.Socket = h
} else {
u.Host = h
}
u.Database = o.Get("dbname")
u.Options = make(map[string]string)
for k := range o {
switch k {
case "user", "password", "host", "port", "dbname":
// Skip
default:
u.Options[k] = o[k]
}
}
return u, err
}
func InitDb() *gorp.DbMap {
url := os.Getenv("HEROKU_POSTGRESQL_COPPER_URL")
conn, _ := pq.ParseURL(url)
conn += " sslmode=require"
// connect to db using standard Go database/sql API
db, err := sql.Open("postgres", conn)
checkErr(err, "sql.Open failed")
// construct a gorp DbMap
dbmap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
// add a table, setting the table name to 'posts' and
// specifying that the Id property is an auto incrementing PK
dbmap.AddTableWithName(email.Email{}, "emails").SetKeys(true, "ID")
// create the table. in a production system you'd generally
// use a migration tool, or create the tables via scripts
err = dbmap.CreateTablesIfNotExists()
checkErr(err, "Create tables failed")
return dbmap
}
// extract configuration details from the given file
func NewDBConf(p, env string, pgschema string, drv, open string) (*DBConf, error) {
// Automatically parse postgres urls
if drv == "postgres" {
// Assumption: If we can parse the URL, we should
if parsedURL, err := pq.ParseURL(open); err == nil && parsedURL != "" {
open = parsedURL
}
}
d := newDBDriver(drv, open)
if !d.IsValid() {
return nil, errors.New(fmt.Sprintf("Invalid DBConf: %v", d))
}
return &DBConf{
MigrationsDir: filepath.Join(p, "cases"),
Env: env,
Driver: d,
PgSchema: pgschema,
}, nil
}
func testAccStepReadCreds(t *testing.T, b logical.Backend, s logical.Storage, name string, connURL string) logicaltest.TestStep {
return logicaltest.TestStep{
Operation: logical.ReadOperation,
Path: "creds/" + name,
Check: func(resp *logical.Response) error {
var d struct {
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
}
if err := mapstructure.Decode(resp.Data, &d); err != nil {
return err
}
log.Printf("[WARN] Generated credentials: %v", d)
conn, err := pq.ParseURL(connURL)
if err != nil {
t.Fatal(err)
}
conn += " timezone=utc"
db, err := sql.Open("postgres", conn)
if err != nil {
t.Fatal(err)
}
returnedRows := func() int {
stmt, err := db.Prepare("SELECT DISTINCT schemaname FROM pg_tables WHERE has_table_privilege($1, 'information_schema.role_column_grants', 'select');")
if err != nil {
return -1
}
defer stmt.Close()
rows, err := stmt.Query(d.Username)
if err != nil {
return -1
}
defer rows.Close()
i := 0
for rows.Next() {
i++
}
return i
}
userRows := returnedRows()
if userRows != 2 {
t.Fatalf("did not get expected number of rows, got %d", userRows)
}
resp, err = b.HandleRequest(&logical.Request{
Operation: logical.RevokeOperation,
Storage: s,
Secret: &logical.Secret{
InternalData: map[string]interface{}{
"secret_type": "creds",
"username": d.Username,
},
},
})
if err != nil {
return err
}
if resp != nil {
if resp.IsError() {
return fmt.Errorf("Error on resp: %#v", *resp)
}
}
userRows = returnedRows()
// User shouldn't exist so returnedRows() should encounter an error and exit with -1
if userRows != -1 {
t.Fatalf("did not get expected number of rows, got %d", userRows)
}
return nil
},
}
}
func testAccStepReadCreds(t *testing.T, b logical.Backend, name string) logicaltest.TestStep {
return logicaltest.TestStep{
Operation: logical.ReadOperation,
Path: "creds/" + name,
Check: func(resp *logical.Response) error {
var d struct {
Username string `mapstructure:"username"`
Password string `mapstructure:"password"`
}
if err := mapstructure.Decode(resp.Data, &d); err != nil {
return err
}
log.Printf("[WARN] Generated credentials: %v", d)
conn, err := pq.ParseURL(os.Getenv("PG_URL"))
if err != nil {
t.Fatal(err)
}
conn += " timezone=utc"
db, err := sql.Open("postgres", conn)
if err != nil {
t.Fatal(err)
}
returnedRows := func() int {
stmt, err := db.Prepare(fmt.Sprintf(
"SELECT DISTINCT table_schema FROM information_schema.role_column_grants WHERE grantee='%s';",
d.Username))
if err != nil {
return -1
}
defer stmt.Close()
rows, err := stmt.Query()
if err != nil {
return -1
}
defer rows.Close()
i := 0
for rows.Next() {
i++
}
return i
}
userRows := returnedRows()
if userRows != 1 {
t.Fatalf("did not get expected number of rows, got %d", userRows)
}
resp, err = b.HandleRequest(&logical.Request{
Operation: logical.RevokeOperation,
Secret: &logical.Secret{
InternalData: map[string]interface{}{
"secret_type": "creds",
"username": d.Username,
},
},
})
if err != nil {
return err
}
if resp != nil {
if resp.IsError() {
return fmt.Errorf("Error on resp: %#v", *resp)
}
}
userRows = returnedRows()
if userRows != 0 {
t.Fatalf("did not get expected number of rows, got %d", userRows)
}
return nil
},
}
}
func (s *Sprint) Run() error {
var err error
dsn, _ := pq.ParseURL(os.Getenv("DATABASE_URL"))
db, err := xorm.NewEngine("postgres", dsn)
if err != nil {
return err
}
err = db.Sync(Project{})
if err != nil {
return err
}
s.Db = db
s.Validator = validator.New(&validator.Config{TagName: "validate"})
port := os.Getenv("PORT")
r := gin.Default()
r.GET("/api/projects", s.GetProjects)
r.GET("/api/projects/:id", s.GetProjectsOne)
r.POST("/api/projects", s.CreateProjects)
r.DELETE("/api/projects/:id", s.DeleteProjects)
r.Run(":" + port)
return nil
}