func getConvertCommand() cli.Command {
return cli.Command{
Name: "convert",
ShortName: "conv",
Usage: "Convert DB",
Description: `
Gohan convert can be used to migrate Gohan resources between different types of databases.
Setting meta-schema option will additionaly convert meta-schema table with schema resources.
Useful for development purposes.`,
Flags: []cli.Flag{
cli.StringFlag{Name: "in-type, it", Value: "", Usage: "Input db type (yaml, json, sqlite3, mysql)"},
cli.StringFlag{Name: "in, i", Value: "", Usage: "Input db connection spec (or filename)"},
cli.StringFlag{Name: "out-type, ot", Value: "", Usage: "Output db type (yaml, json, sqlite3, mysql)"},
cli.StringFlag{Name: "out, o", Value: "", Usage: "Output db connection spec (or filename)"},
cli.StringFlag{Name: "schema, s", Value: "", Usage: "Schema file"},
cli.StringFlag{Name: "meta-schema, m", Value: "", Usage: "Meta-schema file (optional)"},
},
Action: func(c *cli.Context) {
inType, in := c.String("in-type"), c.String("in")
if inType == "" || in == "" {
util.ExitFatal("Need to provide input database specification")
}
outType, out := c.String("out-type"), c.String("out")
if outType == "" || out == "" {
util.ExitFatal("Need to provide output database specification")
}
schemaFile := c.String("schema")
if schemaFile == "" {
util.ExitFatal("Need to provide schema file")
}
metaSchemaFile := c.String("meta-schema")
schemaManager := schema.GetManager()
err := schemaManager.LoadSchemasFromFiles(schemaFile, metaSchemaFile)
if err != nil {
util.ExitFatal("Error loading schema:", err)
}
inDB, err := db.ConnectDB(inType, in)
if err != nil {
util.ExitFatal(err)
}
outDB, err := db.ConnectDB(outType, out)
if err != nil {
util.ExitFatal(err)
}
err = db.CopyDBResources(inDB, outDB)
if err != nil {
util.ExitFatal(err)
}
fmt.Println("Conversion complete")
},
}
}
func newDBConnection(dbfilename string) (db.DB, error) {
connection, err := db.ConnectDB("sqlite3", dbfilename, db.DefaultMaxOpenConn)
if err != nil {
return nil, err
}
return connection, nil
}
func (server *Server) connectDB() error {
config := util.GetConfig()
dbType, dbConnection, _, _ := server.getDatabaseConfig()
maxConn := config.GetInt("database/max_open_conn", db.DefaultMaxOpenConn)
dbConn, err := db.ConnectDB(dbType, dbConnection, maxConn)
if server.sync == nil {
server.db = dbConn
} else {
server.db = &DbSyncWrapper{dbConn}
}
return err
}
RunSpecs(t, "Server Suite")
}
var _ = Describe("Suit set up and tear down", func() {
var conn, dbType string
if os.Getenv("MYSQL_TEST") == "true" {
conn = "root@/gohan_test"
dbType = "mysql"
} else {
conn = "./test.db"
dbType = "sqlite3"
}
var _ = BeforeSuite(func() {
var err error
testDB, err = db.ConnectDB(dbType, conn)
Expect(err).ToNot(HaveOccurred(), "Failed to connect database.")
if os.Getenv("MYSQL_TEST") == "true" {
err = startTestServer("./server_test_mysql_config.yaml")
} else {
err = startTestServer("./server_test_config.yaml")
}
Expect(err).ToNot(HaveOccurred(), "Failed to start test server.")
})
var _ = AfterSuite(func() {
schema.ClearManager()
os.Remove(conn)
})
})
var conn string
var tx transaction.Transaction
var sqlConn *DB
BeforeEach(func() {
var dbType string
if os.Getenv("MYSQL_TEST") == "true" {
conn = "gohan:gohan@/gohan_test"
dbType = "mysql"
} else {
conn = "./test.db"
dbType = "sqlite3"
}
manager := schema.GetManager()
dbc, err := db.ConnectDB(dbType, conn, db.DefaultMaxOpenConn)
sqlConn = dbc.(*DB)
Expect(err).ToNot(HaveOccurred())
Expect(manager.LoadSchemasFromFiles(
"../../etc/schema/gohan.json", "../../tests/test_abstract_schema.yaml", "../../tests/test_schema.yaml")).To(Succeed())
db.InitDBWithSchemas(dbType, conn, true, false)
// Insert fixture data
fixtureDB, err := db.ConnectDB("json", "test_fixture.json", db.DefaultMaxOpenConn)
Expect(err).ToNot(HaveOccurred())
db.CopyDBResources(fixtureDB, dbc, true)
tx, err = dbc.Begin()
Expect(err).ToNot(HaveOccurred())
})
Expect(subnetResource.Path()).To(Equal("/v2.0/subnets/subnetRed"))
server := map[string]interface{}{
"id": "serverRed",
"name": "serverRed",
"tenant_id": "red",
"network_id": "networkRed",
"description": "red server",
"cidr": "10.0.0.0/24"}
serverResource, err = manager.LoadResource("server", server)
Expect(err).ToNot(HaveOccurred())
})
JustBeforeEach(func() {
os.Remove(conn)
dataStore, err = db.ConnectDB(dbType, conn, db.DefaultMaxOpenConn)
Expect(err).ToNot(HaveOccurred())
for _, s := range manager.Schemas() {
Expect(dataStore.RegisterTable(s, false)).To(Succeed())
}
tx, err = dataStore.Begin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
tx.Close()
})
Describe("Using sql", func() {
var (
testDB1 db.DB
testDB2 db.DB
)
func TestExtension(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Extension Suite")
}
var _ = Describe("Suite set up and tear down", func() {
const (
testDBFile1 = "./extensionTest1.db"
testDBFile2 = "./extensionTest2.db"
)
var _ = BeforeSuite(func() {
var err error
testDB1, err = db.ConnectDB("sqlite3", testDBFile1)
Expect(err).NotTo(HaveOccurred())
testDB2, err = db.ConnectDB("sqlite3", testDBFile2)
Expect(err).NotTo(HaveOccurred())
})
var _ = AfterSuite(func() {
os.Remove(testDBFile1)
os.Remove(testDBFile2)
})
})
var (
testDB1 db.DB
testDB2 db.DB
)
func TestExtension(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Extension Suite")
}
var _ = Describe("Suite set up and tear down", func() {
const (
testDBFile1 = "./extensionTest1.db"
testDBFile2 = "./extensionTest2.db"
)
var _ = BeforeSuite(func() {
var err error
testDB1, err = db.ConnectDB("sqlite3", testDBFile1, db.DefaultMaxOpenConn)
Expect(err).NotTo(HaveOccurred())
testDB2, err = db.ConnectDB("sqlite3", testDBFile2, db.DefaultMaxOpenConn)
Expect(err).NotTo(HaveOccurred())
})
var _ = AfterSuite(func() {
os.Remove(testDBFile1)
os.Remove(testDBFile2)
})
})
subnetResource.SetParentID("networkRed")
Expect(subnetResource.Path()).To(Equal("/v2.0/subnets/subnetRed"))
server := map[string]interface{}{
"id": "serverRed",
"name": "serverRed",
"tenant_id": "red",
"network_id": "networkRed",
"cidr": "10.0.0.0/24"}
serverResource, err = manager.LoadResource("server", server)
Expect(err).ToNot(HaveOccurred())
})
JustBeforeEach(func() {
os.Remove(conn)
dataStore, err = db.ConnectDB(dbType, conn)
Expect(err).ToNot(HaveOccurred())
for _, s := range manager.Schemas() {
Expect(dataStore.RegisterTable(s, false)).To(Succeed())
}
tx, err = dataStore.Begin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {
tx.Close()
})
Describe("Using sql", func() {
//NewServer returns new GohanAPIServer
func NewServer(configFile string) (*Server, error) {
manager := schema.GetManager()
config := util.GetConfig()
err := config.ReadConfig(configFile)
err = os.Chdir(path.Dir(configFile))
if err != nil {
return nil, fmt.Errorf("Config load error: %s", err)
}
err = l.SetUpLogging(config)
if err != nil {
return nil, fmt.Errorf("Logging setup error: %s", err)
}
log.Info("logging initialized")
server := &Server{}
m := martini.Classic()
m.Handlers()
m.Use(middleware.Logging())
m.Use(martini.Recovery())
m.Use(middleware.JSONURLs())
m.Use(middleware.WithContext())
server.martini = m
server.address = config.GetString("address", ":9091")
if config.GetBool("tls/enabled", false) {
log.Info("TLS enabled")
server.tls = &tls{
KeyFile: config.GetString("tls/key_file", "./etc/key.pem"),
CertFile: config.GetString("tls/cert_file", "./etc/cert.pem"),
}
}
server.connectDB()
schemaFiles := config.GetStringList("schemas", nil)
if schemaFiles == nil {
log.Fatal("No schema specified in configuraion")
} else {
err = manager.LoadSchemasFromFiles(schemaFiles...)
if err != nil {
return nil, fmt.Errorf("invalid schema: %s", err)
}
}
server.initDB()
etcdServers := config.GetStringList("etcd", nil)
log.Info("etcd servers: %s", etcdServers)
server.sync = etcd.NewSync(etcdServers)
if config.GetList("database/initial_data", nil) != nil {
initialDataList := config.GetList("database/initial_data", nil)
for _, initialData := range initialDataList {
initialDataConfig := initialData.(map[string]interface{})
inType := initialDataConfig["type"].(string)
inConnection := initialDataConfig["connection"].(string)
log.Info("Importing data from %s ...", inConnection)
inDB, err := db.ConnectDB(inType, inConnection)
if err != nil {
log.Fatal(err)
}
db.CopyDBResources(inDB, server.db)
}
}
if config.GetBool("keystone/use_keystone", false) {
//TODO remove this
if config.GetBool("keystone/fake", false) {
server.keystoneIdentity = &middleware.FakeIdentity{}
//TODO(marcin) requests to fake server also get authenticated
// we need a separate routing Group
log.Info("Debug Mode with Fake Keystone Server")
middleware.FakeKeystone(server.martini)
} else {
log.Info("Keystone backend server configured")
server.keystoneIdentity, err = cloud.NewKeystoneIdentity(
config.GetString("keystone/auth_url", "http://localhost:35357/v3"),
config.GetString("keystone/user_name", "admin"),
config.GetString("keystone/password", "password"),
config.GetString("keystone/domain_name", "Default"),
config.GetString("keystone/tenant_name", "admin"),
config.GetString("keystone/version", ""),
)
if err != nil {
log.Fatal(err)
}
}
m.MapTo(server.keystoneIdentity, (*middleware.IdentityService)(nil))
m.Use(middleware.Authentication())
//m.Use(Authorization())
}
if err != nil {
return nil, fmt.Errorf("invalid base dir: %s", err)
}
server.addOptionsRoute()
cors := config.GetString("cors", "")
if cors != "" {
log.Info("Enabling CORS for %s", cors)
if cors == "*" {
log.Warning("cors for * have security issue")
}
server.martini.Use(func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Add("Access-Control-Allow-Origin", cors)
rw.Header().Add("Access-Control-Allow-Headers", "X-Auth-Token, Content-Type")
rw.Header().Add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE")
})
}
documentRoot := config.GetString("document_root", "./")
log.Info("Static file serving from %s", documentRoot)
documentRootABS, err := filepath.Abs(documentRoot)
server.martini.Use(martini.Static(documentRootABS))
server.mapRoutes()
return server, nil
}
func (server *Server) connectDB() error {
dbType, dbConnection, _, _ := server.getDatabaseConfig()
dbConn, err := db.ConnectDB(dbType, dbConnection)
server.db = &DbSyncWrapper{dbConn}
return err
}
func beforeEach() {
os.Remove(conn)
dataStore, _ = db.ConnectDB(dbtype, conn)
}
func beforeEach() {
timelimit = time.Duration(1) * time.Second
os.Remove(conn)
dataStore, _ = db.ConnectDB(dbtype, conn)
}
//ConnectDB start connection to db
func ConnectDB(dbType string, connection string, maxOpenConn int) (db.DB, error) {
return db.ConnectDB(dbType, connection, maxOpenConn)
}
//NewServer returns new GohanAPIServer
func NewServer(configFile string) (*Server, error) {
manager := schema.GetManager()
config := util.GetConfig()
err := config.ReadConfig(configFile)
err = os.Chdir(path.Dir(configFile))
if err != nil {
return nil, fmt.Errorf("Config load error: %s", err)
}
err = l.SetUpLogging(config)
if err != nil {
return nil, fmt.Errorf("Logging setup error: %s", err)
}
log.Info("logging initialized")
server := &Server{}
m := martini.Classic()
m.Handlers()
m.Use(middleware.Logging())
m.Use(martini.Recovery())
m.Use(middleware.JSONURLs())
m.Use(middleware.WithContext())
server.martini = m
port := os.Getenv("PORT")
if port == "" {
port = "9091"
}
setupEditor(server)
server.timelimit = config.GetInt("extension/timelimit", 30)
server.extensions = config.GetStringList("extension/use", []string{
"javascript",
"gohanscript",
"go",
})
schema.DefaultExtension = config.GetString("extension/default", "javascript")
server.address = config.GetString("address", ":"+port)
if config.GetBool("tls/enabled", false) {
log.Info("TLS enabled")
server.tls = &tlsConfig{
KeyFile: config.GetString("tls/key_file", "./etc/key.pem"),
CertFile: config.GetString("tls/cert_file", "./etc/cert.pem"),
}
}
etcdServers := config.GetStringList("etcd", nil)
if etcdServers != nil {
log.Info("etcd servers: %s", etcdServers)
server.sync = etcd.NewSync(etcdServers)
}
server.connectDB()
schemaFiles := config.GetStringList("schemas", nil)
if schemaFiles == nil {
log.Fatal("No schema specified in configuraion")
} else {
err = manager.LoadSchemasFromFiles(schemaFiles...)
if err != nil {
return nil, fmt.Errorf("invalid schema: %s", err)
}
}
if !config.GetBool("database/no_init", false) {
server.initDB()
}
if config.GetList("database/initial_data", nil) != nil {
initialDataList := config.GetList("database/initial_data", nil)
for _, initialData := range initialDataList {
initialDataConfig := initialData.(map[string]interface{})
inType := initialDataConfig["type"].(string)
inConnection := initialDataConfig["connection"].(string)
log.Info("Importing data from %s ...", inConnection)
inDB, err := db.ConnectDB(inType, inConnection, db.DefaultMaxOpenConn)
if err != nil {
log.Fatal(err)
}
db.CopyDBResources(inDB, server.db, false)
}
}
if config.GetBool("keystone/use_keystone", false) {
//TODO remove this
if config.GetBool("keystone/fake", false) {
server.keystoneIdentity = &middleware.FakeIdentity{}
//TODO(marcin) requests to fake server also get authenticated
// we need a separate routing Group
log.Info("Debug Mode with Fake Keystone Server")
} else {
log.Info("Keystone backend server configured")
server.keystoneIdentity, err = cloud.NewKeystoneIdentity(
config.GetString("keystone/auth_url", "http://localhost:35357/v3"),
config.GetString("keystone/user_name", "admin"),
config.GetString("keystone/password", "password"),
config.GetString("keystone/domain_name", "Default"),
config.GetString("keystone/tenant_name", "admin"),
config.GetString("keystone/version", ""),
)
if err != nil {
log.Fatal(err)
}
}
m.MapTo(server.keystoneIdentity, (*middleware.IdentityService)(nil))
m.Use(middleware.Authentication())
//m.Use(Authorization())
} else {
m.MapTo(&middleware.NoIdentityService{}, (*middleware.IdentityService)(nil))
m.Map(schema.NewAuthorization("admin", "admin", "admin_token", []string{"admin"}, nil))
}
if err != nil {
return nil, fmt.Errorf("invalid base dir: %s", err)
}
server.addOptionsRoute()
cors := config.GetString("cors", "")
if cors != "" {
log.Info("Enabling CORS for %s", cors)
if cors == "*" {
log.Warning("cors for * have security issue")
}
server.martini.Use(func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Add("Access-Control-Allow-Origin", cors)
rw.Header().Add("Access-Control-Allow-Headers", "X-Auth-Token, Content-Type")
rw.Header().Add("Access-Control-Expose-Headers", "X-Total-Count")
rw.Header().Add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE")
})
}
documentRoot := config.GetString("document_root", "embed")
if config.GetBool("webui_config/enabled", false) {
m.Use(func(res http.ResponseWriter, req *http.Request, c martini.Context) {
if req.URL.Path != "/webui/config.json" {
c.Next()
return
}
address := config.GetString("webui_config/address", server.address)
if address[0] == ':' {
address = "__HOST__" + address
}
baseURL := "http://" + address
authURL := "http://" + address + "/v2.0"
if config.GetBool("webui_config/tls", config.GetBool("tls/enabled", false)) {
baseURL = "https://" + address
authURL = "https://" + address + "/v2.0"
}
authURL = config.GetString("webui_config/auth_url", authURL)
webUIConfig := map[string]interface{}{
"authUrl": authURL,
"gohan": map[string]interface{}{
"schema": "/gohan/v0.1/schemas",
"url": baseURL,
},
}
routes.ServeJson(res, webUIConfig)
})
}
if documentRoot == "embed" {
m.Use(staticbin.Static("public", util.Asset, staticbin.Options{
SkipLogging: true,
}))
} else {
log.Info("Static file serving from %s", documentRoot)
documentRootABS, err := filepath.Abs(documentRoot)
if err != nil {
return nil, err
}
server.martini.Use(martini.Static(documentRootABS, martini.StaticOptions{
SkipLogging: true,
}))
}
server.mapRoutes()
maxWorkerCount := config.GetInt("workers", 100)
server.queue = job.NewQueue(uint(maxWorkerCount))
return server, nil
}
return err
}
}
return nil
}
func TestOttoExtension(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Otto Extension Suite")
}
var _ = Describe("Suite set up and tear down", func() {
var _ = BeforeSuite(func() {
var err error
Expect(os.Chdir(configDir)).To(Succeed())
testDB, err = db.ConnectDB(dbType, dbFile)
Expect(err).ToNot(HaveOccurred(), "Failed to connect database.")
manager := schema.GetManager()
config := util.GetConfig()
Expect(config.ReadConfig(configFile)).To(Succeed())
schemaFiles := config.GetStringList("schemas", nil)
Expect(schemaFiles).NotTo(BeNil())
Expect(manager.LoadSchemasFromFiles(schemaFiles...)).To(Succeed())
Expect(db.InitDBWithSchemas(dbType, dbFile, false, false)).To(Succeed())
})
var _ = AfterSuite(func() {
schema.ClearManager()
os.Remove(dbFile)
})
})
return err
}
}
return nil
}
func TestOttoExtension(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Otto Extension Suite")
}
var _ = Describe("Suite set up and tear down", func() {
var _ = BeforeSuite(func() {
var err error
Expect(os.Chdir(configDir)).To(Succeed())
testDB, err = db.ConnectDB(dbType, dbFile, db.DefaultMaxOpenConn)
Expect(err).ToNot(HaveOccurred(), "Failed to connect database.")
manager := schema.GetManager()
config := util.GetConfig()
Expect(config.ReadConfig(configFile)).To(Succeed())
schemaFiles := config.GetStringList("schemas", nil)
Expect(schemaFiles).NotTo(BeNil())
Expect(manager.LoadSchemasFromFiles(schemaFiles...)).To(Succeed())
Expect(db.InitDBWithSchemas(dbType, dbFile, false, false)).To(Succeed())
})
var _ = AfterSuite(func() {
schema.ClearManager()
os.Remove(dbFile)
})
})
var conn string
var tx transaction.Transaction
BeforeEach(func() {
var dbType string
if os.Getenv("MYSQL_TEST") == "true" {
conn = "root@/gohan_test"
dbType = "mysql"
} else {
conn = "./test.db"
dbType = "sqlite3"
}
manager := schema.GetManager()
dbc, err := db.ConnectDB(dbType, conn)
Expect(err).ToNot(HaveOccurred())
Expect(manager.LoadSchemasFromFiles(
"../../etc/schema/gohan.json", "../../tests/test_schema.yaml")).To(Succeed())
db.InitDBWithSchemas(dbType, conn, true, false)
// Insert fixture data
fixtureDB, err := db.ConnectDB("json", "test_fixture.json")
Expect(err).ToNot(HaveOccurred())
db.CopyDBResources(fixtureDB, dbc)
tx, err = dbc.Begin()
Expect(err).ToNot(HaveOccurred())
})
AfterEach(func() {