Example #1
0
func getConvertCommand() cli.Command {
	return cli.Command{
		Name:      "convert",
		ShortName: "conv",
		Usage:     "Convert DB",
		Description: `
Gohan convert can be used to migrate Gohan resources between different types of databases.

Setting meta-schema option will additionaly convert meta-schema table with schema resources.
Useful for development purposes.`,
		Flags: []cli.Flag{
			cli.StringFlag{Name: "in-type, it", Value: "", Usage: "Input db type (yaml, json, sqlite3, mysql)"},
			cli.StringFlag{Name: "in, i", Value: "", Usage: "Input db connection spec (or filename)"},
			cli.StringFlag{Name: "out-type, ot", Value: "", Usage: "Output db type (yaml, json, sqlite3, mysql)"},
			cli.StringFlag{Name: "out, o", Value: "", Usage: "Output db connection spec (or filename)"},
			cli.StringFlag{Name: "schema, s", Value: "", Usage: "Schema file"},
			cli.StringFlag{Name: "meta-schema, m", Value: "", Usage: "Meta-schema file (optional)"},
		},
		Action: func(c *cli.Context) {
			inType, in := c.String("in-type"), c.String("in")
			if inType == "" || in == "" {
				util.ExitFatal("Need to provide input database specification")
			}
			outType, out := c.String("out-type"), c.String("out")
			if outType == "" || out == "" {
				util.ExitFatal("Need to provide output database specification")
			}

			schemaFile := c.String("schema")
			if schemaFile == "" {
				util.ExitFatal("Need to provide schema file")
			}
			metaSchemaFile := c.String("meta-schema")

			schemaManager := schema.GetManager()
			err := schemaManager.LoadSchemasFromFiles(schemaFile, metaSchemaFile)
			if err != nil {
				util.ExitFatal("Error loading schema:", err)
			}

			inDB, err := db.ConnectDB(inType, in)
			if err != nil {
				util.ExitFatal(err)
			}
			outDB, err := db.ConnectDB(outType, out)
			if err != nil {
				util.ExitFatal(err)
			}

			err = db.CopyDBResources(inDB, outDB)
			if err != nil {
				util.ExitFatal(err)
			}

			fmt.Println("Conversion complete")
		},
	}
}
Example #2
0
			conn = "./test.db"
			dbType = "sqlite3"
		}

		manager := schema.GetManager()
		dbc, err := db.ConnectDB(dbType, conn, db.DefaultMaxOpenConn)
		sqlConn = dbc.(*DB)
		Expect(err).ToNot(HaveOccurred())
		Expect(manager.LoadSchemasFromFiles(
			"../../etc/schema/gohan.json", "../../tests/test_abstract_schema.yaml", "../../tests/test_schema.yaml")).To(Succeed())
		db.InitDBWithSchemas(dbType, conn, true, false)

		// Insert fixture data
		fixtureDB, err := db.ConnectDB("json", "test_fixture.json", db.DefaultMaxOpenConn)
		Expect(err).ToNot(HaveOccurred())
		db.CopyDBResources(fixtureDB, dbc, true)

		tx, err = dbc.Begin()
		Expect(err).ToNot(HaveOccurred())
	})

	AfterEach(func() {
		schema.ClearManager()
		if os.Getenv("MYSQL_TEST") != "true" {
			os.Remove(conn)
		}
	})

	Describe("Query", func() {
		var s *schema.Schema
Example #3
0
		It("Should do it properly", func() {
			inDB, err := db.ConnectDB("yaml", "test_data/conv_in.yaml", db.DefaultMaxOpenConn)
			Expect(err).ToNot(HaveOccurred())
			defer os.Remove("test_data/conv_in.db")

			db.InitDBWithSchemas("sqlite3", "test_data/conv_out.db", false, false)
			outDB, err := db.ConnectDB("sqlite3", "test_data/conv_out.db", db.DefaultMaxOpenConn)
			Expect(err).ToNot(HaveOccurred())
			defer os.Remove("test_data/conv_out.db")

			db.InitDBWithSchemas("yaml", "test_data/conv_verify.yaml", false, false)
			verifyDB, err := db.ConnectDB("yaml", "test_data/conv_verify.yaml", db.DefaultMaxOpenConn)
			Expect(err).ToNot(HaveOccurred())
			defer os.Remove("test_data/conv_verify.yaml")

			Expect(db.CopyDBResources(inDB, outDB, true)).To(Succeed())

			Expect(db.CopyDBResources(outDB, verifyDB, true)).To(Succeed())

			inTx, err := inDB.Begin()
			Expect(err).ToNot(HaveOccurred())
			defer inTx.Close()

			// SQL returns different types than JSON/YAML Database
			// So we need to move it back again so that DeepEqual would work correctly
			verifyTx, err := verifyDB.Begin()
			Expect(err).ToNot(HaveOccurred())
			defer verifyTx.Close()

			for _, s := range manager.OrderedSchemas() {
				if s.Metadata["type"] == "metaschema" {
Example #4
0
//NewServer returns new GohanAPIServer
func NewServer(configFile string) (*Server, error) {
	manager := schema.GetManager()
	config := util.GetConfig()
	err := config.ReadConfig(configFile)
	err = os.Chdir(path.Dir(configFile))
	if err != nil {
		return nil, fmt.Errorf("Config load error: %s", err)
	}
	err = l.SetUpLogging(config)
	if err != nil {
		return nil, fmt.Errorf("Logging setup error: %s", err)
	}
	log.Info("logging initialized")

	server := &Server{}

	m := martini.Classic()
	m.Handlers()
	m.Use(middleware.Logging())
	m.Use(martini.Recovery())
	m.Use(middleware.JSONURLs())
	m.Use(middleware.WithContext())

	server.martini = m
	server.address = config.GetString("address", ":9091")
	if config.GetBool("tls/enabled", false) {
		log.Info("TLS enabled")
		server.tls = &tls{
			KeyFile:  config.GetString("tls/key_file", "./etc/key.pem"),
			CertFile: config.GetString("tls/cert_file", "./etc/cert.pem"),
		}
	}

	server.connectDB()

	schemaFiles := config.GetStringList("schemas", nil)
	if schemaFiles == nil {
		log.Fatal("No schema specified in configuraion")
	} else {
		err = manager.LoadSchemasFromFiles(schemaFiles...)
		if err != nil {
			return nil, fmt.Errorf("invalid schema: %s", err)
		}
	}
	server.initDB()

	etcdServers := config.GetStringList("etcd", nil)
	log.Info("etcd servers: %s", etcdServers)
	server.sync = etcd.NewSync(etcdServers)

	if config.GetList("database/initial_data", nil) != nil {
		initialDataList := config.GetList("database/initial_data", nil)
		for _, initialData := range initialDataList {
			initialDataConfig := initialData.(map[string]interface{})
			inType := initialDataConfig["type"].(string)
			inConnection := initialDataConfig["connection"].(string)
			log.Info("Importing data from %s ...", inConnection)
			inDB, err := db.ConnectDB(inType, inConnection)
			if err != nil {
				log.Fatal(err)
			}
			db.CopyDBResources(inDB, server.db)
		}
	}

	if config.GetBool("keystone/use_keystone", false) {
		//TODO remove this
		if config.GetBool("keystone/fake", false) {
			server.keystoneIdentity = &middleware.FakeIdentity{}
			//TODO(marcin) requests to fake server also get authenticated
			//             we need a separate routing Group
			log.Info("Debug Mode with Fake Keystone Server")
			middleware.FakeKeystone(server.martini)
		} else {
			log.Info("Keystone backend server configured")
			server.keystoneIdentity, err = cloud.NewKeystoneIdentity(
				config.GetString("keystone/auth_url", "http://localhost:35357/v3"),
				config.GetString("keystone/user_name", "admin"),
				config.GetString("keystone/password", "password"),
				config.GetString("keystone/domain_name", "Default"),
				config.GetString("keystone/tenant_name", "admin"),
				config.GetString("keystone/version", ""),
			)
			if err != nil {
				log.Fatal(err)
			}
		}
		m.MapTo(server.keystoneIdentity, (*middleware.IdentityService)(nil))
		m.Use(middleware.Authentication())
		//m.Use(Authorization())
	}

	if err != nil {
		return nil, fmt.Errorf("invalid base dir: %s", err)
	}

	server.addOptionsRoute()
	cors := config.GetString("cors", "")
	if cors != "" {
		log.Info("Enabling CORS for %s", cors)
		if cors == "*" {
			log.Warning("cors for * have security issue")
		}
		server.martini.Use(func(rw http.ResponseWriter, r *http.Request) {
			rw.Header().Add("Access-Control-Allow-Origin", cors)
			rw.Header().Add("Access-Control-Allow-Headers", "X-Auth-Token, Content-Type")
			rw.Header().Add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE")
		})
	}

	documentRoot := config.GetString("document_root", "./")
	log.Info("Static file serving from %s", documentRoot)
	documentRootABS, err := filepath.Abs(documentRoot)
	server.martini.Use(martini.Static(documentRootABS))

	server.mapRoutes()
	return server, nil
}
Example #5
0
//NewServer returns new GohanAPIServer
func NewServer(configFile string) (*Server, error) {
	manager := schema.GetManager()
	config := util.GetConfig()
	err := config.ReadConfig(configFile)
	err = os.Chdir(path.Dir(configFile))
	if err != nil {
		return nil, fmt.Errorf("Config load error: %s", err)
	}
	err = l.SetUpLogging(config)
	if err != nil {
		return nil, fmt.Errorf("Logging setup error: %s", err)
	}
	log.Info("logging initialized")

	server := &Server{}

	m := martini.Classic()
	m.Handlers()
	m.Use(middleware.Logging())
	m.Use(martini.Recovery())
	m.Use(middleware.JSONURLs())
	m.Use(middleware.WithContext())

	server.martini = m

	port := os.Getenv("PORT")

	if port == "" {
		port = "9091"
	}

	setupEditor(server)

	server.timelimit = config.GetInt("extension/timelimit", 30)
	server.extensions = config.GetStringList("extension/use", []string{
		"javascript",
		"gohanscript",
		"go",
	})
	schema.DefaultExtension = config.GetString("extension/default", "javascript")
	server.address = config.GetString("address", ":"+port)
	if config.GetBool("tls/enabled", false) {
		log.Info("TLS enabled")
		server.tls = &tlsConfig{
			KeyFile:  config.GetString("tls/key_file", "./etc/key.pem"),
			CertFile: config.GetString("tls/cert_file", "./etc/cert.pem"),
		}
	}

	etcdServers := config.GetStringList("etcd", nil)
	if etcdServers != nil {
		log.Info("etcd servers: %s", etcdServers)
		server.sync = etcd.NewSync(etcdServers)
	}

	server.connectDB()

	schemaFiles := config.GetStringList("schemas", nil)
	if schemaFiles == nil {
		log.Fatal("No schema specified in configuraion")
	} else {
		err = manager.LoadSchemasFromFiles(schemaFiles...)
		if err != nil {
			return nil, fmt.Errorf("invalid schema: %s", err)
		}
	}
	if !config.GetBool("database/no_init", false) {
		server.initDB()
	}

	if config.GetList("database/initial_data", nil) != nil {
		initialDataList := config.GetList("database/initial_data", nil)
		for _, initialData := range initialDataList {
			initialDataConfig := initialData.(map[string]interface{})
			inType := initialDataConfig["type"].(string)
			inConnection := initialDataConfig["connection"].(string)
			log.Info("Importing data from %s ...", inConnection)
			inDB, err := db.ConnectDB(inType, inConnection, db.DefaultMaxOpenConn)
			if err != nil {
				log.Fatal(err)
			}
			db.CopyDBResources(inDB, server.db, false)
		}
	}

	if config.GetBool("keystone/use_keystone", false) {
		//TODO remove this
		if config.GetBool("keystone/fake", false) {
			server.keystoneIdentity = &middleware.FakeIdentity{}
			//TODO(marcin) requests to fake server also get authenticated
			//             we need a separate routing Group
			log.Info("Debug Mode with Fake Keystone Server")
		} else {
			log.Info("Keystone backend server configured")
			server.keystoneIdentity, err = cloud.NewKeystoneIdentity(
				config.GetString("keystone/auth_url", "http://localhost:35357/v3"),
				config.GetString("keystone/user_name", "admin"),
				config.GetString("keystone/password", "password"),
				config.GetString("keystone/domain_name", "Default"),
				config.GetString("keystone/tenant_name", "admin"),
				config.GetString("keystone/version", ""),
			)
			if err != nil {
				log.Fatal(err)
			}
		}
		m.MapTo(server.keystoneIdentity, (*middleware.IdentityService)(nil))
		m.Use(middleware.Authentication())
		//m.Use(Authorization())
	} else {
		m.MapTo(&middleware.NoIdentityService{}, (*middleware.IdentityService)(nil))
		m.Map(schema.NewAuthorization("admin", "admin", "admin_token", []string{"admin"}, nil))
	}

	if err != nil {
		return nil, fmt.Errorf("invalid base dir: %s", err)
	}

	server.addOptionsRoute()
	cors := config.GetString("cors", "")
	if cors != "" {
		log.Info("Enabling CORS for %s", cors)
		if cors == "*" {
			log.Warning("cors for * have security issue")
		}
		server.martini.Use(func(rw http.ResponseWriter, r *http.Request) {
			rw.Header().Add("Access-Control-Allow-Origin", cors)
			rw.Header().Add("Access-Control-Allow-Headers", "X-Auth-Token, Content-Type")
			rw.Header().Add("Access-Control-Expose-Headers", "X-Total-Count")
			rw.Header().Add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE")
		})
	}

	documentRoot := config.GetString("document_root", "embed")
	if config.GetBool("webui_config/enabled", false) {
		m.Use(func(res http.ResponseWriter, req *http.Request, c martini.Context) {
			if req.URL.Path != "/webui/config.json" {
				c.Next()
				return
			}
			address := config.GetString("webui_config/address", server.address)
			if address[0] == ':' {
				address = "__HOST__" + address
			}
			baseURL := "http://" + address
			authURL := "http://" + address + "/v2.0"
			if config.GetBool("webui_config/tls", config.GetBool("tls/enabled", false)) {
				baseURL = "https://" + address
				authURL = "https://" + address + "/v2.0"
			}
			authURL = config.GetString("webui_config/auth_url", authURL)
			webUIConfig := map[string]interface{}{
				"authUrl": authURL,
				"gohan": map[string]interface{}{
					"schema": "/gohan/v0.1/schemas",
					"url":    baseURL,
				},
			}
			routes.ServeJson(res, webUIConfig)
		})
	}
	if documentRoot == "embed" {
		m.Use(staticbin.Static("public", util.Asset, staticbin.Options{
			SkipLogging: true,
		}))
	} else {
		log.Info("Static file serving from %s", documentRoot)
		documentRootABS, err := filepath.Abs(documentRoot)
		if err != nil {
			return nil, err
		}
		server.martini.Use(martini.Static(documentRootABS, martini.StaticOptions{
			SkipLogging: true,
		}))
	}
	server.mapRoutes()

	maxWorkerCount := config.GetInt("workers", 100)
	server.queue = job.NewQueue(uint(maxWorkerCount))

	return server, nil
}