// TestExtensions runs extension tests when invoked from Gohan CLI
func TestExtensions(c *cli.Context) {
buflog.SetUpDefaultLogging()
var config *util.Config
configFilePath := c.String("config-file")
if configFilePath != "" && !c.Bool("verbose") {
config = util.GetConfig()
err := config.ReadConfig(configFilePath)
if err != nil {
log.Error(fmt.Sprintf("Failed to read config from path %s: %v", configFilePath, err))
os.Exit(1)
}
err = gohan_log.SetUpLogging(config)
if err != nil {
log.Error(fmt.Sprintf("Failed to set up logging: %v", err))
os.Exit(1)
}
}
testFiles := getTestFiles(c.Args())
//logging from config is a limited printAllLogs option
returnCode := RunTests(testFiles, c.Bool("verbose") || config != nil, c.String("run-test"))
os.Exit(returnCode)
}
//Sync Watch Process
func startSyncWatchProcess(server *Server) {
config := util.GetConfig()
watch := config.GetStringList("watch/keys", nil)
events := config.GetStringList("watch/events", nil)
if watch == nil {
return
}
extensions := map[string]extension.Environment{}
for _, event := range events {
path := "sync://" + event
env, err := server.NewEnvironmentForPath("sync."+event, path)
if err != nil {
log.Fatal(err.Error())
}
extensions[event] = env
}
responseChan := make(chan *gohan_sync.Event)
stopChan := make(chan bool)
for _, path := range watch {
go func(path string) {
defer util.LogFatalPanic(log)
for server.running {
lockKey := lockPath + "watch"
err := server.sync.Lock(lockKey, true)
if err != nil {
log.Warning("Can't start watch process due to lock", err)
time.Sleep(5 * time.Second)
continue
}
defer func() {
server.sync.Unlock(lockKey)
}()
err = server.sync.Watch(path, responseChan, stopChan)
if err != nil {
log.Error(fmt.Sprintf("sync watch error: %s", err))
}
}
}(path)
}
//main response lisnter process
go func() {
defer util.LogFatalPanic(log)
for server.running {
response := <-responseChan
server.queue.Add(job.NewJob(
func() {
defer util.LogPanic(log)
for _, event := range events {
//match extensions
if strings.HasPrefix(response.Key, "/"+event) {
env := extensions[event]
runExtensionOnSync(server, response, env.Clone())
return
}
}
}))
}
}()
}
func getGraceServerCommand() cli.Command {
return cli.Command{
Name: "glace-server",
ShortName: "gsrv",
Usage: "Run API Server with graceful restart support",
Description: "Run Gohan API server with graceful restart support",
Flags: []cli.Flag{
cli.StringFlag{Name: "config-file", Value: defaultConfigFile, Usage: "Server config File"},
},
Action: func(c *cli.Context) {
configFile := c.String("config-file")
loadConfig(configFile)
opts := &options{OptInterval: -1}
opts.OptCommand = os.Args[0]
config := util.GetConfig()
opts.OptPorts = []string{config.GetString("address", ":9091")}
opts.OptArgs = []string{"server", "--config-file", configFile}
s, err := starter.NewStarter(opts)
if err != nil {
fmt.Fprintf(os.Stderr, "error: %s\n", err)
return
}
s.Run()
},
}
}
//AMQP Process
func startAMQPProcess(server *Server) {
config := util.GetConfig()
enableAMQP := config.GetParam("amqp", nil)
if enableAMQP == nil {
return
}
listenAMQP(server)
}
//SNMP Process
//Experimental
func startSNMPProcess(server *Server) {
manager := schema.GetManager()
config := util.GetConfig()
enabled := config.GetParam("snmp", nil)
if enabled == nil {
return
}
host := config.GetString("snmp/address", "localhost:162")
path := "snmp://"
env := newEnvironment(server.db, server.keystoneIdentity, server.timelimit)
err := env.LoadExtensionsForPath(manager.Extensions, path)
if err != nil {
log.Fatal(fmt.Sprintf("Extensions parsing error: %v", err))
}
addr, err := net.ResolveUDPAddr("udp", host)
if err != nil {
log.Fatal(err)
}
conn, err := net.ListenUDP("udp", addr)
if err != nil {
log.Fatal(err)
}
buf := make([]byte, 1024)
go func() {
defer conn.Close()
for server.running {
rlen, remote, err := conn.ReadFromUDP(buf)
if err != nil {
log.Error(fmt.Sprintf("[SNMP] failed read bytes %s", err))
return
}
decoded, err := wapsnmp.DecodeSequence(buf[:rlen])
if err != nil {
log.Error(fmt.Sprintf("[SNMP] failed decode bytes %s", err))
continue
}
infos := decoded[3].([]interface{})[4].([]interface{})[1:]
trap := map[string]string{}
for _, info := range infos {
listInfo := info.([]interface{})
oid := listInfo[1].(wapsnmp.Oid)
trap[oid.String()] = fmt.Sprintf("%v", listInfo[2])
}
context := map[string]interface{}{
"trap": trap,
"remote": remote,
}
if err := env.HandleEvent("notification", context); err != nil {
log.Warning(fmt.Sprintf("extension error: %s", err))
}
}
}()
}
func (server *Server) connectDB() error {
config := util.GetConfig()
dbType, dbConnection, _, _ := server.getDatabaseConfig()
maxConn := config.GetInt("database/max_open_conn", db.DefaultMaxOpenConn)
dbConn, err := db.ConnectDB(dbType, dbConnection, maxConn)
if server.sync == nil {
server.db = dbConn
} else {
server.db = &DbSyncWrapper{dbConn}
}
return err
}
//CRON Process
func startCRONProcess(server *Server) {
manager := schema.GetManager()
config := util.GetConfig()
jobList := config.GetParam("cron", nil)
if jobList == nil {
return
}
log.Info("Started CRON process")
c := cron.New()
for _, rawJob := range jobList.([]interface{}) {
job := rawJob.(map[string]interface{})
path := job["path"].(string)
timing := job["timing"].(string)
env := newEnvironment(server.db, server.keystoneIdentity)
err := env.LoadExtensionsForPath(manager.Extensions, path)
if err != nil {
log.Fatal(fmt.Sprintf("Extensions parsing error: %v", err))
}
log.Info("New job for %s / %s", path, timing)
c.AddFunc(timing, func() {
lockKey := lockPath + "/" + path
err := server.sync.Lock(lockKey, false)
if err != nil {
return
}
defer func() {
server.sync.Unlock(lockKey)
}()
tx, err := server.db.Begin()
defer tx.Close()
context := map[string]interface{}{
"path": path,
}
if err != nil {
log.Warning(fmt.Sprintf("extension error: %s", err))
return
}
if err := env.HandleEvent("notification", context); err != nil {
log.Warning(fmt.Sprintf("extension error: %s", err))
return
}
err = tx.Commit()
if err != nil {
log.Warning(fmt.Sprintf("extension error: %s", err))
return
}
return
})
}
c.Start()
}
func (server *Server) getDatabaseConfig() (string, string, bool, bool) {
config := util.GetConfig()
databaseType := config.GetString("database/type", "sqlite3")
if databaseType == "json" || databaseType == "yaml" {
log.Fatal("json or yaml isn't supported as main db backend")
}
databaseConnection := config.GetString("database/connection", "")
if databaseConnection == "" {
log.Fatal("no database connection specified in the configuraion file.")
}
databaseDropOnCreate := config.GetBool("database/drop_on_create", false)
databaseCascade := config.GetBool("database/cascade_delete", false)
return databaseType, databaseConnection, databaseDropOnCreate, databaseCascade
}
func (server *Server) mapRoutes() {
config := util.GetConfig()
schemaManager := schema.GetManager()
MapNamespacesRoutes(server.martini)
MapRouteBySchemas(server, server.db)
tx, err := server.db.Begin()
if err != nil {
log.Fatal(err)
}
defer tx.Close()
coreSchema, _ := schemaManager.Schema("schema")
if coreSchema == nil {
log.Fatal("Gohan core schema not found")
return
}
policySchema, _ := schemaManager.Schema("policy")
policyList, _, err := tx.List(policySchema, nil, nil)
if err != nil {
log.Info(err.Error())
}
schemaManager.LoadPolicies(policyList)
extensionSchema, _ := schemaManager.Schema("extension")
extensionList, _, err := tx.List(extensionSchema, nil, nil)
if err != nil {
log.Info(err.Error())
}
schemaManager.LoadExtensions(extensionList)
namespaceSchema, _ := schemaManager.Schema("namespace")
if namespaceSchema == nil {
log.Error("No gohan schema. Disabling schema editing mode")
return
}
namespaceList, _, err := tx.List(namespaceSchema, nil, nil)
if err != nil {
log.Info(err.Error())
}
err = tx.Commit()
if err != nil {
log.Info(err.Error())
}
schemaManager.LoadNamespaces(namespaceList)
if config.GetBool("keystone/fake", false) {
middleware.FakeKeystone(server.martini)
}
}
//CRON Process
func startCRONProcess(server *Server) {
config := util.GetConfig()
jobList := config.GetParam("cron", nil)
if jobList == nil {
return
}
if server.sync == nil {
log.Fatalf(fmt.Sprintf("Could not start CRON process because of sync backend misconfiguration."))
util.LogFatalPanic(log)
}
log.Info("Started CRON process")
c := cron.New()
for _, rawJob := range jobList.([]interface{}) {
job := rawJob.(map[string]interface{})
path := job["path"].(string)
timing := job["timing"].(string)
name := strings.TrimPrefix(path, "cron://")
env, err := server.NewEnvironmentForPath(name, path)
if err != nil {
log.Fatal(err.Error())
}
log.Info("New job for %s / %s", path, timing)
c.AddFunc(timing, func() {
lockKey := lockPath + "/" + path
err := server.sync.Lock(lockKey, false)
if err != nil {
return
}
defer func() {
server.sync.Unlock(lockKey)
}()
context := map[string]interface{}{
"path": path,
}
if err != nil {
log.Warning(fmt.Sprintf("extension error: %s", err))
return
}
if err := env.HandleEvent("notification", context); err != nil {
log.Warning(fmt.Sprintf("extension error: %s", err))
return
}
return
})
}
c.Start()
}
func doTemplate(c *cli.Context) {
template := c.String("template")
manager := schema.GetManager()
configFile := c.String("config-file")
config := util.GetConfig()
err := config.ReadConfig(configFile)
if err != nil {
util.ExitFatal(err)
return
}
templateCode, err := util.GetContent(template)
if err != nil {
util.ExitFatal(err)
return
}
pwd, _ := os.Getwd()
os.Chdir(path.Dir(configFile))
schemaFiles := config.GetStringList("schemas", nil)
if schemaFiles == nil {
util.ExitFatal("No schema specified in configuraion")
} else {
err = manager.LoadSchemasFromFiles(schemaFiles...)
if err != nil {
util.ExitFatal(err)
return
}
}
schemas := manager.OrderedSchemas()
if err != nil {
util.ExitFatal(err)
return
}
tpl, err := pongo2.FromString(string(templateCode))
if err != nil {
util.ExitFatal(err)
return
}
output, err := tpl.Execute(pongo2.Context{"schemas": schemas})
if err != nil {
util.ExitFatal(err)
return
}
os.Chdir(pwd)
fmt.Println(output)
}
func loadNPMModules() {
config := util.GetConfig()
npmPath := config.GetString("extension/npm_path", ".")
files, _ := ioutil.ReadDir(npmPath + "/node_modules/")
for _, f := range files {
if f.IsDir() && !strings.HasPrefix(f.Name(), ".") {
module, err := motto.FindFileModule(f.Name(), npmPath, nil)
if err != nil {
log.Error("Finding module failed %s in %s", err, f.Name())
break
}
if !strings.HasSuffix(module, ".js") {
module = module + "/index.js"
}
loader := motto.CreateLoaderFromFile(module)
motto.AddModule(f.Name(), loader)
}
}
}
func loadConfig(configFile string) {
if configFile == "" {
return
}
config := util.GetConfig()
err := config.ReadConfig(configFile)
if err != nil {
if configFile != defaultConfigFile {
fmt.Println(err)
os.Exit(1)
}
return
}
err = l.SetUpLogging(config)
if err != nil {
fmt.Printf("Logging setup error: %s\n", err)
os.Exit(1)
return
}
log.Info("logging initialized")
}
}
}
return nil
}
func TestOttoExtension(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Otto Extension Suite")
}
var _ = Describe("Suite set up and tear down", func() {
var _ = BeforeSuite(func() {
var err error
Expect(os.Chdir(configDir)).To(Succeed())
testDB, err = db.ConnectDB(dbType, dbFile, db.DefaultMaxOpenConn)
Expect(err).ToNot(HaveOccurred(), "Failed to connect database.")
manager := schema.GetManager()
config := util.GetConfig()
Expect(config.ReadConfig(configFile)).To(Succeed())
schemaFiles := config.GetStringList("schemas", nil)
Expect(schemaFiles).NotTo(BeNil())
Expect(manager.LoadSchemasFromFiles(schemaFiles...)).To(Succeed())
Expect(db.InitDBWithSchemas(dbType, dbFile, false, false)).To(Succeed())
})
var _ = AfterSuite(func() {
schema.ClearManager()
os.Remove(dbFile)
})
})
func setupEditor(server *Server) {
manager := schema.GetManager()
config := util.GetConfig()
editableSchemaFile := config.GetString("editable_schema", "")
golang.RegisterGoCallback("handle_schema",
func(event string, context map[string]interface{}) error {
auth := context["auth"].(schema.Authorization)
if event == "pre_list" {
list := []interface{}{}
total := 0
for _, currentSchema := range manager.OrderedSchemas() {
trimmedSchema, err := GetSchema(currentSchema, auth)
if err != nil {
return err
}
if trimmedSchema != nil {
s := trimmedSchema.Data()
s["url"] = currentSchema.URL
list = append(list, s)
total = total + 1
}
}
context["total"] = total
context["response"] = map[string]interface{}{
"schemas": list,
}
return nil
} else if event == "pre_show" {
ID := context["id"].(string)
currentSchema, _ := manager.Schema(ID)
object, _ := GetSchema(currentSchema, auth)
s := object.Data()
s["url"] = currentSchema.URL
context["response"] = map[string]interface{}{
"schema": s,
}
return nil
}
if event != "pre_create" && event != "pre_update" && event != "pre_delete" {
return nil
}
if editableSchemaFile == "" {
return nil
}
ID := context["id"].(string)
schemasInFile, err := util.LoadMap(editableSchemaFile)
if err != nil {
return nil
}
schemas := schemasInFile["schemas"].([]interface{})
updatedSchemas := []interface{}{}
var existingSchema map[string]interface{}
for _, rawSchema := range schemas {
s := rawSchema.(map[string]interface{})
if s["id"] == ID {
existingSchema = s
} else {
updatedSchemas = append(updatedSchemas, s)
}
}
if event == "pre_create" {
if existingSchema != nil {
return fmt.Errorf("ID has been taken")
}
newSchema := context["resource"].(map[string]interface{})
_, err := schema.NewSchemaFromObj(newSchema)
if err != nil {
return err
}
schemasInFile["schemas"] = append(updatedSchemas, context["resource"].(map[string]interface{}))
context["response"] = map[string]interface{}{
"schema": context["resource"],
}
context["exception"] = map[string]interface{}{
"name": "CustomException",
"message": context["response"],
"code": 201,
}
} else if event == "pre_update" {
if existingSchema == nil {
return fmt.Errorf("Not found or Update isn't supported for this schema")
}
for key, value := range context["resource"].(map[string]interface{}) {
existingSchema[key] = value
}
_, err := schema.NewSchemaFromObj(existingSchema)
if err != nil {
return err
}
schemasInFile["schemas"] = append(updatedSchemas, existingSchema)
context["response"] = map[string]interface{}{
"schema": context["resource"],
}
context["exception"] = map[string]interface{}{
"name": "CustomException",
"message": context["response"],
"code": 200,
}
} else if event == "pre_delete" {
if existingSchema == nil {
return fmt.Errorf("Not found or Delete isn't supported for this schema")
}
schemasInFile["schemas"] = updatedSchemas
deletedSchema, ok := manager.Schema(ID)
if ok {
manager.UnRegisterSchema(deletedSchema)
}
context["exception"] = map[string]interface{}{
"name": "CustomException",
"message": map[string]interface{}{"result": "deleted"},
"code": 204,
}
}
util.SaveFile(editableSchemaFile, schemasInFile)
err = manager.LoadSchemaFromFile(editableSchemaFile)
if err != nil {
return err
}
server.initDB()
server.resetRouter()
server.mapRoutes()
return nil
})
}
func listenAMQP(server *Server) {
hostname, _ := os.Hostname()
processID := hostname + uuid.NewV4().String()
config := util.GetConfig()
connection := config.GetString("amqp/connection", "amqp://*****:*****@127.0.0.1:5672/")
queues := config.GetStringList("amqp/queues", []string{"notifications.info", "notifications.error"})
events := config.GetStringList("amqp/events", []string{})
extensions := map[string]extension.Environment{}
for _, event := range events {
path := "amqp://" + event
env, err := server.NewEnvironmentForPath("amqp."+event, path)
if err != nil {
log.Fatal(err.Error())
}
extensions[event] = env
}
for _, queue := range queues {
go func(queue string) {
defer util.LogPanic(log)
for server.running {
conn, err := amqp.Dial(connection)
if err != nil {
log.Error(fmt.Sprintf("[AMQP] connection error: %s", err))
time.Sleep(connectionWait)
continue
}
defer conn.Close()
ch, err := conn.Channel()
if err != nil {
log.Error(fmt.Sprintf("[AMQP] channel: %s", err))
return
}
defer ch.Close()
q, err := ch.QueueDeclare(
queue, // name
false, // durable
false, // delete when usused
false, // exclusive
false, // no-wait
nil, // arguments
)
if err != nil {
log.Error(fmt.Sprintf("[AMQP] queue declare error: %s", err))
return
}
for server.running {
msgs, err := ch.Consume(
q.Name, // queue
"gohan-"+processID+"-"+queue, // consumer
true, // auto-ack
false, // exclusive
false, // no-local
false, // no-wait
nil, // args
)
if err != nil {
log.Error(fmt.Sprintf("[AMQP] consume queue error: %s", err))
break
}
for d := range msgs {
var message map[string]interface{}
err = json.Unmarshal(d.Body, &message)
log.Debug(fmt.Sprintf("Received a message: %s %s", queue, d.Body))
if err != nil {
log.Error(fmt.Sprintf("[AMQP] json decode error: %s", err))
continue
}
eventType, ok := message["event_type"].(string)
if !ok {
log.Error("[AMQP] wrong event type")
continue
}
for _, event := range events {
if strings.HasPrefix(eventType, event) {
env := extensions[event]
context := map[string]interface{}{
"event": message,
}
if err := env.HandleEvent("notification", context); err != nil {
log.Warning(fmt.Sprintf("extension error: %s", err))
}
}
}
}
}
}
}(queue)
}
}
//ReadConfig reads configuraion from file.
func ReadConfig(path string) error {
config := util.GetConfig()
err := config.ReadConfig(path)
return err
}
//Sync Watch Process
func startSyncWatchProcess(server *Server) {
manager := schema.GetManager()
config := util.GetConfig()
watch := config.GetStringList("watch/keys", nil)
events := config.GetStringList("watch/events", nil)
maxWorkerCount := config.GetParam("watch/worker_count", 0).(int)
if watch == nil {
return
}
extensions := map[string]extension.Environment{}
for _, event := range events {
path := "sync://" + event
env := newEnvironment(server.db, server.keystoneIdentity)
err := env.LoadExtensionsForPath(manager.Extensions, path)
if err != nil {
log.Fatal(fmt.Sprintf("Extensions parsing error: %v", err))
}
extensions[event] = env
}
responseChan := make(chan *gohan_sync.Event)
stopChan := make(chan bool)
for _, path := range watch {
go func(path string) {
for server.running {
err := server.sync.Watch(path, responseChan, stopChan)
if err != nil {
log.Error(fmt.Sprintf("sync watch error: %s", err))
}
time.Sleep(5 * time.Second)
}
}(path)
}
//main response lisnter process
go func() {
var wg sync.WaitGroup
workerCount := 0
for server.running {
response := <-responseChan
wg.Add(1)
workerCount++
//spawn workers up to max worker count
go func() {
defer func() {
workerCount--
wg.Done()
}()
for _, event := range events {
//match extensions
if strings.HasPrefix(response.Key, "/"+event) {
env := extensions[event]
runExtensionOnSync(server, response, env)
return
}
}
}()
// Wait if worker pool is full
if workerCount > maxWorkerCount {
wg.Wait()
}
}
stopChan <- true
}()
}
//GetConfig returns config by key.
func GetConfig(key string, defaultValue interface{}) interface{} {
config := util.GetConfig()
return config.GetParam(key, defaultValue)
}
//NewServer returns new GohanAPIServer
func NewServer(configFile string) (*Server, error) {
manager := schema.GetManager()
config := util.GetConfig()
err := config.ReadConfig(configFile)
err = os.Chdir(path.Dir(configFile))
if err != nil {
return nil, fmt.Errorf("Config load error: %s", err)
}
err = l.SetUpLogging(config)
if err != nil {
return nil, fmt.Errorf("Logging setup error: %s", err)
}
log.Info("logging initialized")
server := &Server{}
m := martini.Classic()
m.Handlers()
m.Use(middleware.Logging())
m.Use(martini.Recovery())
m.Use(middleware.JSONURLs())
m.Use(middleware.WithContext())
server.martini = m
port := os.Getenv("PORT")
if port == "" {
port = "9091"
}
setupEditor(server)
server.timelimit = config.GetInt("extension/timelimit", 30)
server.extensions = config.GetStringList("extension/use", []string{
"javascript",
"gohanscript",
"go",
})
schema.DefaultExtension = config.GetString("extension/default", "javascript")
server.address = config.GetString("address", ":"+port)
if config.GetBool("tls/enabled", false) {
log.Info("TLS enabled")
server.tls = &tlsConfig{
KeyFile: config.GetString("tls/key_file", "./etc/key.pem"),
CertFile: config.GetString("tls/cert_file", "./etc/cert.pem"),
}
}
etcdServers := config.GetStringList("etcd", nil)
if etcdServers != nil {
log.Info("etcd servers: %s", etcdServers)
server.sync = etcd.NewSync(etcdServers)
}
server.connectDB()
schemaFiles := config.GetStringList("schemas", nil)
if schemaFiles == nil {
log.Fatal("No schema specified in configuraion")
} else {
err = manager.LoadSchemasFromFiles(schemaFiles...)
if err != nil {
return nil, fmt.Errorf("invalid schema: %s", err)
}
}
if !config.GetBool("database/no_init", false) {
server.initDB()
}
if config.GetList("database/initial_data", nil) != nil {
initialDataList := config.GetList("database/initial_data", nil)
for _, initialData := range initialDataList {
initialDataConfig := initialData.(map[string]interface{})
inType := initialDataConfig["type"].(string)
inConnection := initialDataConfig["connection"].(string)
log.Info("Importing data from %s ...", inConnection)
inDB, err := db.ConnectDB(inType, inConnection, db.DefaultMaxOpenConn)
if err != nil {
log.Fatal(err)
}
db.CopyDBResources(inDB, server.db, false)
}
}
if config.GetBool("keystone/use_keystone", false) {
//TODO remove this
if config.GetBool("keystone/fake", false) {
server.keystoneIdentity = &middleware.FakeIdentity{}
//TODO(marcin) requests to fake server also get authenticated
// we need a separate routing Group
log.Info("Debug Mode with Fake Keystone Server")
} else {
log.Info("Keystone backend server configured")
server.keystoneIdentity, err = cloud.NewKeystoneIdentity(
config.GetString("keystone/auth_url", "http://localhost:35357/v3"),
config.GetString("keystone/user_name", "admin"),
config.GetString("keystone/password", "password"),
config.GetString("keystone/domain_name", "Default"),
config.GetString("keystone/tenant_name", "admin"),
config.GetString("keystone/version", ""),
)
if err != nil {
log.Fatal(err)
}
}
m.MapTo(server.keystoneIdentity, (*middleware.IdentityService)(nil))
m.Use(middleware.Authentication())
//m.Use(Authorization())
} else {
m.MapTo(&middleware.NoIdentityService{}, (*middleware.IdentityService)(nil))
m.Map(schema.NewAuthorization("admin", "admin", "admin_token", []string{"admin"}, nil))
}
if err != nil {
return nil, fmt.Errorf("invalid base dir: %s", err)
}
server.addOptionsRoute()
cors := config.GetString("cors", "")
if cors != "" {
log.Info("Enabling CORS for %s", cors)
if cors == "*" {
log.Warning("cors for * have security issue")
}
server.martini.Use(func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Add("Access-Control-Allow-Origin", cors)
rw.Header().Add("Access-Control-Allow-Headers", "X-Auth-Token, Content-Type")
rw.Header().Add("Access-Control-Expose-Headers", "X-Total-Count")
rw.Header().Add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE")
})
}
documentRoot := config.GetString("document_root", "embed")
if config.GetBool("webui_config/enabled", false) {
m.Use(func(res http.ResponseWriter, req *http.Request, c martini.Context) {
if req.URL.Path != "/webui/config.json" {
c.Next()
return
}
address := config.GetString("webui_config/address", server.address)
if address[0] == ':' {
address = "__HOST__" + address
}
baseURL := "http://" + address
authURL := "http://" + address + "/v2.0"
if config.GetBool("webui_config/tls", config.GetBool("tls/enabled", false)) {
baseURL = "https://" + address
authURL = "https://" + address + "/v2.0"
}
authURL = config.GetString("webui_config/auth_url", authURL)
webUIConfig := map[string]interface{}{
"authUrl": authURL,
"gohan": map[string]interface{}{
"schema": "/gohan/v0.1/schemas",
"url": baseURL,
},
}
routes.ServeJson(res, webUIConfig)
})
}
if documentRoot == "embed" {
m.Use(staticbin.Static("public", util.Asset, staticbin.Options{
SkipLogging: true,
}))
} else {
log.Info("Static file serving from %s", documentRoot)
documentRootABS, err := filepath.Abs(documentRoot)
if err != nil {
return nil, err
}
server.martini.Use(martini.Static(documentRootABS, martini.StaticOptions{
SkipLogging: true,
}))
}
server.mapRoutes()
maxWorkerCount := config.GetInt("workers", 100)
server.queue = job.NewQueue(uint(maxWorkerCount))
return server, nil
}
//NewServer returns new GohanAPIServer
func NewServer(configFile string) (*Server, error) {
manager := schema.GetManager()
config := util.GetConfig()
err := config.ReadConfig(configFile)
err = os.Chdir(path.Dir(configFile))
if err != nil {
return nil, fmt.Errorf("Config load error: %s", err)
}
err = l.SetUpLogging(config)
if err != nil {
return nil, fmt.Errorf("Logging setup error: %s", err)
}
log.Info("logging initialized")
server := &Server{}
m := martini.Classic()
m.Handlers()
m.Use(middleware.Logging())
m.Use(martini.Recovery())
m.Use(middleware.JSONURLs())
m.Use(middleware.WithContext())
server.martini = m
server.address = config.GetString("address", ":9091")
if config.GetBool("tls/enabled", false) {
log.Info("TLS enabled")
server.tls = &tls{
KeyFile: config.GetString("tls/key_file", "./etc/key.pem"),
CertFile: config.GetString("tls/cert_file", "./etc/cert.pem"),
}
}
server.connectDB()
schemaFiles := config.GetStringList("schemas", nil)
if schemaFiles == nil {
log.Fatal("No schema specified in configuraion")
} else {
err = manager.LoadSchemasFromFiles(schemaFiles...)
if err != nil {
return nil, fmt.Errorf("invalid schema: %s", err)
}
}
server.initDB()
etcdServers := config.GetStringList("etcd", nil)
log.Info("etcd servers: %s", etcdServers)
server.sync = etcd.NewSync(etcdServers)
if config.GetList("database/initial_data", nil) != nil {
initialDataList := config.GetList("database/initial_data", nil)
for _, initialData := range initialDataList {
initialDataConfig := initialData.(map[string]interface{})
inType := initialDataConfig["type"].(string)
inConnection := initialDataConfig["connection"].(string)
log.Info("Importing data from %s ...", inConnection)
inDB, err := db.ConnectDB(inType, inConnection)
if err != nil {
log.Fatal(err)
}
db.CopyDBResources(inDB, server.db)
}
}
if config.GetBool("keystone/use_keystone", false) {
//TODO remove this
if config.GetBool("keystone/fake", false) {
server.keystoneIdentity = &middleware.FakeIdentity{}
//TODO(marcin) requests to fake server also get authenticated
// we need a separate routing Group
log.Info("Debug Mode with Fake Keystone Server")
middleware.FakeKeystone(server.martini)
} else {
log.Info("Keystone backend server configured")
server.keystoneIdentity, err = cloud.NewKeystoneIdentity(
config.GetString("keystone/auth_url", "http://localhost:35357/v3"),
config.GetString("keystone/user_name", "admin"),
config.GetString("keystone/password", "password"),
config.GetString("keystone/domain_name", "Default"),
config.GetString("keystone/tenant_name", "admin"),
config.GetString("keystone/version", ""),
)
if err != nil {
log.Fatal(err)
}
}
m.MapTo(server.keystoneIdentity, (*middleware.IdentityService)(nil))
m.Use(middleware.Authentication())
//m.Use(Authorization())
}
if err != nil {
return nil, fmt.Errorf("invalid base dir: %s", err)
}
server.addOptionsRoute()
cors := config.GetString("cors", "")
if cors != "" {
log.Info("Enabling CORS for %s", cors)
if cors == "*" {
log.Warning("cors for * have security issue")
}
server.martini.Use(func(rw http.ResponseWriter, r *http.Request) {
rw.Header().Add("Access-Control-Allow-Origin", cors)
rw.Header().Add("Access-Control-Allow-Headers", "X-Auth-Token, Content-Type")
rw.Header().Add("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE")
})
}
documentRoot := config.GetString("document_root", "./")
log.Info("Static file serving from %s", documentRoot)
documentRootABS, err := filepath.Abs(documentRoot)
server.martini.Use(martini.Static(documentRootABS))
server.mapRoutes()
return server, nil
}
//SetUp sets up vm to with environment
func init() {
gohanRemoteInit := func(env *Environment) {
vm := env.VM
builtins := map[string]interface{}{
"gohan_netconf_open": func(call otto.FunctionCall) otto.Value {
if len(call.ArgumentList) != 2 {
panic("Wrong number of arguments in gohan_netconf_open call.")
}
rawHost, _ := call.Argument(0).Export()
host, ok := rawHost.(string)
if !ok {
return otto.NullValue()
}
rawUserName, _ := call.Argument(1).Export()
userName, ok := rawUserName.(string)
if !ok {
return otto.NullValue()
}
config := util.GetConfig()
publicKeyFile := config.GetString("ssh/key_file", "")
if publicKeyFile == "" {
return otto.NullValue()
}
sshConfig := &ssh.ClientConfig{
User: userName,
Auth: []ssh.AuthMethod{
util.PublicKeyFile(publicKeyFile),
},
}
s, err := netconf.DialSSH(host, sshConfig)
if err != nil {
ThrowOttoException(&call, "Error during gohan_netconf_open: %s", err.Error())
}
value, _ := vm.ToValue(s)
return value
},
"gohan_netconf_close": func(call otto.FunctionCall) otto.Value {
if len(call.ArgumentList) != 1 {
panic("Wrong number of arguments in gohan_netconf_close call.")
}
rawSession, _ := call.Argument(0).Export()
s, ok := rawSession.(*netconf.Session)
if !ok {
ThrowOttoException(&call, "Error during gohan_netconf_close")
}
s.Close()
return otto.NullValue()
},
"gohan_netconf_exec": func(call otto.FunctionCall) otto.Value {
if len(call.ArgumentList) != 2 {
panic("Wrong number of arguments in gohan_netconf_exec call.")
}
rawSession, _ := call.Argument(0).Export()
s, ok := rawSession.(*netconf.Session)
if !ok {
return otto.NullValue()
}
rawCommand, _ := call.Argument(1).Export()
command, ok := rawCommand.(string)
if !ok {
return otto.NullValue()
}
reply, err := s.Exec(netconf.RawMethod(command))
resp := map[string]interface{}{}
if err != nil {
resp["status"] = "error"
resp["output"] = err.Error()
} else {
resp["status"] = "success"
resp["output"] = reply
}
value, _ := vm.ToValue(resp)
return value
},
"gohan_ssh_open": func(call otto.FunctionCall) otto.Value {
if len(call.ArgumentList) != 2 {
panic("Wrong number of arguments in gohan_netconf_open call.")
}
rawHost, _ := call.Argument(0).Export()
host, ok := rawHost.(string)
if !ok {
return otto.NullValue()
}
rawUserName, _ := call.Argument(1).Export()
userName, ok := rawUserName.(string)
if !ok {
return otto.NullValue()
}
config := util.GetConfig()
publicKeyFile := config.GetString("ssh/key_file", "")
if publicKeyFile == "" {
return otto.NullValue()
}
sshConfig := &ssh.ClientConfig{
User: userName,
Auth: []ssh.AuthMethod{
util.PublicKeyFile(publicKeyFile),
},
}
conn, err := ssh.Dial("tcp", host, sshConfig)
if err != nil {
ThrowOttoException(&call, "Error during gohan_ssh_open %s", err)
}
session, err := conn.NewSession()
if err != nil {
ThrowOttoException(&call, "Error during gohan_ssh_open %s", err)
}
value, _ := vm.ToValue(session)
return value
},
"gohan_ssh_close": func(call otto.FunctionCall) otto.Value {
if len(call.ArgumentList) != 1 {
panic("Wrong number of arguments in gohan_netconf_close call.")
}
rawSession, _ := call.Argument(0).Export()
s, ok := rawSession.(*ssh.Session)
if !ok {
ThrowOttoException(&call, "Error during gohan_ssh_close")
}
s.Close()
return otto.NullValue()
},
"gohan_ssh_exec": func(call otto.FunctionCall) otto.Value {
if len(call.ArgumentList) != 2 {
panic("Wrong number of arguments in gohan_netconf_exec call.")
}
rawSession, _ := call.Argument(0).Export()
s, ok := rawSession.(*ssh.Session)
if !ok {
return otto.NullValue()
}
rawCommand, _ := call.Argument(1).Export()
command, ok := rawCommand.(string)
if !ok {
return otto.NullValue()
}
var stdoutBuf bytes.Buffer
s.Stdout = &stdoutBuf
err := s.Run(command)
resp := map[string]interface{}{}
if err != nil {
resp["status"] = "error"
resp["output"] = err.Error()
} else {
resp["status"] = "success"
resp["output"] = stdoutBuf.String()
}
value, _ := vm.ToValue(resp)
return value
},
}
for name, object := range builtins {
vm.Set(name, object)
}
}
RegistInit(gohanRemoteInit)
}
func init() {
gohanUtilInit := func(env *Environment) {
vm := env.VM
builtins := map[string]interface{}{
"gohan_http": func(call otto.FunctionCall) otto.Value {
if len(call.ArgumentList) == 4 {
defaultOpaque, _ := otto.ToValue(false)
call.ArgumentList = append(call.ArgumentList, defaultOpaque)
}
VerifyCallArguments(&call, "gohan_http", 5)
method, err := GetString(call.Argument(0))
if err != nil {
ThrowOttoException(&call, err.Error())
}
url, err := GetString(call.Argument(1))
if err != nil {
ThrowOttoException(&call, err.Error())
}
rawHeaders, err := GetMap(call.Argument(2))
if err != nil {
ThrowOttoException(&call, err.Error())
}
// A string or a map[string]interface{}
data := ConvertOttoToGo(call.Argument(3))
opaque, err := GetBool(call.Argument(4))
if err != nil {
ThrowOttoException(&call, err.Error())
}
log.Debug("gohan_http [%s] %s %s %s", method, rawHeaders, url, opaque)
code, headers, body, err := gohanHTTP(method, url, rawHeaders, data, opaque)
log.Debug("response code %d", code)
resp := map[string]interface{}{}
if err != nil {
resp["status"] = "err"
resp["error"] = err.Error()
} else {
resp["status"] = "success"
resp["status_code"] = fmt.Sprint(code)
resp["body"] = body
resp["headers"] = headers
}
log.Debug("response code %d", code)
value, _ := vm.ToValue(resp)
return value
},
"gohan_schemas": func(call otto.FunctionCall) otto.Value {
VerifyCallArguments(&call, "gohan_schemas", 0)
manager := schema.GetManager()
response := []interface{}{}
for _, schema := range manager.OrderedSchemas() {
response = append(response, schema)
}
value, _ := vm.ToValue(response)
return value
},
"gohan_schema_url": func(call otto.FunctionCall) otto.Value {
VerifyCallArguments(&call, "gohan_schema_url", 1)
schemaID, err := GetString(call.Argument(0))
if err != nil {
ThrowOttoException(&call, err.Error())
}
schema, err := getSchema(schemaID)
if err != nil {
ThrowOttoException(&call, err.Error())
}
value, _ := vm.ToValue(schema.URL)
return value
},
"gohan_policies": func(call otto.FunctionCall) otto.Value {
VerifyCallArguments(&call, "gohan_policies", 0)
manager := schema.GetManager()
response := []interface{}{}
for _, policy := range manager.Policies() {
response = append(response, policy.RawData)
}
value, _ := vm.ToValue(response)
return value
},
"gohan_uuid": func(call otto.FunctionCall) otto.Value {
value, _ := vm.ToValue(uuid.NewV4().String())
return value
},
"gohan_sleep": func(call otto.FunctionCall) otto.Value {
VerifyCallArguments(&call, "gohan_sleep", 1)
rawSleep, _ := call.Argument(0).Export()
var sleep time.Duration
switch rawSleep.(type) {
case int:
sleep = time.Duration(rawSleep.(int))
case int64:
sleep = time.Duration(rawSleep.(int64))
}
time.Sleep(sleep * time.Millisecond)
return otto.NullValue()
},
"gohan_template": func(call otto.FunctionCall) otto.Value {
VerifyCallArguments(&call, "gohan_template", 2)
templateString, err := GetString(call.Argument(0))
if err != nil {
return call.Argument(0)
}
data := ConvertOttoToGo(call.Argument(1))
t := template.Must(template.New("tmpl").Parse(templateString))
b := bytes.NewBuffer(make([]byte, 0, 100))
t.Execute(b, data)
value, _ := vm.ToValue(b.String())
return value
},
"gohan_exec": func(call otto.FunctionCall) otto.Value {
VerifyCallArguments(&call, "gohan_exec", 2)
command, err := GetString(call.Argument(0))
if err != nil {
ThrowOttoException(&call, err.Error())
}
stringArgs, err := GetStringList(call.Argument(1))
if err != nil {
ThrowOttoException(&call, err.Error())
}
out, err := exec.Command(command, stringArgs...).Output()
resp := map[string]string{}
if err != nil {
resp["status"] = "error"
resp["output"] = err.Error()
} else {
resp["status"] = "success"
resp["output"] = string(out)
}
value, _ := vm.ToValue(resp)
return value
},
"gohan_config": func(call otto.FunctionCall) otto.Value {
VerifyCallArguments(&call, "gohan_exec", 2)
configKey, err := GetString(call.Argument(0))
if err != nil {
ThrowOttoException(&call, err.Error())
}
defaultValue, err := call.Argument(1).Export()
if err != nil {
ThrowOttoException(&call, err.Error())
}
config := util.GetConfig()
result := config.GetParam(configKey, defaultValue)
value, _ := vm.ToValue(result)
return value
},
}
for name, object := range builtins {
vm.Set(name, object)
}
}
RegisterInit(gohanUtilInit)
}