func Initialize(args []string) []string { // options fs := flag.FlagSet{} Flags["attributes"] = fs.String("attributes", "", "") Flags["full"] = fs.String("full", "", "") Flags["parts"] = fs.String("parts", "", "") Flags["part"] = fs.String("part", "", "") Flags["file"] = fs.String("file", "", "") Flags["threads"] = fs.String("threads", "", "") Flags["virtual_file"] = fs.String("virtual_file", "", "") Flags["remote_path"] = fs.String("remote_path", "", "") Flags["index"] = fs.String("index", "", "") Flags["index_options"] = fs.String("index_options", "", "") fs.StringVar(&ConfFile, "conf", DefaultPath(), "path to config file") fs.Parse(args) c, err := config.ReadDefault(ConfFile) handle(err) // Cache Cache.Dir, err = c.String("Cache", "dir") Cache.MaxConnections, err = c.Int("Cache", "max_connections") // Server Server.Url, err = c.String("Server", "url") handle(err) // Auth Auth.Type, _ = c.String("Auth", "type") switch Auth.Type { case "globus": Auth.TokenUrl, _ = c.String("Auth", "token_url") Auth.ProfileUrl, _ = c.String("Auth", "profile_url") case "basic": // nothing yet } return fs.Args() }
func init() { flag.StringVar(&CONFIG_FILE, "conf", "", "path to config file") flag.StringVar(&RELOAD, "reload", "", "path or url to awe job data. WARNING this will drop all current jobs.") flag.BoolVar(&RECOVER, "recover", false, "path to awe job data.") flag.BoolVar(&SHOW_VERSION, "version", false, "show version.") flag.IntVar(&DEBUG_LEVEL, "debug", -1, "debug level: 0-3") flag.BoolVar(&DEV_MODE, "dev", false, "dev or demo mode, print some msgs on screen") // TODO only on client flag.StringVar(&CGROUP_MEMORY_DOCKER_DIR, "cgroup_memory_docker_dir", "/sys/fs/cgroup/memory/docker/", "path to cgroup directory for docker") flag.StringVar(&SERVER_URL, "server_url", "", "URL of AWE server, including API port") flag.StringVar(&CLIENT_GROUP, "client_group", "", "name of client group") flag.Parse() // fmt.Printf("in conf.init(), flag=%v", flag) if SHOW_VERSION { PrintVersionMsg() os.Exit(0) } if len(CONFIG_FILE) == 0 { fmt.Fprintf(os.Stderr, "ERROR: conf file not specified\n") INIT_SUCCESS = false return } c, err := config.ReadDefault(CONFIG_FILE) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: %v\n", err) INIT_SUCCESS = false return } // Ports SITE_PORT, _ = c.Int("Ports", "site-port") API_PORT, _ = c.Int("Ports", "api-port") // URLs SITE_URL, _ = c.String("External", "site-url") API_URL, _ = c.String("External", "api-url") // SSL SSL_ENABLED, _ = c.Bool("SSL", "enable") if SSL_ENABLED { SSL_KEY_FILE, _ = c.String("SSL", "key") SSL_CERT_FILE, _ = c.String("SSL", "cert") } // Access-Control ANON_WRITE, _ = c.Bool("Anonymous", "write") ANON_READ, _ = c.Bool("Anonymous", "read") ANON_DELETE, _ = c.Bool("Anonymous", "delete") ANON_CG_WRITE, _ = c.Bool("Anonymous", "cg_write") ANON_CG_READ, _ = c.Bool("Anonymous", "cg_read") ANON_CG_DELETE, _ = c.Bool("Anonymous", "cg_delete") // Auth if basic_auth, err := c.Bool("Auth", "basic"); err == nil { BASIC_AUTH = basic_auth } GLOBUS_TOKEN_URL, _ = c.String("Auth", "globus_token_url") GLOBUS_PROFILE_URL, _ = c.String("Auth", "globus_profile_url") MGRAST_OAUTH_URL, _ = c.String("Auth", "mgrast_oauth_url") if GLOBUS_TOKEN_URL != "" && GLOBUS_PROFILE_URL != "" { GLOBUS_OAUTH = true } if MGRAST_OAUTH_URL != "" { MGRAST_OAUTH = true } CLIENT_AUTH_REQ, _ = c.Bool("Auth", "client_auth_required") // Admin if admin_users, err := c.String("Admin", "users"); err == nil { for _, name := range strings.Split(admin_users, ",") { Admin_Users[strings.TrimSpace(name)] = true } } ADMIN_EMAIL, _ = c.String("Admin", "email") SECRET_KEY, _ = c.String("Admin", "secretkey") // Directories SITE_PATH, _ = c.String("Directories", "site") DATA_PATH, _ = c.String("Directories", "data") LOGS_PATH, _ = c.String("Directories", "logs") AWF_PATH, _ = c.String("Directories", "awf") // Paths PID_FILE_PATH, _ = c.String("Paths", "pidfile") if PID_FILE_PATH == "" { PID_FILE_PATH = DATA_PATH + "/pidfile" } // Mongodb MONGODB_HOST, _ = c.String("Mongodb", "hosts") MONGODB_DATABASE, _ = c.String("Mongodb", "database") MONGODB_USER, _ = c.String("Mongodb", "user") MONGODB_PASSWD, _ = c.String("Mongodb", "password") if MONGODB_DATABASE == "" { MONGODB_DATABASE = "AWEDB" } // APP APP_REGISTRY_URL, _ = c.String("App", "app_registry_url") // Server options if perf_log_workunit, err := c.Bool("Server", "perf_log_workunit"); err == nil { PERF_LOG_WORKUNIT = perf_log_workunit } if big_data_size, err := c.Int("Server", "big_data_size"); err == nil { BIG_DATA_SIZE = int64(big_data_size) } if max_work_failure, err := c.Int("Server", "max_work_failure"); err == nil { MAX_WORK_FAILURE = max_work_failure } if max_client_failure, err := c.Int("Server", "max_client_failure"); err == nil { MAX_CLIENT_FAILURE = max_client_failure } if go_max_procs, err := c.Int("Server", "go_max_procs"); err == nil { GOMAXPROCS = go_max_procs } // Client WORK_PATH, _ = c.String("Client", "workpath") APP_PATH, _ = c.String("Client", "app_path") if SERVER_URL == "" { SERVER_URL, _ = c.String("Client", "serverurl") } OPENSTACK_METADATA_URL, _ = c.String("Client", "openstack_metadata_url") SUPPORTED_APPS, _ = c.String("Client", "supported_apps") if clientname, err := c.String("Client", "name"); err == nil { CLIENT_NAME = clientname } if pre_work_script, err := c.String("Client", "pre_work_script"); err == nil { PRE_WORK_SCRIPT = pre_work_script } if pre_work_script_args, err := c.String("Client", "pre_work_script_args"); err == nil { PRE_WORK_SCRIPT_ARGS = strings.Split(pre_work_script_args, ",") } if CLIENT_NAME == "" || CLIENT_NAME == "default" || CLIENT_NAME == "hostname" { hostname, err := os.Hostname() if err == nil { CLIENT_NAME = hostname } } if CLIENT_GROUP == "" { if clientgroup, err := c.String("Client", "group"); err == nil { CLIENT_GROUP = clientgroup } } if clientdomain, err := c.String("Client", "domain"); err == nil { CLIENT_DOMAIN = clientdomain } if print_app_msg, err := c.Bool("Client", "print_app_msg"); err == nil { PRINT_APP_MSG = print_app_msg } if worker_overlap, err := c.Bool("Client", "worker_overlap"); err == nil { WORKER_OVERLAP = worker_overlap } if auto_clean_dir, err := c.Bool("Client", "auto_clean_dir"); err == nil { AUTO_CLEAN_DIR = auto_clean_dir } if cache_enabled, err := c.Bool("Client", "cache_enabled"); err == nil { CACHE_ENABLED = cache_enabled } CLIENT_GROUP_TOKEN, _ = c.String("Client", "clientgroup_token") //Proxy P_SITE_PORT, _ = c.Int("Proxy", "p-site-port") P_API_PORT, _ = c.Int("Proxy", "p-api-port") //Args if DEBUG_LEVEL == -1 { //if no debug level set in cmd line args, find values in config file. if dlevel, err := c.Int("Args", "debuglevel"); err == nil { DEBUG_LEVEL = dlevel } else { DEBUG_LEVEL = 0 } } }
func Init_conf(mode string) (err error) { // the flag libary needs to parse after config file has been read, thus the conf file is parsed manually here CONFIG_FILE = "" for i, elem := range os.Args { if strings.HasPrefix(elem, "-conf") || strings.HasPrefix(elem, "--conf") { parts := strings.SplitN(elem, "=", 2) if len(parts) == 2 { CONFIG_FILE = parts[1] } else if i+1 < len(os.Args) { CONFIG_FILE = os.Args[i+1] } else { return errors.New("ERROR: parsing command options, missing conf file") } } } var c *config.Config = nil if CONFIG_FILE != "" { c, err = config.ReadDefault(CONFIG_FILE) if err != nil { return errors.New("ERROR: error reading conf file: " + err.Error()) } } c_store, err := getConfiguration(c, mode) // from config file and command line arguments if err != nil { fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: %v\n", err) return } // ####### at this point configuration variables are set ######## if FAKE_VAR == false { fmt.Fprintf(os.Stderr, "ERROR: config was not parsed\n") os.Exit(1) } if PRINT_HELP || SHOW_HELP { c_store.PrintHelp() os.Exit(0) } if SHOW_VERSION { PrintVersionMsg() os.Exit(0) } if SHOW_GIT_COMMIT_HASH { fmt.Fprintf(os.Stdout, "GIT_COMMIT_HASH=%s\n", GIT_COMMIT_HASH) os.Exit(0) } // configuration post processing if mode == "client" { if CLIENT_NAME == "" || CLIENT_NAME == "default" || CLIENT_NAME == "hostname" { hostname, err := os.Hostname() if err == nil { CLIENT_NAME = hostname } } if MEM_CHECK_INTERVAL_SECONDS > 0 { MEM_CHECK_INTERVAL = time.Duration(MEM_CHECK_INTERVAL_SECONDS) * time.Second // TODO } } if GLOBUS_TOKEN_URL != "" && GLOBUS_PROFILE_URL != "" { GLOBUS_OAUTH = true AUTH_DEFAULT = "KBase" AUTH_RESOURCES["KBase"] = AuthResource{ Icon: "KBase_favicon.ico", Prefix: "kbgo4711", Keyword: "auth", Url: MGRAST_LOGIN_URL, UseHeader: false, } } if MGRAST_OAUTH_URL != "" { MGRAST_OAUTH = true AUTH_DEFAULT = "MG-RAST" AUTH_RESOURCES["MG-RAST"] = AuthResource{ Icon: "MGRAST_favicon.ico", Prefix: "mggo4711", Keyword: "auth", Url: MGRAST_LOGIN_URL, UseHeader: false, } } if ADMIN_USERS_VAR != "" { for _, name := range strings.Split(ADMIN_USERS_VAR, ",") { Admin_Users[strings.TrimSpace(name)] = true } } if mode == "server" { if PIPELINE_EXPIRE != "" { for _, set := range strings.Split(PIPELINE_EXPIRE, ",") { parts := strings.Split(set, "=") if valid, _, _ := parseExpiration(parts[1]); !valid { return errors.New("expiration format in pipeline_expire is invalid") } PIPELINE_EXPIRE_MAP[parts[0]] = parts[1] } } if GLOBAL_EXPIRE != "" { if valid, _, _ := parseExpiration(GLOBAL_EXPIRE); !valid { return errors.New("expiration format in global_expire is invalid") } } } if PID_FILE_PATH == "" { PID_FILE_PATH = DATA_PATH + "/pidfile" } if PRE_WORK_SCRIPT_ARGS_STRING != "" { PRE_WORK_SCRIPT_ARGS = strings.Split(PRE_WORK_SCRIPT_ARGS_STRING, ",") } vaildLogout := false for _, logout := range LOG_OUTPUTS { if LOG_OUTPUT == logout { vaildLogout = true } } if !vaildLogout { return errors.New("invalid option for logoutput, use one of: file, console, both") } WORK_PATH, _ = filepath.Abs(WORK_PATH) APP_PATH, _ = filepath.Abs(APP_PATH) SITE_PATH, _ = filepath.Abs(SITE_PATH) DATA_PATH, _ = filepath.Abs(DATA_PATH) LOGS_PATH, _ = filepath.Abs(LOGS_PATH) AWF_PATH, _ = filepath.Abs(AWF_PATH) VERSIONS["Job"] = 2 return }
// Initialize is an explicit init. Enables outside use // of shock-server packages. Parses config and populates // the conf variables. func Initialize() { gopath := os.Getenv("GOPATH") flag.StringVar(&CONFIG_FILE, "conf", gopath+"/src/github.com/MG-RAST/Shock/shock-server.conf.template", "path to config file") flag.StringVar(&RELOAD, "reload", "", "path or url to shock data. WARNING this will drop all current data.") flag.Parse() c, err := config.ReadDefault(CONFIG_FILE) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: %v\n", err) os.Exit(1) } // Admin ADMIN_EMAIL, _ = c.String("Admin", "email") ADMIN_USERS, _ = c.String("Admin", "users") if ADMIN_USERS != "" { for _, name := range strings.Split(ADMIN_USERS, ",") { AdminUsers = append(AdminUsers, strings.TrimSpace(name)) } } // Access-Control ANON_READ, _ = c.Bool("Anonymous", "read") ANON_WRITE, _ = c.Bool("Anonymous", "write") ANON_DELETE, _ = c.Bool("Anonymous", "delete") // Address API_IP, _ = c.String("Address", "api-ip") API_PORT, _ = c.String("Address", "api-port") // URLs API_URL, _ = c.String("External", "api-url") // Auth AUTH_BASIC, _ = c.Bool("Auth", "basic") AUTH_GLOBUS_TOKEN_URL, _ = c.String("Auth", "globus_token_url") AUTH_GLOBUS_PROFILE_URL, _ = c.String("Auth", "globus_profile_url") AUTH_MGRAST_OAUTH_URL, _ = c.String("Auth", "mgrast_oauth_url") // Runtime EXPIRE_WAIT, _ = c.Int("Runtime", "expire_wait") GOMAXPROCS, _ = c.String("Runtime", "GOMAXPROCS") LOG_PERF, _ = c.Bool("Log", "perf_log") LOG_ROTATE, _ = c.Bool("Log", "rotate") // Mongodb MONGODB_ATTRIBUTE_INDEXES, _ = c.String("Mongodb", "attribute_indexes") if MONGODB_DATABASE, err = c.String("Mongodb", "database"); err != nil { fmt.Fprintf(os.Stderr, "ERROR: Mongodb database must be set in config file.") os.Exit(1) } MONGODB_HOSTS, _ = c.String("Mongodb", "hosts") MONGODB_PASSWORD, _ = c.String("Mongodb", "password") MONGODB_USER, _ = c.String("Mongodb", "user") // parse Node-Indices NODE_IDXS = map[string]idxOpts{} nodeIdx, _ := c.Options("Node-Indices") for _, opt := range nodeIdx { val, _ := c.String("Node-Indices", opt) opts := idxOpts{} for _, parts := range strings.Split(val, ",") { p := strings.Split(parts, ":") if p[0] == "unique" { if p[1] == "true" { opts.unique = true } else { opts.unique = false } } else if p[0] == "dropDups" { if p[1] == "true" { opts.dropDups = true } else { opts.dropDups = false } } else if p[0] == "sparse" { if p[1] == "true" { opts.sparse = true } else { opts.sparse = false } } } NODE_IDXS[opt] = opts } // Paths PATH_SITE, _ = c.String("Paths", "site") PATH_DATA, _ = c.String("Paths", "data") PATH_LOGS, _ = c.String("Paths", "logs") PATH_LOCAL, _ = c.String("Paths", "local_paths") PATH_PIDFILE, _ = c.String("Paths", "pidfile") // SSL SSL, _ = c.Bool("SSL", "enable") if SSL { SSL_KEY, _ = c.String("SSL", "key") SSL_CERT, _ = c.String("SSL", "cert") } VERSIONS["ACL"] = 2 VERSIONS["Auth"] = 1 VERSIONS["Node"] = 4 }
func init() { flag.StringVar(&CONFIG_FILE, "conf", "", "path to config file") flag.StringVar(&RELOAD, "reload", "", "path or url to awe job data. WARNING this will drop all current jobs.") flag.BoolVar(&RECOVER, "recover", false, "path to awe job data.") flag.StringVar(&CLIENT_PROFILE, "profile", "", "path to awe client profile.") flag.IntVar(&DEBUG_LEVEL, "debug", 0, "debug level: 0-3") flag.BoolVar(&DEV_MODE, "dev", false, "dev or demo mode, print some msgs on screen") flag.Parse() // fmt.Printf("in conf.init(), flag=%v", flag) if len(CONFIG_FILE) == 0 { fmt.Fprintf(os.Stderr, "ERROR: conf file not specified\n") INIT_SUCCESS = false return } c, err := config.ReadDefault(CONFIG_FILE) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: %v\n", err) INIT_SUCCESS = false return } // Ports SITE_PORT, _ = c.Int("Ports", "site-port") API_PORT, _ = c.Int("Ports", "api-port") // URLs SITE_URL, _ = c.String("External", "site-url") API_URL, _ = c.String("External", "api-url") // SSL SSL_ENABLED, _ = c.Bool("SSL", "enable") if SSL_ENABLED { SSL_KEY_FILE, _ = c.String("SSL", "key") SSL_CERT_FILE, _ = c.String("SSL", "cert") } // Access-Control ANON_WRITE, _ = c.Bool("Anonymous", "write") ANON_READ, _ = c.Bool("Anonymous", "read") ANON_CREATEUSER, _ = c.Bool("Anonymous", "create-user") // Auth if basic_auth, err := c.Bool("Auth", "basic"); err == nil { BASIC_AUTH = basic_auth } GLOBUS_TOKEN_URL, _ = c.String("Auth", "globus_token_url") GLOBUS_PROFILE_URL, _ = c.String("Auth", "globus_profile_url") MGRAST_OAUTH_URL, _ = c.String("Auth", "mgrast_oauth_url") // Admin ADMIN_EMAIL, _ = c.String("Admin", "email") SECRET_KEY, _ = c.String("Admin", "secretkey") // Directories SITE_PATH, _ = c.String("Directories", "site") DATA_PATH, _ = c.String("Directories", "data") LOGS_PATH, _ = c.String("Directories", "logs") AWF_PATH, _ = c.String("Directories", "awf") // Mongodb MONGODB_HOST, _ = c.String("Mongodb", "hosts") MONGODB_DATABASE, _ = c.String("Mongodb", "database") MONGODB_USER, _ = c.String("Mongodb", "user") MONGODB_PASSWD, _ = c.String("Mongodb", "password") if MONGODB_DATABASE == "" { MONGODB_DATABASE = "AWEDB" } // Server options if perf_log_workunit, err := c.Bool("Server", "perf_log_workunit"); err == nil { PERF_LOG_WORKUNIT = perf_log_workunit } if big_data_size, err := c.Int("Server", "big_data_size"); err == nil { BIG_DATA_SIZE = int64(big_data_size) } /* if default_index, err := c.String("Server", "default_index"); err == nil { DEFAULT_INDEX = default_index } */ // Client WORK_PATH, _ = c.String("Client", "workpath") APP_PATH, _ = c.String("Client", "app_path") SERVER_URL, _ = c.String("Client", "serverurl") SUPPORTED_APPS, _ = c.String("Client", "supported_apps") if clientname, err := c.String("Client", "name"); err == nil { CLIENT_NAME = clientname } if clientgroup, err := c.String("Client", "group"); err == nil { CLIENT_GROUP = clientgroup } if clientprofile, err := c.String("Client", "clientprofile"); err == nil { CLIENT_PROFILE = clientprofile } if print_app_msg, err := c.Bool("Client", "print_app_msg"); err == nil { PRINT_APP_MSG = print_app_msg } if worker_overlap, err := c.Bool("Client", "worker_overlap"); err == nil { WORKER_OVERLAP = worker_overlap } if auto_clean_dir, err := c.Bool("Client", "auto_clean_dir"); err == nil { AUTO_CLEAN_DIR = auto_clean_dir } CLIENT_USERNAME, _ = c.String("Client", "username") CLIENT_PASSWORD, _ = c.String("Client", "password") if CLIENT_USERNAME == "" { CLIENT_USERNAME = "******" } if CLIENT_PASSWORD == "" { CLIENT_PASSWORD = "******" } //Proxy P_SITE_PORT, _ = c.Int("Proxy", "p-site-port") P_API_PORT, _ = c.Int("Proxy", "p-api-port") //Auth if GLOBUS_TOKEN_URL != "" && GLOBUS_PROFILE_URL != "" { GLOBUS_OAUTH = true } if MGRAST_OAUTH_URL != "" { MGRAST_OAUTH = true } }
// Initialize is an explicit init. Enables outside use // of shock-server packages. Parses config and populates // the Conf variable. func Initialize() { flag.StringVar(&CONFIG_FILE, "conf", "/usr/local/shock/conf/shock.cfg", "path to config file") flag.StringVar(&RELOAD, "reload", "", "path or url to shock data. WARNING this will drop all current data.") flag.Parse() c, err := config.ReadDefault(CONFIG_FILE) if err != nil { fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: %v\n", err) os.Exit(1) } // Address Conf["api-ip"], _ = c.String("Address", "api-ip") Conf["api-port"], _ = c.String("Address", "api-port") // URLs Conf["api-url"], _ = c.String("External", "api-url") // SSL Conf["ssl"], _ = c.String("SSL", "enable") if Bool(Conf["ssl"]) { Conf["ssl-key"], _ = c.String("SSL", "key") Conf["ssl-cert"], _ = c.String("SSL", "cert") } // Access-Control Conf["anon-write"], _ = c.String("Anonymous", "write") Conf["anon-read"], _ = c.String("Anonymous", "read") Conf["anon-user"], _ = c.String("Anonymous", "create-user") // Auth Conf["basic_auth"], _ = c.String("Auth", "basic") Conf["globus_token_url"], _ = c.String("Auth", "globus_token_url") Conf["globus_profile_url"], _ = c.String("Auth", "globus_profile_url") Conf["mgrast_oauth_url"], _ = c.String("Auth", "mgrast_oauth_url") // Admin Conf["admin-email"], _ = c.String("Admin", "email") Conf["admin-users"], _ = c.String("Admin", "users") // Paths Conf["site-path"], _ = c.String("Paths", "site") Conf["data-path"], _ = c.String("Paths", "data") Conf["logs-path"], _ = c.String("Paths", "logs") Conf["local-paths"], _ = c.String("Paths", "local_paths") Conf["pidfile"], _ = c.String("Paths", "pidfile") // Runtime Conf["GOMAXPROCS"], _ = c.String("Runtime", "GOMAXPROCS") // Mongodb Conf["mongodb-hosts"], _ = c.String("Mongodb", "hosts") if Conf["mongodb-database"], err = c.String("Mongodb", "database"); err != nil { fmt.Fprintf(os.Stderr, "ERROR: Mongodb database must be set in config file.") os.Exit(1) } Conf["mongodb-user"], _ = c.String("Mongodb", "user") Conf["mongodb-password"], _ = c.String("Mongodb", "password") Conf["mongodb-attribute-indexes"], _ = c.String("Mongodb", "attribute_indexes") // parse Node-Indices NODE_IDXS = map[string]idxOpts{} nodeIdx, _ := c.Options("Node-Indices") for _, opt := range nodeIdx { val, _ := c.String("Node-Indices", opt) opts := idxOpts{} for _, parts := range strings.Split(val, ",") { p := strings.Split(parts, ":") if p[0] == "unique" { if p[1] == "true" { opts.unique = true } else { opts.unique = false } } else if p[0] == "dropDups" { if p[1] == "true" { opts.dropDups = true } else { opts.dropDups = false } } else if p[0] == "sparse" { if p[1] == "true" { opts.sparse = true } else { opts.sparse = false } } } NODE_IDXS[opt] = opts } Conf["perf-log"], _ = c.String("Log", "perf_log") }