//convenience function for requiring a string in the config file
func GetRequiredString(config *goconf.ConfigFile, section string, key string) (value string) {
value, err := config.GetString(section, key)
if err != nil {
logger.Fatalf("[CONFIG] %v is required: [section=%v]", key, section)
}
return
}
func parseConfig(conf *goconf.ConfigFile) (*config, error) {
host, err := conf.GetString("database", "host")
if err != nil || len(host) == 0 {
return nil, errors.New("empty database host in the config file")
}
port, err := conf.GetInt("database", "port")
if err != nil || port <= 0 || port > 0xFFFF {
return nil, errors.New("invalid database port in the config file")
}
user, err := conf.GetString("database", "user")
if err != nil || len(user) == 0 {
return nil, errors.New("empty database user in the config file")
}
password, err := conf.GetString("database", "password")
if err != nil || len(password) == 0 {
return nil, errors.New("empty database password in the config file")
}
dbname, err := conf.GetString("database", "name")
if err != nil || len(dbname) == 0 {
return nil, errors.New("empty database name in the config file")
}
v := &config{
hosts: strings.Split(strings.Replace(host, " ", "", -1), ","),
port: uint16(port),
username: user,
password: password,
dbName: dbname,
}
return v, nil
}
// sets global logger based on verbosity level in configuration
// optional parameter: default.verbose (defaults to true if not present or incorrectly set)
func GlobalLogger(configFile *goconf.ConfigFile) {
verbose, err := configFile.GetBool("default", "verbose")
logger = log4go.NewVerboseLogger(verbose, nil, "")
if err != nil {
logger.Warn(err)
verbose = true
}
logger.Printf("verbose set [%v]", verbose)
}
// Sets global variable to configure buffersize for channels wrapping connections between worker and master (stdout, stderr)
// optional parameters: master.buffersize
func ConBufferSize(section string, config *goconf.ConfigFile) {
bufsize, err := config.GetInt(section, "conbuffersize")
if err != nil {
logger.Warn(err)
} else {
conbuffersize = bufsize
logger.Printf("conbuffersize=[%v]", conbuffersize)
}
}
//get the number of processors to use for golem itself
func GoMaxProc(section string, config *goconf.ConfigFile) {
gomaxproc, err := config.GetInt(section, "gomaxproc")
if err != nil {
logger.Warn(err)
} else {
runtime.GOMAXPROCS(gomaxproc)
logger.Printf("gomaxproc=[%v]", gomaxproc)
}
}
// cfgOpt returns the configuration option from the specified section. If the
// option does not exist an empty string is returned.
func cfgOpt(cfg *conf.ConfigFile, section, option string) string {
if !cfg.HasOption(section, option) {
return ""
}
s, err := cfg.GetString(section, option)
if err != nil {
log.Exitf("Failed to get %s for %s: %v", option, section, err)
}
return s
}
// Sets global variable to configure buffersize for master submission channels (stdout, stderr)
// optional parameters: master.buffersize
func SubIOBufferSize(section string, configFile *goconf.ConfigFile) {
bufsize, err := configFile.GetInt(section, "subiobuffersize")
if err != nil {
logger.Warn(err)
} else {
iobuffersize = bufsize
}
logger.Printf("buffersize=[%v]", iobuffersize)
}
//get the number of IO monitors to run per node
func IOMOnitors(config *goconf.ConfigFile) {
iomons, err := config.GetInt("master", "iomonitors")
if err != nil {
logger.Warn(err)
} else {
if iomons > 0 {
iomonitors = iomons
}
}
logger.Printf("iomonitors=[%v]", iomonitors)
}
// starts http handlers for HTML content based on the given configuration file
// optional parameters: default.contentDirectory (location of html content to be served at https://example.com/ or https://example.com/html/index.html
func StartHtmlHandler(configFile *goconf.ConfigFile) {
if contentDir, _ := configFile.GetString("default", "contentDirectory"); contentDir != "" {
logger.Printf("StartHtmlHandler(): serving HTML content from [%v]", contentDir)
http.Handle("/html/", http.StripPrefix("/html/", http.FileServer(http.Dir(contentDir))))
http.Handle("/", http.RedirectHandler("/html/index.html", http.StatusTemporaryRedirect))
wd, err := os.Getwd()
if err == nil {
http.Handle("/output/", http.StripPrefix("/output/", http.FileServer(http.Dir(wd))))
}
}
}
// starts worker based on the given configuration file
// required parameters: worker.masterhost
// optional parameters: worker.processes
func StartWorker(configFile *goconf.ConfigFile) {
GoMaxProc("worker", configFile)
ConBufferSize("worker", configFile)
processes, err := configFile.GetInt("worker", "processes")
if err != nil {
logger.Warn(err)
processes = 3
}
masterhost := GetRequiredString(configFile, "worker", "masterhost")
logger.Printf("StartWorker() [%v, %d]", masterhost, processes)
RunNode(processes, masterhost)
}
func MongoLogger(configFile *goconf.ConfigFile) {
verbose, err := configFile.GetBool("mgodb", "verbose")
if err != nil {
logger.Warn(err)
verbose = false
}
logger.Printf("mongo logger verbose set [%v]", verbose)
if verbose {
mgo.SetLogger(logger)
mgo.SetDebug(verbose)
}
}
func (c *Config) readDefaultConfig(readConfigData *goconf.ConfigFile) error {
var err error
c.BaseDir, err = readConfigData.GetString("default", "base_dir")
if err != nil || len(c.BaseDir) == 0 {
return errors.New("invalid BaseDir directory")
}
if c.BaseDir[0] != '/' {
return errors.New("BaseDir directory should be specified as an absolute path")
}
return nil
}
// Initializes a new connection to the given account in a separate goroutine
func initConnection(notify chan bool, conf *goconf.ConfigFile, account string) {
hostname, _ := conf.GetString(account, "hostname")
username, _ := conf.GetString(account, "username")
passexec, _ := conf.GetString(account, "password")
pw := exec.Command("/bin/sh", "-c", passexec)
pw.Env = os.Environ()
pwbytes, err := pw.Output()
if err != nil {
fmt.Printf("%s: password command failed: %s\n", account, err)
return
}
password := strings.TrimRight(string(pwbytes), "\n")
if password == "" {
fmt.Printf("%s: password command returned an empty string\n", account)
return
}
folder, e := conf.GetString(account, "folder")
if e != nil {
folder = "INBOX"
}
poll, e := conf.GetInt(account, "poll")
if e != nil {
poll = 29
}
polli := time.Duration(poll) * time.Minute
go parts.Connect(notify, account, hostname, username, password, folder, polli)
}
func parseRESTConfig(conf *goconf.ConfigFile) (*restConfig, error) {
var err error
c := &restConfig{}
c.tls.enable, err = conf.GetBool("rest", "tls")
if err != nil {
return nil, errors.New("invalid rest/tls value")
}
port, err := conf.GetInt("rest", "port")
if err != nil || port <= 0 || port > 65535 {
return nil, errors.New("empty or invalid rest/port value")
}
c.port = uint16(port)
c.tls.certFile, err = conf.GetString("rest", "cert_file")
if err != nil || len(c.tls.certFile) == 0 {
return nil, errors.New("empty rest/cert_file value")
}
if c.tls.certFile[0] != '/' {
return nil, errors.New("rest/cert_file should be specified as an absolute path")
}
c.tls.keyFile, err = conf.GetString("rest", "key_file")
if err != nil || len(c.tls.keyFile) == 0 {
return nil, errors.New("empty rest/key_file value")
}
if c.tls.keyFile[0] != '/' {
return nil, errors.New("rest/key_file should be specified as an absolute path")
}
return c, nil
}
func (c *Config) readDefaultConfig(conf *goconf.ConfigFile) error {
var err error
c.Port, err = conf.GetInt("default", "port")
if err != nil || c.Port <= 0 || c.Port > 0xFFFF {
return errors.New("invalid port in the config file")
}
logLevel, err := conf.GetString("default", "log_level")
if err != nil || len(logLevel) == 0 {
return errors.New("invalid log level in the config file")
}
if err := c.parseLogLevel(logLevel); err != nil {
return err
}
apps, err := conf.GetString("default", "applications")
if err != nil || len(apps) == 0 {
return errors.New("empty applications in the config file")
}
if err := c.parseApplications(apps); err != nil {
return fmt.Errorf("invalid applications config: %v", err)
}
return nil
}
// Sets global variable to enable TLS communications and other related variables (certificate path, organization)
// optional parameters: default.certpath, default.organization, default.tls
func GlobalTls(configFile *goconf.ConfigFile) {
certificatepath, err := configFile.GetString("default", "certpath")
if err != nil {
logger.Warn(err)
} else {
certpath = certificatepath
}
logger.Printf("certpath=[%v]", certpath)
certificateorg, err := configFile.GetString("default", "organization")
if err != nil {
logger.Warn(err)
if certificateorg, err = os.Hostname(); err != nil {
logger.Warn(err)
certificateorg = "golem.googlecode.com"
}
}
certorg = certificateorg
logger.Printf("certorg=[%v]", certorg)
useTlsl, err := configFile.GetBool("default", "tls")
if err != nil {
logger.Warn(err)
useTls = true
} else {
useTls = useTlsl
}
logger.Printf("TLS=[%v]", useTls)
}
func setupAuthHandler(cfg *goconf.ConfigFile) martini.Handler {
auth_url, _ := cfg.GetString("Auth", "endpoint")
hostname, _ := cfg.GetString("Cache", "hostname")
port, _ := cfg.GetString("Cache", "port")
password, _ := cfg.GetString("Cache", "password")
return auth.Keystone(
auth.IdentityValidator{AuthUrl: auth_url},
auth.Redis{
Hostname: hostname,
Port: port,
Password: password,
Database: int64(0)})
}
// starts scribe service based on the given configuration file
// required parameters: default.hostname, default.password, scribe.target, mgodb.server, mgodb.store, mgodb.jobcollection, mgodb.taskcollection
func StartScribe(configFile *goconf.ConfigFile) {
MongoLogger(configFile)
GoMaxProc("scribe", configFile)
hostname := GetRequiredString(configFile, "default", "hostname")
apikey := GetRequiredString(configFile, "default", "password")
target := GetRequiredString(configFile, "scribe", "target")
dbhost := GetRequiredString(configFile, "mgodb", "server")
dbstore := GetRequiredString(configFile, "mgodb", "store")
url, err := url.Parse(target)
if err != nil {
panic(err)
}
go LaunchScribe(NewMongoJobStore(dbhost, dbstore), target, apikey)
rest.Resource("jobs", ScribeJobController{NewMongoJobStore(dbhost, dbstore), url, apikey})
rest.ResourceContentType("jobs", "application/json")
rest.Resource("nodes", ProxyNodeController{url, apikey})
rest.ResourceContentType("nodes", "application/json")
rest.Resource("cluster", ScribeClusterController{NewMongoJobStore(dbhost, dbstore), url})
rest.ResourceContentType("cluster", "application/json")
var numberOfSeconds int = 10
if pollingSecs, oops := configFile.GetInt("scribe", "clusterStatsPollingInSeconds"); oops == nil {
numberOfSeconds = pollingSecs
}
logger.Printf("polling for cluster stats every %d secs", numberOfSeconds)
go MonitorClusterStats(NewMongoJobStore(dbhost, dbstore), target, int64(numberOfSeconds))
ListenAndServeTLSorNot(hostname)
}
func setValues(d *schema.ResourceData, c *goconf.ConfigFile) error {
buildval, err := c.GetString("default", "COREOS_BUILD")
if err != nil {
return err
}
branchval, err := c.GetString("default", "COREOS_BRANCH")
if err != nil {
return err
}
patchval, err := c.GetString("default", "COREOS_PATCH")
if err != nil {
return err
}
versionval, err := c.GetString("default", "COREOS_VERSION")
if err != nil {
return err
}
version_idval, err := c.GetString("default", "COREOS_VERSION_ID")
if err != nil {
return err
}
build_idval, err := c.GetString("default", "COREOS_BUILD_ID")
if err != nil {
return err
}
sdk_versionval, err := c.GetString("default", "COREOS_SDK_VERSION")
if err != nil {
return err
}
d.Set("build", buildval)
d.Set("branch", branchval)
d.Set("patch", patchval)
d.Set("version", versionval)
d.Set("version_id", version_idval)
d.Set("build_id", build_idval)
d.Set("sdk_version", sdk_versionval)
return nil
}