// Initializes a new connection to the given account in a separate goroutine
func initConnection(notify chan bool, conf *goconf.ConfigFile, account string) {
hostname, _ := conf.GetString(account, "hostname")
username, _ := conf.GetString(account, "username")
passexec, _ := conf.GetString(account, "password")
pw := exec.Command("/bin/sh", "-c", passexec)
pw.Env = os.Environ()
pwbytes, err := pw.Output()
if err != nil {
fmt.Printf("%s: password command failed: %s\n", account, err)
return
}
password := strings.TrimRight(string(pwbytes), "\n")
if password == "" {
fmt.Printf("%s: password command returned an empty string\n", account)
return
}
folder, e := conf.GetString(account, "folder")
if e != nil {
folder = "INBOX"
}
poll, e := conf.GetInt(account, "poll")
if e != nil {
poll = 29
}
polli := time.Duration(poll) * time.Minute
go parts.Connect(notify, account, hostname, username, password, folder, polli)
}
func parseRESTConfig(conf *goconf.ConfigFile) (*restConfig, error) {
var err error
c := &restConfig{}
c.tls.enable, err = conf.GetBool("rest", "tls")
if err != nil {
return nil, errors.New("invalid rest/tls value")
}
port, err := conf.GetInt("rest", "port")
if err != nil || port <= 0 || port > 65535 {
return nil, errors.New("empty or invalid rest/port value")
}
c.port = uint16(port)
c.tls.certFile, err = conf.GetString("rest", "cert_file")
if err != nil || len(c.tls.certFile) == 0 {
return nil, errors.New("empty rest/cert_file value")
}
if c.tls.certFile[0] != '/' {
return nil, errors.New("rest/cert_file should be specified as an absolute path")
}
c.tls.keyFile, err = conf.GetString("rest", "key_file")
if err != nil || len(c.tls.keyFile) == 0 {
return nil, errors.New("empty rest/key_file value")
}
if c.tls.keyFile[0] != '/' {
return nil, errors.New("rest/key_file should be specified as an absolute path")
}
return c, nil
}
func (c *Config) readDefaultConfig(conf *goconf.ConfigFile) error {
var err error
c.Port, err = conf.GetInt("default", "port")
if err != nil || c.Port <= 0 || c.Port > 0xFFFF {
return errors.New("invalid port in the config file")
}
logLevel, err := conf.GetString("default", "log_level")
if err != nil || len(logLevel) == 0 {
return errors.New("invalid log level in the config file")
}
if err := c.parseLogLevel(logLevel); err != nil {
return err
}
apps, err := conf.GetString("default", "applications")
if err != nil || len(apps) == 0 {
return errors.New("empty applications in the config file")
}
if err := c.parseApplications(apps); err != nil {
return fmt.Errorf("invalid applications config: %v", err)
}
return nil
}
func parseConfig(conf *goconf.ConfigFile) (*config, error) {
host, err := conf.GetString("database", "host")
if err != nil || len(host) == 0 {
return nil, errors.New("empty database host in the config file")
}
port, err := conf.GetInt("database", "port")
if err != nil || port <= 0 || port > 0xFFFF {
return nil, errors.New("invalid database port in the config file")
}
user, err := conf.GetString("database", "user")
if err != nil || len(user) == 0 {
return nil, errors.New("empty database user in the config file")
}
password, err := conf.GetString("database", "password")
if err != nil || len(password) == 0 {
return nil, errors.New("empty database password in the config file")
}
dbname, err := conf.GetString("database", "name")
if err != nil || len(dbname) == 0 {
return nil, errors.New("empty database name in the config file")
}
v := &config{
hosts: strings.Split(strings.Replace(host, " ", "", -1), ","),
port: uint16(port),
username: user,
password: password,
dbName: dbname,
}
return v, nil
}
// Sets global variable to configure buffersize for channels wrapping connections between worker and master (stdout, stderr)
// optional parameters: master.buffersize
func ConBufferSize(section string, config *goconf.ConfigFile) {
bufsize, err := config.GetInt(section, "conbuffersize")
if err != nil {
logger.Warn(err)
} else {
conbuffersize = bufsize
logger.Printf("conbuffersize=[%v]", conbuffersize)
}
}
//get the number of processors to use for golem itself
func GoMaxProc(section string, config *goconf.ConfigFile) {
gomaxproc, err := config.GetInt(section, "gomaxproc")
if err != nil {
logger.Warn(err)
} else {
runtime.GOMAXPROCS(gomaxproc)
logger.Printf("gomaxproc=[%v]", gomaxproc)
}
}
// Sets global variable to configure buffersize for master submission channels (stdout, stderr)
// optional parameters: master.buffersize
func SubIOBufferSize(section string, configFile *goconf.ConfigFile) {
bufsize, err := configFile.GetInt(section, "subiobuffersize")
if err != nil {
logger.Warn(err)
} else {
iobuffersize = bufsize
}
logger.Printf("buffersize=[%v]", iobuffersize)
}
//get the number of IO monitors to run per node
func IOMOnitors(config *goconf.ConfigFile) {
iomons, err := config.GetInt("master", "iomonitors")
if err != nil {
logger.Warn(err)
} else {
if iomons > 0 {
iomonitors = iomons
}
}
logger.Printf("iomonitors=[%v]", iomonitors)
}
// starts worker based on the given configuration file
// required parameters: worker.masterhost
// optional parameters: worker.processes
func StartWorker(configFile *goconf.ConfigFile) {
GoMaxProc("worker", configFile)
ConBufferSize("worker", configFile)
processes, err := configFile.GetInt("worker", "processes")
if err != nil {
logger.Warn(err)
processes = 3
}
masterhost := GetRequiredString(configFile, "worker", "masterhost")
logger.Printf("StartWorker() [%v, %d]", masterhost, processes)
RunNode(processes, masterhost)
}
// starts scribe service based on the given configuration file
// required parameters: default.hostname, default.password, scribe.target, mgodb.server, mgodb.store, mgodb.jobcollection, mgodb.taskcollection
func StartScribe(configFile *goconf.ConfigFile) {
MongoLogger(configFile)
GoMaxProc("scribe", configFile)
hostname := GetRequiredString(configFile, "default", "hostname")
apikey := GetRequiredString(configFile, "default", "password")
target := GetRequiredString(configFile, "scribe", "target")
dbhost := GetRequiredString(configFile, "mgodb", "server")
dbstore := GetRequiredString(configFile, "mgodb", "store")
url, err := url.Parse(target)
if err != nil {
panic(err)
}
go LaunchScribe(NewMongoJobStore(dbhost, dbstore), target, apikey)
rest.Resource("jobs", ScribeJobController{NewMongoJobStore(dbhost, dbstore), url, apikey})
rest.ResourceContentType("jobs", "application/json")
rest.Resource("nodes", ProxyNodeController{url, apikey})
rest.ResourceContentType("nodes", "application/json")
rest.Resource("cluster", ScribeClusterController{NewMongoJobStore(dbhost, dbstore), url})
rest.ResourceContentType("cluster", "application/json")
var numberOfSeconds int = 10
if pollingSecs, oops := configFile.GetInt("scribe", "clusterStatsPollingInSeconds"); oops == nil {
numberOfSeconds = pollingSecs
}
logger.Printf("polling for cluster stats every %d secs", numberOfSeconds)
go MonitorClusterStats(NewMongoJobStore(dbhost, dbstore), target, int64(numberOfSeconds))
ListenAndServeTLSorNot(hostname)
}