func Setup(dsn string, dbMaxConns uint32) error {
var err error
db, err = sql.Open("mysql", dsn)
if err != nil {
return err
}
db.SetMaxOpenConns(int(dbMaxConns))
parts := strings.Split(dsn, ")")
if len(parts) != 2 {
log.Fatalf("Unexpected MySQL DSN format, expected it to in form '<login>:<password>@<transport>(...)/<db>'")
}
parts = strings.Split(parts[1], "/")
if len(parts) != 2 {
log.Fatalf("Unexpected MySQL DSN format, expected it to be in form of '<host_settings>/<db>'")
}
qParts := strings.Split(parts[1], "?")
dbName = qParts[0]
return nil
}
func GenerateJobsCycle() {
hostname, err := os.Hostname()
if err != nil {
log.Fatalf("Could not get hostname: %s", err.Error())
}
log.Print("Initial select from RunQueue and starting launcher goroutines")
if err := selectRQAndNotify(); err != nil {
log.Fatalf("Could not do initial select run queue: %s", err.Error())
}
log.Print("Initial select from Timetable")
ttRows, err := selectTimetable()
if err != nil {
log.Fatalf("Could not do initial select timetable: %s", err.Error())
}
log.Print("Starting jobgen goroutines")
if err := notifyForFullTTSelect(ttRows, false); err != nil {
log.Fatalf("Could notify about timetable: %s", err.Error())
}
for {
res, err := db.LockCycle(getLockName(), hostname)
if err != nil || !res {
if err == nil {
log.Println("Could not get lock, another host holds it? Retrying in 10 seconds")
} else {
log.Warningf("Could not get lock, got DB error: ", err.Error())
}
time.Sleep(time.Second * 10)
continue
}
// timer := pinba.TimerStart(map[string]string{"group": "jobgenerator"})
startTs := time.Now().UnixNano()
db.LogCycleStart(CYCLE_CLASS_NAME, hostname, 0)
log.Debug("Cycle started")
success := doCycle()
log.Debug("Cycle finished")
successInt := 1
if !success {
successInt = 0
}
db.LogCycleStop(CYCLE_CLASS_NAME, hostname, 0, successInt)
passedMs := int64((time.Now().UnixNano() - startTs) / 1e6)
if passedMs < cycleMs {
time.Sleep(time.Duration(cycleMs-passedMs) * time.Millisecond)
}
}
}
func Setup(config common.FullConfig) {
isDevelServer = config.GetIsDevel()
dispatchThreads.v = make(map[string]map[string]*DispatcherData)
launcherThreads.v = make(map[string]*LauncherData)
killerThreads.v = make(map[string]map[string]bool)
throttle.c = make(chan bool, THROTTLE_CHAN_CAPACITY)
throttle.setIntervalCh = make(chan time.Duration, 1)
go throttleThread()
rusageInfo.groupsMaxParrots = make(map[string]uint64)
rusageInfo.groupHosts = make(map[string][]string)
rusageInfo.hostsInfo = make(map[string]*ServerInfo)
rusageInfo.loadEstimate = make(map[string]*ScriptRusageEntry)
rusageInfo.timetableRusage = make(map[uint64]*ScriptRusageEntry)
rusageInfo.groupIdx = make(map[string]uint64)
def := config.GetDefault()
defaultParasiteMemory = def.GetParasiteMemory()
defaultMinIdleCpu = def.GetMinIdleCpu()
defaultMinMemory = def.GetMinMemory()
defaultMinMemoryRatio = def.GetMinMemoryRatio()
defaultMaxMemory = def.GetMaxMemory()
defaultRusage = def.GetRusage()
cycleMs = config.GetCycleMs()
ttReloadIntervalMs = config.GetFullTimetableReloadIntervalMs()
autoIncrementIncrement = getAutoIncrementIncrement()
launcherConf := config.GetLauncher()
hostSuffix = launcherConf.GetHostSuffix()
basePath = launcherConf.GetBasePath()
if launcherConf.DeveloperPath != nil {
haveDeveloper = true
developerPath = launcherConf.GetDeveloperPath()
log.Printf("We have developer dir: %s", developerPath)
}
log.Printf("Updating hosts")
updateHosts()
log.Printf("Launching update hosts thread")
go updateHostsThread()
log.Printf("Clearing old heartbeats")
if err := clearOldHeartbeats(); err != nil {
log.Fatalf("Could not clear old heartbeats: %s", err.Error())
}
log.Printf("Launching periodic run queue select thread")
go runqueuePeriodicSelectThread()
go forceCheckDeletedThread()
}
func Initialize(default_config_path string, service_conf Config) {
flag.StringVar(&flags.ConfFile, "c", default_config_path, "path to config file")
flag.StringVar(&flags.LogFile, "l", "", "path to log file, special value '-' means 'stdout'")
flag.StringVar(&flags.PidFile, "p", "", "path to pid file. if empty, pidfile will not be created")
flag.BoolVar(&flags.Testconf, "t", false, "test configuration and exit")
flag.BoolVar(&flags.Version, "v", false, "print version")
flag.BoolVar(&flags.FullVersion, "V", false, "print full version info")
flag.BoolVar(&flags.Debug, "debug", false, "force DEBUG log level")
flag.Parse()
if flags.Version {
fmt.Printf("%s\n", VersionInfo.GetVersion())
os.Exit(0)
}
if flags.FullVersion {
data, _ := json.MarshalIndent(VersionInfo, "", " ")
fmt.Printf("%s\n", data)
os.Exit(0)
}
var err error
config = service_conf // save a pointer to service's config (NOT a copy, mon!)
commandLine = strings.Join(os.Args, " ") // XXX(antoxa): couldn't think of a better way
hostname = getHostname() // get hostname early
// moved here from init(), just importing a package should not publish expvars
initExpvars()
// current executable full path (symlinks and shit sometimes complicate things)
binaryPath = func() string {
path, err := osutil.GetCurrentBinary()
if err != nil {
// log as notice, non-critical error (only stats affected)
log.Infof("couldn't get current binary (using argv[0] = %q): %v", os.Args[0], err)
return os.Args[0]
}
return path
}()
// config path
confPath := func() string {
if flags.ConfFile != "" {
return flags.ConfFile
}
return default_config_path
}()
// resolve absolute config path, convenient for stats
configPath = func(path string) string {
var err error
if path, err = filepath.Abs(path); err != nil {
return path
}
if path, err = filepath.EvalSymlinks(path); err != nil {
return path
}
return path
}(confPath)
// parse config and construct final config merged with command line flags
// use path as supplied to us in args (i.e. unresolved), just to avoid 'too smart, outsmarted yourself' gotchas
err = ParseConfigFromFile(confPath, service_conf)
if err != nil {
err_message := func(err error) string {
switch real_err := err.(type) {
case nil:
return "syntax is ok"
case *json.SyntaxError:
return fmt.Sprintf("%v at offset %d", real_err, real_err.Offset)
case *os.PathError:
return fmt.Sprintf("%v", real_err)
default:
return fmt.Sprintf("(%T) %v", real_err, real_err)
}
}(err)
stderrLogger.Fatalf("Error in config: %s", err_message)
} else {
if flags.Testconf {
fmt.Printf("testconf %s: syntax is ok\n", configPath)
}
}
mergeCommandlineFlagsToConfig(flags, config)
daemonConfig := config.GetDaemonConfig()
// antoxa: need the fancy wrapper function to have testconf behave properly
// FIXME: testconf should check more stuff (below) and graceful restart also
initPidfileLogfile := func() error {
// FIXME(antoxa): this testconf thingy is everywhere! must... resist... full rewrite
if flags.Testconf {
err = PidfileTest(daemonConfig.GetPidFile())
} else {
pidfile, err = PidfileOpen(daemonConfig.GetPidFile())
}
if err != nil {
return fmt.Errorf("can't open pidfile: %s", err)
}
// FIXME: this is shit ugly
// need better integration between logger and daemon-config
// or 1-to-1 mapping
// or better log package :)
log_level := daemonConfig.GetLogLevel()
if log_level == 0 {
return fmt.Errorf("unknown log_level, supported: %v", badoo_config.ServiceConfigDaemonConfigTLogLevels_name)
}
err = reopenLogfile(daemonConfig.GetLogFile(), log.Level(log_level))
if err != nil {
return fmt.Errorf("can't open logfile: %s", err)
}
return nil
}
err = initPidfileLogfile()
if err != nil {
if flags.Testconf {
stderrLogger.Errorf("%v", err)
fmt.Printf("testconf failed\n")
} else {
stderrLogger.Errorf("%v", err) // always pidfile/logfile errors to stderr
}
os.Exit(1)
} else {
if flags.Testconf {
fmt.Printf("testconf successful\n")
os.Exit(0)
}
}
// log some version info like libangel does
versionString := func() string {
vi := &VersionInfo
version := func() string {
if vi.GetAutoBuildTag() != "" {
return fmt.Sprintf("%s-%s", vi.GetVersion(), vi.GetAutoBuildTag())
} else {
return vi.GetVersion()
}
}()
return fmt.Sprintf("%s version %s, git %s, built %s on %s",
vi.GetVcsBasename(), version, vi.GetVcsShortHash(), vi.GetBuildDate(), vi.GetBuildHost())
}()
log.Infof("%s", versionString)
// max cpus, 0 = all of them
numCPU := func() int {
maxCpus := int(daemonConfig.GetMaxCpus())
if maxCpus <= 0 || maxCpus > runtime.NumCPU() {
maxCpus = runtime.NumCPU()
}
return maxCpus
}()
runtime.GOMAXPROCS(numCPU)
// gc percent, <0 - disables GC
if daemonConfig.GcPercent != nil {
debug.SetGCPercent(int(daemonConfig.GetGcPercent()))
}
// process pinba configuration and related stuff
pinbaSender, err = func() (*PinbaSender, error) { // assigns a global
if daemonConfig.GetPinbaAddress() == "" {
return nil, nil // user doesn't want pinba configured
}
pi, err := PinbaInfoFromConfig(config)
if err != nil {
return nil, err
}
return NewPinbaSender(pi), nil
}()
if err != nil {
log.Fatalf("pinba config error: %v", err)
}
// graceful restart handling
// see restart.go and signals.go for more details
restartData, err = ParseRestartDataFromEnv()
if err != nil {
log.Fatalf("can't parse restart data: %v", err)
}
if restartData != nil {
log.Debugf("[CHILD] this is a restart, parent: %d, me: %d", restartData.PPid, os.Getpid())
}
// start http pprof server (possibly - inherit fd from parent if this is a restart)
err = func() (err error) {
HttpServer, err = newHttpServer(config, restartData) // assigning a global here
if err != nil {
return err
}
if HttpServer != nil { // nil here means it has not been configured
go HttpServer.Serve()
}
return nil
}()
if err != nil {
log.Fatalf("can't start http_pprof server: %v", err)
}
}