// Monitors that the tunnel is alive by periodically querying the node status off Jenkins. // Timeout, hanging connections or connection errors lead to a restart of the current execution mode (which implicitly closes SSH tunnel as well). func (self *SSHTunnelEstablisher) startAliveStateMonitoring(config *util.Config) { // Periodically check the node status and increment lastAliveTick on success go func() { for _ = range self.aliveTicker.C { if !self.tunnelConnected.Get() { continue } if _, err := GetJenkinsNodeStatus(config); err == nil { self.lastAliveTick.Set(self.expectedAliveTick.Get()) } } }() // Periodically check that lastAliveTick was incremented. go func() { for _ = range self.aliveTickEvaluator.C { if !self.tunnelConnected.Get() { continue } if math.Abs(float64(self.expectedAliveTick.Get()-self.lastAliveTick.Get())) > 1 { util.GOut("ssh-tunnel", "WARN: The SSH tunnel appears to be dead or Jenkins is gone. Forcing restart of client and SSH tunnel.") modes.GetConfiguredMode(config).Stop() } else { self.expectedAliveTick.AddAndGet(1) } } }() }
// Listens for key codes. func listenForKeyboardInput(config *util.Config, subsequentRestarts *int64) { var keyCode = make([]byte, 1) util.Out("Listening for keys: [%s]: Print Stacktrace | [%s]: Restart client.", "D+Return", "R+Return") for { if n, err := os.Stdin.Read(keyCode); err == nil && n == 1 { switch keyCode[0] { case 'r', 'R': *subsequentRestarts = 0 modes.GetConfiguredMode(config).Stop() case 'd', 'D': util.PrintAllStackTraces() } } else { return } } }
func (self *PeriodicRestarter) Prepare(config *util.Config) { if self.ticker != nil { self.ticker.Stop() } if !config.PeriodicClientRestartEnabled || config.PeriodicClientRestartIntervalHours <= 0 { return } self.ticker = time.NewTicker(time.Hour * time.Duration(config.PeriodicClientRestartIntervalHours)) go func() { // Run in schedule for time := range self.ticker.C { util.GOut("periodic", "Triggering periodic restart.", time) self.waitForIdleIfRequired(config) // Stopping the mode as this will automatically do a restart. modes.GetConfiguredMode(config).Stop() } }() }
// Forces a reconnect with Jenkins by stopping the current mode. func (self *JenkinsNodeMonitor) forceReconnect(config *util.Config) { if self.isThisSideConnected(config) { util.GOut("monitor", "WARN: This node appears dead in Jenkins, forcing a reconnect.") modes.GetConfiguredMode(config).Stop() } }
// Checks if the run mode is in started state. // Also updates the global "util.NodeIsIdle" state to true if run mode is not in started state. func (self *JenkinsNodeMonitor) isThisSideConnected(config *util.Config) bool { return modes.GetConfiguredMode(config).Status().Get() == modes.ModeStarted }
// Is the applications main run loop. func Run() { util.FlatOut("\n%s %s\n---------------------------", AppName, AppVersion) AppImagePath, _ = filepath.Abs(os.Args[0]) modeNames := make([]string, len(modes.AllModes)) for i, m := range modes.AllModes { modeNames[i] = m.Name() } help := flag.Bool("help", false, "Show this help.") autoStart := flag.Bool("autostart", false, "Automatically start this app when the OS runs (implies '-persist=true').") runMode := flag.String("runMode", "client", "Specifies the mode in which the launcher operates if not already set "+ "inside '"+ConfigName+"'. Supporte modes are "+fmt.Sprintf("%v", modeNames)+".") url := flag.String("url", "", "Specifies the URL to Jenkins server if not already set inside '"+ConfigName+"'.") name := flag.String("name", "", "Specifies the name of this node in Jenkins (defaults to [hostname] if not specified).") create := flag.Bool("create", false, "Enables the auto creation of a Jenkins node if it is missing.") secretKey := flag.String("secret", "", "Specifies the secret key to use in client mode when starting the Jenkins client.") acceptAnyCert := flag.Bool("anyCert", false, "Disabled cert verification for TLS connections with Jenkins (this is not secure at all).") dir := flag.String("directory", "", "Changes the current working directory before performing any other operations.") saveChanges := flag.Bool("persist", false, "Stores any CLI config overrides inside '"+ConfigName+"'.") defaultConfig := flag.String("defaultConfig", "", "Loads the initial config from the specified path or URL (http[s]). "+ "Does nothing when '"+ConfigName+"' exists already.") overwrite := flag.Bool("overwrite", false, "Overwrites '"+ConfigName+"' with the content from initial config "+ "(requires '-defaultConfig=...', implies '-persist=true').") flag.CommandLine.Init(AppName, flag.ContinueOnError) flag.CommandLine.SetOutput(os.Stdout) flag.Parse() handleWorkingDirectory(*dir) if alreadyRunning := CheckIfAlreadyRunning(); alreadyRunning { util.Out("WARN: Another launcher is already running with the same configuration. Exiting...") return } else { defer alreadyRunning.Close() } config := loadConfig(*defaultConfig, *overwrite) if len(*runMode) > 0 { config.RunMode = *runMode } if len(*url) > 0 { config.CIHostURI = *url } if len(*secretKey) > 0 { config.SecretKey = *secretKey } if len(*name) > 0 { config.ClientName = *name } if *create { config.CreateClientIfMissing = true } if *acceptAnyCert { config.CIAcceptAnyCert = true } saveConfigIfRequired := func() { if config.NeedsSave || *saveChanges || *autoStart || *overwrite { config.Save(ConfigName) } } if *help { saveConfigIfRequired() fmt.Printf(AppDescription, ConfigName, filepath.Base(os.Args[0])) flag.PrintDefaults() return } environment.RunPreparers(config) abort := !modes.GetConfiguredMode(config).IsConfigAcceptable(config) saveConfigIfRequired() if abort { return } restartCount := int64(0) sleepTimePerRestart := int64(time.Second * time.Duration(config.SleepTimeSecondsBetweenFailures)) runTimeAfterResettingRestartCount := time.Hour * 2 timeOfLastStart := time.Now() go listenForKeyboardInput(config, &restartCount) for modes.RunConfiguredMode(config) { util.FlatOut("\n:::::::::::::::::::::::::::::::::\n:: %25s ::\n:::::::::::::::::::::::::::::::::\n", "Restarting Jenkins Client") if timeOfLastStart.Before(time.Now().Add(-runTimeAfterResettingRestartCount)) { restartCount = 0 } if sleepTime := time.Duration(restartCount * sleepTimePerRestart); sleepTime > 0 { util.FlatOut("Sleeping %v seconds before restarting the client.\n\n", sleepTime.Seconds()) time.Sleep(sleepTime) } restartCount++ timeOfLastStart = time.Now() } }