// Monitors that the tunnel is alive by periodically querying the node status off Jenkins.
// Timeout, hanging connections or connection errors lead to a restart of the current execution mode (which implicitly closes SSH tunnel as well).
func (self *SSHTunnelEstablisher) startAliveStateMonitoring(config *util.Config) {
// Periodically check the node status and increment lastAliveTick on success
go func() {
for _ = range self.aliveTicker.C {
if !self.tunnelConnected.Get() {
continue
}
if _, err := GetJenkinsNodeStatus(config); err == nil {
self.lastAliveTick.Set(self.expectedAliveTick.Get())
}
}
}()
// Periodically check that lastAliveTick was incremented.
go func() {
for _ = range self.aliveTickEvaluator.C {
if !self.tunnelConnected.Get() {
continue
}
if math.Abs(float64(self.expectedAliveTick.Get()-self.lastAliveTick.Get())) > 1 {
util.GOut("ssh-tunnel", "WARN: The SSH tunnel appears to be dead or Jenkins is gone. Forcing restart of client and SSH tunnel.")
modes.GetConfiguredMode(config).Stop()
} else {
self.expectedAliveTick.AddAndGet(1)
}
}
}()
}
// Listens for key codes.
func listenForKeyboardInput(config *util.Config, subsequentRestarts *int64) {
var keyCode = make([]byte, 1)
util.Out("Listening for keys: [%s]: Print Stacktrace | [%s]: Restart client.", "D+Return", "R+Return")
for {
if n, err := os.Stdin.Read(keyCode); err == nil && n == 1 {
switch keyCode[0] {
case 'r', 'R':
*subsequentRestarts = 0
modes.GetConfiguredMode(config).Stop()
case 'd', 'D':
util.PrintAllStackTraces()
}
} else {
return
}
}
}
func (self *PeriodicRestarter) Prepare(config *util.Config) {
if self.ticker != nil {
self.ticker.Stop()
}
if !config.PeriodicClientRestartEnabled || config.PeriodicClientRestartIntervalHours <= 0 {
return
}
self.ticker = time.NewTicker(time.Hour * time.Duration(config.PeriodicClientRestartIntervalHours))
go func() {
// Run in schedule
for time := range self.ticker.C {
util.GOut("periodic", "Triggering periodic restart.", time)
self.waitForIdleIfRequired(config)
// Stopping the mode as this will automatically do a restart.
modes.GetConfiguredMode(config).Stop()
}
}()
}
// Forces a reconnect with Jenkins by stopping the current mode.
func (self *JenkinsNodeMonitor) forceReconnect(config *util.Config) {
if self.isThisSideConnected(config) {
util.GOut("monitor", "WARN: This node appears dead in Jenkins, forcing a reconnect.")
modes.GetConfiguredMode(config).Stop()
}
}
// Checks if the run mode is in started state.
// Also updates the global "util.NodeIsIdle" state to true if run mode is not in started state.
func (self *JenkinsNodeMonitor) isThisSideConnected(config *util.Config) bool {
return modes.GetConfiguredMode(config).Status().Get() == modes.ModeStarted
}
// Is the applications main run loop.
func Run() {
util.FlatOut("\n%s %s\n---------------------------", AppName, AppVersion)
AppImagePath, _ = filepath.Abs(os.Args[0])
modeNames := make([]string, len(modes.AllModes))
for i, m := range modes.AllModes {
modeNames[i] = m.Name()
}
help := flag.Bool("help", false, "Show this help.")
autoStart := flag.Bool("autostart", false, "Automatically start this app when the OS runs (implies '-persist=true').")
runMode := flag.String("runMode", "client", "Specifies the mode in which the launcher operates if not already set "+
"inside '"+ConfigName+"'. Supporte modes are "+fmt.Sprintf("%v", modeNames)+".")
url := flag.String("url", "", "Specifies the URL to Jenkins server if not already set inside '"+ConfigName+"'.")
name := flag.String("name", "", "Specifies the name of this node in Jenkins (defaults to [hostname] if not specified).")
create := flag.Bool("create", false, "Enables the auto creation of a Jenkins node if it is missing.")
secretKey := flag.String("secret", "", "Specifies the secret key to use in client mode when starting the Jenkins client.")
acceptAnyCert := flag.Bool("anyCert", false, "Disabled cert verification for TLS connections with Jenkins (this is not secure at all).")
dir := flag.String("directory", "", "Changes the current working directory before performing any other operations.")
saveChanges := flag.Bool("persist", false, "Stores any CLI config overrides inside '"+ConfigName+"'.")
defaultConfig := flag.String("defaultConfig", "", "Loads the initial config from the specified path or URL (http[s]). "+
"Does nothing when '"+ConfigName+"' exists already.")
overwrite := flag.Bool("overwrite", false, "Overwrites '"+ConfigName+"' with the content from initial config "+
"(requires '-defaultConfig=...', implies '-persist=true').")
flag.CommandLine.Init(AppName, flag.ContinueOnError)
flag.CommandLine.SetOutput(os.Stdout)
flag.Parse()
handleWorkingDirectory(*dir)
if alreadyRunning := CheckIfAlreadyRunning(); alreadyRunning {
util.Out("WARN: Another launcher is already running with the same configuration. Exiting...")
return
} else {
defer alreadyRunning.Close()
}
config := loadConfig(*defaultConfig, *overwrite)
if len(*runMode) > 0 {
config.RunMode = *runMode
}
if len(*url) > 0 {
config.CIHostURI = *url
}
if len(*secretKey) > 0 {
config.SecretKey = *secretKey
}
if len(*name) > 0 {
config.ClientName = *name
}
if *create {
config.CreateClientIfMissing = true
}
if *acceptAnyCert {
config.CIAcceptAnyCert = true
}
saveConfigIfRequired := func() {
if config.NeedsSave || *saveChanges || *autoStart || *overwrite {
config.Save(ConfigName)
}
}
if *help {
saveConfigIfRequired()
fmt.Printf(AppDescription, ConfigName, filepath.Base(os.Args[0]))
flag.PrintDefaults()
return
}
environment.RunPreparers(config)
abort := !modes.GetConfiguredMode(config).IsConfigAcceptable(config)
saveConfigIfRequired()
if abort {
return
}
restartCount := int64(0)
sleepTimePerRestart := int64(time.Second * time.Duration(config.SleepTimeSecondsBetweenFailures))
runTimeAfterResettingRestartCount := time.Hour * 2
timeOfLastStart := time.Now()
go listenForKeyboardInput(config, &restartCount)
for modes.RunConfiguredMode(config) {
util.FlatOut("\n:::::::::::::::::::::::::::::::::\n:: %25s ::\n:::::::::::::::::::::::::::::::::\n", "Restarting Jenkins Client")
if timeOfLastStart.Before(time.Now().Add(-runTimeAfterResettingRestartCount)) {
restartCount = 0
}
if sleepTime := time.Duration(restartCount * sleepTimePerRestart); sleepTime > 0 {
util.FlatOut("Sleeping %v seconds before restarting the client.\n\n", sleepTime.Seconds())
time.Sleep(sleepTime)
}
restartCount++
timeOfLastStart = time.Now()
}
}