// RunUntil starts a watch and handles watch events. Will restart the watch if it is closed. // RunUntil starts a goroutine and returns immediately. It will exit when stopCh is closed. func (r *Reflector) RunUntil(stopCh <-chan struct{}) { glog.V(3).Infof("Starting reflector %v (%s) from %s", r.expectedType, r.resyncPeriod, r.name) go wait.Until(func() { if err := r.ListAndWatch(stopCh); err != nil { utilruntime.HandleError(err) } }, r.period, stopCh) }
// Run begins quota controller using the specified number of workers func (rq *ResourceQuotaController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() go rq.rqController.Run(stopCh) // the controllers that replenish other resources to respond rapidly to state changes for _, replenishmentController := range rq.replenishmentControllers { go replenishmentController.Run(stopCh) } // the workers that chug through the quota calculation backlog for i := 0; i < workers; i++ { go wait.Until(rq.worker, time.Second, stopCh) } // the timer for how often we do a full recalculation across all quotas go wait.Until(func() { rq.enqueueAll() }, rq.resyncPeriod(), stopCh) <-stopCh glog.Infof("Shutting down ResourceQuotaController") rq.queue.ShutDown() }
// Run starts a watch and handles watch events. Will restart the watch if it is closed. // Run starts a goroutine and returns immediately. func (r *Reflector) Run() { glog.V(3).Infof("Starting reflector %v (%s) from %s", r.expectedType, r.resyncPeriod, r.name) go wait.Until(func() { if err := r.ListAndWatch(wait.NeverStop); err != nil { utilruntime.HandleError(err) } }, r.period, wait.NeverStop) }
func NewSourceFile(path string, nodeName string, period time.Duration, updates chan<- interface{}) { config := &sourceFile{ path: path, nodeName: nodeName, updates: updates, } glog.V(1).Infof("Watching path %q", path) go wait.Until(config.run, period, wait.NeverStop) }
// NewSerializedImagePuller takes an event recorder and container runtime to create a // image puller that wraps the container runtime's PullImage interface. // Pulls one image at a time. // Issue #10959 has the rationale behind serializing image pulls. func NewSerializedImagePuller(recorder record.EventRecorder, runtime Runtime, imageBackOff *flowcontrol.Backoff) ImagePuller { imagePuller := &serializedImagePuller{ recorder: recorder, runtime: runtime, backOff: imageBackOff, pullRequests: make(chan *imagePullRequest, 10), } go wait.Until(imagePuller.pullImages, time.Second, wait.NeverStop) return imagePuller }
// Run begins watching and syncing. func (rsc *ReplicaSetController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() go rsc.rsController.Run(stopCh) go rsc.podController.Run(stopCh) for i := 0; i < workers; i++ { go wait.Until(rsc.worker, time.Second, stopCh) } <-stopCh glog.Infof("Shutting down ReplicaSet Controller") rsc.queue.ShutDown() }
// nodesSyncLoop lists nodes ever 15 seconds, calling Update() on the TunnelList // each time (Update() is a noop if no changes are necessary). func (c *SSHTunneler) nodesSyncLoop() { // TODO (cjcullen) make this watch. go wait.Until(func() { addrs, err := c.getAddresses() glog.Infof("Calling update w/ addrs: %v", addrs) if err != nil { glog.Errorf("Failed to getAddresses: %v", err) } c.tunnels.Update(addrs) atomic.StoreInt64(&c.lastSync, c.clock.Now().Unix()) }, 15*time.Second, c.stopChan) }
// Run runs the petset controller. func (psc *PetSetController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() glog.Infof("Starting petset controller") go psc.podController.Run(stopCh) go psc.psController.Run(stopCh) for i := 0; i < workers; i++ { go wait.Until(psc.worker, time.Second, stopCh) } <-stopCh glog.Infof("Shutting down petset controller") psc.queue.ShutDown() }
func newPortRangeAllocator(r net.PortRange) PortAllocator { if r.Base == 0 || r.Size == 0 { panic("illegal argument: may not specify an empty port range") } ra := &rangeAllocator{ PortRange: r, ports: make(chan int, portsBufSize), rand: rand.New(rand.NewSource(time.Now().UnixNano())), } go wait.Until(func() { ra.fillPorts(wait.NeverStop) }, nextFreePortCooldown, wait.NeverStop) return ra }
func (cm *containerManagerImpl) Start() error { // Setup the node if err := cm.setupNode(); err != nil { return err } // Don't run a background thread if there are no ensureStateFuncs. numEnsureStateFuncs := 0 for _, cont := range cm.systemContainers { if cont.ensureStateFunc != nil { numEnsureStateFuncs++ } } if numEnsureStateFuncs >= 0 { // Run ensure state functions every minute. go wait.Until(func() { for _, cont := range cm.systemContainers { if cont.ensureStateFunc != nil { if err := cont.ensureStateFunc(cont.manager); err != nil { glog.Warningf("[ContainerManager] Failed to ensure state of %q: %v", cont.name, err) } } } }, time.Minute, wait.NeverStop) } if len(cm.periodicTasks) > 0 { go wait.Until(func() { for _, task := range cm.periodicTasks { if task != nil { task() } } }, 5*time.Minute, wait.NeverStop) } return nil }
// acquire loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew succeeds. func (le *LeaderElector) acquire() { stop := make(chan struct{}) wait.Until(func() { succeeded := le.tryAcquireOrRenew() le.maybeReportTransition() if !succeeded { glog.V(4).Infof("failed to renew lease %v/%v", le.config.EndpointsMeta.Namespace, le.config.EndpointsMeta.Name) time.Sleep(wait.Jitter(le.config.RetryPeriod, JitterFactor)) return } le.config.EventRecorder.Eventf(&api.Endpoints{ObjectMeta: le.config.EndpointsMeta}, api.EventTypeNormal, "%v became leader", le.config.Identity) glog.Infof("sucessfully acquired lease %v/%v", le.config.EndpointsMeta.Namespace, le.config.EndpointsMeta.Name) close(stop) }, 0, stop) }
// Run the main goroutine responsible for watching and syncing jobs. func (jm *JobController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() go jm.jobController.Run(stopCh) for i := 0; i < workers; i++ { go wait.Until(jm.worker, time.Second, stopCh) } if jm.internalPodInformer != nil { go jm.internalPodInformer.Run(stopCh) } <-stopCh glog.Infof("Shutting down Job Manager") jm.queue.ShutDown() }
// renew loops calling tryAcquireOrRenew and returns immediately when tryAcquireOrRenew fails. func (le *LeaderElector) renew() { stop := make(chan struct{}) wait.Until(func() { err := wait.Poll(le.config.RetryPeriod, le.config.RenewDeadline, func() (bool, error) { return le.tryAcquireOrRenew(), nil }) le.maybeReportTransition() if err == nil { glog.V(4).Infof("succesfully renewed lease %v/%v", le.config.EndpointsMeta.Namespace, le.config.EndpointsMeta.Name) return } le.config.EventRecorder.Eventf(&api.Endpoints{ObjectMeta: le.config.EndpointsMeta}, api.EventTypeNormal, "%v stopped leading", le.config.Identity) glog.Infof("failed to renew lease %v/%v", le.config.EndpointsMeta.Namespace, le.config.EndpointsMeta.Name) close(stop) }, 0, stop) }
func (im *realImageManager) Start() error { go wait.Until(func() { // Initial detection make detected time "unknown" in the past. var ts time.Time if im.initialized { ts = time.Now() } err := im.detectImages(ts) if err != nil { glog.Warningf("[ImageManager] Failed to monitor images: %v", err) } else { im.initialized = true } }, 5*time.Minute, wait.NeverStop) return nil }
// Run begins watching and syncing daemon sets. func (dsc *DaemonSetsController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() glog.Infof("Starting Daemon Sets controller manager") go dsc.dsController.Run(stopCh) go dsc.podController.Run(stopCh) go dsc.nodeController.Run(stopCh) for i := 0; i < workers; i++ { go wait.Until(dsc.runWorker, time.Second, stopCh) } if dsc.internalPodInformer != nil { go dsc.internalPodInformer.Run(stopCh) } <-stopCh glog.Infof("Shutting down Daemon Set Controller") dsc.queue.ShutDown() }
// Create a new Cacher responsible from service WATCH and LIST requests from its // internal cache and updating its cache in the background based on the given // configuration. func NewCacherFromConfig(config CacherConfig) *Cacher { watchCache := newWatchCache(config.CacheCapacity) listerWatcher := newCacherListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc) // Give this error when it is constructed rather than when you get the // first watch item, because it's much easier to track down that way. if obj, ok := config.Type.(runtime.Object); ok { if err := runtime.CheckCodec(config.Storage.Codec(), obj); err != nil { panic("storage codec doesn't seem to match given type: " + err.Error()) } } cacher := &Cacher{ storage: config.Storage, watchCache: watchCache, reflector: cache.NewReflector(listerWatcher, config.Type, watchCache, 0), watchers: make(map[int]*cacheWatcher), versioner: config.Versioner, keyFunc: config.KeyFunc, // We need to (potentially) stop both: // - wait.Until go-routine // - reflector.ListAndWatch // and there are no guarantees on the order that they will stop. // So we will be simply closing the channel, and synchronizing on the WaitGroup. stopCh: make(chan struct{}), } // See startCaching method for explanation and where this is unlocked. cacher.usable.Lock() watchCache.SetOnEvent(cacher.processEvent) stopCh := cacher.stopCh cacher.stopWg.Add(1) go func() { defer cacher.stopWg.Done() wait.Until( func() { if !cacher.isStopped() { cacher.startCaching(stopCh) } }, time.Second, stopCh, ) }() return cacher }
// Runs e; will not return until stopCh is closed. workers determines how many // endpoints will be handled in parallel. func (e *EndpointController) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() go e.serviceController.Run(stopCh) go e.podController.Run(stopCh) for i := 0; i < workers; i++ { go wait.Until(e.worker, time.Second, stopCh) } go func() { defer utilruntime.HandleCrash() time.Sleep(5 * time.Minute) // give time for our cache to fill e.checkLeftoverEndpoints() }() if e.internalPodInformer != nil { go e.internalPodInformer.Run(stopCh) } <-stopCh e.queue.ShutDown() }
func NewSSHTunnelList(user, keyfile string, healthCheckURL *url.URL, stopChan chan struct{}) *SSHTunnelList { l := &SSHTunnelList{ adding: make(map[string]bool), tunnelCreator: &realTunnelCreator{}, user: user, keyfile: keyfile, healthCheckURL: healthCheckURL, } healthCheckPoll := 1 * time.Minute go wait.Until(func() { l.tunnelsLock.Lock() defer l.tunnelsLock.Unlock() // Healthcheck each tunnel every minute numTunnels := len(l.entries) for i, entry := range l.entries { // Stagger healthchecks evenly across duration of healthCheckPoll. delay := healthCheckPoll * time.Duration(i) / time.Duration(numTunnels) l.delayedHealthCheck(entry, delay) } }, healthCheckPoll, stopChan) return l }
func (c *SSHTunneler) installSSHKeySyncLoop(user, publicKeyfile string) { go wait.Until(func() { if c.InstallSSHKey == nil { glog.Error("Won't attempt to install ssh key: InstallSSHKey function is nil") return } key, err := ssh.ParsePublicKeyFromFile(publicKeyfile) if err != nil { glog.Errorf("Failed to load public key: %v", err) return } keyData, err := ssh.EncodeSSHKey(key) if err != nil { glog.Errorf("Failed to encode public key: %v", err) return } if err := c.InstallSSHKey(user, keyData); err != nil { glog.Errorf("Failed to install ssh key: %v", err) return } atomic.StoreInt64(&c.lastSSHKeySync, c.clock.Now().Unix()) }, 5*time.Minute, c.stopChan) }
// InitLogs initializes logs the way we want for kubernetes. func InitLogs() { log.SetOutput(GlogWriter{}) log.SetFlags(0) // The default glog flush interval is 30 seconds, which is frighteningly long. go wait.Until(glog.Flush, *logFlushFreq, wait.NeverStop) }
// Start starts the control loop to observe and response to low compute resources. func (m *managerImpl) Start(podFunc ActivePodsFunc, monitoringInterval time.Duration) { go wait.Until(func() { m.synchronize(podFunc) }, monitoringInterval, wait.NeverStop) }
// Run starts an asynchronous loop that monitors the status of cluster nodes. func (nc *NodeController) Run(period time.Duration) { if nc.allocateNodeCIDRs { if nc.serviceCIDR != nil { nc.filterOutServiceRange() } else { glog.Info("No Service CIDR provided. Skipping filtering out service addresses.") } } go nc.nodeController.Run(wait.NeverStop) go nc.podController.Run(wait.NeverStop) go nc.daemonSetController.Run(wait.NeverStop) // Incorporate the results of node status pushed from kubelet to master. go wait.Until(func() { if err := nc.monitorNodeStatus(); err != nil { glog.Errorf("Error monitoring node status: %v", err) } }, nc.nodeMonitorPeriod, wait.NeverStop) // Managing eviction of nodes: // 1. when we delete pods off a node, if the node was not empty at the time we then // queue a termination watcher // a. If we hit an error, retry deletion // 2. The terminator loop ensures that pods are eventually cleaned and we never // terminate a pod in a time period less than nc.maximumGracePeriod. AddedAt // is the time from which we measure "has this pod been terminating too long", // after which we will delete the pod with grace period 0 (force delete). // a. If we hit errors, retry instantly // b. If there are no pods left terminating, exit // c. If there are pods still terminating, wait for their estimated completion // before retrying go wait.Until(func() { nc.evictorLock.Lock() defer nc.evictorLock.Unlock() nc.podEvictor.Try(func(value TimedValue) (bool, time.Duration) { remaining, err := nc.deletePods(value.Value) if err != nil { utilruntime.HandleError(fmt.Errorf("unable to evict node %q: %v", value.Value, err)) return false, 0 } if remaining { nc.terminationEvictor.Add(value.Value) } return true, 0 }) }, nodeEvictionPeriod, wait.NeverStop) // TODO: replace with a controller that ensures pods that are terminating complete // in a particular time period go wait.Until(func() { nc.evictorLock.Lock() defer nc.evictorLock.Unlock() nc.terminationEvictor.Try(func(value TimedValue) (bool, time.Duration) { completed, remaining, err := nc.terminatePods(value.Value, value.AddedAt) if err != nil { utilruntime.HandleError(fmt.Errorf("unable to terminate pods on node %q: %v", value.Value, err)) return false, 0 } if completed { glog.V(2).Infof("All pods terminated on %s", value.Value) nc.recordNodeEvent(value.Value, api.EventTypeNormal, "TerminatedAllPods", fmt.Sprintf("Terminated all Pods on Node %s.", value.Value)) return true, 0 } glog.V(2).Infof("Pods terminating since %s on %q, estimated completion %s", value.AddedAt, value.Value, remaining) // clamp very short intervals if remaining < nodeEvictionPeriod { remaining = nodeEvictionPeriod } return false, remaining }) }, nodeEvictionPeriod, wait.NeverStop) go wait.Until(nc.cleanupOrphanedPods, 30*time.Second, wait.NeverStop) for i := 0; i < cidrUpdateWorkers; i++ { go func(stopChan <-chan struct{}) { for { select { case workItem, ok := <-nc.nodeCIDRUpdateChannel: if !ok { glog.Warning("NodeCIDRUpdateChannel read returned false.") return } nc.updateCIDRAllocation(workItem) case <-stopChan: glog.V(0).Info("StopChannel is closed.") return } } }(wait.NeverStop) } }
func (dswp *desiredStateOfWorldPopulator) Run(stopCh <-chan struct{}) { wait.Until(dswp.populatorLoopFunc(), dswp.loopSleepDuration, stopCh) }
// Start spawns a goroutine to relist periodically. func (g *GenericPLEG) Start() { go wait.Until(g.relist, g.relistPeriod, wait.NeverStop) }
func (gcc *PodGCController) Run(stop <-chan struct{}) { go gcc.podStoreSyncer.Run(stop) go wait.Until(gcc.gc, gcCheckPeriod, stop) <-stop }