func (rc *RouteController) Run(syncPeriod time.Duration) {
go util.Forever(func() {
if err := rc.reconcileNodeRoutes(); err != nil {
glog.Errorf("Couldn't reconcile node routes: %v", err)
}
}, syncPeriod)
}
func startQinglet(k QingletBootstrap, podCfg *config.PodConfig, kc *QingletConfig) {
// start the qinglet
go util.Forever(func() { k.Run(podCfg.Updates()) }, 0)
// start the qinglet server
if kc.EnableServer {
go util.Forever(func() {
k.ListenAndServe(net.IP(kc.Address), kc.Port, kc.TLSOptions, kc.EnableDebuggingHandlers)
}, 0)
}
if kc.ReadOnlyPort > 0 {
go util.Forever(func() {
k.ListenAndServeReadOnly(net.IP(kc.Address), kc.ReadOnlyPort)
}, 0)
}
}
func NewSourceFile(path string, nodeName string, period time.Duration, updates chan<- interface{}) {
config := &sourceFile{
path: path,
nodeName: nodeName,
updates: updates,
}
glog.V(1).Infof("Watching path %q", path)
go util.Forever(config.run, period)
}
func (s *statusManager) Start() {
// syncBatch blocks when no updates are available, we can run it in a tight loop.
glog.Info("Starting to sync pod status with apiserver")
go util.Forever(func() {
err := s.syncBatch()
if err != nil {
glog.Warningf("Failed to updated pod status: %v", err)
}
}, 0)
}
// NewSourceAPI creates a config source that watches for changes to the services and endpoints.
func NewSourceAPI(servicesWatcher ServicesWatcher, endpointsWatcher EndpointsWatcher, period time.Duration, services chan<- ServiceUpdate, endpoints chan<- EndpointsUpdate) *SourceAPI {
config := &SourceAPI{
s: servicesReflector{
watcher: servicesWatcher,
services: services,
resourceVersion: "",
waitDuration: period,
// prevent hot loops if the server starts to misbehave
reconnectDuration: time.Second * 1,
},
e: endpointsReflector{
watcher: endpointsWatcher,
endpoints: endpoints,
resourceVersion: "",
waitDuration: period,
// prevent hot loops if the server starts to misbehave
reconnectDuration: time.Second * 1,
},
}
go util.Forever(func() { config.s.listAndWatch() }, period)
go util.Forever(func() { config.e.listAndWatch() }, period)
return config
}
// Channel returns a channel where a configuration source
// can send updates of new configurations. Multiple calls with the same
// source will return the same channel. This allows change and state based sources
// to use the same channel. Different source names however will be treated as a
// union.
func (m *Mux) Channel(source string) chan interface{} {
if len(source) == 0 {
panic("Channel given an empty name")
}
m.sourceLock.Lock()
defer m.sourceLock.Unlock()
channel, exists := m.sources[source]
if exists {
return channel
}
newChannel := make(chan interface{})
m.sources[source] = newChannel
go util.Forever(func() { m.listen(source, newChannel) }, 0)
return newChannel
}
func (im *realImageManager) Start() error {
// Initial detection make detected time "unknown" in the past.
var zero time.Time
err := im.detectImages(zero)
if err != nil {
return err
}
go util.Forever(func() {
err := im.detectImages(time.Now())
if err != nil {
glog.Warningf("[ImageManager] Failed to monitor images: %v", err)
}
}, 5*time.Minute)
return nil
}
// Elect implements the election.MasterElector interface.
func (e *etcdMasterElector) Elect(path, id string) watch.Interface {
e.done = make(chan empty)
e.events = make(chan watch.Event)
go util.Forever(func() { e.run(path, id) }, time.Second*5)
return e
}
// Run begins polling. It starts a goroutine and returns immediately.
func (p *Poller) Run() {
go util.Forever(p.run, p.period)
}
// Run starts a watch and handles watch events. Will restart the watch if it is closed.
// Run starts a goroutine and returns immediately.
func (r *Reflector) Run() {
go util.Forever(func() { r.listAndWatch(util.NeverStop) }, r.period)
}
// Run runs the specified QingletExecutorServer.
func (s *QingletExecutorServer) Run(hks hyperqing.Interface, _ []string) error {
rand.Seed(time.Now().UTC().UnixNano())
if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
log.Info(err)
}
var apiclient *client.Client
clientConfig, err := s.CreateAPIServerClientConfig()
if err == nil {
apiclient, err = client.New(clientConfig)
}
if err != nil {
// required for k8sm since we need to send api.Binding information
// back to the apiserver
log.Fatalf("No API client: %v", err)
}
log.Infof("Using root directory: %v", s.RootDirectory)
credentialprovider.SetPreferredDockercfgPath(s.RootDirectory)
shutdownCloser, err := s.syncExternalShutdownWatcher()
if err != nil {
return err
}
cadvisorInterface, err := cadvisor.New(s.CadvisorPort)
if err != nil {
return err
}
imageGCPolicy := qinglet.ImageGCPolicy{
HighThresholdPercent: s.ImageGCHighThresholdPercent,
LowThresholdPercent: s.ImageGCLowThresholdPercent,
}
diskSpacePolicy := qinglet.DiskSpacePolicy{
DockerFreeDiskMB: s.LowDiskSpaceThresholdMB,
RootFreeDiskMB: s.LowDiskSpaceThresholdMB,
}
//TODO(jdef) intentionally NOT initializing a cloud provider here since:
//(a) the qinglet doesn't actually use it
//(b) we don't need to create N-qinglet connections to zookeeper for no good reason
//cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
//log.Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
hostNetworkSources, err := qinglet.GetValidatedSources(strings.Split(s.HostNetworkSources, ","))
if err != nil {
return err
}
tlsOptions, err := s.InitializeTLS()
if err != nil {
return err
}
mounter := mount.New()
if s.Containerized {
log.V(2).Info("Running qinglet in containerized mode (experimental)")
mounter = &mount.NsenterMounter{}
}
var dockerExecHandler dockertools.ExecHandler
switch s.DockerExecHandlerName {
case "native":
dockerExecHandler = &dockertools.NativeExecHandler{}
case "nsenter":
dockerExecHandler = &dockertools.NsenterExecHandler{}
default:
log.Warningf("Unknown Docker exec handler %q; defaulting to native", s.DockerExecHandlerName)
dockerExecHandler = &dockertools.NativeExecHandler{}
}
kcfg := app.QingletConfig{
Address: s.Address,
AllowPrivileged: s.AllowPrivileged,
HostNetworkSources: hostNetworkSources,
HostnameOverride: s.HostnameOverride,
RootDirectory: s.RootDirectory,
// ConfigFile: ""
// ManifestURL: ""
FileCheckFrequency: s.FileCheckFrequency,
// HTTPCheckFrequency
PodInfraContainerImage: s.PodInfraContainerImage,
SyncFrequency: s.SyncFrequency,
RegistryPullQPS: s.RegistryPullQPS,
RegistryBurst: s.RegistryBurst,
MinimumGCAge: s.MinimumGCAge,
MaxPerPodContainerCount: s.MaxPerPodContainerCount,
MaxContainerCount: s.MaxContainerCount,
RegisterNode: s.RegisterNode,
// StandaloneMode: false
ClusterDomain: s.ClusterDomain,
ClusterDNS: s.ClusterDNS,
Runonce: s.RunOnce,
Port: s.Port,
ReadOnlyPort: s.ReadOnlyPort,
CadvisorInterface: cadvisorInterface,
EnableServer: s.EnableServer,
EnableDebuggingHandlers: s.EnableDebuggingHandlers,
DockerClient: dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
QingClient: apiclient,
MasterServiceNamespace: s.MasterServiceNamespace,
VolumePlugins: app.ProbeVolumePlugins(),
NetworkPlugins: app.ProbeNetworkPlugins(),
NetworkPluginName: s.NetworkPluginName,
StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout,
TLSOptions: tlsOptions,
ImageGCPolicy: imageGCPolicy,
DiskSpacePolicy: diskSpacePolicy,
Cloud: nil, // TODO(jdef) Cloud, specifying null here because we don't want all qinglets polling mesos-master; need to account for this in the cloudprovider impl
NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency,
ResourceContainer: s.ResourceContainer,
CgroupRoot: s.CgroupRoot,
ContainerRuntime: s.ContainerRuntime,
Mounter: mounter,
DockerDaemonContainer: s.DockerDaemonContainer,
SystemContainer: s.SystemContainer,
ConfigureCBR0: s.ConfigureCBR0,
MaxPods: s.MaxPods,
DockerExecHandler: dockerExecHandler,
}
kcfg.NodeName = kcfg.Hostname
err = app.RunQinglet(&kcfg, app.QingletBuilder(func(kc *app.QingletConfig) (app.QingletBootstrap, *kconfig.PodConfig, error) {
return s.createAndInitQinglet(kc, hks, clientConfig, shutdownCloser)
}))
if err != nil {
return err
}
if s.HealthzPort > 0 {
healthz.DefaultHealthz()
go util.Forever(func() {
err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress.String(), strconv.Itoa(s.HealthzPort)), nil)
if err != nil {
log.Errorf("Starting health server failed: %v", err)
}
}, 5*time.Second)
}
// block until executor is shut down or commits shutdown
select {}
}
// Run runs the specified QingletServer. This should never exit.
func (s *QingletServer) Run(_ []string) error {
util.ReallyCrash = s.ReallyCrashForTesting
rand.Seed(time.Now().UTC().UnixNano())
// TODO(vmarmol): Do this through container config.
if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
glog.Warning(err)
}
var apiclient *client.Client
clientConfig, err := s.CreateAPIServerClientConfig()
if err == nil {
apiclient, err = client.New(clientConfig)
}
if err != nil && len(s.APIServerList) > 0 {
glog.Warningf("No API client: %v", err)
}
glog.V(2).Infof("Using root directory: %v", s.RootDirectory)
credentialprovider.SetPreferredDockercfgPath(s.RootDirectory)
cadvisorInterface, err := cadvisor.New(s.CadvisorPort)
if err != nil {
return err
}
imageGCPolicy := qinglet.ImageGCPolicy{
HighThresholdPercent: s.ImageGCHighThresholdPercent,
LowThresholdPercent: s.ImageGCLowThresholdPercent,
}
diskSpacePolicy := qinglet.DiskSpacePolicy{
DockerFreeDiskMB: s.LowDiskSpaceThresholdMB,
RootFreeDiskMB: s.LowDiskSpaceThresholdMB,
}
cloud := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
hostNetworkSources, err := qinglet.GetValidatedSources(strings.Split(s.HostNetworkSources, ","))
if err != nil {
return err
}
tlsOptions, err := s.InitializeTLS()
if err != nil {
return err
}
mounter := mount.New()
if s.Containerized {
glog.V(2).Info("Running qinglet in containerized mode (experimental)")
mounter = &mount.NsenterMounter{}
}
var dockerExecHandler dockertools.ExecHandler
switch s.DockerExecHandlerName {
case "native":
dockerExecHandler = &dockertools.NativeExecHandler{}
case "nsenter":
dockerExecHandler = &dockertools.NsenterExecHandler{}
default:
glog.Warningf("Unknown Docker exec handler %q; defaulting to native", s.DockerExecHandlerName)
dockerExecHandler = &dockertools.NativeExecHandler{}
}
kcfg := QingletConfig{
Address: s.Address,
AllowPrivileged: s.AllowPrivileged,
HostNetworkSources: hostNetworkSources,
HostnameOverride: s.HostnameOverride,
RootDirectory: s.RootDirectory,
ConfigFile: s.Config,
ManifestURL: s.ManifestURL,
FileCheckFrequency: s.FileCheckFrequency,
HTTPCheckFrequency: s.HTTPCheckFrequency,
PodInfraContainerImage: s.PodInfraContainerImage,
SyncFrequency: s.SyncFrequency,
RegistryPullQPS: s.RegistryPullQPS,
RegistryBurst: s.RegistryBurst,
MinimumGCAge: s.MinimumGCAge,
MaxPerPodContainerCount: s.MaxPerPodContainerCount,
MaxContainerCount: s.MaxContainerCount,
RegisterNode: s.RegisterNode,
StandaloneMode: (len(s.APIServerList) == 0),
ClusterDomain: s.ClusterDomain,
ClusterDNS: s.ClusterDNS,
Runonce: s.RunOnce,
Port: s.Port,
ReadOnlyPort: s.ReadOnlyPort,
CadvisorInterface: cadvisorInterface,
EnableServer: s.EnableServer,
EnableDebuggingHandlers: s.EnableDebuggingHandlers,
DockerClient: dockertools.ConnectToDockerOrDie(s.DockerEndpoint),
QingClient: apiclient,
MasterServiceNamespace: s.MasterServiceNamespace,
VolumePlugins: ProbeVolumePlugins(),
NetworkPlugins: ProbeNetworkPlugins(),
NetworkPluginName: s.NetworkPluginName,
StreamingConnectionIdleTimeout: s.StreamingConnectionIdleTimeout,
TLSOptions: tlsOptions,
ImageGCPolicy: imageGCPolicy,
DiskSpacePolicy: diskSpacePolicy,
Cloud: cloud,
NodeStatusUpdateFrequency: s.NodeStatusUpdateFrequency,
ResourceContainer: s.ResourceContainer,
CgroupRoot: s.CgroupRoot,
ContainerRuntime: s.ContainerRuntime,
Mounter: mounter,
DockerDaemonContainer: s.DockerDaemonContainer,
SystemContainer: s.SystemContainer,
ConfigureCBR0: s.ConfigureCBR0,
PodCIDR: s.PodCIDR,
MaxPods: s.MaxPods,
DockerExecHandler: dockerExecHandler,
}
if err := RunQinglet(&kcfg, nil); err != nil {
return err
}
if s.HealthzPort > 0 {
healthz.DefaultHealthz()
go util.Forever(func() {
err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress.String(), strconv.Itoa(s.HealthzPort)), nil)
if err != nil {
glog.Errorf("Starting health server failed: %v", err)
}
}, 5*time.Second)
}
if s.RunOnce {
return nil
}
// run forever
select {}
}
// Run begins watching and syncing.
func (rm *ResourceQuotaManager) Run(period time.Duration) {
rm.syncTime = time.Tick(period)
go util.Forever(func() { rm.synchronize() }, period)
}
// Run runs the specified ProxyServer. This should never exit.
func (s *ProxyServer) Run(_ []string) error {
// TODO(vmarmol): Use container config for this.
if err := util.ApplyOomScoreAdj(0, s.OOMScoreAdj); err != nil {
glog.V(2).Info(err)
}
// Run in its own container.
if err := util.RunInResourceContainer(s.ResourceContainer); err != nil {
glog.Warningf("Failed to start in resource-only container %q: %v", s.ResourceContainer, err)
} else {
glog.V(2).Infof("Running in resource-only container %q", s.ResourceContainer)
}
serviceConfig := config.NewServiceConfig()
endpointsConfig := config.NewEndpointsConfig()
protocol := iptables.ProtocolIpv4
if net.IP(s.BindAddress).To4() == nil {
protocol = iptables.ProtocolIpv6
}
loadBalancer := proxy.NewLoadBalancerRR()
proxier, err := proxy.NewProxier(loadBalancer, net.IP(s.BindAddress), iptables.New(exec.New(), protocol), s.PortRange)
if err != nil {
glog.Fatalf("Unable to create proxer: %v", err)
}
// Wire proxier to handle changes to services
serviceConfig.RegisterHandler(proxier)
// And wire loadBalancer to handle changes to endpoints to services
endpointsConfig.RegisterHandler(loadBalancer)
// Note: RegisterHandler() calls need to happen before creation of Sources because sources
// only notify on changes, and the initial update (on process start) may be lost if no handlers
// are registered yet.
// define api config source
if s.Qingconfig == "" && s.Master == "" {
glog.Warningf("Neither --qingconfig nor --master was specified. Using default API client. This might not work.")
}
// This creates a client, first loading any specified qingconfig
// file, and then overriding the Master flag, if non-empty.
qingconfig, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
&clientcmd.ClientConfigLoadingRules{ExplicitPath: s.Qingconfig},
&clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: s.Master}}).ClientConfig()
if err != nil {
return err
}
client, err := client.New(qingconfig)
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
}
config.NewSourceAPI(
client.Services(api.NamespaceAll),
client.Endpoints(api.NamespaceAll),
30*time.Second,
serviceConfig.Channel("api"),
endpointsConfig.Channel("api"),
)
if s.HealthzPort > 0 {
go util.Forever(func() {
err := http.ListenAndServe(s.HealthzBindAddress.String()+":"+strconv.Itoa(s.HealthzPort), nil)
if err != nil {
glog.Errorf("Starting health server failed: %v", err)
}
}, 5*time.Second)
}
// Just loop forever for now...
proxier.SyncLoop()
return nil
}