// issue: https://github.com/kubernetes/kubernetes/issues/23218 func TestDeploymentController_dontSyncDeploymentsWithEmptyPodSelector(t *testing.T) { fake := &fake.Clientset{} informers := informers.NewSharedInformerFactory(fake, nil, controller.NoResyncPeriodFunc()) controller := NewDeploymentController(informers.Deployments(), informers.ReplicaSets(), informers.Pods(), fake) controller.eventRecorder = &record.FakeRecorder{} controller.dListerSynced = alwaysReady controller.rsListerSynced = alwaysReady controller.podListerSynced = alwaysReady stopCh := make(chan struct{}) defer close(stopCh) informers.Start(stopCh) d := newDeployment("foo", 1, nil, nil, nil, map[string]string{"foo": "bar"}) empty := metav1.LabelSelector{} d.Spec.Selector = &empty controller.dLister.Indexer.Add(d) // We expect the deployment controller to not take action here since it's configuration // is invalid, even though no replicasets exist that match it's selector. controller.syncDeployment(fmt.Sprintf("%s/%s", d.ObjectMeta.Namespace, d.ObjectMeta.Name)) filteredActions := filterInformerActions(fake.Actions()) if len(filteredActions) == 0 { return } for _, action := range filteredActions { t.Logf("unexpected action: %#v", action) } t.Errorf("expected deployment controller to not take action") }
// NewclusterController returns a new cluster controller func NewclusterController(federationClient federationclientset.Interface, clusterMonitorPeriod time.Duration) *ClusterController { cc := &ClusterController{ knownClusterSet: make(sets.String), federationClient: federationClient, clusterMonitorPeriod: clusterMonitorPeriod, clusterClusterStatusMap: make(map[string]federationv1beta1.ClusterStatus), clusterKubeClientMap: make(map[string]ClusterClient), } cc.clusterStore.Store, cc.clusterController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (runtime.Object, error) { return cc.federationClient.Federation().Clusters().List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return cc.federationClient.Federation().Clusters().Watch(options) }, }, &federationv1beta1.Cluster{}, controller.NoResyncPeriodFunc(), cache.ResourceEventHandlerFuncs{ DeleteFunc: cc.delFromClusterSet, AddFunc: cc.addToClusterSet, }, ) return cc }
func NewPodGC(kubeClient clientset.Interface, podInformer cache.SharedIndexInformer, terminatedPodThreshold int) *PodGCController { if kubeClient != nil && kubeClient.Core().RESTClient().GetRateLimiter() != nil { metrics.RegisterMetricAndTrackRateLimiterUsage("gc_controller", kubeClient.Core().RESTClient().GetRateLimiter()) } gcc := &PodGCController{ kubeClient: kubeClient, terminatedPodThreshold: terminatedPodThreshold, deletePod: func(namespace, name string) error { glog.Infof("PodGC is force deleting Pod: %v:%v", namespace, name) return kubeClient.Core().Pods(namespace).Delete(name, v1.NewDeleteOptions(0)) }, } gcc.podStore.Indexer = podInformer.GetIndexer() gcc.podController = podInformer.GetController() gcc.nodeStore.Store, gcc.nodeController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (runtime.Object, error) { return gcc.kubeClient.Core().Nodes().List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return gcc.kubeClient.Core().Nodes().Watch(options) }, }, &v1.Node{}, controller.NoResyncPeriodFunc(), cache.ResourceEventHandlerFuncs{}, ) return gcc }
func NewNodeControllerFromClient( cloud cloudprovider.Interface, kubeClient clientset.Interface, podEvictionTimeout time.Duration, evictionLimiterQPS float32, secondaryEvictionLimiterQPS float32, largeClusterThreshold int32, unhealthyZoneThreshold float32, nodeMonitorGracePeriod time.Duration, nodeStartupGracePeriod time.Duration, nodeMonitorPeriod time.Duration, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, nodeCIDRMaskSize int, allocateNodeCIDRs bool) (*NodeController, error) { podInformer := informers.NewPodInformer(kubeClient, controller.NoResyncPeriodFunc()) nc, err := NewNodeController(podInformer, cloud, kubeClient, podEvictionTimeout, evictionLimiterQPS, secondaryEvictionLimiterQPS, largeClusterThreshold, unhealthyZoneThreshold, nodeMonitorGracePeriod, nodeStartupGracePeriod, nodeMonitorPeriod, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, allocateNodeCIDRs) if err != nil { return nil, err } nc.internalPodInformer = podInformer return nc, nil }
func New(routes cloudprovider.Routes, kubeClient clientset.Interface, clusterName string, clusterCIDR *net.IPNet) *RouteController { if kubeClient != nil && kubeClient.Core().RESTClient().GetRateLimiter() != nil { metrics.RegisterMetricAndTrackRateLimiterUsage("route_controller", kubeClient.Core().RESTClient().GetRateLimiter()) } rc := &RouteController{ routes: routes, kubeClient: kubeClient, clusterName: clusterName, clusterCIDR: clusterCIDR, } rc.nodeStore.Store, rc.nodeController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (runtime.Object, error) { return rc.kubeClient.Core().Nodes().List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return rc.kubeClient.Core().Nodes().Watch(options) }, }, &v1.Node{}, controller.NoResyncPeriodFunc(), cache.ResourceEventHandlerFuncs{}, ) return rc }
func NewFromClient( kubeClient clientset.Interface, terminatedPodThreshold int, ) *PodGCController { podInformer := informers.NewPodInformer(kubeClient, controller.NoResyncPeriodFunc()) controller := NewPodGC(kubeClient, podInformer, terminatedPodThreshold) controller.internalPodInformer = podInformer return controller }
func TestScaleDownOldReplicaSets(t *testing.T) { tests := []struct { oldRSSizes []int d *extensions.Deployment }{ { oldRSSizes: []int{3}, d: newDeployment("foo", 3, nil, nil, nil, map[string]string{"foo": "bar"}), }, } for i := range tests { t.Logf("running scenario %d", i) test := tests[i] var oldRSs []*extensions.ReplicaSet var expected []runtime.Object for n, size := range test.oldRSSizes { rs := newReplicaSet(test.d, fmt.Sprintf("%s-%d", test.d.Name, n), size) oldRSs = append(oldRSs, rs) objCopy, err := api.Scheme.Copy(rs) if err != nil { t.Errorf("unexpected error while deep-copying: %v", err) continue } rsCopy := objCopy.(*extensions.ReplicaSet) zero := int32(0) rsCopy.Spec.Replicas = &zero expected = append(expected, rsCopy) if *(oldRSs[n].Spec.Replicas) == *(expected[n].(*extensions.ReplicaSet).Spec.Replicas) { t.Errorf("broken test - original and expected RS have the same size") } } kc := fake.NewSimpleClientset(expected...) informers := informers.NewSharedInformerFactory(kc, nil, controller.NoResyncPeriodFunc()) c := NewDeploymentController(informers.Deployments(), informers.ReplicaSets(), informers.Pods(), kc) c.scaleDownOldReplicaSetsForRecreate(oldRSs, test.d) for j := range oldRSs { rs := oldRSs[j] if *rs.Spec.Replicas != 0 { t.Errorf("rs %q has non-zero replicas", rs.Name) } } } }
func newTestController(initialObjects ...runtime.Object) (*DaemonSetsController, *controller.FakePodControl, *fake.Clientset) { clientset := fake.NewSimpleClientset(initialObjects...) informerFactory := informers.NewSharedInformerFactory(clientset, nil, controller.NoResyncPeriodFunc()) manager := NewDaemonSetsController(informerFactory.DaemonSets(), informerFactory.Pods(), informerFactory.Nodes(), clientset, 0) manager.podStoreSynced = alwaysReady manager.nodeStoreSynced = alwaysReady manager.dsStoreSynced = alwaysReady podControl := &controller.FakePodControl{} manager.podControl = podControl return manager, podControl, clientset }
func newTestController() (*DaemonSetsController, *controller.FakePodControl) { clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: "", ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) informerFactory := informers.NewSharedInformerFactory(clientset, controller.NoResyncPeriodFunc()) manager := NewDaemonSetsController(informerFactory.DaemonSets(), informerFactory.Pods(), informerFactory.Nodes(), clientset, 0) informerFactory.Start(wait.NeverStop) manager.podStoreSynced = alwaysReady manager.nodeStoreSynced = alwaysReady podControl := &controller.FakePodControl{} manager.podControl = podControl return manager, podControl }
func (f *fixture) run(deploymentName string) { f.client = fake.NewSimpleClientset(f.objects...) informers := informers.NewSharedInformerFactory(f.client, nil, controller.NoResyncPeriodFunc()) c := NewDeploymentController(informers.Deployments(), informers.ReplicaSets(), informers.Pods(), f.client) c.eventRecorder = &record.FakeRecorder{} c.dListerSynced = alwaysReady c.rsListerSynced = alwaysReady c.podListerSynced = alwaysReady for _, d := range f.dLister { c.dLister.Indexer.Add(d) } for _, rs := range f.rsLister { c.rsLister.Indexer.Add(rs) } for _, pod := range f.podLister { c.podLister.Indexer.Add(pod) } stopCh := make(chan struct{}) defer close(stopCh) informers.Start(stopCh) err := c.syncDeployment(deploymentName) if err != nil { f.t.Errorf("error syncing deployment: %v", err) } actions := filterInformerActions(f.client.Actions()) for i, action := range actions { if len(f.actions) < i+1 { f.t.Errorf("%d unexpected actions: %+v", len(actions)-len(f.actions), actions[i:]) break } expectedAction := f.actions[i] if !expectedAction.Matches(action.GetVerb(), action.GetResource().Resource) { f.t.Errorf("Expected\n\t%#v\ngot\n\t%#v", expectedAction, action) continue } } if len(f.actions) > len(actions) { f.t.Errorf("%d additional expected actions:%+v", len(f.actions)-len(actions), f.actions[len(actions):]) } }
func (f *fixture) newController() (*DeploymentController, informers.SharedInformerFactory) { f.client = fake.NewSimpleClientset(f.objects...) informers := informers.NewSharedInformerFactory(f.client, nil, controller.NoResyncPeriodFunc()) c := NewDeploymentController(informers.Deployments(), informers.ReplicaSets(), informers.Pods(), f.client) c.eventRecorder = &record.FakeRecorder{} c.dListerSynced = alwaysReady c.rsListerSynced = alwaysReady c.podListerSynced = alwaysReady for _, d := range f.dLister { c.dLister.Indexer.Add(d) } for _, rs := range f.rsLister { c.rsLister.Indexer.Add(rs) } for _, pod := range f.podLister { c.podLister.Indexer.Add(pod) } return c, informers }
func startComponents(firstManifestURL, secondManifestURL string) (string, string) { // Setup handler := delegateHandler{} apiServer := httptest.NewServer(&handler) cfg := etcd.Config{ Endpoints: []string{"http://127.0.0.1:4001"}, } etcdClient, err := etcd.New(cfg) if err != nil { glog.Fatalf("Error creating etcd client: %v", err) } glog.Infof("Creating etcd client pointing to %v", cfg.Endpoints) keysAPI := etcd.NewKeysAPI(etcdClient) sleep := 4 * time.Second ok := false for i := 0; i < 3; i++ { keys, err := keysAPI.Get(context.TODO(), "/", nil) if err != nil { glog.Warningf("Unable to list root etcd keys: %v", err) if i < 2 { time.Sleep(sleep) sleep = sleep * sleep } continue } for _, node := range keys.Node.Nodes { if _, err := keysAPI.Delete(context.TODO(), node.Key, &etcd.DeleteOptions{Recursive: true}); err != nil { glog.Fatalf("Unable delete key: %v", err) } } ok = true break } if !ok { glog.Fatalf("Failed to connect to etcd") } cl := client.NewOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}}) // TODO: caesarxuchao: hacky way to specify version of Experimental client. // We will fix this by supporting multiple group versions in Config cl.ExtensionsClient = client.NewExtensionsOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Extensions.GroupVersion()}}) // Master host, port, err := net.SplitHostPort(strings.TrimLeft(apiServer.URL, "http://")) if err != nil { glog.Fatalf("Unable to parse URL '%v': %v", apiServer.URL, err) } portNumber, err := strconv.Atoi(port) if err != nil { glog.Fatalf("Nonnumeric port? %v", err) } publicAddress := net.ParseIP(host) if publicAddress == nil { glog.Fatalf("No public address for %s", host) } // The caller of master.New should guarantee pulicAddress is properly set hostIP, err := utilnet.ChooseBindAddress(publicAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Fail to get a valid public address for master.", err) } masterConfig := framework.NewMasterConfig() masterConfig.EnableCoreControllers = true masterConfig.EnableProfiling = true masterConfig.ReadWritePort = portNumber masterConfig.PublicAddress = hostIP masterConfig.CacheTimeout = 2 * time.Second masterConfig.EnableWatchCache = watchCache // Create a master and install handlers into mux. m, err := master.New(masterConfig) if err != nil { glog.Fatalf("Error in bringing up the master: %v", err) } handler.delegate = m.Handler // Scheduler schedulerConfigFactory := factory.NewConfigFactory(cl, api.DefaultSchedulerName) schedulerConfig, err := schedulerConfigFactory.Create() if err != nil { glog.Fatalf("Couldn't create scheduler config: %v", err) } eventBroadcaster := record.NewBroadcaster() schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName}) eventBroadcaster.StartLogging(glog.Infof) eventBroadcaster.StartRecordingToSink(cl.Events("")) scheduler.New(schedulerConfig).Run() podInformer := informers.CreateSharedPodInformer(clientset, controller.NoResyncPeriodFunc()) // ensure the service endpoints are sync'd several times within the window that the integration tests wait go endpointcontroller.NewEndpointController(podInformer, clientset). Run(3, wait.NeverStop) // TODO: Write an integration test for the replication controllers watch. go replicationcontroller.NewReplicationManager(podInformer, clientset, controller.NoResyncPeriodFunc, replicationcontroller.BurstReplicas, 4096). Run(3, wait.NeverStop) go podInformer.Run(wait.NeverStop) nodeController := nodecontroller.NewNodeController(nil, clientset, 5*time.Minute, flowcontrol.NewFakeAlwaysRateLimiter(), flowcontrol.NewFakeAlwaysRateLimiter(), 40*time.Second, 60*time.Second, 5*time.Second, nil, false) nodeController.Run(5 * time.Second) cadvisorInterface := new(cadvisortest.Fake) // Kubelet (localhost) testRootDir := integration.MakeTempDirOrDie("kubelet_integ_1.", "") configFilePath := integration.MakeTempDirOrDie("config", testRootDir) glog.Infof("Using %s as root dir for kubelet #1", testRootDir) cm := cm.NewStubContainerManager() kcfg := kubeletapp.SimpleKubelet( clientset, fakeDocker1, "localhost", testRootDir, firstManifestURL, "127.0.0.1", 10250, /* KubeletPort */ 0, /* ReadOnlyPort */ api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, configFilePath, nil, containertest.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ 1*time.Second, /* HTTPCheckFrequency */ 10*time.Second, /* MinimumGCAge */ 3*time.Second, /* NodeStatusUpdateFrequency */ 10*time.Second, /* SyncFrequency */ 10*time.Second, /* OutOfDiskTransitionFrequency */ 40, /* MaxPods */ cm, net.ParseIP("127.0.0.1")) kubeletapp.RunKubelet(kcfg) // Kubelet (machine) // Create a second kubelet so that the guestbook example's two redis slaves both // have a place they can schedule. testRootDir = integration.MakeTempDirOrDie("kubelet_integ_2.", "") glog.Infof("Using %s as root dir for kubelet #2", testRootDir) kcfg = kubeletapp.SimpleKubelet( clientset, fakeDocker2, "127.0.0.1", testRootDir, secondManifestURL, "127.0.0.1", 10251, /* KubeletPort */ 0, /* ReadOnlyPort */ api.NamespaceDefault, empty_dir.ProbeVolumePlugins(), nil, cadvisorInterface, "", nil, containertest.FakeOS{}, 1*time.Second, /* FileCheckFrequency */ 1*time.Second, /* HTTPCheckFrequency */ 10*time.Second, /* MinimumGCAge */ 3*time.Second, /* NodeStatusUpdateFrequency */ 10*time.Second, /* SyncFrequency */ 10*time.Second, /* OutOfDiskTransitionFrequency */ 40, /* MaxPods */ cm, net.ParseIP("127.0.0.1")) kubeletapp.RunKubelet(kcfg) return apiServer.URL, configFilePath }
// NewclusterController returns a new cluster controller func NewReplicaSetController(federationClient fedclientset.Interface) *ReplicaSetController { frsc := &ReplicaSetController{ fedClient: federationClient, replicasetDeliverer: fedutil.NewDelayingDeliverer(), clusterDeliverer: fedutil.NewDelayingDeliverer(), replicasetWorkQueue: workqueue.New(), replicaSetBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), defaultPlanner: planner.NewPlanner(&fed.FederatedReplicaSetPreferences{ Clusters: map[string]fed.ClusterReplicaSetPreferences{ "*": {Weight: 1}, }, }), } replicaSetFedInformerFactory := func(cluster *fedv1.Cluster, clientset kubeclientset.Interface) (cache.Store, framework.ControllerInterface) { return framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return clientset.Extensions().ReplicaSets(apiv1.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return clientset.Extensions().ReplicaSets(apiv1.NamespaceAll).Watch(options) }, }, &extensionsv1.ReplicaSet{}, controller.NoResyncPeriodFunc(), fedutil.NewTriggerOnAllChanges( func(obj runtime.Object) { frsc.deliverLocalReplicaSet(obj, allReplicaSetReviewDealy) }, ), ) } clusterLifecycle := fedutil.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *fedv1.Cluster) { frsc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterUnavailableDelay) }, ClusterUnavailable: func(cluster *fedv1.Cluster, _ []interface{}) { frsc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterUnavailableDelay) }, } frsc.fedReplicaSetInformer = fedutil.NewFederatedInformer(federationClient, replicaSetFedInformerFactory, &clusterLifecycle) podFedInformerFactory := func(cluster *fedv1.Cluster, clientset kubeclientset.Interface) (cache.Store, framework.ControllerInterface) { return framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return clientset.Core().Pods(apiv1.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return clientset.Core().Pods(apiv1.NamespaceAll).Watch(options) }, }, &apiv1.Pod{}, controller.NoResyncPeriodFunc(), fedutil.NewTriggerOnAllChanges( func(obj runtime.Object) { frsc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterUnavailableDelay) }, ), ) } frsc.fedPodInformer = fedutil.NewFederatedInformer(federationClient, podFedInformerFactory, &fedutil.ClusterLifecycleHandlerFuncs{}) frsc.replicaSetStore.Store, frsc.replicaSetController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return frsc.fedClient.Extensions().ReplicaSets(apiv1.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return frsc.fedClient.Extensions().ReplicaSets(apiv1.NamespaceAll).Watch(options) }, }, &extensionsv1.ReplicaSet{}, controller.NoResyncPeriodFunc(), fedutil.NewTriggerOnMetaAndSpecChanges( func(obj runtime.Object) { frsc.deliverFedReplicaSetObj(obj, replicaSetReviewDelay) }, ), ) return frsc }
// NewNodeController returns a new node controller to sync instances from cloudprovider. func NewNodeController( cloud cloudprovider.Interface, kubeClient clientset.Interface, podEvictionTimeout time.Duration, deletionEvictionLimiter flowcontrol.RateLimiter, terminationEvictionLimiter flowcontrol.RateLimiter, nodeMonitorGracePeriod time.Duration, nodeStartupGracePeriod time.Duration, nodeMonitorPeriod time.Duration, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, nodeCIDRMaskSize int, allocateNodeCIDRs bool) *NodeController { eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "controllermanager"}) eventBroadcaster.StartLogging(glog.Infof) if kubeClient != nil { glog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: kubeClient.Core().Events("")}) } else { glog.V(0).Infof("No api server defined - no events will be sent to API server.") } if kubeClient != nil && kubeClient.Core().GetRESTClient().GetRateLimiter() != nil { metrics.RegisterMetricAndTrackRateLimiterUsage("node_controller", kubeClient.Core().GetRESTClient().GetRateLimiter()) } if allocateNodeCIDRs { if clusterCIDR == nil { glog.Fatal("NodeController: Must specify clusterCIDR if allocateNodeCIDRs == true.") } mask := clusterCIDR.Mask if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize { glog.Fatal("NodeController: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.") } } evictorLock := sync.Mutex{} nc := &NodeController{ cloud: cloud, knownNodeSet: make(sets.String), kubeClient: kubeClient, recorder: recorder, podEvictionTimeout: podEvictionTimeout, maximumGracePeriod: 5 * time.Minute, evictorLock: &evictorLock, podEvictor: NewRateLimitedTimedQueue(deletionEvictionLimiter), terminationEvictor: NewRateLimitedTimedQueue(terminationEvictionLimiter), nodeStatusMap: make(map[string]nodeStatusData), nodeMonitorGracePeriod: nodeMonitorGracePeriod, nodeMonitorPeriod: nodeMonitorPeriod, nodeStartupGracePeriod: nodeStartupGracePeriod, lookupIP: net.LookupIP, now: unversioned.Now, clusterCIDR: clusterCIDR, serviceCIDR: serviceCIDR, allocateNodeCIDRs: allocateNodeCIDRs, forcefullyDeletePod: func(p *api.Pod) error { return forcefullyDeletePod(kubeClient, p) }, nodeExistsInCloudProvider: func(nodeName string) (bool, error) { return nodeExistsInCloudProvider(cloud, nodeName) }, nodeCIDRUpdateChannel: make(chan nodeAndCIDR, cidrUpdateQueueSize), } nc.podStore.Indexer, nc.podController = framework.NewIndexerInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).Watch(options) }, }, &api.Pod{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{ AddFunc: nc.maybeDeleteTerminatingPod, UpdateFunc: func(_, obj interface{}) { nc.maybeDeleteTerminatingPod(obj) }, }, // We don't need to build a index for podStore here actually, but build one for consistency. // It will ensure that if people start making use of the podStore in more specific ways, // they'll get the benefits they expect. It will also reserve the name for future refactorings. cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, ) nodeEventHandlerFuncs := framework.ResourceEventHandlerFuncs{} if nc.allocateNodeCIDRs { nodeEventHandlerFuncs = framework.ResourceEventHandlerFuncs{ AddFunc: nc.allocateOrOccupyCIDR, DeleteFunc: nc.recycleCIDR, } } nc.nodeStore.Store, nc.nodeController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Nodes().List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Nodes().Watch(options) }, }, &api.Node{}, controller.NoResyncPeriodFunc(), nodeEventHandlerFuncs, ) nc.daemonSetStore.Store, nc.daemonSetController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).Watch(options) }, }, &extensions.DaemonSet{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{}, ) if allocateNodeCIDRs { nc.cidrAllocator = NewCIDRRangeAllocator(clusterCIDR, nodeCIDRMaskSize) } return nc }
func New(federationClient fedclientset.Interface, dns dnsprovider.Interface, federationName, serviceDnsSuffix, zoneName string, zoneID string) *ServiceController { broadcaster := record.NewBroadcaster() // federationClient event is not supported yet // broadcaster.StartRecordingToSink(&unversioned_core.EventSinkImpl{Interface: kubeClient.Core().Events("")}) recorder := broadcaster.NewRecorder(v1.EventSource{Component: UserAgentName}) s := &ServiceController{ dns: dns, federationClient: federationClient, federationName: federationName, serviceDnsSuffix: serviceDnsSuffix, zoneName: zoneName, zoneID: zoneID, serviceCache: &serviceCache{fedServiceMap: make(map[string]*cachedService)}, clusterCache: &clusterClientCache{ rwlock: sync.Mutex{}, clientMap: make(map[string]*clusterCache), }, eventBroadcaster: broadcaster, eventRecorder: recorder, queue: workqueue.New(), knownClusterSet: make(sets.String), } s.clusterDeliverer = util.NewDelayingDeliverer() s.serviceStore.Indexer, s.serviceController = cache.NewIndexerInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (pkgruntime.Object, error) { return s.federationClient.Core().Services(v1.NamespaceAll).List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return s.federationClient.Core().Services(v1.NamespaceAll).Watch(options) }, }, &v1.Service{}, serviceSyncPeriod, cache.ResourceEventHandlerFuncs{ AddFunc: s.enqueueService, UpdateFunc: func(old, cur interface{}) { // there is case that old and new are equals but we still catch the event now. if !reflect.DeepEqual(old, cur) { s.enqueueService(cur) } }, DeleteFunc: s.enqueueService, }, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, ) s.clusterStore.Store, s.clusterController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (pkgruntime.Object, error) { return s.federationClient.Federation().Clusters().List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return s.federationClient.Federation().Clusters().Watch(options) }, }, &v1beta1.Cluster{}, clusterSyncPeriod, cache.ResourceEventHandlerFuncs{ DeleteFunc: s.clusterCache.delFromClusterSet, AddFunc: s.clusterCache.addToClientMap, UpdateFunc: func(old, cur interface{}) { oldCluster, ok := old.(*v1beta1.Cluster) if !ok { return } curCluster, ok := cur.(*v1beta1.Cluster) if !ok { return } if !reflect.DeepEqual(oldCluster.Spec, curCluster.Spec) { // update when spec is changed s.clusterCache.addToClientMap(cur) } pred := getClusterConditionPredicate() // only update when condition changed to ready from not-ready if !pred(*oldCluster) && pred(*curCluster) { s.clusterCache.addToClientMap(cur) } // did not handle ready -> not-ready // how could we stop a controller? }, }, ) clusterLifecycle := fedutil.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *v1beta1.Cluster) { s.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterAvailableDelay) }, } fedInformerFactory := func(cluster *v1beta1.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.Controller) { return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (pkgruntime.Object, error) { return targetClient.Core().Services(v1.NamespaceAll).List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return targetClient.Core().Services(v1.NamespaceAll).Watch(options) }, }, &v1.Service{}, controller.NoResyncPeriodFunc(), // Trigger reconciliation whenever something in federated cluster is changed. In most cases it // would be just confirmation that some service operation succeeded. util.NewTriggerOnAllChanges( func(obj pkgruntime.Object) { // TODO: Use this to enque services. }, )) } s.federatedInformer = fedutil.NewFederatedInformer(federationClient, fedInformerFactory, &clusterLifecycle) federatedUpdater := fedutil.NewFederatedUpdater(s.federatedInformer, func(client kubeclientset.Interface, obj pkgruntime.Object) error { svc := obj.(*v1.Service) _, err := client.Core().Services(svc.Namespace).Create(svc) return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { svc := obj.(*v1.Service) _, err := client.Core().Services(svc.Namespace).Update(svc) return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { svc := obj.(*v1.Service) err := client.Core().Services(svc.Namespace).Delete(svc.Name, &v1.DeleteOptions{}) return err }) s.deletionHelper = deletionhelper.NewDeletionHelper( s.hasFinalizerFunc, s.removeFinalizerFunc, s.addFinalizerFunc, // objNameFunc func(obj pkgruntime.Object) string { service := obj.(*v1.Service) return service.Name }, updateTimeout, s.eventRecorder, s.federatedInformer, federatedUpdater, ) s.endpointWorkerMap = make(map[string]bool) s.serviceWorkerMap = make(map[string]bool) s.endpointWorkerDoneChan = make(chan string, maxNoOfClusters) s.serviceWorkerDoneChan = make(chan string, maxNoOfClusters) return s }
// NewDaemonSetController returns a new daemonset controller func NewDaemonSetController(client federationclientset.Interface) *DaemonSetController { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client)) recorder := broadcaster.NewRecorder(api_v1.EventSource{Component: "federated-daemonset-controller"}) daemonsetcontroller := &DaemonSetController{ federatedApiClient: client, daemonsetReviewDelay: time.Second * 10, clusterAvailableDelay: time.Second * 20, smallDelay: time.Second * 3, updateTimeout: time.Second * 30, daemonsetBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), eventRecorder: recorder, } // Build deliverers for triggering reconciliations. daemonsetcontroller.daemonsetDeliverer = util.NewDelayingDeliverer() daemonsetcontroller.clusterDeliverer = util.NewDelayingDeliverer() // Start informer in federated API servers on daemonsets that should be federated. daemonsetcontroller.daemonsetInformerStore, daemonsetcontroller.daemonsetInformerController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api_v1.ListOptions) (pkg_runtime.Object, error) { return client.Extensions().DaemonSets(api_v1.NamespaceAll).List(options) }, WatchFunc: func(options api_v1.ListOptions) (watch.Interface, error) { return client.Extensions().DaemonSets(api_v1.NamespaceAll).Watch(options) }, }, &extensionsv1.DaemonSet{}, controller.NoResyncPeriodFunc(), util.NewTriggerOnAllChanges(func(obj pkg_runtime.Object) { daemonsetcontroller.deliverDaemonSetObj(obj, 0, false) })) // Federated informer on daemonsets in members of federation. daemonsetcontroller.daemonsetFederatedInformer = util.NewFederatedInformer( client, func(cluster *federation_api.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.ControllerInterface) { return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api_v1.ListOptions) (pkg_runtime.Object, error) { return targetClient.Extensions().DaemonSets(api_v1.NamespaceAll).List(options) }, WatchFunc: func(options api_v1.ListOptions) (watch.Interface, error) { return targetClient.Extensions().DaemonSets(api_v1.NamespaceAll).Watch(options) }, }, &extensionsv1.DaemonSet{}, controller.NoResyncPeriodFunc(), // Trigger reconciliation whenever something in federated cluster is changed. In most cases it // would be just confirmation that some daemonset opration succeeded. util.NewTriggerOnAllChanges( func(obj pkg_runtime.Object) { daemonsetcontroller.deliverDaemonSetObj(obj, daemonsetcontroller.daemonsetReviewDelay, false) }, )) }, &util.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *federation_api.Cluster) { // When new cluster becomes available process all the daemonsets again. daemonsetcontroller.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(daemonsetcontroller.clusterAvailableDelay)) }, }, ) // Federated updater along with Create/Update/Delete operations. daemonsetcontroller.federatedUpdater = util.NewFederatedUpdater(daemonsetcontroller.daemonsetFederatedInformer, func(client kubeclientset.Interface, obj pkg_runtime.Object) error { daemonset := obj.(*extensionsv1.DaemonSet) glog.V(4).Infof("Attempting to create daemonset: %s/%s", daemonset.Namespace, daemonset.Name) _, err := client.Extensions().DaemonSets(daemonset.Namespace).Create(daemonset) if err != nil { glog.Errorf("Error creating daemonset %s/%s/: %v", daemonset.Namespace, daemonset.Name, err) } else { glog.V(4).Infof("Successfully created deamonset %s/%s", daemonset.Namespace, daemonset.Name) } return err }, func(client kubeclientset.Interface, obj pkg_runtime.Object) error { daemonset := obj.(*extensionsv1.DaemonSet) glog.V(4).Infof("Attempting to update daemonset: %s/%s", daemonset.Namespace, daemonset.Name) _, err := client.Extensions().DaemonSets(daemonset.Namespace).Update(daemonset) if err != nil { glog.Errorf("Error updating daemonset %s/%s/: %v", daemonset.Namespace, daemonset.Name, err) } else { glog.V(4).Infof("Successfully updating deamonset %s/%s", daemonset.Namespace, daemonset.Name) } return err }, func(client kubeclientset.Interface, obj pkg_runtime.Object) error { daemonset := obj.(*extensionsv1.DaemonSet) glog.V(4).Infof("Attempting to delete daemonset: %s/%s", daemonset.Namespace, daemonset.Name) err := client.Extensions().DaemonSets(daemonset.Namespace).Delete(daemonset.Name, &api_v1.DeleteOptions{}) if err != nil { glog.Errorf("Error deleting daemonset %s/%s/: %v", daemonset.Namespace, daemonset.Name, err) } else { glog.V(4).Infof("Successfully deleting deamonset %s/%s", daemonset.Namespace, daemonset.Name) } return err }) daemonsetcontroller.deletionHelper = deletionhelper.NewDeletionHelper( daemonsetcontroller.hasFinalizerFunc, daemonsetcontroller.removeFinalizerFunc, daemonsetcontroller.addFinalizerFunc, // objNameFunc func(obj pkg_runtime.Object) string { daemonset := obj.(*extensionsv1.DaemonSet) return daemonset.Name }, daemonsetcontroller.updateTimeout, daemonsetcontroller.eventRecorder, daemonsetcontroller.daemonsetFederatedInformer, daemonsetcontroller.federatedUpdater, ) return daemonsetcontroller }
// startServiceAccountTestServer returns a started server // It is the responsibility of the caller to ensure the returned stopFunc is called func startServiceAccountTestServer(t *testing.T) (*clientset.Clientset, restclient.Config, func()) { // Listener h := &framework.MasterHolder{Initialized: make(chan struct{})} apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { <-h.Initialized h.M.GenericAPIServer.Handler.ServeHTTP(w, req) })) // Anonymous client config clientConfig := restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(v1.GroupName).GroupVersion}} // Root client // TODO: remove rootClient after we refactor pkg/admission to use the clientset. rootClientset := clientset.NewForConfigOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(v1.GroupName).GroupVersion}, BearerToken: rootToken}) internalRootClientset := internalclientset.NewForConfigOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(v1.GroupName).GroupVersion}, BearerToken: rootToken}) // Set up two authenticators: // 1. A token authenticator that maps the rootToken to the "root" user // 2. A ServiceAccountToken authenticator that validates ServiceAccount tokens rootTokenAuth := authenticator.TokenFunc(func(token string) (user.Info, bool, error) { if token == rootToken { return &user.DefaultInfo{Name: rootUserName}, true, nil } return nil, false, nil }) serviceAccountKey, _ := rsa.GenerateKey(rand.Reader, 2048) serviceAccountTokenGetter := serviceaccountcontroller.NewGetterFromClient(rootClientset) serviceAccountTokenAuth := serviceaccount.JWTTokenAuthenticator([]interface{}{&serviceAccountKey.PublicKey}, true, serviceAccountTokenGetter) authenticator := union.New( bearertoken.New(rootTokenAuth), bearertoken.New(serviceAccountTokenAuth), ) // Set up a stub authorizer: // 1. The "root" user is allowed to do anything // 2. ServiceAccounts named "ro" are allowed read-only operations in their namespace // 3. ServiceAccounts named "rw" are allowed any operation in their namespace authorizer := authorizer.AuthorizerFunc(func(attrs authorizer.Attributes) (bool, string, error) { username := "" if user := attrs.GetUser(); user != nil { username = user.GetName() } ns := attrs.GetNamespace() // If the user is "root"... if username == rootUserName { // allow them to do anything return true, "", nil } // If the user is a service account... if serviceAccountNamespace, serviceAccountName, err := serviceaccount.SplitUsername(username); err == nil { // Limit them to their own namespace if serviceAccountNamespace == ns { switch serviceAccountName { case readOnlyServiceAccountName: if attrs.IsReadOnly() { return true, "", nil } case readWriteServiceAccountName: return true, "", nil } } } return false, fmt.Sprintf("User %s is denied (ns=%s, readonly=%v, resource=%s)", username, ns, attrs.IsReadOnly(), attrs.GetResource()), nil }) // Set up admission plugin to auto-assign serviceaccounts to pods serviceAccountAdmission := serviceaccountadmission.NewServiceAccount(internalRootClientset) masterConfig := framework.NewMasterConfig() masterConfig.GenericConfig.EnableIndex = true masterConfig.GenericConfig.Authenticator = authenticator masterConfig.GenericConfig.Authorizer = authorizer masterConfig.GenericConfig.AdmissionControl = serviceAccountAdmission framework.RunAMasterUsingServer(masterConfig, apiServer, h) // Start the service account and service account token controllers stopCh := make(chan struct{}) tokenController := serviceaccountcontroller.NewTokensController(rootClientset, serviceaccountcontroller.TokensControllerOptions{TokenGenerator: serviceaccount.JWTTokenGenerator(serviceAccountKey)}) go tokenController.Run(1, stopCh) informers := informers.NewSharedInformerFactory(rootClientset, nil, controller.NoResyncPeriodFunc()) serviceAccountController := serviceaccountcontroller.NewServiceAccountsController(informers.ServiceAccounts(), informers.Namespaces(), rootClientset, serviceaccountcontroller.DefaultServiceAccountsControllerOptions()) informers.Start(stopCh) go serviceAccountController.Run(5, stopCh) // Start the admission plugin reflectors serviceAccountAdmission.Run() stop := func() { close(stopCh) serviceAccountAdmission.Stop() apiServer.Close() } return rootClientset, clientConfig, stop }
// NewNodeController returns a new node controller to sync instances from cloudprovider. // This method returns an error if it is unable to initialize the CIDR bitmap with // podCIDRs it has already allocated to nodes. Since we don't allow podCIDR changes // currently, this should be handled as a fatal error. func NewNodeController( cloud cloudprovider.Interface, kubeClient clientset.Interface, podEvictionTimeout time.Duration, evictionLimiterQPS float32, nodeMonitorGracePeriod time.Duration, nodeStartupGracePeriod time.Duration, nodeMonitorPeriod time.Duration, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, nodeCIDRMaskSize int, allocateNodeCIDRs bool) (*NodeController, error) { eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "controllermanager"}) eventBroadcaster.StartLogging(glog.Infof) if kubeClient != nil { glog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: kubeClient.Core().Events("")}) } else { glog.V(0).Infof("No api server defined - no events will be sent to API server.") } if kubeClient != nil && kubeClient.Core().GetRESTClient().GetRateLimiter() != nil { metrics.RegisterMetricAndTrackRateLimiterUsage("node_controller", kubeClient.Core().GetRESTClient().GetRateLimiter()) } if allocateNodeCIDRs { if clusterCIDR == nil { glog.Fatal("NodeController: Must specify clusterCIDR if allocateNodeCIDRs == true.") } mask := clusterCIDR.Mask if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize { glog.Fatal("NodeController: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.") } } evictorLock := sync.Mutex{} nc := &NodeController{ cloud: cloud, knownNodeSet: make(map[string]*api.Node), kubeClient: kubeClient, recorder: recorder, podEvictionTimeout: podEvictionTimeout, maximumGracePeriod: 5 * time.Minute, evictorLock: &evictorLock, zonePodEvictor: make(map[string]*RateLimitedTimedQueue), zoneTerminationEvictor: make(map[string]*RateLimitedTimedQueue), nodeStatusMap: make(map[string]nodeStatusData), nodeMonitorGracePeriod: nodeMonitorGracePeriod, nodeMonitorPeriod: nodeMonitorPeriod, nodeStartupGracePeriod: nodeStartupGracePeriod, lookupIP: net.LookupIP, now: unversioned.Now, clusterCIDR: clusterCIDR, serviceCIDR: serviceCIDR, allocateNodeCIDRs: allocateNodeCIDRs, forcefullyDeletePod: func(p *api.Pod) error { return forcefullyDeletePod(kubeClient, p) }, nodeExistsInCloudProvider: func(nodeName string) (bool, error) { return nodeExistsInCloudProvider(cloud, nodeName) }, computeZoneStateFunc: ComputeZoneState, evictionLimiterQPS: evictionLimiterQPS, zoneStates: make(map[string]zoneState), } nc.podStore.Indexer, nc.podController = framework.NewIndexerInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).Watch(options) }, }, &api.Pod{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { nc.maybeDeleteTerminatingPod(obj, nc.nodeStore.Store, nc.forcefullyDeletePod) }, UpdateFunc: func(_, obj interface{}) { nc.maybeDeleteTerminatingPod(obj, nc.nodeStore.Store, nc.forcefullyDeletePod) }, }, // We don't need to build a index for podStore here actually, but build one for consistency. // It will ensure that if people start making use of the podStore in more specific ways, // they'll get the benefits they expect. It will also reserve the name for future refactorings. cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, ) nodeEventHandlerFuncs := framework.ResourceEventHandlerFuncs{} if nc.allocateNodeCIDRs { nodeEventHandlerFuncs = framework.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { node := obj.(*api.Node) err := nc.cidrAllocator.AllocateOrOccupyCIDR(node) if err != nil { glog.Errorf("Error allocating CIDR: %v", err) } }, DeleteFunc: func(obj interface{}) { node := obj.(*api.Node) err := nc.cidrAllocator.ReleaseCIDR(node) if err != nil { glog.Errorf("Error releasing CIDR: %v", err) } }, } } nc.nodeStore.Store, nc.nodeController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Nodes().List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Nodes().Watch(options) }, }, &api.Node{}, controller.NoResyncPeriodFunc(), nodeEventHandlerFuncs, ) nc.daemonSetStore.Store, nc.daemonSetController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).Watch(options) }, }, &extensions.DaemonSet{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{}, ) if allocateNodeCIDRs { var nodeList *api.NodeList var err error // We must poll because apiserver might not be up. This error causes // controller manager to restart. if pollErr := wait.Poll(10*time.Second, apiserverStartupGracePeriod, func() (bool, error) { nodeList, err = kubeClient.Core().Nodes().List(api.ListOptions{ FieldSelector: fields.Everything(), LabelSelector: labels.Everything(), }) if err != nil { glog.Errorf("Failed to list all nodes: %v", err) return false, nil } return true, nil }); pollErr != nil { return nil, fmt.Errorf("Failed to list all nodes in %v, cannot proceed without updating CIDR map", apiserverStartupGracePeriod) } nc.cidrAllocator, err = NewCIDRRangeAllocator(kubeClient, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList) if err != nil { return nil, err } } return nc, nil }
// NewNodeController returns a new node controller to sync instances from cloudprovider. // This method returns an error if it is unable to initialize the CIDR bitmap with // podCIDRs it has already allocated to nodes. Since we don't allow podCIDR changes // currently, this should be handled as a fatal error. func NewNodeController( podInformer cache.SharedIndexInformer, cloud cloudprovider.Interface, kubeClient clientset.Interface, podEvictionTimeout time.Duration, evictionLimiterQPS float32, secondaryEvictionLimiterQPS float32, largeClusterThreshold int32, unhealthyZoneThreshold float32, nodeMonitorGracePeriod time.Duration, nodeStartupGracePeriod time.Duration, nodeMonitorPeriod time.Duration, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, nodeCIDRMaskSize int, allocateNodeCIDRs bool) (*NodeController, error) { eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "controllermanager"}) eventBroadcaster.StartLogging(glog.Infof) if kubeClient != nil { glog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: kubeClient.Core().Events("")}) } else { glog.V(0).Infof("No api server defined - no events will be sent to API server.") } if kubeClient != nil && kubeClient.Core().GetRESTClient().GetRateLimiter() != nil { metrics.RegisterMetricAndTrackRateLimiterUsage("node_controller", kubeClient.Core().GetRESTClient().GetRateLimiter()) } if allocateNodeCIDRs { if clusterCIDR == nil { glog.Fatal("NodeController: Must specify clusterCIDR if allocateNodeCIDRs == true.") } mask := clusterCIDR.Mask if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize { glog.Fatal("NodeController: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.") } } nc := &NodeController{ cloud: cloud, knownNodeSet: make(map[string]*api.Node), kubeClient: kubeClient, recorder: recorder, podEvictionTimeout: podEvictionTimeout, maximumGracePeriod: 5 * time.Minute, zonePodEvictor: make(map[string]*RateLimitedTimedQueue), zoneTerminationEvictor: make(map[string]*RateLimitedTimedQueue), nodeStatusMap: make(map[string]nodeStatusData), nodeMonitorGracePeriod: nodeMonitorGracePeriod, nodeMonitorPeriod: nodeMonitorPeriod, nodeStartupGracePeriod: nodeStartupGracePeriod, lookupIP: net.LookupIP, now: unversioned.Now, clusterCIDR: clusterCIDR, serviceCIDR: serviceCIDR, allocateNodeCIDRs: allocateNodeCIDRs, forcefullyDeletePod: func(p *api.Pod) error { return forcefullyDeletePod(kubeClient, p) }, nodeExistsInCloudProvider: func(nodeName string) (bool, error) { return nodeExistsInCloudProvider(cloud, nodeName) }, evictionLimiterQPS: evictionLimiterQPS, secondaryEvictionLimiterQPS: secondaryEvictionLimiterQPS, largeClusterThreshold: largeClusterThreshold, unhealthyZoneThreshold: unhealthyZoneThreshold, zoneStates: make(map[string]zoneState), } nc.enterPartialDisruptionFunc = nc.ReducedQPSFunc nc.enterFullDisruptionFunc = nc.HealthyQPSFunc nc.computeZoneStateFunc = nc.ComputeZoneState podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: nc.maybeDeleteTerminatingPod, UpdateFunc: func(_, obj interface{}) { nc.maybeDeleteTerminatingPod(obj) }, }) nc.podStore.Indexer = podInformer.GetIndexer() nc.podController = podInformer.GetController() nodeEventHandlerFuncs := cache.ResourceEventHandlerFuncs{} if nc.allocateNodeCIDRs { nodeEventHandlerFuncs = cache.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { node := obj.(*api.Node) err := nc.cidrAllocator.AllocateOrOccupyCIDR(node) if err != nil { glog.Errorf("Error allocating CIDR: %v", err) } }, UpdateFunc: func(_, obj interface{}) { node := obj.(*api.Node) // If the PodCIDR is not empty we either: // - already processed a Node that already had a CIDR after NC restarted // (cidr is marked as used), // - already processed a Node successfully and allocated a CIDR for it // (cidr is marked as used), // - already processed a Node but we did saw a "timeout" response and // request eventually got through in this case we haven't released // the allocated CIDR (cidr is still marked as used). // There's a possible error here: // - NC sees a new Node and assigns a CIDR X to it, // - Update Node call fails with a timeout, // - Node is updated by some other component, NC sees an update and // assigns CIDR Y to the Node, // - Both CIDR X and CIDR Y are marked as used in the local cache, // even though Node sees only CIDR Y // The problem here is that in in-memory cache we see CIDR X as marked, // which prevents it from being assigned to any new node. The cluster // state is correct. // Restart of NC fixes the issue. if node.Spec.PodCIDR == "" { err := nc.cidrAllocator.AllocateOrOccupyCIDR(node) if err != nil { glog.Errorf("Error allocating CIDR: %v", err) } } }, DeleteFunc: func(obj interface{}) { node, isNode := obj.(*api.Node) // We can get DeletedFinalStateUnknown instead of *api.Node here and we need to handle that correctly. #34692 if !isNode { deletedState, ok := obj.(cache.DeletedFinalStateUnknown) if !ok { glog.Errorf("Received unexpected object: %v", obj) return } node, ok = deletedState.Obj.(*api.Node) if !ok { glog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj) return } } if err := nc.cidrAllocator.ReleaseCIDR(node); err != nil { glog.Errorf("Error releasing CIDR: %v", err) } }, } } nc.nodeStore.Store, nc.nodeController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Nodes().List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Nodes().Watch(options) }, }, &api.Node{}, controller.NoResyncPeriodFunc(), nodeEventHandlerFuncs, ) nc.daemonSetStore.Store, nc.daemonSetController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).Watch(options) }, }, &extensions.DaemonSet{}, controller.NoResyncPeriodFunc(), cache.ResourceEventHandlerFuncs{}, ) if allocateNodeCIDRs { var nodeList *api.NodeList var err error // We must poll because apiserver might not be up. This error causes // controller manager to restart. if pollErr := wait.Poll(10*time.Second, apiserverStartupGracePeriod, func() (bool, error) { nodeList, err = kubeClient.Core().Nodes().List(api.ListOptions{ FieldSelector: fields.Everything(), LabelSelector: labels.Everything(), }) if err != nil { glog.Errorf("Failed to list all nodes: %v", err) return false, nil } return true, nil }); pollErr != nil { return nil, fmt.Errorf("Failed to list all nodes in %v, cannot proceed without updating CIDR map", apiserverStartupGracePeriod) } nc.cidrAllocator, err = NewCIDRRangeAllocator(kubeClient, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList) if err != nil { return nil, err } } return nc, nil }
// NewNodeController returns a new node controller to sync instances from cloudprovider. func NewNodeController( cloud cloudprovider.Interface, kubeClient clientset.Interface, podEvictionTimeout time.Duration, deletionEvictionLimiter util.RateLimiter, terminationEvictionLimiter util.RateLimiter, nodeMonitorGracePeriod time.Duration, nodeStartupGracePeriod time.Duration, nodeMonitorPeriod time.Duration, clusterCIDR *net.IPNet, allocateNodeCIDRs bool) *NodeController { eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "controllermanager"}) eventBroadcaster.StartLogging(glog.Infof) if kubeClient != nil { glog.Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&unversioned_core.EventSinkImpl{kubeClient.Core().Events("")}) } else { glog.Infof("No api server defined - no events will be sent to API server.") } if allocateNodeCIDRs && clusterCIDR == nil { glog.Fatal("NodeController: Must specify clusterCIDR if allocateNodeCIDRs == true.") } evictorLock := sync.Mutex{} nc := &NodeController{ cloud: cloud, knownNodeSet: make(sets.String), kubeClient: kubeClient, recorder: recorder, podEvictionTimeout: podEvictionTimeout, maximumGracePeriod: 5 * time.Minute, evictorLock: &evictorLock, podEvictor: NewRateLimitedTimedQueue(deletionEvictionLimiter), terminationEvictor: NewRateLimitedTimedQueue(terminationEvictionLimiter), nodeStatusMap: make(map[string]nodeStatusData), nodeMonitorGracePeriod: nodeMonitorGracePeriod, nodeMonitorPeriod: nodeMonitorPeriod, nodeStartupGracePeriod: nodeStartupGracePeriod, lookupIP: net.LookupIP, now: unversioned.Now, clusterCIDR: clusterCIDR, allocateNodeCIDRs: allocateNodeCIDRs, forcefullyDeletePod: func(p *api.Pod) { forcefullyDeletePod(kubeClient, p) }, } nc.podStore.Store, nc.podController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).Watch(options) }, }, &api.Pod{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{ AddFunc: nc.maybeDeleteTerminatingPod, UpdateFunc: func(_, obj interface{}) { nc.maybeDeleteTerminatingPod(obj) }, }, ) nc.nodeStore.Store, nc.nodeController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Nodes().List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Nodes().Watch(options) }, }, &api.Node{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{}, ) nc.daemonSetStore.Store, nc.daemonSetController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).Watch(options) }, }, &api.Node{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{}, ) return nc }
// NewConfigMapController returns a new configmap controller func NewConfigMapController(client federationclientset.Interface) *ConfigMapController { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client)) recorder := broadcaster.NewRecorder(apiv1.EventSource{Component: "federated-configmaps-controller"}) configmapcontroller := &ConfigMapController{ federatedApiClient: client, configmapReviewDelay: time.Second * 10, clusterAvailableDelay: time.Second * 20, smallDelay: time.Second * 3, updateTimeout: time.Second * 30, configmapBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), eventRecorder: recorder, } // Build delivereres for triggering reconciliations. configmapcontroller.configmapDeliverer = util.NewDelayingDeliverer() configmapcontroller.clusterDeliverer = util.NewDelayingDeliverer() // Start informer on federated API servers on configmaps that should be federated. configmapcontroller.configmapInformerStore, configmapcontroller.configmapInformerController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options apiv1.ListOptions) (pkgruntime.Object, error) { return client.Core().ConfigMaps(apiv1.NamespaceAll).List(options) }, WatchFunc: func(options apiv1.ListOptions) (watch.Interface, error) { return client.Core().ConfigMaps(apiv1.NamespaceAll).Watch(options) }, }, &apiv1.ConfigMap{}, controller.NoResyncPeriodFunc(), util.NewTriggerOnAllChanges(func(obj pkgruntime.Object) { configmapcontroller.deliverConfigMapObj(obj, 0, false) })) // Federated informer on configmaps in members of federation. configmapcontroller.configmapFederatedInformer = util.NewFederatedInformer( client, func(cluster *federationapi.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.ControllerInterface) { return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options apiv1.ListOptions) (pkgruntime.Object, error) { return targetClient.Core().ConfigMaps(apiv1.NamespaceAll).List(options) }, WatchFunc: func(options apiv1.ListOptions) (watch.Interface, error) { return targetClient.Core().ConfigMaps(apiv1.NamespaceAll).Watch(options) }, }, &apiv1.ConfigMap{}, controller.NoResyncPeriodFunc(), // Trigger reconciliation whenever something in federated cluster is changed. In most cases it // would be just confirmation that some configmap opration succeeded. util.NewTriggerOnAllChanges( func(obj pkgruntime.Object) { configmapcontroller.deliverConfigMapObj(obj, configmapcontroller.configmapReviewDelay, false) }, )) }, &util.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *federationapi.Cluster) { // When new cluster becomes available process all the configmaps again. configmapcontroller.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(configmapcontroller.clusterAvailableDelay)) }, }, ) // Federated updater along with Create/Update/Delete operations. configmapcontroller.federatedUpdater = util.NewFederatedUpdater(configmapcontroller.configmapFederatedInformer, func(client kubeclientset.Interface, obj pkgruntime.Object) error { configmap := obj.(*apiv1.ConfigMap) _, err := client.Core().ConfigMaps(configmap.Namespace).Create(configmap) return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { configmap := obj.(*apiv1.ConfigMap) _, err := client.Core().ConfigMaps(configmap.Namespace).Update(configmap) return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { configmap := obj.(*apiv1.ConfigMap) err := client.Core().ConfigMaps(configmap.Namespace).Delete(configmap.Name, &apiv1.DeleteOptions{}) return err }) return configmapcontroller }
// NewclusterController returns a new cluster controller func NewReplicaSetController(federationClient fedclientset.Interface) *ReplicaSetController { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(federationClient)) recorder := broadcaster.NewRecorder(api.EventSource{Component: "federated-replicaset-controller"}) frsc := &ReplicaSetController{ fedClient: federationClient, replicasetDeliverer: fedutil.NewDelayingDeliverer(), clusterDeliverer: fedutil.NewDelayingDeliverer(), replicasetWorkQueue: workqueue.New(), replicaSetBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), defaultPlanner: planner.NewPlanner(&fed.FederatedReplicaSetPreferences{ Clusters: map[string]fed.ClusterReplicaSetPreferences{ "*": {Weight: 1}, }, }), eventRecorder: recorder, } replicaSetFedInformerFactory := func(cluster *fedv1.Cluster, clientset kubeclientset.Interface) (cache.Store, cache.ControllerInterface) { return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { versionedOptions := fedutil.VersionizeV1ListOptions(options) return clientset.Extensions().ReplicaSets(apiv1.NamespaceAll).List(versionedOptions) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { versionedOptions := fedutil.VersionizeV1ListOptions(options) return clientset.Extensions().ReplicaSets(apiv1.NamespaceAll).Watch(versionedOptions) }, }, &extensionsv1.ReplicaSet{}, controller.NoResyncPeriodFunc(), fedutil.NewTriggerOnAllChanges( func(obj runtime.Object) { frsc.deliverLocalReplicaSet(obj, replicaSetReviewDelay) }, ), ) } clusterLifecycle := fedutil.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *fedv1.Cluster) { frsc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterAvailableDelay) }, ClusterUnavailable: func(cluster *fedv1.Cluster, _ []interface{}) { frsc.clusterDeliverer.DeliverAfter(allClustersKey, nil, clusterUnavailableDelay) }, } frsc.fedReplicaSetInformer = fedutil.NewFederatedInformer(federationClient, replicaSetFedInformerFactory, &clusterLifecycle) podFedInformerFactory := func(cluster *fedv1.Cluster, clientset kubeclientset.Interface) (cache.Store, cache.ControllerInterface) { return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { versionedOptions := fedutil.VersionizeV1ListOptions(options) return clientset.Core().Pods(apiv1.NamespaceAll).List(versionedOptions) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { versionedOptions := fedutil.VersionizeV1ListOptions(options) return clientset.Core().Pods(apiv1.NamespaceAll).Watch(versionedOptions) }, }, &apiv1.Pod{}, controller.NoResyncPeriodFunc(), fedutil.NewTriggerOnAllChanges( func(obj runtime.Object) { frsc.clusterDeliverer.DeliverAfter(allClustersKey, nil, allReplicaSetReviewDelay) }, ), ) } frsc.fedPodInformer = fedutil.NewFederatedInformer(federationClient, podFedInformerFactory, &fedutil.ClusterLifecycleHandlerFuncs{}) frsc.replicaSetStore.Indexer, frsc.replicaSetController = cache.NewIndexerInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { versionedOptions := fedutil.VersionizeV1ListOptions(options) return frsc.fedClient.Extensions().ReplicaSets(apiv1.NamespaceAll).List(versionedOptions) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { versionedOptions := fedutil.VersionizeV1ListOptions(options) return frsc.fedClient.Extensions().ReplicaSets(apiv1.NamespaceAll).Watch(versionedOptions) }, }, &extensionsv1.ReplicaSet{}, controller.NoResyncPeriodFunc(), fedutil.NewTriggerOnMetaAndSpecChanges( func(obj runtime.Object) { frsc.deliverFedReplicaSetObj(obj, replicaSetReviewDelay) }, ), cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, ) frsc.fedUpdater = fedutil.NewFederatedUpdater(frsc.fedReplicaSetInformer, func(client kubeclientset.Interface, obj runtime.Object) error { rs := obj.(*extensionsv1.ReplicaSet) _, err := client.Extensions().ReplicaSets(rs.Namespace).Create(rs) return err }, func(client kubeclientset.Interface, obj runtime.Object) error { rs := obj.(*extensionsv1.ReplicaSet) _, err := client.Extensions().ReplicaSets(rs.Namespace).Update(rs) return err }, func(client kubeclientset.Interface, obj runtime.Object) error { rs := obj.(*extensionsv1.ReplicaSet) err := client.Extensions().ReplicaSets(rs.Namespace).Delete(rs.Name, &apiv1.DeleteOptions{}) return err }) frsc.deletionHelper = deletionhelper.NewDeletionHelper( frsc.hasFinalizerFunc, frsc.removeFinalizerFunc, frsc.addFinalizerFunc, // objNameFunc func(obj runtime.Object) string { replicaset := obj.(*extensionsv1.ReplicaSet) return replicaset.Name }, updateTimeout, frsc.eventRecorder, frsc.fedReplicaSetInformer, frsc.fedUpdater, ) return frsc }
// NewIngressController returns a new ingress controller func NewIngressController(client federation_release_1_4.Interface) *IngressController { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client)) recorder := broadcaster.NewRecorder(api.EventSource{Component: "federated-ingress-controller"}) ic := &IngressController{ federatedApiClient: client, ingressReviewDelay: time.Second * 10, clusterAvailableDelay: time.Second * 20, smallDelay: time.Second * 3, updateTimeout: time.Second * 30, ingressBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), eventRecorder: recorder, } // Build deliverers for triggering reconcilations. ic.ingressDeliverer = util.NewDelayingDeliverer() ic.clusterDeliverer = util.NewDelayingDeliverer() // Start informer in federated API servers on ingresses that should be federated. ic.ingressInformerStore, ic.ingressInformerController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (pkg_runtime.Object, error) { return client.Extensions().Ingresses(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return client.Extensions().Ingresses(api.NamespaceAll).Watch(options) }, }, &extensions_v1beta1.Ingress{}, controller.NoResyncPeriodFunc(), util.NewTriggerOnAllChanges( func(obj pkg_runtime.Object) { ic.deliverIngressObj(obj, 0, false) }, )) // Federated informer on ingresses in members of federation. ic.ingressFederatedInformer = util.NewFederatedInformer( client, func(cluster *federation_api.Cluster, targetClient kube_release_1_4.Interface) (cache.Store, framework.ControllerInterface) { return framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (pkg_runtime.Object, error) { return targetClient.Extensions().Ingresses(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return targetClient.Extensions().Ingresses(api.NamespaceAll).Watch(options) }, }, &extensions_v1beta1.Ingress{}, controller.NoResyncPeriodFunc(), // Trigger reconcilation whenever something in federated cluster is changed. In most cases it // would be just confirmation that some ingress operation suceeded. util.NewTriggerOnAllChanges( func(obj pkg_runtime.Object) { ic.deliverIngressObj(obj, ic.ingressReviewDelay, false) }, )) }, &util.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *federation_api.Cluster) { // When new cluster becomes available process all the ingresses again. ic.clusterDeliverer.DeliverAt(allClustersKey, nil, time.Now().Add(ic.clusterAvailableDelay)) }, }, ) // Federated updater along with Create/Update/Delete operations. ic.federatedUpdater = util.NewFederatedUpdater(ic.ingressFederatedInformer, func(client kube_release_1_4.Interface, obj pkg_runtime.Object) error { ingress := obj.(*extensions_v1beta1.Ingress) glog.V(4).Infof("Attempting to create Ingress: %v", ingress) _, err := client.Extensions().Ingresses(ingress.Namespace).Create(ingress) return err }, func(client kube_release_1_4.Interface, obj pkg_runtime.Object) error { ingress := obj.(*extensions_v1beta1.Ingress) glog.V(4).Infof("Attempting to update Ingress: %v", ingress) _, err := client.Extensions().Ingresses(ingress.Namespace).Update(ingress) return err }, func(client kube_release_1_4.Interface, obj pkg_runtime.Object) error { ingress := obj.(*extensions_v1beta1.Ingress) glog.V(4).Infof("Attempting to delete Ingress: %v", ingress) err := client.Extensions().Ingresses(ingress.Namespace).Delete(ingress.Name, &api.DeleteOptions{}) return err }) return ic }
// NewNodeController returns a new node controller to sync instances from cloudprovider. // This method returns an error if it is unable to initialize the CIDR bitmap with // podCIDRs it has already allocated to nodes. Since we don't allow podCIDR changes // currently, this should be handled as a fatal error. func NewNodeController( cloud cloudprovider.Interface, kubeClient clientset.Interface, podEvictionTimeout time.Duration, deletionEvictionLimiter flowcontrol.RateLimiter, terminationEvictionLimiter flowcontrol.RateLimiter, nodeMonitorGracePeriod time.Duration, nodeStartupGracePeriod time.Duration, nodeMonitorPeriod time.Duration, clusterCIDR *net.IPNet, serviceCIDR *net.IPNet, nodeCIDRMaskSize int, allocateNodeCIDRs bool) (*NodeController, error) { eventBroadcaster := record.NewBroadcaster() recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "controllermanager"}) eventBroadcaster.StartLogging(glog.Infof) if kubeClient != nil { glog.V(0).Infof("Sending events to api server.") eventBroadcaster.StartRecordingToSink(&unversionedcore.EventSinkImpl{Interface: kubeClient.Core().Events("")}) } else { glog.V(0).Infof("No api server defined - no events will be sent to API server.") } if kubeClient != nil && kubeClient.Core().GetRESTClient().GetRateLimiter() != nil { metrics.RegisterMetricAndTrackRateLimiterUsage("node_controller", kubeClient.Core().GetRESTClient().GetRateLimiter()) } if allocateNodeCIDRs { if clusterCIDR == nil { glog.Fatal("NodeController: Must specify clusterCIDR if allocateNodeCIDRs == true.") } mask := clusterCIDR.Mask if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize { glog.Fatal("NodeController: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.") } } evictorLock := sync.Mutex{} nc := &NodeController{ cloud: cloud, knownNodeSet: make(sets.String), kubeClient: kubeClient, recorder: recorder, podEvictionTimeout: podEvictionTimeout, maximumGracePeriod: 5 * time.Minute, evictorLock: &evictorLock, podEvictor: NewRateLimitedTimedQueue(deletionEvictionLimiter), terminationEvictor: NewRateLimitedTimedQueue(terminationEvictionLimiter), nodeStatusMap: make(map[string]nodeStatusData), nodeMonitorGracePeriod: nodeMonitorGracePeriod, nodeMonitorPeriod: nodeMonitorPeriod, nodeStartupGracePeriod: nodeStartupGracePeriod, lookupIP: net.LookupIP, now: unversioned.Now, clusterCIDR: clusterCIDR, serviceCIDR: serviceCIDR, allocateNodeCIDRs: allocateNodeCIDRs, forcefullyDeletePod: func(p *api.Pod) error { return forcefullyDeletePod(kubeClient, p) }, nodeExistsInCloudProvider: func(nodeName string) (bool, error) { return nodeExistsInCloudProvider(cloud, nodeName) }, } nc.podStore.Indexer, nc.podController = framework.NewIndexerInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Pods(api.NamespaceAll).Watch(options) }, }, &api.Pod{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{ AddFunc: nc.maybeDeleteTerminatingPod, UpdateFunc: func(_, obj interface{}) { nc.maybeDeleteTerminatingPod(obj) }, }, // We don't need to build a index for podStore here actually, but build one for consistency. // It will ensure that if people start making use of the podStore in more specific ways, // they'll get the benefits they expect. It will also reserve the name for future refactorings. cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, ) nodeEventHandlerFuncs := framework.ResourceEventHandlerFuncs{} if nc.allocateNodeCIDRs { nodeEventHandlerFuncs = framework.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { node := obj.(*api.Node) err := nc.cidrAllocator.AllocateOrOccupyCIDR(node) if err != nil { glog.Errorf("Error allocating CIDR: %v", err) } }, UpdateFunc: func(_, obj interface{}) { node := obj.(*api.Node) // If the PodCIDR is not empty we either: // - already processed a Node that already had a CIDR after NC restarted // (cidr is marked as used), // - already processed a Node successfully and allocated a CIDR for it // (cidr is marked as used), // - already processed a Node but we did saw a "timeout" response and // request eventually got through in this case we haven't released // the allocated CIDR (cidr is still marked as used). // There's a possible error here: // - NC sees a new Node and assigns a CIDR X to it, // - Update Node call fails with a timeout, // - Node is updated by some other component, NC sees an update and // assigns CIDR Y to the Node, // - Both CIDR X and CIDR Y are marked as used in the local cache, // even though Node sees only CIDR Y // The problem here is that in in-memory cache we see CIDR X as marked, // which prevents it from being assigned to any new node. The cluster // state is correct. // Restart of NC fixes the issue. if node.Spec.PodCIDR == "" { err := nc.cidrAllocator.AllocateOrOccupyCIDR(node) if err != nil { glog.Errorf("Error allocating CIDR: %v", err) } } }, DeleteFunc: func(obj interface{}) { node := obj.(*api.Node) err := nc.cidrAllocator.ReleaseCIDR(node) if err != nil { glog.Errorf("Error releasing CIDR: %v", err) } }, } } nc.nodeStore.Store, nc.nodeController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Core().Nodes().List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Core().Nodes().Watch(options) }, }, &api.Node{}, controller.NoResyncPeriodFunc(), nodeEventHandlerFuncs, ) nc.daemonSetStore.Store, nc.daemonSetController = framework.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (runtime.Object, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return nc.kubeClient.Extensions().DaemonSets(api.NamespaceAll).Watch(options) }, }, &extensions.DaemonSet{}, controller.NoResyncPeriodFunc(), framework.ResourceEventHandlerFuncs{}, ) if allocateNodeCIDRs { var nodeList *api.NodeList var err error // We must poll because apiserver might not be up. This error causes // controller manager to restart. if pollErr := wait.Poll(10*time.Second, apiserverStartupGracePeriod, func() (bool, error) { nodeList, err = kubeClient.Core().Nodes().List(api.ListOptions{ FieldSelector: fields.Everything(), LabelSelector: labels.Everything(), }) if err != nil { glog.Errorf("Failed to list all nodes: %v", err) return false, nil } return true, nil }); pollErr != nil { return nil, fmt.Errorf("Failed to list all nodes in %v, cannot proceed without updating CIDR map", apiserverStartupGracePeriod) } nc.cidrAllocator, err = NewCIDRRangeAllocator(kubeClient, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList) if err != nil { return nil, err } } return nc, nil }
// NewIngressController returns a new ingress controller func NewIngressController(client federationclientset.Interface) *IngressController { glog.V(4).Infof("->NewIngressController V(4)") broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client)) recorder := broadcaster.NewRecorder(v1.EventSource{Component: "federated-ingress-controller"}) ic := &IngressController{ federatedApiClient: client, ingressReviewDelay: time.Second * 10, configMapReviewDelay: time.Second * 10, clusterAvailableDelay: time.Second * 20, smallDelay: time.Second * 3, updateTimeout: time.Second * 30, ingressBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), eventRecorder: recorder, configMapBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), } // Build deliverers for triggering reconciliations. ic.ingressDeliverer = util.NewDelayingDeliverer() ic.clusterDeliverer = util.NewDelayingDeliverer() ic.configMapDeliverer = util.NewDelayingDeliverer() // Start informer in federated API servers on ingresses that should be federated. ic.ingressInformerStore, ic.ingressInformerController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (pkgruntime.Object, error) { return client.Extensions().Ingresses(api.NamespaceAll).List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return client.Extensions().Ingresses(api.NamespaceAll).Watch(options) }, }, &extensionsv1beta1.Ingress{}, controller.NoResyncPeriodFunc(), util.NewTriggerOnAllChanges( func(obj pkgruntime.Object) { ic.deliverIngressObj(obj, 0, false) }, )) // Federated informer on ingresses in members of federation. ic.ingressFederatedInformer = util.NewFederatedInformer( client, func(cluster *federationapi.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.ControllerInterface) { return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (pkgruntime.Object, error) { return targetClient.Extensions().Ingresses(api.NamespaceAll).List(options) }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { return targetClient.Extensions().Ingresses(api.NamespaceAll).Watch(options) }, }, &extensionsv1beta1.Ingress{}, controller.NoResyncPeriodFunc(), // Trigger reconciliation whenever something in federated cluster is changed. In most cases it // would be just confirmation that some ingress operation succeeded. util.NewTriggerOnAllChanges( func(obj pkgruntime.Object) { ic.deliverIngressObj(obj, ic.ingressReviewDelay, false) }, )) }, &util.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *federationapi.Cluster) { // When new cluster becomes available process all the ingresses again, and configure it's ingress controller's configmap with the correct UID ic.clusterDeliverer.DeliverAfter(cluster.Name, cluster, ic.clusterAvailableDelay) }, }, ) // Federated informer on configmaps for ingress controllers in members of the federation. ic.configMapFederatedInformer = util.NewFederatedInformer( client, func(cluster *federationapi.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.ControllerInterface) { glog.V(4).Infof("Returning new informer for cluster %q", cluster.Name) return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options v1.ListOptions) (pkgruntime.Object, error) { if targetClient == nil { glog.Errorf("Internal error: targetClient is nil") } return targetClient.Core().ConfigMaps(uidConfigMapNamespace).List(options) // we only want to list one by name - unfortunately Kubernetes don't have a selector for that. }, WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { if targetClient == nil { glog.Errorf("Internal error: targetClient is nil") } return targetClient.Core().ConfigMaps(uidConfigMapNamespace).Watch(options) // as above }, }, &v1.ConfigMap{}, controller.NoResyncPeriodFunc(), // Trigger reconcilation whenever the ingress controller's configmap in a federated cluster is changed. In most cases it // would be just confirmation that the configmap for the ingress controller is correct. util.NewTriggerOnAllChanges( func(obj pkgruntime.Object) { ic.deliverConfigMapObj(cluster.Name, obj, ic.configMapReviewDelay, false) }, )) }, &util.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *federationapi.Cluster) { ic.clusterDeliverer.DeliverAfter(cluster.Name, cluster, ic.clusterAvailableDelay) }, }, ) // Federated ingress updater along with Create/Update/Delete operations. ic.federatedIngressUpdater = util.NewFederatedUpdater(ic.ingressFederatedInformer, func(client kubeclientset.Interface, obj pkgruntime.Object) error { ingress := obj.(*extensionsv1beta1.Ingress) glog.V(4).Infof("Attempting to create Ingress: %v", ingress) _, err := client.Extensions().Ingresses(ingress.Namespace).Create(ingress) if err != nil { glog.Errorf("Error creating ingress %q: %v", types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}, err) } else { glog.V(4).Infof("Successfully created ingress %q", types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}) } return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { ingress := obj.(*extensionsv1beta1.Ingress) glog.V(4).Infof("Attempting to update Ingress: %v", ingress) _, err := client.Extensions().Ingresses(ingress.Namespace).Update(ingress) if err != nil { glog.V(4).Infof("Failed to update Ingress: %v", err) } else { glog.V(4).Infof("Successfully updated Ingress: %q", types.NamespacedName{Name: ingress.Name, Namespace: ingress.Namespace}) } return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { ingress := obj.(*extensionsv1beta1.Ingress) glog.V(4).Infof("Attempting to delete Ingress: %v", ingress) err := client.Extensions().Ingresses(ingress.Namespace).Delete(ingress.Name, &v1.DeleteOptions{}) return err }) // Federated configmap updater along with Create/Update/Delete operations. Only Update should ever be called. ic.federatedConfigMapUpdater = util.NewFederatedUpdater(ic.configMapFederatedInformer, func(client kubeclientset.Interface, obj pkgruntime.Object) error { configMap := obj.(*v1.ConfigMap) configMapName := types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace} glog.Errorf("Internal error: Incorrectly attempting to create ConfigMap: %q", configMapName) _, err := client.Core().ConfigMaps(configMap.Namespace).Create(configMap) return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { configMap := obj.(*v1.ConfigMap) configMapName := types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace} glog.V(4).Infof("Attempting to update ConfigMap: %v", configMap) _, err := client.Core().ConfigMaps(configMap.Namespace).Update(configMap) if err == nil { glog.V(4).Infof("Successfully updated ConfigMap %q", configMapName) } else { glog.V(4).Infof("Failed to update ConfigMap %q: %v", configMapName, err) } return err }, func(client kubeclientset.Interface, obj pkgruntime.Object) error { configMap := obj.(*v1.ConfigMap) configMapName := types.NamespacedName{Name: configMap.Name, Namespace: configMap.Namespace} glog.Errorf("Internal error: Incorrectly attempting to delete ConfigMap: %q", configMapName) err := client.Core().ConfigMaps(configMap.Namespace).Delete(configMap.Name, &v1.DeleteOptions{}) return err }) ic.deletionHelper = deletionhelper.NewDeletionHelper( ic.hasFinalizerFunc, ic.removeFinalizerFunc, ic.addFinalizerFunc, // objNameFunc func(obj pkgruntime.Object) string { ingress := obj.(*extensionsv1beta1.Ingress) return ingress.Name }, ic.updateTimeout, ic.eventRecorder, ic.ingressFederatedInformer, ic.federatedIngressUpdater, ) return ic }
func TestDeploymentController_cleanupDeployment(t *testing.T) { selector := map[string]string{"foo": "bar"} tests := []struct { oldRSs []*extensions.ReplicaSet revisionHistoryLimit int32 expectedDeletions int }{ { oldRSs: []*extensions.ReplicaSet{ newRSWithStatus("foo-1", 0, 0, selector), newRSWithStatus("foo-2", 0, 0, selector), newRSWithStatus("foo-3", 0, 0, selector), }, revisionHistoryLimit: 1, expectedDeletions: 2, }, { // Only delete the replica set with Spec.Replicas = Status.Replicas = 0. oldRSs: []*extensions.ReplicaSet{ newRSWithStatus("foo-1", 0, 0, selector), newRSWithStatus("foo-2", 0, 1, selector), newRSWithStatus("foo-3", 1, 0, selector), newRSWithStatus("foo-4", 1, 1, selector), }, revisionHistoryLimit: 0, expectedDeletions: 1, }, { oldRSs: []*extensions.ReplicaSet{ newRSWithStatus("foo-1", 0, 0, selector), newRSWithStatus("foo-2", 0, 0, selector), }, revisionHistoryLimit: 0, expectedDeletions: 2, }, { oldRSs: []*extensions.ReplicaSet{ newRSWithStatus("foo-1", 1, 1, selector), newRSWithStatus("foo-2", 1, 1, selector), }, revisionHistoryLimit: 0, expectedDeletions: 0, }, } for i := range tests { test := tests[i] fake := &fake.Clientset{} informers := informers.NewSharedInformerFactory(fake, nil, controller.NoResyncPeriodFunc()) controller := NewDeploymentController(informers.Deployments(), informers.ReplicaSets(), informers.Pods(), fake) controller.eventRecorder = &record.FakeRecorder{} controller.dListerSynced = alwaysReady controller.rsListerSynced = alwaysReady controller.podListerSynced = alwaysReady for _, rs := range test.oldRSs { controller.rsLister.Indexer.Add(rs) } stopCh := make(chan struct{}) defer close(stopCh) informers.Start(stopCh) d := newDeployment("foo", 1, &test.revisionHistoryLimit, nil, nil, map[string]string{"foo": "bar"}) controller.cleanupDeployment(test.oldRSs, d) gotDeletions := 0 for _, action := range fake.Actions() { if "delete" == action.GetVerb() { gotDeletions++ } } if gotDeletions != test.expectedDeletions { t.Errorf("expect %v old replica sets been deleted, but got %v", test.expectedDeletions, gotDeletions) continue } } }
// 1.2 code gets: // quota_test.go:95: Took 4.218619579s to scale up without quota // quota_test.go:199: unexpected error: timed out waiting for the condition, ended with 342 pods (1 minute) // 1.3+ code gets: // quota_test.go:100: Took 4.196205966s to scale up without quota // quota_test.go:115: Took 12.021640372s to scale up with quota func TestQuota(t *testing.T) { // Set up a master h := &framework.MasterHolder{Initialized: make(chan struct{})} s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { <-h.Initialized h.M.GenericAPIServer.Handler.ServeHTTP(w, req) })) defer s.Close() admissionCh := make(chan struct{}) clientset := clientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &api.Registry.GroupOrDie(v1.GroupName).GroupVersion}}) internalClientset := internalclientset.NewForConfigOrDie(&restclient.Config{QPS: -1, Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: &api.Registry.GroupOrDie(v1.GroupName).GroupVersion}}) admission, err := resourcequota.NewResourceQuota(quotainstall.NewRegistry(nil, nil), 5, admissionCh) if err != nil { t.Fatalf("unexpected error: %v", err) } admission.(kubeadmission.WantsInternalClientSet).SetInternalClientSet(internalClientset) defer close(admissionCh) masterConfig := framework.NewIntegrationTestMasterConfig() masterConfig.GenericConfig.AdmissionControl = admission framework.RunAMasterUsingServer(masterConfig, s, h) ns := framework.CreateTestingNamespace("quotaed", s, t) defer framework.DeleteTestingNamespace(ns, s, t) ns2 := framework.CreateTestingNamespace("non-quotaed", s, t) defer framework.DeleteTestingNamespace(ns2, s, t) controllerCh := make(chan struct{}) defer close(controllerCh) informers := informers.NewSharedInformerFactory(clientset, nil, controller.NoResyncPeriodFunc()) podInformer := informers.Pods().Informer() rcInformer := informers.ReplicationControllers().Informer() rm := replicationcontroller.NewReplicationManager(podInformer, rcInformer, clientset, replicationcontroller.BurstReplicas, 4096, false) rm.SetEventRecorder(&record.FakeRecorder{}) informers.Start(controllerCh) go rm.Run(3, controllerCh) resourceQuotaRegistry := quotainstall.NewRegistry(clientset, nil) groupKindsToReplenish := []schema.GroupKind{ api.Kind("Pod"), } resourceQuotaControllerOptions := &resourcequotacontroller.ResourceQuotaControllerOptions{ KubeClient: clientset, ResyncPeriod: controller.NoResyncPeriodFunc, Registry: resourceQuotaRegistry, GroupKindsToReplenish: groupKindsToReplenish, ReplenishmentResyncPeriod: controller.NoResyncPeriodFunc, ControllerFactory: resourcequotacontroller.NewReplenishmentControllerFactoryFromClient(clientset), } go resourcequotacontroller.NewResourceQuotaController(resourceQuotaControllerOptions).Run(2, controllerCh) startTime := time.Now() scale(t, ns2.Name, clientset) endTime := time.Now() t.Logf("Took %v to scale up without quota", endTime.Sub(startTime)) quota := &v1.ResourceQuota{ ObjectMeta: metav1.ObjectMeta{ Name: "quota", Namespace: ns.Name, }, Spec: v1.ResourceQuotaSpec{ Hard: v1.ResourceList{ v1.ResourcePods: resource.MustParse("1000"), }, }, } waitForQuota(t, quota, clientset) startTime = time.Now() scale(t, "quotaed", clientset) endTime = time.Now() t.Logf("Took %v to scale up with quota", endTime.Sub(startTime)) }
// This test checks that the node is deleted when kubelet stops reporting // and cloud provider says node is gone func TestNodeDeleted(t *testing.T) { pod0 := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pod0", }, Spec: v1.PodSpec{ NodeName: "node0", }, Status: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type: v1.PodReady, Status: v1.ConditionTrue, }, }, }, } pod1 := &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: "default", Name: "pod1", }, Spec: v1.PodSpec{ NodeName: "node0", }, Status: v1.PodStatus{ Conditions: []v1.PodCondition{ { Type: v1.PodReady, Status: v1.ConditionTrue, }, }, }, } fnh := &testutil.FakeNodeHandler{ Existing: []*v1.Node{ { ObjectMeta: metav1.ObjectMeta{ Name: "node0", CreationTimestamp: metav1.Date(2012, 1, 1, 0, 0, 0, 0, time.UTC), }, Status: v1.NodeStatus{ Conditions: []v1.NodeCondition{ { Type: v1.NodeReady, Status: v1.ConditionUnknown, LastHeartbeatTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), LastTransitionTime: metav1.Date(2015, 1, 1, 12, 0, 0, 0, time.UTC), }, }, }, }, }, Clientset: fake.NewSimpleClientset(&v1.PodList{Items: []v1.Pod{*pod0, *pod1}}), DeleteWaitChan: make(chan struct{}), } factory := informers.NewSharedInformerFactory(fnh, nil, controller.NoResyncPeriodFunc()) eventBroadcaster := record.NewBroadcaster() cloudNodeController := &CloudNodeController{ kubeClient: fnh, nodeInformer: factory.Nodes(), cloud: &fakecloud.FakeCloud{Err: cloudprovider.InstanceNotFound}, nodeMonitorPeriod: 5 * time.Second, recorder: eventBroadcaster.NewRecorder(v1.EventSource{Component: "controllermanager"}), } eventBroadcaster.StartLogging(glog.Infof) cloudNodeController.Run() select { case <-fnh.DeleteWaitChan: case <-time.After(wait.ForeverTestTimeout): t.Errorf("Timed out waiting %v for node to be deleted", wait.ForeverTestTimeout) } if len(fnh.DeletedNodes) != 1 || fnh.DeletedNodes[0].Name != "node0" { t.Errorf("Node was not deleted") } }
// NewNamespaceController returns a new namespace controller func NewNamespaceController(client federationclientset.Interface) *NamespaceController { broadcaster := record.NewBroadcaster() broadcaster.StartRecordingToSink(eventsink.NewFederatedEventSink(client)) recorder := broadcaster.NewRecorder(api.EventSource{Component: "federated-namespace-controller"}) nc := &NamespaceController{ federatedApiClient: client, namespaceReviewDelay: time.Second * 10, clusterAvailableDelay: time.Second * 20, smallDelay: time.Second * 3, updateTimeout: time.Second * 30, namespaceBackoff: flowcontrol.NewBackOff(5*time.Second, time.Minute), eventRecorder: recorder, } // Build delivereres for triggering reconciliations. nc.namespaceDeliverer = util.NewDelayingDeliverer() nc.clusterDeliverer = util.NewDelayingDeliverer() // Start informer in federated API servers on namespaces that should be federated. nc.namespaceInformerStore, nc.namespaceInformerController = cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (pkg_runtime.Object, error) { return client.Core().Namespaces().List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return client.Core().Namespaces().Watch(options) }, }, &api_v1.Namespace{}, controller.NoResyncPeriodFunc(), util.NewTriggerOnAllChanges(func(obj pkg_runtime.Object) { nc.deliverNamespaceObj(obj, 0, false) })) // Federated informer on namespaces in members of federation. nc.namespaceFederatedInformer = util.NewFederatedInformer( client, func(cluster *federation_api.Cluster, targetClient kubeclientset.Interface) (cache.Store, cache.ControllerInterface) { return cache.NewInformer( &cache.ListWatch{ ListFunc: func(options api.ListOptions) (pkg_runtime.Object, error) { return targetClient.Core().Namespaces().List(options) }, WatchFunc: func(options api.ListOptions) (watch.Interface, error) { return targetClient.Core().Namespaces().Watch(options) }, }, &api_v1.Namespace{}, controller.NoResyncPeriodFunc(), // Trigger reconciliation whenever something in federated cluster is changed. In most cases it // would be just confirmation that some namespace opration succeeded. util.NewTriggerOnMetaAndSpecChanges( func(obj pkg_runtime.Object) { nc.deliverNamespaceObj(obj, nc.namespaceReviewDelay, false) }, )) }, &util.ClusterLifecycleHandlerFuncs{ ClusterAvailable: func(cluster *federation_api.Cluster) { // When new cluster becomes available process all the namespaces again. nc.clusterDeliverer.DeliverAfter(allClustersKey, nil, nc.clusterAvailableDelay) }, }, ) // Federated updeater along with Create/Update/Delete operations. nc.federatedUpdater = util.NewFederatedUpdater(nc.namespaceFederatedInformer, func(client kubeclientset.Interface, obj pkg_runtime.Object) error { namespace := obj.(*api_v1.Namespace) _, err := client.Core().Namespaces().Create(namespace) return err }, func(client kubeclientset.Interface, obj pkg_runtime.Object) error { namespace := obj.(*api_v1.Namespace) _, err := client.Core().Namespaces().Update(namespace) return err }, func(client kubeclientset.Interface, obj pkg_runtime.Object) error { namespace := obj.(*api_v1.Namespace) err := client.Core().Namespaces().Delete(namespace.Name, &api.DeleteOptions{}) return err }) return nc }
func TestServiceAccountCreation(t *testing.T) { ns := api.NamespaceDefault defaultName := "default" managedName := "managed" activeNS := &api.Namespace{ ObjectMeta: api.ObjectMeta{Name: ns}, Status: api.NamespaceStatus{ Phase: api.NamespaceActive, }, } terminatingNS := &api.Namespace{ ObjectMeta: api.ObjectMeta{Name: ns}, Status: api.NamespaceStatus{ Phase: api.NamespaceTerminating, }, } defaultServiceAccount := &api.ServiceAccount{ ObjectMeta: api.ObjectMeta{ Name: defaultName, Namespace: ns, ResourceVersion: "1", }, } managedServiceAccount := &api.ServiceAccount{ ObjectMeta: api.ObjectMeta{ Name: managedName, Namespace: ns, ResourceVersion: "1", }, } unmanagedServiceAccount := &api.ServiceAccount{ ObjectMeta: api.ObjectMeta{ Name: "other-unmanaged", Namespace: ns, ResourceVersion: "1", }, } testcases := map[string]struct { ExistingNamespace *api.Namespace ExistingServiceAccounts []*api.ServiceAccount AddedNamespace *api.Namespace UpdatedNamespace *api.Namespace DeletedServiceAccount *api.ServiceAccount ExpectCreatedServiceAccounts []string }{ "new active namespace missing serviceaccounts": { ExistingServiceAccounts: []*api.ServiceAccount{}, AddedNamespace: activeNS, ExpectCreatedServiceAccounts: sets.NewString(defaultName, managedName).List(), }, "new active namespace missing serviceaccount": { ExistingServiceAccounts: []*api.ServiceAccount{managedServiceAccount}, AddedNamespace: activeNS, ExpectCreatedServiceAccounts: []string{defaultName}, }, "new active namespace with serviceaccounts": { ExistingServiceAccounts: []*api.ServiceAccount{defaultServiceAccount, managedServiceAccount}, AddedNamespace: activeNS, ExpectCreatedServiceAccounts: []string{}, }, "new terminating namespace": { ExistingServiceAccounts: []*api.ServiceAccount{}, AddedNamespace: terminatingNS, ExpectCreatedServiceAccounts: []string{}, }, "updated active namespace missing serviceaccounts": { ExistingServiceAccounts: []*api.ServiceAccount{}, UpdatedNamespace: activeNS, ExpectCreatedServiceAccounts: sets.NewString(defaultName, managedName).List(), }, "updated active namespace missing serviceaccount": { ExistingServiceAccounts: []*api.ServiceAccount{defaultServiceAccount}, UpdatedNamespace: activeNS, ExpectCreatedServiceAccounts: []string{managedName}, }, "updated active namespace with serviceaccounts": { ExistingServiceAccounts: []*api.ServiceAccount{defaultServiceAccount, managedServiceAccount}, UpdatedNamespace: activeNS, ExpectCreatedServiceAccounts: []string{}, }, "updated terminating namespace": { ExistingServiceAccounts: []*api.ServiceAccount{}, UpdatedNamespace: terminatingNS, ExpectCreatedServiceAccounts: []string{}, }, "deleted serviceaccount without namespace": { DeletedServiceAccount: defaultServiceAccount, ExpectCreatedServiceAccounts: []string{}, }, "deleted serviceaccount with active namespace": { ExistingServiceAccounts: []*api.ServiceAccount{managedServiceAccount}, ExistingNamespace: activeNS, DeletedServiceAccount: defaultServiceAccount, ExpectCreatedServiceAccounts: []string{defaultName}, }, "deleted serviceaccount with terminating namespace": { ExistingNamespace: terminatingNS, DeletedServiceAccount: defaultServiceAccount, ExpectCreatedServiceAccounts: []string{}, }, "deleted unmanaged serviceaccount with active namespace": { ExistingServiceAccounts: []*api.ServiceAccount{defaultServiceAccount, managedServiceAccount}, ExistingNamespace: activeNS, DeletedServiceAccount: unmanagedServiceAccount, ExpectCreatedServiceAccounts: []string{}, }, "deleted unmanaged serviceaccount with terminating namespace": { ExistingNamespace: terminatingNS, DeletedServiceAccount: unmanagedServiceAccount, ExpectCreatedServiceAccounts: []string{}, }, } for k, tc := range testcases { client := fake.NewSimpleClientset(defaultServiceAccount, managedServiceAccount) informers := informers.NewSharedInformerFactory(fake.NewSimpleClientset(), controller.NoResyncPeriodFunc()) options := DefaultServiceAccountsControllerOptions() options.ServiceAccounts = []api.ServiceAccount{ {ObjectMeta: api.ObjectMeta{Name: defaultName}}, {ObjectMeta: api.ObjectMeta{Name: managedName}}, } controller := NewServiceAccountsController(informers.ServiceAccounts(), informers.Namespaces(), client, options) controller.saLister = &cache.StoreToServiceAccountLister{Indexer: cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})} controller.nsLister = &cache.IndexerToNamespaceLister{Indexer: cache.NewIndexer(cache.DeletionHandlingMetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc})} controller.saSynced = alwaysReady controller.nsSynced = alwaysReady syncCalls := make(chan struct{}) controller.syncHandler = func(key string) error { err := controller.syncNamespace(key) if err != nil { t.Logf("%s: %v", k, err) } syncCalls <- struct{}{} return err } stopCh := make(chan struct{}) defer close(stopCh) go controller.Run(1, stopCh) if tc.ExistingNamespace != nil { controller.nsLister.Add(tc.ExistingNamespace) } for _, s := range tc.ExistingServiceAccounts { controller.saLister.Indexer.Add(s) } if tc.AddedNamespace != nil { controller.nsLister.Add(tc.AddedNamespace) controller.namespaceAdded(tc.AddedNamespace) } if tc.UpdatedNamespace != nil { controller.nsLister.Add(tc.UpdatedNamespace) controller.namespaceUpdated(nil, tc.UpdatedNamespace) } if tc.DeletedServiceAccount != nil { controller.serviceAccountDeleted(tc.DeletedServiceAccount) } // wait to be called select { case <-syncCalls: case <-time.After(10 * time.Second): t.Errorf("%s: took too long", k) } actions := client.Actions() if len(tc.ExpectCreatedServiceAccounts) != len(actions) { t.Errorf("%s: Expected to create accounts %#v. Actual actions were: %#v", k, tc.ExpectCreatedServiceAccounts, actions) continue } for i, expectedName := range tc.ExpectCreatedServiceAccounts { action := actions[i] if !action.Matches("create", "serviceaccounts") { t.Errorf("%s: Unexpected action %s", k, action) break } createdAccount := action.(core.CreateAction).GetObject().(*api.ServiceAccount) if createdAccount.Name != expectedName { t.Errorf("%s: Expected %s to be created, got %s", k, expectedName, createdAccount.Name) } } } }