// NewSourceAPI creates config source that watches for changes to the services and endpoints.
func NewSourceAPI(c cache.Getter, period time.Duration, servicesChan chan<- ServiceUpdate, endpointsChan chan<- EndpointsUpdate) {
servicesLW := cache.NewListWatchFromClient(c, "services", api.NamespaceAll, fields.Everything())
cache.NewReflector(servicesLW, &api.Service{}, NewServiceStore(nil, servicesChan), period).Run()
endpointsLW := cache.NewListWatchFromClient(c, "endpoints", api.NamespaceAll, fields.Everything())
cache.NewReflector(endpointsLW, &api.Endpoints{}, NewEndpointsStore(nil, endpointsChan), period).Run()
}
func ExtractFromListOptions(opts interface{}) (labelSelector labels.Selector, fieldSelector fields.Selector, resourceVersion string) {
var err error
switch t := opts.(type) {
case api.ListOptions:
labelSelector = t.LabelSelector
fieldSelector = t.FieldSelector
resourceVersion = t.ResourceVersion
case v1.ListOptions:
labelSelector, err = labels.Parse(t.LabelSelector)
if err != nil {
panic(err)
}
fieldSelector, err = fields.ParseSelector(t.FieldSelector)
if err != nil {
panic(err)
}
resourceVersion = t.ResourceVersion
default:
panic(fmt.Errorf("expect a ListOptions"))
}
if labelSelector == nil {
labelSelector = labels.Everything()
}
if fieldSelector == nil {
fieldSelector = fields.Everything()
}
return labelSelector, fieldSelector, resourceVersion
}
func main() {
cc, err := restclient.InClusterConfig()
if err != nil {
log.Fatalf("Failed to create client: %v", err)
}
kubeClient, err := clientset.NewForConfig(cc)
if err != nil {
log.Fatalf("Failed to create client: %v", err)
}
listAll := api.ListOptions{LabelSelector: labels.Everything(), FieldSelector: fields.Everything()}
nodes, err := kubeClient.Core().Nodes().List(listAll)
if err != nil {
log.Fatalf("Failed to list nodes: %v", err)
}
log.Printf("Nodes:")
for _, node := range nodes.Items {
log.Printf("\t%v", node.Name)
}
services, err := kubeClient.Core().Services(api.NamespaceDefault).List(listAll)
if err != nil {
log.Fatalf("Failed to list services: %v", err)
}
log.Printf("Services:")
for _, svc := range services.Items {
log.Printf("\t%v", svc.Name)
}
log.Printf("Success")
http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintf(w, "Ok")
})
log.Fatal(http.ListenAndServe(":8080", nil))
}
func (c *FakeEvents) GetFieldSelector(involvedObjectName, involvedObjectNamespace, involvedObjectKind, involvedObjectUID *string) fields.Selector {
action := core.GenericActionImpl{}
action.Verb = "get-field-selector"
action.Resource = eventsResource
c.Fake.Invokes(action, nil)
return fields.Everything()
}
func matchEverything() storage.SelectionPredicate {
return storage.SelectionPredicate{
Label: labels.Everything(),
Field: fields.Everything(),
GetAttrs: func(obj runtime.Object) (label labels.Set, field fields.Set, err error) {
return nil, nil, nil
},
}
}
func addDefaultingFuncs(scheme *runtime.Scheme) error {
return scheme.AddDefaultingFuncs(
func(obj *ListOptions) {
if obj.LabelSelector == nil {
obj.LabelSelector = labels.Everything()
}
if obj.FieldSelector == nil {
obj.FieldSelector = fields.Everything()
}
},
)
}
// matchPodName returns selection predicate that matches any pod with name in the set.
// Makes testing simpler.
func matchPodName(names ...string) storage.SelectionPredicate {
// Note: even if pod name is a field, we have to use labels,
// because field selector doesn't support "IN" operator.
l, err := labels.NewRequirement("name", selection.In, names)
if err != nil {
panic("Labels requirement must validate successfully")
}
return storage.SelectionPredicate{
Label: labels.Everything().Add(*l),
Field: fields.Everything(),
GetAttrs: getPodAttrs,
}
}
// Watch makes a matcher for the given label and field, and calls
// WatchPredicate. If possible, you should customize PredicateFunc to produre a
// matcher that matches by key. SelectionPredicate does this for you
// automatically.
func (e *Store) Watch(ctx genericapirequest.Context, options *api.ListOptions) (watch.Interface, error) {
label := labels.Everything()
if options != nil && options.LabelSelector != nil {
label = options.LabelSelector
}
field := fields.Everything()
if options != nil && options.FieldSelector != nil {
field = options.FieldSelector
}
resourceVersion := ""
if options != nil {
resourceVersion = options.ResourceVersion
}
return e.WatchPredicate(ctx, e.PredicateFunc(label, field), resourceVersion)
}
// Benchmark pod listing by waiting on `Tasks` listers to list `Pods` pods via `Workers`.
func BenchmarkPodListEtcd(b *testing.B) {
b.StopTimer()
m := framework.NewMasterComponents(&framework.Config{nil, true, false, 250.0, 500})
defer m.Stop(true, true)
ns := framework.CreateTestingNamespace("benchmark-pod-list-etcd", s, t)
defer framework.DeleteTestingNamespace(ns, s, t)
numPods, numTasks, iter := getPods(b.N), getTasks(b.N), getIterations(b.N)
podsPerNode := numPods / numTasks
if podsPerNode < 1 {
podsPerNode = 1
}
startPodsOnNodes(ns.Name, numPods, numTasks, m.RestClient)
// Stop the rc manager so it doesn't steal resources
m.Stop(false, true)
glog.Infof("Starting benchmark: b.N %d, pods %d, workers %d, podsPerNode %d",
b.N, numPods, numTasks, podsPerNode)
b.StartTimer()
for i := 0; i < iter; i++ {
framework.RunParallel(func(id int) error {
now := time.Now()
defer func() {
glog.V(3).Infof("Worker %d: listing pods took %v", id, time.Since(now))
}()
pods, err := m.ClientSet.Core().Pods(ns.Name).List(api.ListOptions{
LabelSelector: labels.Everything(),
FieldSelector: fields.Everything(),
})
if err != nil {
return err
}
if len(pods.Items) < numPods {
glog.Fatalf("List retrieved %d pods, which is less than %d", len(pods.Items), numPods)
}
return nil
}, numTasks, Workers)
}
b.StopTimer()
}
// List returns a list of items matching labels and field
func (e *Store) List(ctx genericapirequest.Context, options *api.ListOptions) (runtime.Object, error) {
label := labels.Everything()
if options != nil && options.LabelSelector != nil {
label = options.LabelSelector
}
field := fields.Everything()
if options != nil && options.FieldSelector != nil {
field = options.FieldSelector
}
out, err := e.ListPredicate(ctx, e.PredicateFunc(label, field), options)
if err != nil {
return nil, err
}
if e.Decorator != nil {
if err := e.Decorator(out); err != nil {
return nil, err
}
}
return out, nil
}
func logPodStartupStatus(c clientset.Interface, expectedPods int, observedLabels map[string]string, period time.Duration, stopCh chan struct{}) {
label := labels.SelectorFromSet(labels.Set(observedLabels))
podStore := testutils.NewPodStore(c, v1.NamespaceAll, label, fields.Everything())
defer podStore.Stop()
ticker := time.NewTicker(period)
defer ticker.Stop()
for {
select {
case <-ticker.C:
pods := podStore.List()
startupStatus := testutils.ComputeRCStartupStatus(pods, expectedPods)
framework.Logf(startupStatus.String("Density"))
case <-stopCh:
pods := podStore.List()
startupStatus := testutils.ComputeRCStartupStatus(pods, expectedPods)
framework.Logf(startupStatus.String("Density"))
return
}
}
}
// Wait up to 10 minutes for all matching pods to become Running and at least one
// matching pod exists.
func WaitForPodsWithLabelRunning(c clientset.Interface, ns string, label labels.Selector) error {
running := false
PodStore := NewPodStore(c, ns, label, fields.Everything())
defer PodStore.Stop()
waitLoop:
for start := time.Now(); time.Since(start) < 10*time.Minute; time.Sleep(5 * time.Second) {
pods := PodStore.List()
if len(pods) == 0 {
continue waitLoop
}
for _, p := range pods {
if p.Status.Phase != v1.PodRunning {
continue waitLoop
}
}
running = true
break
}
if !running {
return fmt.Errorf("Timeout while waiting for pods with labels %q to be running", label.String())
}
return nil
}
func TestWatchParamParsing(t *testing.T) {
simpleStorage := &SimpleRESTStorage{}
handler := handle(map[string]rest.Storage{
"simples": simpleStorage,
"simpleroots": simpleStorage,
})
server := httptest.NewServer(handler)
defer server.Close()
dest, _ := url.Parse(server.URL)
rootPath := "/" + prefix + "/" + testGroupVersion.Group + "/" + testGroupVersion.Version + "/watch/simples"
namespacedPath := "/" + prefix + "/" + testGroupVersion.Group + "/" + testGroupVersion.Version + "/watch/namespaces/other/simpleroots"
table := []struct {
path string
rawQuery string
resourceVersion string
labelSelector string
fieldSelector string
namespace string
}{
{
path: rootPath,
rawQuery: "resourceVersion=1234",
resourceVersion: "1234",
labelSelector: "",
fieldSelector: "",
namespace: api.NamespaceAll,
}, {
path: rootPath,
rawQuery: "resourceVersion=314159&fieldSelector=Host%3D&labelSelector=name%3Dfoo",
resourceVersion: "314159",
labelSelector: "name=foo",
fieldSelector: "Host=",
namespace: api.NamespaceAll,
}, {
path: rootPath,
rawQuery: "fieldSelector=id%3dfoo&resourceVersion=1492",
resourceVersion: "1492",
labelSelector: "",
fieldSelector: "id=foo",
namespace: api.NamespaceAll,
}, {
path: rootPath,
rawQuery: "",
resourceVersion: "",
labelSelector: "",
fieldSelector: "",
namespace: api.NamespaceAll,
},
{
path: namespacedPath,
rawQuery: "resourceVersion=1234",
resourceVersion: "1234",
labelSelector: "",
fieldSelector: "",
namespace: "other",
}, {
path: namespacedPath,
rawQuery: "resourceVersion=314159&fieldSelector=Host%3D&labelSelector=name%3Dfoo",
resourceVersion: "314159",
labelSelector: "name=foo",
fieldSelector: "Host=",
namespace: "other",
}, {
path: namespacedPath,
rawQuery: "fieldSelector=id%3dfoo&resourceVersion=1492",
resourceVersion: "1492",
labelSelector: "",
fieldSelector: "id=foo",
namespace: "other",
}, {
path: namespacedPath,
rawQuery: "",
resourceVersion: "",
labelSelector: "",
fieldSelector: "",
namespace: "other",
},
}
for _, item := range table {
simpleStorage.requestedLabelSelector = labels.Everything()
simpleStorage.requestedFieldSelector = fields.Everything()
simpleStorage.requestedResourceVersion = "5" // Prove this is set in all cases
simpleStorage.requestedResourceNamespace = ""
dest.Path = item.path
dest.RawQuery = item.rawQuery
resp, err := http.Get(dest.String())
if err != nil {
t.Errorf("%v: unexpected error: %v", item.rawQuery, err)
continue
}
resp.Body.Close()
if e, a := item.namespace, simpleStorage.requestedResourceNamespace; e != a {
t.Errorf("%v: expected %v, got %v", item.rawQuery, e, a)
}
if e, a := item.resourceVersion, simpleStorage.requestedResourceVersion; e != a {
t.Errorf("%v: expected %v, got %v", item.rawQuery, e, a)
}
if e, a := item.labelSelector, simpleStorage.requestedLabelSelector.String(); e != a {
t.Errorf("%v: expected %v, got %v", item.rawQuery, e, a)
}
if e, a := item.fieldSelector, simpleStorage.requestedFieldSelector.String(); e != a {
t.Errorf("%v: expected %v, got %v", item.rawQuery, e, a)
}
}
}
// Run starts a background goroutine that watches for changes to services that
// have (or had) LoadBalancers=true and ensures that they have
// load balancers created and deleted appropriately.
// serviceSyncPeriod controls how often we check the cluster's services to
// ensure that the correct load balancers exist.
// nodeSyncPeriod controls how often we check the cluster's nodes to determine
// if load balancers need to be updated to point to a new set.
//
// It's an error to call Run() more than once for a given ServiceController
// object.
func (s *ServiceController) Run(workers int) {
defer runtime.HandleCrash()
go s.serviceController.Run(wait.NeverStop)
for i := 0; i < workers; i++ {
go wait.Until(s.worker, time.Second, wait.NeverStop)
}
nodeLW := cache.NewListWatchFromClient(s.kubeClient.Core().RESTClient(), "nodes", v1.NamespaceAll, fields.Everything())
cache.NewReflector(nodeLW, &v1.Node{}, s.nodeLister.Store, 0).Run()
go wait.Until(s.nodeSyncLoop, nodeSyncPeriod, wait.NeverStop)
}
Value string
}
// TriggerPublisherFunc is a function that takes an object, and returns a list of pairs
// (<index name>, <index value for the given object>) for all indexes known
// to that function.
type TriggerPublisherFunc func(obj runtime.Object) []MatchValue
// FilterFunc takes an API object and returns true if the object satisfies some requirements.
// TODO: We will remove this type and use SelectionPredicate everywhere.
type FilterFunc func(obj runtime.Object) bool
// Everything accepts all objects.
var Everything = SelectionPredicate{
Label: labels.Everything(),
Field: fields.Everything(),
}
// Pass an UpdateFunc to Interface.GuaranteedUpdate to make an update
// that is guaranteed to succeed.
// See the comment for GuaranteedUpdate for more details.
type UpdateFunc func(input runtime.Object, res ResponseMeta) (output runtime.Object, ttl *uint64, err error)
// Preconditions must be fulfilled before an operation (update, delete, etc.) is carried out.
type Preconditions struct {
// Specifies the target UID.
// +optional
UID *types.UID `json:"uid,omitempty"`
}
// NewUIDPreconditions returns a Preconditions with UID set.
func (config *RCConfig) start() error {
// Don't force tests to fail if they don't care about containers restarting.
var maxContainerFailures int
if config.MaxContainerFailures == nil {
maxContainerFailures = int(math.Max(1.0, float64(config.Replicas)*.01))
} else {
maxContainerFailures = *config.MaxContainerFailures
}
label := labels.SelectorFromSet(labels.Set(map[string]string{"name": config.Name}))
PodStore := NewPodStore(config.Client, config.Namespace, label, fields.Everything())
defer PodStore.Stop()
interval := config.PollInterval
if interval <= 0 {
interval = 10 * time.Second
}
timeout := config.Timeout
if timeout <= 0 {
timeout = 5 * time.Minute
}
oldPods := make([]*v1.Pod, 0)
oldRunning := 0
lastChange := time.Now()
for oldRunning != config.Replicas {
time.Sleep(interval)
pods := PodStore.List()
startupStatus := ComputeRCStartupStatus(pods, config.Replicas)
pods = startupStatus.Created
if config.CreatedPods != nil {
*config.CreatedPods = pods
}
if !config.Silent {
config.RCConfigLog(startupStatus.String(config.Name))
}
if config.PodStatusFile != nil {
fmt.Fprintf(config.PodStatusFile, "%d, running, %d, pending, %d, waiting, %d, inactive, %d, unknown, %d, runningButNotReady\n", startupStatus.Running, startupStatus.Pending, startupStatus.Waiting, startupStatus.Inactive, startupStatus.Unknown, startupStatus.RunningButNotReady)
}
if startupStatus.FailedContainers > maxContainerFailures {
if config.NodeDumpFunc != nil {
config.NodeDumpFunc(config.Client, startupStatus.ContainerRestartNodes.List(), config.RCConfigLog)
}
if config.ContainerDumpFunc != nil {
// Get the logs from the failed containers to help diagnose what caused them to fail
config.ContainerDumpFunc(config.Client, config.Namespace, config.RCConfigLog)
}
return fmt.Errorf("%d containers failed which is more than allowed %d", startupStatus.FailedContainers, maxContainerFailures)
}
if len(pods) < len(oldPods) || len(pods) > config.Replicas {
// This failure mode includes:
// kubelet is dead, so node controller deleted pods and rc creates more
// - diagnose by noting the pod diff below.
// pod is unhealthy, so replication controller creates another to take its place
// - diagnose by comparing the previous "2 Pod states" lines for inactive pods
errorStr := fmt.Sprintf("Number of reported pods for %s changed: %d vs %d", config.Name, len(pods), len(oldPods))
config.RCConfigLog("%v, pods that changed since the last iteration:", errorStr)
config.RCConfigLog(Diff(oldPods, pods).String(sets.NewString()))
return fmt.Errorf(errorStr)
}
if len(pods) > len(oldPods) || startupStatus.Running > oldRunning {
lastChange = time.Now()
}
oldPods = pods
oldRunning = startupStatus.Running
if time.Since(lastChange) > timeout {
break
}
}
if oldRunning != config.Replicas {
// List only pods from a given replication controller.
options := v1.ListOptions{LabelSelector: label.String()}
if pods, err := config.Client.Core().Pods(v1.NamespaceAll).List(options); err == nil {
for _, pod := range pods.Items {
config.RCConfigLog("Pod %s\t%s\t%s\t%s", pod.Name, pod.Spec.NodeName, pod.Status.Phase, pod.DeletionTimestamp)
}
} else {
config.RCConfigLog("Can't list pod debug info: %v", err)
}
return fmt.Errorf("Only %d pods started out of %d", oldRunning, config.Replicas)
}
return nil
}
func (config *DaemonConfig) Run() error {
if config.Image == "" {
config.Image = "kubernetes/pause"
}
nameLabel := map[string]string{
"name": config.Name + "-daemon",
}
daemon := &extensions.DaemonSet{
ObjectMeta: metav1.ObjectMeta{
Name: config.Name,
},
Spec: extensions.DaemonSetSpec{
Template: v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: nameLabel,
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: config.Name,
Image: config.Image,
},
},
},
},
},
}
_, err := config.Client.Extensions().DaemonSets(config.Namespace).Create(daemon)
if err != nil {
return fmt.Errorf("Error creating DaemonSet %v: %v", config.Name, err)
}
var nodes *v1.NodeList
for i := 0; i < retries; i++ {
// Wait for all daemons to be running
nodes, err = config.Client.Core().Nodes().List(v1.ListOptions{ResourceVersion: "0"})
if err == nil {
break
} else if i+1 == retries {
return fmt.Errorf("Error listing Nodes while waiting for DaemonSet %v: %v", config.Name, err)
}
}
timeout := config.Timeout
if timeout <= 0 {
timeout = 5 * time.Minute
}
podStore := NewPodStore(config.Client, config.Namespace, labels.SelectorFromSet(nameLabel), fields.Everything())
defer podStore.Stop()
err = wait.Poll(time.Second, timeout, func() (bool, error) {
pods := podStore.List()
nodeHasDaemon := sets.NewString()
for _, pod := range pods {
podReady, _ := PodRunningReady(pod)
if pod.Spec.NodeName != "" && podReady {
nodeHasDaemon.Insert(pod.Spec.NodeName)
}
}
running := len(nodeHasDaemon)
config.LogFunc("Found %v/%v Daemons %v running", running, config.Name, len(nodes.Items))
return running == len(nodes.Items), nil
})
if err != nil {
config.LogFunc("Timed out while waiting for DaemonsSet %v/%v to be running.", config.Namespace, config.Name)
} else {
config.LogFunc("Created Daemon %v/%v", config.Namespace, config.Name)
}
return err
}
func scale(t *testing.T, namespace string, clientset *clientset.Clientset) {
target := int32(100)
rc := &v1.ReplicationController{
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
Namespace: namespace,
},
Spec: v1.ReplicationControllerSpec{
Replicas: &target,
Selector: map[string]string{"foo": "bar"},
Template: &v1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"foo": "bar",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "container",
Image: "busybox",
},
},
},
},
},
}
w, err := clientset.Core().ReplicationControllers(namespace).Watch(v1.SingleObject(metav1.ObjectMeta{Name: rc.Name}))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := clientset.Core().ReplicationControllers(namespace).Create(rc); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = watch.Until(3*time.Minute, w, func(event watch.Event) (bool, error) {
switch event.Type {
case watch.Modified:
default:
return false, nil
}
switch cast := event.Object.(type) {
case *v1.ReplicationController:
fmt.Printf("Found %v of %v replicas\n", int(cast.Status.Replicas), target)
if cast.Status.Replicas == target {
return true, nil
}
}
return false, nil
})
if err != nil {
pods, _ := clientset.Core().Pods(namespace).List(v1.ListOptions{LabelSelector: labels.Everything().String(), FieldSelector: fields.Everything().String()})
t.Fatalf("unexpected error: %v, ended with %v pods", err, len(pods.Items))
}
}
}
results = append(results, p)
}
return results
}
var _ = framework.KubeDescribe("Restart [Disruptive]", func() {
f := framework.NewDefaultFramework("restart")
var ps *testutils.PodStore
BeforeEach(func() {
// This test requires the ability to restart all nodes, so the provider
// check must be identical to that call.
framework.SkipUnlessProviderIs("gce", "gke")
ps = testutils.NewPodStore(f.ClientSet, api.NamespaceSystem, labels.Everything(), fields.Everything())
})
AfterEach(func() {
if ps != nil {
ps.Stop()
}
})
It("should restart all nodes and ensure all nodes and pods recover", func() {
nn := framework.TestContext.CloudConfig.NumNodes
By("ensuring all nodes are ready")
nodeNamesBefore, err := framework.CheckNodesReady(f.ClientSet, framework.NodeReadyInitialTimeout, nn)
Expect(err).NotTo(HaveOccurred())
framework.Logf("Got the following nodes before restart: %v", nodeNamesBefore)
// NewNodeController returns a new node controller to sync instances from cloudprovider.
// This method returns an error if it is unable to initialize the CIDR bitmap with
// podCIDRs it has already allocated to nodes. Since we don't allow podCIDR changes
// currently, this should be handled as a fatal error.
func NewNodeController(
podInformer informers.PodInformer,
nodeInformer informers.NodeInformer,
daemonSetInformer informers.DaemonSetInformer,
cloud cloudprovider.Interface,
kubeClient clientset.Interface,
podEvictionTimeout time.Duration,
evictionLimiterQPS float32,
secondaryEvictionLimiterQPS float32,
largeClusterThreshold int32,
unhealthyZoneThreshold float32,
nodeMonitorGracePeriod time.Duration,
nodeStartupGracePeriod time.Duration,
nodeMonitorPeriod time.Duration,
clusterCIDR *net.IPNet,
serviceCIDR *net.IPNet,
nodeCIDRMaskSize int,
allocateNodeCIDRs bool) (*NodeController, error) {
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(v1.EventSource{Component: "controllermanager"})
eventBroadcaster.StartLogging(glog.Infof)
if kubeClient != nil {
glog.V(0).Infof("Sending events to api server.")
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.Core().Events("")})
} else {
glog.V(0).Infof("No api server defined - no events will be sent to API server.")
}
if kubeClient != nil && kubeClient.Core().RESTClient().GetRateLimiter() != nil {
metrics.RegisterMetricAndTrackRateLimiterUsage("node_controller", kubeClient.Core().RESTClient().GetRateLimiter())
}
if allocateNodeCIDRs {
if clusterCIDR == nil {
glog.Fatal("NodeController: Must specify clusterCIDR if allocateNodeCIDRs == true.")
}
mask := clusterCIDR.Mask
if maskSize, _ := mask.Size(); maskSize > nodeCIDRMaskSize {
glog.Fatal("NodeController: Invalid clusterCIDR, mask size of clusterCIDR must be less than nodeCIDRMaskSize.")
}
}
nc := &NodeController{
cloud: cloud,
knownNodeSet: make(map[string]*v1.Node),
kubeClient: kubeClient,
recorder: recorder,
podEvictionTimeout: podEvictionTimeout,
maximumGracePeriod: 5 * time.Minute,
zonePodEvictor: make(map[string]*RateLimitedTimedQueue),
nodeStatusMap: make(map[string]nodeStatusData),
nodeMonitorGracePeriod: nodeMonitorGracePeriod,
nodeMonitorPeriod: nodeMonitorPeriod,
nodeStartupGracePeriod: nodeStartupGracePeriod,
lookupIP: net.LookupIP,
now: metav1.Now,
clusterCIDR: clusterCIDR,
serviceCIDR: serviceCIDR,
allocateNodeCIDRs: allocateNodeCIDRs,
forcefullyDeletePod: func(p *v1.Pod) error { return forcefullyDeletePod(kubeClient, p) },
nodeExistsInCloudProvider: func(nodeName types.NodeName) (bool, error) { return nodeExistsInCloudProvider(cloud, nodeName) },
evictionLimiterQPS: evictionLimiterQPS,
secondaryEvictionLimiterQPS: secondaryEvictionLimiterQPS,
largeClusterThreshold: largeClusterThreshold,
unhealthyZoneThreshold: unhealthyZoneThreshold,
zoneStates: make(map[string]zoneState),
podInformer: podInformer,
nodeInformer: nodeInformer,
daemonSetInformer: daemonSetInformer,
}
nc.enterPartialDisruptionFunc = nc.ReducedQPSFunc
nc.enterFullDisruptionFunc = nc.HealthyQPSFunc
nc.computeZoneStateFunc = nc.ComputeZoneState
podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
AddFunc: nc.maybeDeleteTerminatingPod,
UpdateFunc: func(_, obj interface{}) { nc.maybeDeleteTerminatingPod(obj) },
})
nc.podStore = *podInformer.Lister()
nodeEventHandlerFuncs := cache.ResourceEventHandlerFuncs{}
if nc.allocateNodeCIDRs {
var nodeList *v1.NodeList
var err error
// We must poll because apiserver might not be up. This error causes
// controller manager to restart.
if pollErr := wait.Poll(10*time.Second, apiserverStartupGracePeriod, func() (bool, error) {
nodeList, err = kubeClient.Core().Nodes().List(v1.ListOptions{
FieldSelector: fields.Everything().String(),
LabelSelector: labels.Everything().String(),
})
if err != nil {
glog.Errorf("Failed to list all nodes: %v", err)
return false, nil
}
return true, nil
}); pollErr != nil {
return nil, fmt.Errorf("Failed to list all nodes in %v, cannot proceed without updating CIDR map", apiserverStartupGracePeriod)
}
nc.cidrAllocator, err = NewCIDRRangeAllocator(kubeClient, clusterCIDR, serviceCIDR, nodeCIDRMaskSize, nodeList)
if err != nil {
return nil, err
}
nodeEventHandlerFuncs = cache.ResourceEventHandlerFuncs{
AddFunc: func(originalObj interface{}) {
obj, err := api.Scheme.DeepCopy(originalObj)
if err != nil {
utilruntime.HandleError(err)
return
}
node := obj.(*v1.Node)
if err := nc.cidrAllocator.AllocateOrOccupyCIDR(node); err != nil {
utilruntime.HandleError(fmt.Errorf("Error allocating CIDR: %v", err))
}
},
UpdateFunc: func(_, obj interface{}) {
node := obj.(*v1.Node)
// If the PodCIDR is not empty we either:
// - already processed a Node that already had a CIDR after NC restarted
// (cidr is marked as used),
// - already processed a Node successfully and allocated a CIDR for it
// (cidr is marked as used),
// - already processed a Node but we did saw a "timeout" response and
// request eventually got through in this case we haven't released
// the allocated CIDR (cidr is still marked as used).
// There's a possible error here:
// - NC sees a new Node and assigns a CIDR X to it,
// - Update Node call fails with a timeout,
// - Node is updated by some other component, NC sees an update and
// assigns CIDR Y to the Node,
// - Both CIDR X and CIDR Y are marked as used in the local cache,
// even though Node sees only CIDR Y
// The problem here is that in in-memory cache we see CIDR X as marked,
// which prevents it from being assigned to any new node. The cluster
// state is correct.
// Restart of NC fixes the issue.
if node.Spec.PodCIDR == "" {
nodeCopy, err := api.Scheme.Copy(node)
if err != nil {
utilruntime.HandleError(err)
return
}
if err := nc.cidrAllocator.AllocateOrOccupyCIDR(nodeCopy.(*v1.Node)); err != nil {
utilruntime.HandleError(fmt.Errorf("Error allocating CIDR: %v", err))
}
}
},
DeleteFunc: func(originalObj interface{}) {
obj, err := api.Scheme.DeepCopy(originalObj)
if err != nil {
utilruntime.HandleError(err)
return
}
node, isNode := obj.(*v1.Node)
// We can get DeletedFinalStateUnknown instead of *v1.Node here and we need to handle that correctly. #34692
if !isNode {
deletedState, ok := obj.(cache.DeletedFinalStateUnknown)
if !ok {
glog.Errorf("Received unexpected object: %v", obj)
return
}
node, ok = deletedState.Obj.(*v1.Node)
if !ok {
glog.Errorf("DeletedFinalStateUnknown contained non-Node object: %v", deletedState.Obj)
return
}
}
if err := nc.cidrAllocator.ReleaseCIDR(node); err != nil {
glog.Errorf("Error releasing CIDR: %v", err)
}
},
}
}
nodeInformer.Informer().AddEventHandler(nodeEventHandlerFuncs)
nc.nodeStore = *nodeInformer.Lister()
nc.daemonSetStore = *daemonSetInformer.Lister()
return nc, nil
}
func TestFiltering(t *testing.T) {
server, etcdStorage := newEtcdTestStorage(t, testapi.Default.Codec(), etcdtest.PathPrefix())
defer server.Terminate(t)
cacher := newTestCacher(etcdStorage, 10)
defer cacher.Stop()
// Ensure that the cacher is initialized, before creating any pods,
// so that we are sure that all events will be present in cacher.
syncWatcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", "0", storage.Everything)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
syncWatcher.Stop()
podFoo := makeTestPod("foo")
podFoo.Labels = map[string]string{"filter": "foo"}
podFooFiltered := makeTestPod("foo")
podFooPrime := makeTestPod("foo")
podFooPrime.Labels = map[string]string{"filter": "foo"}
podFooPrime.Spec.NodeName = "fakeNode"
podFooNS2 := makeTestPod("foo")
podFooNS2.Namespace += "2"
podFooNS2.Labels = map[string]string{"filter": "foo"}
// Create in another namespace first to make sure events from other namespaces don't get delivered
updatePod(t, etcdStorage, podFooNS2, nil)
fooCreated := updatePod(t, etcdStorage, podFoo, nil)
fooFiltered := updatePod(t, etcdStorage, podFooFiltered, fooCreated)
fooUnfiltered := updatePod(t, etcdStorage, podFoo, fooFiltered)
_ = updatePod(t, etcdStorage, podFooPrime, fooUnfiltered)
deleted := api.Pod{}
if err := etcdStorage.Delete(context.TODO(), "pods/ns/foo", &deleted, nil); err != nil {
t.Errorf("Unexpected error: %v", err)
}
// Set up Watch for object "podFoo" with label filter set.
pred := storage.SelectionPredicate{
Label: labels.SelectorFromSet(labels.Set{"filter": "foo"}),
Field: fields.Everything(),
GetAttrs: func(obj runtime.Object) (label labels.Set, field fields.Set, err error) {
metadata, err := meta.Accessor(obj)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
return labels.Set(metadata.GetLabels()), nil, nil
},
}
watcher, err := cacher.Watch(context.TODO(), "pods/ns/foo", fooCreated.ResourceVersion, pred)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
defer watcher.Stop()
verifyWatchEvent(t, watcher, watch.Deleted, podFooFiltered)
verifyWatchEvent(t, watcher, watch.Added, podFoo)
verifyWatchEvent(t, watcher, watch.Modified, podFooPrime)
verifyWatchEvent(t, watcher, watch.Deleted, podFooPrime)
}