// TestPolicyBindingListRespectingFields tests that a List() call for some namespace, filtered with a field to the ReadOnlyPolicyBindingCache
// will return all policyBindings in that namespace matching that field
func TestPolicyBindingListRespectingFields(t *testing.T) {
testCache, cacheChannel, testChannel := beforeTestingSetup_readonlypolicybindingcache()
defer close(cacheChannel)
var policyBindings *authorizationapi.PolicyBindingList
var err error
name := "uniquePolicyBindingName"
namespace := "namespaceTwo"
label := labels.Everything()
field := fields.OneTermEqualSelector("metadata.name", name)
util.Until(func() {
policyBindings, err = testCache.List(label, field, namespace)
if (err == nil) &&
(policyBindings != nil) &&
(len(policyBindings.Items) == 1) &&
(policyBindings.Items[0].Name == name) {
close(testChannel)
}
}, 1*time.Millisecond, testChannel)
switch {
case err != nil:
t.Errorf("Error getting policyBindingList using ReadOnlyPolicyBindingCache: %v", err)
case policyBindings == nil:
t.Error("PolicyBindingList is nil.")
case len(policyBindings.Items) != 1:
t.Errorf("Expected policyBindingList to have 1 item, had %d", len(policyBindings.Items))
case policyBindings.Items[0].Name != name:
t.Errorf("Expected policyBinding name to be '%s', was '%s'", name, policyBindings.Items[0].Name)
}
}
// WaitForRunningBuild waits until the specified build is no longer New or Pending. Returns true if
// the build ran within timeout, false if it did not, and an error if any other error state occurred.
// The last observed Build state is returned.
func WaitForRunningBuild(watcher rest.Watcher, ctx kapi.Context, build *api.Build, timeout time.Duration) (*api.Build, bool, error) {
fieldSelector := fields.OneTermEqualSelector("metadata.name", build.Name)
options := &kapi.ListOptions{FieldSelector: fieldSelector, ResourceVersion: build.ResourceVersion}
w, err := watcher.Watch(ctx, options)
if err != nil {
return nil, false, err
}
defer w.Stop()
ch := w.ResultChan()
observed := build
expire := time.After(timeout)
for {
select {
case event := <-ch:
obj, ok := event.Object.(*api.Build)
if !ok {
return observed, false, fmt.Errorf("received unknown object while watching for builds")
}
observed = obj
switch obj.Status.Phase {
case api.BuildPhaseRunning, api.BuildPhaseComplete, api.BuildPhaseFailed, api.BuildPhaseError, api.BuildPhaseCancelled:
return observed, true, nil
case api.BuildPhaseNew, api.BuildPhasePending:
default:
return observed, false, ErrUnknownBuildPhase
}
case <-expire:
return observed, false, nil
}
}
}
// deletePods will delete all pods from master running on given node, and return true
// if any pods were deleted.
func (nc *NodeController) deletePods(nodeName string) (bool, error) {
remaining := false
pods, err := nc.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(), fields.OneTermEqualSelector(client.PodHost, nodeName))
if err != nil {
return remaining, err
}
if len(pods.Items) > 0 {
nc.recordNodeEvent(nodeName, "DeletingAllPods", fmt.Sprintf("Deleting all Pods from Node %v.", nodeName))
}
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeName {
continue
}
// if the pod has already been deleted, ignore it
if pod.DeletionGracePeriodSeconds != nil {
continue
}
glog.V(2).Infof("Starting deletion of pod %v", pod.Name)
nc.recorder.Eventf(&pod, "NodeControllerEviction", "Marking for deletion Pod %s from Node %s", pod.Name, nodeName)
if err := nc.kubeClient.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
return false, err
}
remaining = true
}
return remaining, nil
}
// TestClusterPolicyBindingListRespectingFields tests that a List() call, filtered with a field to the ReadOnlyClusterPolicyBindingCache
// will return all clusterPolicyBindings matching that field
func TestClusterPolicyBindingListRespectingFields(t *testing.T) {
testCache, cacheChannel, testChannel := beforeTestingSetup_readonlyclusterpolicybindingcache()
defer close(cacheChannel)
var clusterPolicyBindings *authorizationapi.ClusterPolicyBindingList
var err error
name := "uniqueClusterPolicyBindingName"
field := fields.OneTermEqualSelector("metadata.name", name)
util.Until(func() {
clusterPolicyBindings, err = testCache.List(&unversioned.ListOptions{FieldSelector: unversioned.FieldSelector{Selector: field}})
if (err == nil) &&
(clusterPolicyBindings != nil) &&
(len(clusterPolicyBindings.Items) == 1) &&
(clusterPolicyBindings.Items[0].Name == name) {
close(testChannel)
}
}, 1*time.Millisecond, testChannel)
switch {
case err != nil:
t.Errorf("Error getting clusterPolicyBinding with fieldSelector using ReadOnlyClusterBindingCache: %v", err)
case clusterPolicyBindings == nil:
t.Error("ClusterPolicyBindingList using fieldSelector is nil")
case len(clusterPolicyBindings.Items) != 1:
t.Errorf("Expected clusterPolicyBindingList using fieldSelector to contain 1 items, had %d", len(clusterPolicyBindings.Items))
case clusterPolicyBindings.Items[0].Name != name:
t.Errorf("Expected clusterPolicyBinding to have name '%s', had '%s'", name, clusterPolicyBindings.Items[0].Name)
}
}
// NewDockercfgDeletedController returns a new *DockercfgDeletedController.
func NewDockercfgDeletedController(cl client.Interface, options DockercfgDeletedControllerOptions) *DockercfgDeletedController {
e := &DockercfgDeletedController{
client: cl,
}
dockercfgSelector := fields.OneTermEqualSelector(client.SecretType, string(api.SecretTypeDockercfg))
_, e.secretController = framework.NewInformer(
&cache.ListWatch{
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
opts := api.ListOptions{FieldSelector: dockercfgSelector}
return e.client.Secrets(api.NamespaceAll).List(opts)
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
opts := api.ListOptions{FieldSelector: dockercfgSelector, ResourceVersion: options.ResourceVersion}
return e.client.Secrets(api.NamespaceAll).Watch(opts)
},
},
&api.Secret{},
options.Resync,
framework.ResourceEventHandlerFuncs{
DeleteFunc: e.secretDeleted,
},
)
return e
}
// update ready status of all pods running on given node from master
// return true if success
func (nc *NodeController) markAllPodsNotReady(nodeName string) error {
glog.V(2).Infof("Update ready status of pods on node [%v]", nodeName)
opts := api.ListOptions{FieldSelector: fields.OneTermEqualSelector(client.PodHost, nodeName)}
pods, err := nc.kubeClient.Core().Pods(api.NamespaceAll).List(opts)
if err != nil {
return err
}
errMsg := []string{}
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeName {
continue
}
for i, cond := range pod.Status.Conditions {
if cond.Type == api.PodReady {
pod.Status.Conditions[i].Status = api.ConditionFalse
glog.V(2).Infof("Updating ready status of pod %v to false", pod.Name)
pod, err := nc.kubeClient.Core().Pods(pod.Namespace).UpdateStatus(&pod)
if err != nil {
glog.Warningf("Failed to updated status for pod %q: %v", format.Pod(pod), err)
errMsg = append(errMsg, fmt.Sprintf("%v", err))
}
break
}
}
}
if len(errMsg) == 0 {
return nil
}
return fmt.Errorf("%v", strings.Join(errMsg, "; "))
}
// TestPolicyListRespectingFields tests that a List() call for some namespace, filtered with a field to the ReadOnlyPolicyCache
// will return all policies in that namespace matching that field
func TestPolicyListRespectingFields(t *testing.T) {
testCache, cacheChannel, testChannel := beforeTestingSetup_readonlypolicycache()
defer close(cacheChannel)
var policies *authorizationapi.PolicyList
var err error
name := "uniquePolicyName"
namespace := "namespaceTwo"
field := fields.OneTermEqualSelector("metadata.name", name)
utilwait.Until(func() {
policies, err = testCache.List(&kapi.ListOptions{FieldSelector: field}, namespace)
if (err == nil) &&
(policies != nil) &&
(len(policies.Items) == 1) &&
(policies.Items[0].Name == name) {
close(testChannel)
}
}, 1*time.Millisecond, testChannel)
switch {
case err != nil:
t.Errorf("Error getting policies using ReadOnlyPolicyCache: %v", err)
case policies == nil:
t.Error("PoliciesList is nil")
case len(policies.Items) != 1:
t.Errorf("Expected policyList to have 1 policy, had %d", len(policies.Items))
case policies.Items[0].Name != name:
t.Errorf("Expected policy name to be '%s', was '%s'", name, policies.Items[0].Name)
}
}
// Scale updates a ReplicationController to a new size, with optional precondition check (if preconditions is not nil),
// optional retries (if retry is not nil), and then optionally waits for it's replica count to reach the new value
// (if wait is not nil).
func (scaler *ReplicationControllerScaler) Scale(namespace, name string, newSize uint, preconditions *ScalePrecondition, retry, waitForReplicas *RetryParams) error {
if preconditions == nil {
preconditions = &ScalePrecondition{-1, ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond}
}
cond := ScaleCondition(scaler, preconditions, namespace, name, newSize)
if err := wait.PollImmediate(retry.Interval, retry.Timeout, cond); err != nil {
return err
}
if waitForReplicas != nil {
watchOptions := api.ListOptions{FieldSelector: fields.OneTermEqualSelector("metadata.name", name), ResourceVersion: "0"}
watcher, err := scaler.c.ReplicationControllers(namespace).Watch(watchOptions)
if err != nil {
return err
}
_, err = watch.Until(waitForReplicas.Timeout, watcher, func(event watch.Event) (bool, error) {
if event.Type != watch.Added && event.Type != watch.Modified {
return false, nil
}
rc := event.Object.(*api.ReplicationController)
return rc.Status.ObservedGeneration >= rc.Generation && rc.Status.Replicas == rc.Spec.Replicas, nil
})
if err == wait.ErrWaitTimeout {
return fmt.Errorf("timed out waiting for %q to be synced", name)
}
return err
}
return nil
}
// TestClusterPolicyListRespectingFields tests that a List() call, filtered with a field to the ReadOnlyClusterPolicyCache
// will return all clusterPolicies matching that field
func TestClusterPolicyListRespectingFields(t *testing.T) {
testCache, cacheChannel, testChannel := beforeTestingSetup_readonlyclusterpolicycache()
defer close(cacheChannel)
var clusterPolicies *authorizationapi.ClusterPolicyList
var err error
name := "uniqueClusterPolicyName"
label := labels.Everything()
field := fields.OneTermEqualSelector("metadata.name", name)
util.Until(func() {
clusterPolicies, err = testCache.List(label, field)
if (err == nil) &&
(clusterPolicies != nil) &&
(len(clusterPolicies.Items) == 1) &&
(clusterPolicies.Items[0].Name == name) {
close(testChannel)
}
}, 1*time.Millisecond, testChannel)
switch {
case err != nil:
t.Errorf("Error getting clusterPolicyList with fieldSelector using ReadOnlyClusterPolicyCache: %v", err)
case clusterPolicies == nil:
t.Error("ClusterPolicyList is nil.")
case len(clusterPolicies.Items) != 1:
t.Errorf("Expected clusterPolicyList to contain 2 clusterPolicies, contained %d", len(clusterPolicies.Items))
case clusterPolicies.Items[0].Name != name:
t.Errorf("Expected field-selected clusterPolicy name to be '%s', was '%s'", name, clusterPolicies.Items[0].Name)
}
}
// StartPods check for numPods in TestNS. If they exist, it no-ops, otherwise it starts up
// a temp rc, scales it to match numPods, then deletes the rc leaving behind the pods.
func StartPods(numPods int, host string, restClient *client.Client) error {
start := time.Now()
defer func() {
glog.Infof("StartPods took %v with numPods %d", time.Since(start), numPods)
}()
hostField := fields.OneTermEqualSelector(client.PodHost, host)
pods, err := restClient.Pods(TestNS).List(labels.Everything(), hostField)
if err != nil || len(pods.Items) == numPods {
return err
}
glog.Infof("Found %d pods that match host %v, require %d", len(pods.Items), hostField, numPods)
// For the sake of simplicity, assume all pods in TestNS have selectors matching TestRCManifest.
controller := RCFromManifest(TestRCManifest)
// Make the rc unique to the given host.
controller.Spec.Replicas = numPods
controller.Spec.Template.Spec.NodeName = host
controller.Name = controller.Name + host
controller.Spec.Selector["host"] = host
controller.Spec.Template.Labels["host"] = host
if rc, err := StartRC(controller, restClient); err != nil {
return err
} else {
// Delete the rc, otherwise when we restart master components for the next benchmark
// the rc controller will race with the pods controller in the rc manager.
return restClient.ReplicationControllers(TestNS).Delete(rc.Name)
}
}
// NewDockercfgController returns a new *DockercfgController.
func NewDockercfgController(cl kclientset.Interface, options DockercfgControllerOptions) *DockercfgController {
e := &DockercfgController{
client: cl,
queue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
dockerURLsIntialized: options.DockerURLsIntialized,
}
var serviceAccountCache cache.Store
serviceAccountCache, e.serviceAccountController = cache.NewInformer(
&cache.ListWatch{
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
return e.client.Core().ServiceAccounts(api.NamespaceAll).List(options)
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
return e.client.Core().ServiceAccounts(api.NamespaceAll).Watch(options)
},
},
&api.ServiceAccount{},
options.Resync,
cache.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
serviceAccount := obj.(*api.ServiceAccount)
glog.V(5).Infof("Adding service account %s", serviceAccount.Name)
e.enqueueServiceAccount(serviceAccount)
},
UpdateFunc: func(old, cur interface{}) {
serviceAccount := cur.(*api.ServiceAccount)
glog.V(5).Infof("Updating service account %s", serviceAccount.Name)
// Resync on service object relist.
e.enqueueServiceAccount(serviceAccount)
},
},
)
e.serviceAccountCache = NewEtcdMutationCache(serviceAccountCache)
tokenSecretSelector := fields.OneTermEqualSelector(api.SecretTypeField, string(api.SecretTypeServiceAccountToken))
e.secretCache, e.secretController = cache.NewInformer(
&cache.ListWatch{
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
options.FieldSelector = tokenSecretSelector
return e.client.Core().Secrets(api.NamespaceAll).List(options)
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
options.FieldSelector = tokenSecretSelector
return e.client.Core().Secrets(api.NamespaceAll).Watch(options)
},
},
&api.Secret{},
options.Resync,
cache.ResourceEventHandlerFuncs{
AddFunc: func(cur interface{}) { e.handleTokenSecretUpdate(nil, cur) },
UpdateFunc: func(old, cur interface{}) { e.handleTokenSecretUpdate(old, cur) },
DeleteFunc: e.handleTokenSecretDelete,
},
)
e.syncHandler = e.syncServiceAccount
return e
}
// returns true if the provided node still has pods scheduled to it, or an error if
// the server could not be contacted.
func (nc *NodeController) hasPods(nodeName string) (bool, error) {
pods, err := nc.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(), fields.OneTermEqualSelector(client.PodHost, nodeName))
if err != nil {
return false, err
}
return len(pods.Items) > 0, nil
}
// returns true if the provided node still has pods scheduled to it, or an error if
// the server could not be contacted.
func (nc *NodeController) hasPods(nodeName string) (bool, error) {
selector := fields.OneTermEqualSelector(client.PodHost, nodeName)
options := api.ListOptions{FieldSelector: selector}
pods, err := nc.kubeClient.Core().Pods(api.NamespaceAll).List(options)
if err != nil {
return false, err
}
return len(pods.Items) > 0, nil
}
// Scale updates a ReplicationController to a new size, with optional precondition check (if preconditions is not nil),
// optional retries (if retry is not nil), and then optionally waits for it's replica count to reach the new value
// (if wait is not nil).
func (scaler *ReplicationControllerScaler) Scale(namespace, name string, newSize uint, preconditions *ScalePrecondition, retry, waitForReplicas *RetryParams) error {
if preconditions == nil {
preconditions = &ScalePrecondition{-1, ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond}
}
var updatedResourceVersion string
cond := ScaleCondition(scaler, preconditions, namespace, name, newSize, &updatedResourceVersion)
if err := wait.PollImmediate(retry.Interval, retry.Timeout, cond); err != nil {
return err
}
if waitForReplicas != nil {
checkRC := func(rc *api.ReplicationController) bool {
if uint(rc.Spec.Replicas) != newSize {
// the size is changed by other party. Don't need to wait for the new change to complete.
return true
}
return rc.Status.ObservedGeneration >= rc.Generation && rc.Status.Replicas == rc.Spec.Replicas
}
// If number of replicas doesn't change, then the update may not event
// be sent to underlying databse (we don't send no-op changes).
// In such case, <updatedResourceVersion> will have value of the most
// recent update (which may be far in the past) so we may get "too old
// RV" error from watch or potentially no ReplicationController events
// will be deliver, since it may already be in the expected state.
// To protect from these two, we first issue Get() to ensure that we
// are not already in the expected state.
currentRC, err := scaler.c.ReplicationControllers(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return err
}
if !checkRC(currentRC) {
watchOptions := api.ListOptions{
FieldSelector: fields.OneTermEqualSelector("metadata.name", name),
ResourceVersion: updatedResourceVersion,
}
watcher, err := scaler.c.ReplicationControllers(namespace).Watch(watchOptions)
if err != nil {
return err
}
_, err = watch.Until(waitForReplicas.Timeout, watcher, func(event watch.Event) (bool, error) {
if event.Type != watch.Added && event.Type != watch.Modified {
return false, nil
}
return checkRC(event.Object.(*api.ReplicationController)), nil
})
if err == wait.ErrWaitTimeout {
return fmt.Errorf("timed out waiting for %q to be synced", name)
}
return err
}
}
return nil
}
// Returns true if a node update matching the predicate was emitted from the
// system after performing the supplied action.
func observeNodeUpdateAfterAction(f *framework.Framework, nodeName string, nodePredicate func(*api.Node) bool, action func() error) (bool, error) {
observedMatchingNode := false
nodeSelector := fields.OneTermEqualSelector("metadata.name", nodeName)
informerStartedChan := make(chan struct{})
var informerStartedGuard sync.Once
_, controller := cache.NewInformer(
&cache.ListWatch{
ListFunc: func(options api.ListOptions) (runtime.Object, error) {
options.FieldSelector = nodeSelector
ls, err := f.ClientSet.Core().Nodes().List(options)
return ls, err
},
WatchFunc: func(options api.ListOptions) (watch.Interface, error) {
options.FieldSelector = nodeSelector
w, err := f.ClientSet.Core().Nodes().Watch(options)
// Signal parent goroutine that watching has begun.
informerStartedGuard.Do(func() { close(informerStartedChan) })
return w, err
},
},
&api.Node{},
0,
cache.ResourceEventHandlerFuncs{
UpdateFunc: func(oldObj, newObj interface{}) {
n, ok := newObj.(*api.Node)
Expect(ok).To(Equal(true))
if nodePredicate(n) {
observedMatchingNode = true
}
},
},
)
// Start the informer and block this goroutine waiting for the started signal.
informerStopChan := make(chan struct{})
defer func() { close(informerStopChan) }()
go controller.Run(informerStopChan)
<-informerStartedChan
// Invoke the action function.
err := action()
if err != nil {
return false, err
}
// Poll whether the informer has found a matching node update with a timeout.
// Wait up 2 minutes polling every second.
timeout := 2 * time.Minute
interval := 1 * time.Second
err = wait.Poll(interval, timeout, func() (bool, error) {
return observedMatchingNode, nil
})
return err == nil, err
}
// deletePods will delete all pods from master running on given node, and return true
// if any pods were deleted, or were found pending deletion.
func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, nodeName, nodeUID string, daemonStore cache.StoreToDaemonSetLister) (bool, error) {
remaining := false
selector := fields.OneTermEqualSelector(api.PodHostField, nodeName)
options := api.ListOptions{FieldSelector: selector}
pods, err := kubeClient.Core().Pods(api.NamespaceAll).List(options)
var updateErrList []error
if err != nil {
return remaining, err
}
if len(pods.Items) > 0 {
recordNodeEvent(recorder, nodeName, nodeUID, api.EventTypeNormal, "DeletingAllPods", fmt.Sprintf("Deleting all Pods from Node %v.", nodeName))
}
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeName {
continue
}
// Set reason and message in the pod object.
if _, err = setPodTerminationReason(kubeClient, &pod, nodeName); err != nil {
if errors.IsConflict(err) {
updateErrList = append(updateErrList,
fmt.Errorf("update status failed for pod %q: %v", format.Pod(&pod), err))
continue
}
}
// if the pod has already been marked for deletion, we still return true that there are remaining pods.
if pod.DeletionGracePeriodSeconds != nil {
remaining = true
continue
}
// if the pod is managed by a daemonset, ignore it
_, err := daemonStore.GetPodDaemonSets(&pod)
if err == nil { // No error means at least one daemonset was found
continue
}
glog.V(2).Infof("Starting deletion of pod %v", pod.Name)
recorder.Eventf(&pod, api.EventTypeNormal, "NodeControllerEviction", "Marking for deletion Pod %s from Node %s", pod.Name, nodeName)
if err := kubeClient.Core().Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
return false, err
}
remaining = true
}
if len(updateErrList) > 0 {
return false, utilerrors.NewAggregate(updateErrList)
}
return remaining, nil
}
// terminatePods will ensure all pods on the given node that are in terminating state are eventually
// cleaned up. Returns true if the node has no pods in terminating state, a duration that indicates how
// long before we should check again (the next deadline for a pod to complete), or an error.
func terminatePods(kubeClient clientset.Interface, recorder record.EventRecorder, nodeName string, nodeUID string, since time.Time, maxGracePeriod time.Duration) (bool, time.Duration, error) {
// the time before we should try again
nextAttempt := time.Duration(0)
// have we deleted all pods
complete := true
selector := fields.OneTermEqualSelector(api.PodHostField, nodeName)
options := api.ListOptions{FieldSelector: selector}
pods, err := kubeClient.Core().Pods(api.NamespaceAll).List(options)
if err != nil {
return false, nextAttempt, err
}
now := time.Now()
elapsed := now.Sub(since)
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeName {
continue
}
// only clean terminated pods
if pod.DeletionGracePeriodSeconds == nil {
continue
}
// the user's requested grace period
grace := time.Duration(*pod.DeletionGracePeriodSeconds) * time.Second
if grace > maxGracePeriod {
grace = maxGracePeriod
}
// the time remaining before the pod should have been deleted
remaining := grace - elapsed
if remaining < 0 {
remaining = 0
glog.V(2).Infof("Removing pod %v after %s grace period", pod.Name, grace)
recordNodeEvent(recorder, nodeName, nodeUID, api.EventTypeNormal, "TerminatingEvictedPod", fmt.Sprintf("Pod %s has exceeded the grace period for deletion after being evicted from Node %q and is being force killed", pod.Name, nodeName))
if err := kubeClient.Core().Pods(pod.Namespace).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil {
glog.Errorf("Error completing deletion of pod %s: %v", pod.Name, err)
complete = false
}
} else {
glog.V(2).Infof("Pod %v still terminating, requested grace period %s, %s remaining", pod.Name, grace, remaining)
complete = false
}
if nextAttempt < remaining {
nextAttempt = remaining
}
}
return complete, nextAttempt, nil
}
func GetClientForServiceAccount(adminClient *kclientset.Clientset, clientConfig restclient.Config, namespace, name string) (*client.Client, *kclientset.Clientset, *restclient.Config, error) {
_, err := adminClient.Core().Namespaces().Create(&kapi.Namespace{ObjectMeta: kapi.ObjectMeta{Name: namespace}})
if err != nil && !kerrs.IsAlreadyExists(err) {
return nil, nil, nil, err
}
sa, err := adminClient.Core().ServiceAccounts(namespace).Create(&kapi.ServiceAccount{ObjectMeta: kapi.ObjectMeta{Name: name}})
if kerrs.IsAlreadyExists(err) {
sa, err = adminClient.Core().ServiceAccounts(namespace).Get(name)
}
if err != nil {
return nil, nil, nil, err
}
token := ""
err = wait.Poll(time.Second, 30*time.Second, func() (bool, error) {
selector := fields.OneTermEqualSelector(kapi.SecretTypeField, string(kapi.SecretTypeServiceAccountToken))
secrets, err := adminClient.Core().Secrets(namespace).List(kapi.ListOptions{FieldSelector: selector})
if err != nil {
return false, err
}
for _, secret := range secrets.Items {
if serviceaccounts.IsValidServiceAccountToken(sa, &secret) {
token = string(secret.Data[kapi.ServiceAccountTokenKey])
return true, nil
}
}
return false, nil
})
if err != nil {
return nil, nil, nil, err
}
saClientConfig := clientcmd.AnonymousClientConfig(&clientConfig)
saClientConfig.BearerToken = token
kubeClient, err := kclient.New(&saClientConfig)
if err != nil {
return nil, nil, nil, err
}
kubeClientset := adapter.FromUnversionedClient(kubeClient)
osClient, err := client.New(&saClientConfig)
if err != nil {
return nil, nil, nil, err
}
return osClient, kubeClientset, &saClientConfig, nil
}
// findDockercfgSecret checks all the secrets in the namespace to see if the token secret has any existing dockercfg secrets that reference it
func (e *DockercfgTokenDeletedController) findDockercfgSecrets(tokenSecret *api.Secret) ([]*api.Secret, error) {
dockercfgSecrets := []*api.Secret{}
options := api.ListOptions{FieldSelector: fields.OneTermEqualSelector(client.SecretType, string(api.SecretTypeDockercfg))}
potentialSecrets, err := e.client.Secrets(tokenSecret.Namespace).List(options)
if err != nil {
return nil, err
}
for i, currSecret := range potentialSecrets.Items {
if currSecret.Annotations[ServiceAccountTokenSecretNameKey] == tokenSecret.Name {
dockercfgSecrets = append(dockercfgSecrets, &potentialSecrets.Items[i])
}
}
return dockercfgSecrets, nil
}
// Run runs the specified KubeletExecutorServer.
func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
// create shared channels
kubeletFinished := make(chan struct{})
nodeInfos := make(chan executor.NodeInfo, 1)
// create static pods directory
staticPodsConfigPath := filepath.Join(s.RootDirectory, "static-pods")
err := os.Mkdir(staticPodsConfigPath, 0750)
if err != nil {
return err
}
// we're expecting that either Mesos or the minion process will set this for us
s.containerID = os.Getenv(envContainerID)
if s.containerID == "" {
log.Warningf("missing expected environment variable %q", envContainerID)
}
// create apiserver client
var apiclient *clientset.Clientset
clientConfig, err := kubeletapp.CreateAPIServerClientConfig(s.KubeletServer)
if err == nil {
apiclient, err = clientset.NewForConfig(clientConfig)
}
if err != nil {
// required for k8sm since we need to send api.Binding information back to the apiserver
return fmt.Errorf("cannot create API client: %v", err)
}
var (
pw = cache.NewListWatchFromClient(apiclient.CoreClient, "pods", api.NamespaceAll,
fields.OneTermEqualSelector(client.PodHost, s.HostnameOverride),
)
reg = executor.NewRegistry(apiclient)
)
// start executor
var executorDone <-chan struct{}
executorDone, err = s.runExecutor(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, reg)
if err != nil {
return err
}
// start kubelet, blocking
return s.runKubelet(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, pw, reg, executorDone)
}
func runBuildPodAdmissionTest(t *testing.T, client *client.Client, kclient *kclient.Client, build *buildapi.Build) (*buildapi.Build, *kapi.Pod) {
ns := testutil.Namespace()
_, err := client.Builds(ns).Create(build)
if err != nil {
t.Fatalf("%v", err)
}
watchOpt := kapi.ListOptions{
FieldSelector: fields.OneTermEqualSelector(
"metadata.name",
buildutil.GetBuildPodName(build),
),
}
podWatch, err := kclient.Pods(ns).Watch(watchOpt)
if err != nil {
t.Fatalf("%v", err)
}
type resultObjs struct {
build *buildapi.Build
pod *kapi.Pod
}
result := make(chan resultObjs)
defer podWatch.Stop()
go func() {
for e := range podWatch.ResultChan() {
if e.Type == watchapi.Added {
pod, ok := e.Object.(*kapi.Pod)
if !ok {
t.Fatalf("unexpected object: %v", e.Object)
}
build := (*buildtestutil.TestPod)(pod).GetBuild(t)
result <- resultObjs{build: build, pod: pod}
}
}
}()
select {
case <-time.After(buildPodAdmissionTestTimeout):
t.Fatalf("timed out after %v", buildPodAdmissionTestTimeout)
case objs := <-result:
return objs.build, objs.pod
}
return nil, nil
}
// NewPodWatch creates a pod watching function which is backed by a
// FIFO/reflector pair. This avoids managing watches directly.
// A stop channel to close the watch's reflector is also returned.
// It is the caller's responsibility to defer closing the stop channel to prevent leaking resources.
func NewPodWatch(client kclient.Interface, namespace, name, resourceVersion string, stopChannel chan struct{}) func() *kapi.Pod {
fieldSelector := fields.OneTermEqualSelector("metadata.name", name)
podLW := &deployutil.ListWatcherImpl{
ListFunc: func() (runtime.Object, error) {
return client.Pods(namespace).List(labels.Everything(), fieldSelector)
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return client.Pods(namespace).Watch(labels.Everything(), fieldSelector, resourceVersion)
},
}
queue := cache.NewFIFO(cache.MetaNamespaceKeyFunc)
cache.NewReflector(podLW, &kapi.Pod{}, queue, 1*time.Minute).RunUntil(stopChannel)
return func() *kapi.Pod {
obj := queue.Pop()
return obj.(*kapi.Pod)
}
}
// terminatePods will ensure all pods on the given node that are in terminating state are eventually
// cleaned up
func (nc *NodeController) terminatePods(nodeID string, since time.Time) (time.Duration, error) {
remaining := time.Duration(0)
glog.V(2).Infof("Terminating all pods on %s", nodeID)
pods, err := nc.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(),
fields.OneTermEqualSelector(client.PodHost, nodeID))
if err != nil {
return remaining, err
}
nc.recordNodeEvent(nodeID, "TerminatingAllPods", fmt.Sprintf("Terminating all Pods on Node %s.", nodeID))
now := time.Now()
elapsed := now.Sub(since)
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeID {
continue
}
// only clean terminated pods
if pod.DeletionGracePeriodSeconds == nil {
continue
}
grace := time.Duration(*pod.DeletionGracePeriodSeconds) * time.Second
if grace > nc.maximumGracePeriod {
grace = nc.maximumGracePeriod
}
next := grace - elapsed
if next < 0 {
glog.V(2).Infof("Removing pod %v after %s grace period", pod.Name, grace)
nc.recordNodeEvent(nodeID, "TerminatingEvictedPod", fmt.Sprintf("Pod %s has exceeded the grace period for deletion after being evicted from Node %q and is being force killed", pod.Name, nodeID))
if err := nc.kubeClient.Pods(pod.Namespace).Delete(pod.Name, api.NewDeleteOptions(0)); err != nil {
glog.Errorf("Error completing deletion of pod %s: %v", pod.Name, err)
next = 1
}
}
if remaining < next {
remaining = next
}
}
return remaining, nil
}
// forcefullyDeleteNode immediately deletes all pods on the node, and then
// deletes the node itself.
func forcefullyDeleteNode(kubeClient clientset.Interface, nodeName string, forcefulDeletePodFunc func(*api.Pod) error) error {
selector := fields.OneTermEqualSelector(api.PodHostField, nodeName)
options := api.ListOptions{FieldSelector: selector}
pods, err := kubeClient.Core().Pods(api.NamespaceAll).List(options)
if err != nil {
return fmt.Errorf("unable to list pods on node %q: %v", nodeName, err)
}
for _, pod := range pods.Items {
if pod.Spec.NodeName != nodeName {
continue
}
if err := forcefulDeletePodFunc(&pod); err != nil {
return fmt.Errorf("unable to delete pod %q on node %q: %v", pod.Name, nodeName, err)
}
}
if err := kubeClient.Core().Nodes().Delete(nodeName, nil); err != nil {
return fmt.Errorf("unable to delete node %q: %v", nodeName, err)
}
return nil
}
// update ready status of all pods running on given node from master
// return true if success
func markAllPodsNotReady(kubeClient clientset.Interface, node *api.Node) error {
// Don't set pods to NotReady if the kubelet is running a version that
// doesn't understand how to correct readiness.
// TODO: Remove this check when we no longer guarantee backward compatibility
// with node versions < 1.2.0.
if nodeRunningOutdatedKubelet(node) {
return nil
}
nodeName := node.Name
glog.V(2).Infof("Update ready status of pods on node [%v]", nodeName)
opts := api.ListOptions{FieldSelector: fields.OneTermEqualSelector(api.PodHostField, nodeName)}
pods, err := kubeClient.Core().Pods(api.NamespaceAll).List(opts)
if err != nil {
return err
}
errMsg := []string{}
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeName {
continue
}
for i, cond := range pod.Status.Conditions {
if cond.Type == api.PodReady {
pod.Status.Conditions[i].Status = api.ConditionFalse
glog.V(2).Infof("Updating ready status of pod %v to false", pod.Name)
_, err := kubeClient.Core().Pods(pod.Namespace).UpdateStatus(&pod)
if err != nil {
glog.Warningf("Failed to update status for pod %q: %v", format.Pod(&pod), err)
errMsg = append(errMsg, fmt.Sprintf("%v", err))
}
break
}
}
}
if len(errMsg) == 0 {
return nil
}
return fmt.Errorf("%v", strings.Join(errMsg, "; "))
}
// Benchmark pod listing by waiting on `Tasks` listers to list `Pods` pods via `Workers`.
func BenchmarkPodList(b *testing.B) {
b.StopTimer()
m := framework.NewMasterComponents(&framework.Config{nil, true, false, 250.0, 500})
defer m.Stop(true, true)
ns := framework.CreateTestingNamespace("benchmark-pod-list", s, t)
defer framework.DeleteTestingNamespace(ns, s, t)
numPods, numTasks, iter := getPods(b.N), getTasks(b.N), getIterations(b.N)
podsPerNode := numPods / numTasks
if podsPerNode < 1 {
podsPerNode = 1
}
glog.Infof("Starting benchmark: b.N %d, pods %d, workers %d, podsPerNode %d",
b.N, numPods, numTasks, podsPerNode)
startPodsOnNodes(ns.Name, numPods, numTasks, m.RestClient)
// Stop the rc manager so it doesn't steal resources
m.Stop(false, true)
b.StartTimer()
for i := 0; i < iter; i++ {
framework.RunParallel(func(id int) error {
host := fmt.Sprintf("host.%d", id)
now := time.Now()
defer func() {
glog.V(3).Infof("Worker %d: Node %v listing pods took %v", id, host, time.Since(now))
}()
if pods, err := m.ClientSet.Core().Pods(ns.Name).List(api.ListOptions{
LabelSelector: labels.Everything(),
FieldSelector: fields.OneTermEqualSelector(api.PodHostField, host),
}); err != nil {
return err
} else if len(pods.Items) < podsPerNode {
glog.Fatalf("List retrieved %d pods, which is less than %d", len(pods.Items), podsPerNode)
}
return nil
}, numTasks, Workers)
}
b.StopTimer()
}
// NewPodWatch creates a pod watching function which is backed by a
// FIFO/reflector pair. This avoids managing watches directly.
// A stop channel to close the watch's reflector is also returned.
// It is the caller's responsibility to defer closing the stop channel to prevent leaking resources.
func NewPodWatch(client kcoreclient.PodInterface, namespace, name, resourceVersion string, stopChannel chan struct{}) func() *kapi.Pod {
fieldSelector := fields.OneTermEqualSelector("metadata.name", name)
podLW := &cache.ListWatch{
ListFunc: func(options kapi.ListOptions) (runtime.Object, error) {
options.FieldSelector = fieldSelector
return client.List(options)
},
WatchFunc: func(options kapi.ListOptions) (watch.Interface, error) {
options.FieldSelector = fieldSelector
return client.Watch(options)
},
}
queue := cache.NewResyncableFIFO(cache.MetaNamespaceKeyFunc)
cache.NewReflector(podLW, &kapi.Pod{}, queue, 1*time.Minute).RunUntil(stopChannel)
return func() *kapi.Pod {
obj := cache.Pop(queue)
return obj.(*kapi.Pod)
}
}
// Run runs the specified KubeletExecutorServer.
func (s *KubeletExecutorServer) Run(hks hyperkube.Interface, _ []string) error {
// create shared channels
kubeletFinished := make(chan struct{})
nodeInfos := make(chan executor.NodeInfo, 1)
// create static pods directory
staticPodsConfigPath := filepath.Join(s.RootDirectory, "static-pods")
err := os.Mkdir(staticPodsConfigPath, 0750)
if err != nil {
return err
}
// create apiserver client
var apiclient *client.Client
clientConfig, err := kubeletapp.CreateAPIServerClientConfig(s.KubeletServer)
if err == nil {
apiclient, err = client.New(clientConfig)
}
if err != nil {
// required for k8sm since we need to send api.Binding information back to the apiserver
return fmt.Errorf("cannot create API client: %v", err)
}
var (
pw = cache.NewListWatchFromClient(apiclient, "pods", api.NamespaceAll,
fields.OneTermEqualSelector(client.PodHost, s.HostnameOverride),
)
reg = executor.NewRegistry(apiclient)
)
// start executor
var executorDone <-chan struct{}
executorDone, err = s.runExecutor(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, reg)
if err != nil {
return err
}
// start kubelet, blocking
return s.runKubelet(nodeInfos, kubeletFinished, staticPodsConfigPath, apiclient, pw, reg, executorDone)
}
// deletePods will delete all pods from master running on given node.
func (nc *NodeController) deletePods(nodeID string) error {
glog.V(2).Infof("Delete all pods from %v", nodeID)
pods, err := nc.kubeClient.Pods(api.NamespaceAll).List(labels.Everything(),
fields.OneTermEqualSelector(client.PodHost, nodeID))
if err != nil {
return err
}
nc.recordNodeEvent(nodeID, "DeletingAllPods", fmt.Sprintf("Deleting all Pods from Node %v.", nodeID))
for _, pod := range pods.Items {
// Defensive check, also needed for tests.
if pod.Spec.NodeName != nodeID {
continue
}
glog.V(2).Infof("Delete pod %v", pod.Name)
nc.recorder.Eventf(&pod, "NodeControllerEviction", "Deleting Pod %s from Node %s", pod.Name, nodeID)
if err := nc.kubeClient.Pods(pod.Namespace).Delete(pod.Name, nil); err != nil {
glog.Errorf("Error deleting pod %v: %v", pod.Name, err)
}
}
return nil
}
// NewDockerRegistryServiceController returns a new *DockerRegistryServiceController.
func NewDockerRegistryServiceController(cl client.Interface, options DockerRegistryServiceControllerOptions) *DockerRegistryServiceController {
e := &DockerRegistryServiceController{
client: cl,
dockercfgController: options.DockercfgController,
registryLocationQueue: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
secretsToUpdate: workqueue.NewRateLimitingQueue(workqueue.DefaultControllerRateLimiter()),
serviceName: options.RegistryServiceName,
serviceNamespace: options.RegistryNamespace,
dockerURLsIntialized: options.DockerURLsIntialized,
}
e.serviceCache, e.serviceController = framework.NewInformer(
&cache.ListWatch{
ListFunc: func(opts kapi.ListOptions) (runtime.Object, error) {
opts.FieldSelector = fields.OneTermEqualSelector("metadata.name", options.RegistryServiceName)
return e.client.Services(options.RegistryNamespace).List(opts)
},
WatchFunc: func(opts kapi.ListOptions) (watch.Interface, error) {
opts.FieldSelector = fields.OneTermEqualSelector("metadata.name", options.RegistryServiceName)
return e.client.Services(options.RegistryNamespace).Watch(opts)
},
},
&kapi.Service{},
options.Resync,
framework.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
e.enqueueRegistryLocationQueue()
},
UpdateFunc: func(old, cur interface{}) {
e.enqueueRegistryLocationQueue()
},
DeleteFunc: func(obj interface{}) {
e.enqueueRegistryLocationQueue()
},
},
)
e.servicesSynced = e.serviceController.HasSynced
e.syncRegistryLocationHandler = e.syncRegistryLocationChange
dockercfgOptions := kapi.ListOptions{FieldSelector: fields.SelectorFromSet(map[string]string{kapi.SecretTypeField: string(kapi.SecretTypeDockercfg)})}
e.secretCache, e.secretController = framework.NewInformer(
&cache.ListWatch{
ListFunc: func(opts kapi.ListOptions) (runtime.Object, error) {
return e.client.Secrets(kapi.NamespaceAll).List(dockercfgOptions)
},
WatchFunc: func(opts kapi.ListOptions) (watch.Interface, error) {
return e.client.Secrets(kapi.NamespaceAll).Watch(dockercfgOptions)
},
},
&kapi.Secret{},
options.Resync,
framework.ResourceEventHandlerFuncs{},
)
e.secretsSynced = e.secretController.HasSynced
e.syncSecretHandler = e.syncSecretUpdate
return e
}