func (o *DrainOptions) evictPods(pods []api.Pod, policyGroupVersion string, getPodFn func(namespace, name string) (*api.Pod, error)) error { doneCh := make(chan bool, len(pods)) errCh := make(chan error, 1) for _, pod := range pods { go func(pod api.Pod, doneCh chan bool, errCh chan error) { var err error for { err = o.evictPod(pod, policyGroupVersion) if err == nil { break } else if apierrors.IsTooManyRequests(err) { time.Sleep(5 * time.Second) } else { errCh <- fmt.Errorf("error when evicting pod %q: %v", pod.Name, err) return } } podArray := []api.Pod{pod} _, err = o.waitForDelete(podArray, kubectl.Interval, time.Duration(math.MaxInt64), true, getPodFn) if err == nil { doneCh <- true } else { errCh <- fmt.Errorf("error when waiting for pod %q terminating: %v", pod.Name, err) } }(pod, doneCh, errCh) } doneCount := 0 // 0 timeout means infinite, we use MaxInt64 to represent it. var globalTimeout time.Duration if o.Timeout == 0 { globalTimeout = time.Duration(math.MaxInt64) } else { globalTimeout = o.Timeout } for { select { case err := <-errCh: return err case <-doneCh: doneCount++ if doneCount == len(pods) { return nil } case <-time.After(globalTimeout): return fmt.Errorf("Drain did not complete within %v", globalTimeout) } } }
// TestConcurrentEvictionRequests is to make sure pod disruption budgets (PDB) controller is able to // handle concurrent eviction requests. Original issue:#37605 func TestConcurrentEvictionRequests(t *testing.T) { podNameFormat := "test-pod-%d" s, rm, podInformer, clientSet := rmSetup(t) defer s.Close() ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t) defer framework.DeleteTestingNamespace(ns, s, t) stopCh := make(chan struct{}) go podInformer.Run(stopCh) go rm.Run(stopCh) defer close(stopCh) config := restclient.Config{Host: s.URL} clientSet, err := clientset.NewForConfig(&config) if err != nil { t.Fatalf("Failed to create clientset: %v", err) } var gracePeriodSeconds int64 = 30 deleteOption := &v1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, } // Generate numOfEvictions pods to evict for i := 0; i < numOfEvictions; i++ { podName := fmt.Sprintf(podNameFormat, i) pod := newPod(podName) if _, err := clientSet.Core().Pods(ns.Name).Create(pod); err != nil { t.Errorf("Failed to create pod: %v", err) } addPodConditionReady(pod) if _, err := clientSet.Core().Pods(ns.Name).UpdateStatus(pod); err != nil { t.Fatal(err) } } waitToObservePods(t, podInformer, numOfEvictions) pdb := newPDB() if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil { t.Errorf("Failed to create PodDisruptionBudget: %v", err) } waitPDBStable(t, clientSet, numOfEvictions, ns.Name, pdb.Name) var numberPodsEvicted uint32 = 0 errCh := make(chan error, 3*numOfEvictions) var wg sync.WaitGroup // spawn numOfEvictions goroutines to concurrently evict the pods for i := 0; i < numOfEvictions; i++ { wg.Add(1) go func(id int, errCh chan error) { defer wg.Done() podName := fmt.Sprintf(podNameFormat, id) eviction := newEviction(ns.Name, podName, deleteOption) err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) { e := clientSet.Policy().Evictions(ns.Name).Evict(eviction) switch { case errors.IsTooManyRequests(e): return false, nil case errors.IsConflict(e): return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e) case e == nil: return true, nil default: return false, e } }) if err != nil { errCh <- err // should not return here otherwise we would leak the pod } _, err = clientSet.Core().Pods(ns.Name).Get(podName, metav1.GetOptions{}) switch { case errors.IsNotFound(err): atomic.AddUint32(&numberPodsEvicted, 1) // pod was evicted and deleted so return from goroutine immediately return case err == nil: // this shouldn't happen if the pod was evicted successfully errCh <- fmt.Errorf("Pod %q is expected to be evicted", podName) default: errCh <- err } // delete pod which still exists due to error e := clientSet.Core().Pods(ns.Name).Delete(podName, deleteOption) if e != nil { errCh <- e } }(i, errCh) } wg.Wait() close(errCh) var errList []error if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil { errList = append(errList, fmt.Errorf("Failed to delete PodDisruptionBudget: %v", err)) } for err := range errCh { errList = append(errList, err) } if len(errList) > 0 { t.Fatal(utilerrors.NewAggregate(errList)) } if atomic.LoadUint32(&numberPodsEvicted) != numOfEvictions { t.Fatalf("fewer number of successful evictions than expected :", numberPodsEvicted) } }