func (o *DrainOptions) evictPods(pods []api.Pod, policyGroupVersion string, getPodFn func(namespace, name string) (*api.Pod, error)) error {
doneCh := make(chan bool, len(pods))
errCh := make(chan error, 1)
for _, pod := range pods {
go func(pod api.Pod, doneCh chan bool, errCh chan error) {
var err error
for {
err = o.evictPod(pod, policyGroupVersion)
if err == nil {
break
} else if apierrors.IsTooManyRequests(err) {
time.Sleep(5 * time.Second)
} else {
errCh <- fmt.Errorf("error when evicting pod %q: %v", pod.Name, err)
return
}
}
podArray := []api.Pod{pod}
_, err = o.waitForDelete(podArray, kubectl.Interval, time.Duration(math.MaxInt64), true, getPodFn)
if err == nil {
doneCh <- true
} else {
errCh <- fmt.Errorf("error when waiting for pod %q terminating: %v", pod.Name, err)
}
}(pod, doneCh, errCh)
}
doneCount := 0
// 0 timeout means infinite, we use MaxInt64 to represent it.
var globalTimeout time.Duration
if o.Timeout == 0 {
globalTimeout = time.Duration(math.MaxInt64)
} else {
globalTimeout = o.Timeout
}
for {
select {
case err := <-errCh:
return err
case <-doneCh:
doneCount++
if doneCount == len(pods) {
return nil
}
case <-time.After(globalTimeout):
return fmt.Errorf("Drain did not complete within %v", globalTimeout)
}
}
}
// TestConcurrentEvictionRequests is to make sure pod disruption budgets (PDB) controller is able to
// handle concurrent eviction requests. Original issue:#37605
func TestConcurrentEvictionRequests(t *testing.T) {
podNameFormat := "test-pod-%d"
s, rm, podInformer, clientSet := rmSetup(t)
defer s.Close()
ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t)
defer framework.DeleteTestingNamespace(ns, s, t)
stopCh := make(chan struct{})
go podInformer.Run(stopCh)
go rm.Run(stopCh)
defer close(stopCh)
config := restclient.Config{Host: s.URL}
clientSet, err := clientset.NewForConfig(&config)
if err != nil {
t.Fatalf("Failed to create clientset: %v", err)
}
var gracePeriodSeconds int64 = 30
deleteOption := &v1.DeleteOptions{
GracePeriodSeconds: &gracePeriodSeconds,
}
// Generate numOfEvictions pods to evict
for i := 0; i < numOfEvictions; i++ {
podName := fmt.Sprintf(podNameFormat, i)
pod := newPod(podName)
if _, err := clientSet.Core().Pods(ns.Name).Create(pod); err != nil {
t.Errorf("Failed to create pod: %v", err)
}
addPodConditionReady(pod)
if _, err := clientSet.Core().Pods(ns.Name).UpdateStatus(pod); err != nil {
t.Fatal(err)
}
}
waitToObservePods(t, podInformer, numOfEvictions)
pdb := newPDB()
if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
t.Errorf("Failed to create PodDisruptionBudget: %v", err)
}
waitPDBStable(t, clientSet, numOfEvictions, ns.Name, pdb.Name)
var numberPodsEvicted uint32 = 0
errCh := make(chan error, 3*numOfEvictions)
var wg sync.WaitGroup
// spawn numOfEvictions goroutines to concurrently evict the pods
for i := 0; i < numOfEvictions; i++ {
wg.Add(1)
go func(id int, errCh chan error) {
defer wg.Done()
podName := fmt.Sprintf(podNameFormat, id)
eviction := newEviction(ns.Name, podName, deleteOption)
err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
e := clientSet.Policy().Evictions(ns.Name).Evict(eviction)
switch {
case errors.IsTooManyRequests(e):
return false, nil
case errors.IsConflict(e):
return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
case e == nil:
return true, nil
default:
return false, e
}
})
if err != nil {
errCh <- err
// should not return here otherwise we would leak the pod
}
_, err = clientSet.Core().Pods(ns.Name).Get(podName, metav1.GetOptions{})
switch {
case errors.IsNotFound(err):
atomic.AddUint32(&numberPodsEvicted, 1)
// pod was evicted and deleted so return from goroutine immediately
return
case err == nil:
// this shouldn't happen if the pod was evicted successfully
errCh <- fmt.Errorf("Pod %q is expected to be evicted", podName)
default:
errCh <- err
}
// delete pod which still exists due to error
e := clientSet.Core().Pods(ns.Name).Delete(podName, deleteOption)
if e != nil {
errCh <- e
}
}(i, errCh)
}
wg.Wait()
close(errCh)
var errList []error
if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
errList = append(errList, fmt.Errorf("Failed to delete PodDisruptionBudget: %v", err))
}
for err := range errCh {
errList = append(errList, err)
}
if len(errList) > 0 {
t.Fatal(utilerrors.NewAggregate(errList))
}
if atomic.LoadUint32(&numberPodsEvicted) != numOfEvictions {
t.Fatalf("fewer number of successful evictions than expected :", numberPodsEvicted)
}
}