// GetSelfURL executes a curl against the given path via kubectl exec into a // test container running with host networking, and fails if the output // doesn't match the expected string. func (config *NetworkingTestConfig) GetSelfURL(path string, expected string) { cmd := fmt.Sprintf("curl -q -s --connect-timeout 1 http://localhost:10249%s", path) By(fmt.Sprintf("Getting kube-proxy self URL %s", path)) // These are arbitrary timeouts. The curl command should pass on first try, // unless kubeproxy is starved/bootstrapping/restarting etc. const retryInterval = 1 * time.Second const retryTimeout = 30 * time.Second podName := config.HostTestContainerPod.Name var msg string if pollErr := wait.PollImmediate(retryInterval, retryTimeout, func() (bool, error) { stdout, err := RunHostCmd(config.Namespace, podName, cmd) if err != nil { msg = fmt.Sprintf("failed executing cmd %v in %v/%v: %v", cmd, config.Namespace, podName, err) Logf(msg) return false, nil } if !strings.Contains(stdout, expected) { msg = fmt.Sprintf("successfully executed %v in %v/%v, but output '%v' doesn't contain expected string '%v'", cmd, config.Namespace, podName, stdout, expected) Logf(msg) return false, nil } return true, nil }); pollErr != nil { Logf("\nOutput of kubectl describe pod %v/%v:\n", config.Namespace, podName) desc, _ := RunKubectl( "describe", "pod", podName, fmt.Sprintf("--namespace=%v", config.Namespace)) Logf("%s", desc) Failf("Timed out in %v: %v", retryTimeout, msg) } }
func waitForPodsOrDie(cs *kubernetes.Clientset, ns string, n int) { By("Waiting for all pods to be running") err := wait.PollImmediate(framework.Poll, schedulingTimeout, func() (bool, error) { pods, err := cs.Core().Pods(ns).List(v1.ListOptions{LabelSelector: "foo=bar"}) if err != nil { return false, err } if pods == nil { return false, fmt.Errorf("pods is nil") } if len(pods.Items) < n { framework.Logf("pods: %v < %v", len(pods.Items), n) return false, nil } ready := 0 for i := 0; i < n; i++ { if pods.Items[i].Status.Phase == v1.PodRunning { ready++ } } if ready < n { framework.Logf("running pods: %v < %v", ready, n) return false, nil } return true, nil }) framework.ExpectNoError(err, "Waiting for pods in namespace %q to be ready", ns) }
func (s *statefulSetTester) waitForRunning(numStatefulPods int32, ss *apps.StatefulSet, shouldBeReady bool) { pollErr := wait.PollImmediate(statefulsetPoll, statefulsetTimeout, func() (bool, error) { podList := s.getPodList(ss) if int32(len(podList.Items)) < numStatefulPods { framework.Logf("Found %d stateful pods, waiting for %d", len(podList.Items), numStatefulPods) return false, nil } if int32(len(podList.Items)) > numStatefulPods { return false, fmt.Errorf("Too many pods scheduled, expected %d got %d", numStatefulPods, len(podList.Items)) } for _, p := range podList.Items { isReady := v1.IsPodReady(&p) desiredReadiness := shouldBeReady == isReady framework.Logf("Waiting for pod %v to enter %v - Ready=%v, currently %v - Ready=%v", p.Name, v1.PodRunning, shouldBeReady, p.Status.Phase, isReady) if p.Status.Phase != v1.PodRunning || !desiredReadiness { return false, nil } } return true, nil }) if pollErr != nil { framework.Failf("Failed waiting for pods to enter running: %v", pollErr) } }
func (f *Framework) deleteFederationNs() { ns := f.FederationNamespace By(fmt.Sprintf("Destroying federation namespace %q for this suite.", ns.Name)) timeout := 5 * time.Minute if f.NamespaceDeletionTimeout != 0 { timeout = f.NamespaceDeletionTimeout } clientset := f.FederationClientset // First delete the namespace from federation apiserver. // Also delete the corresponding namespaces from underlying clusters. orphanDependents := false if err := clientset.Core().Namespaces().Delete(ns.Name, &v1.DeleteOptions{OrphanDependents: &orphanDependents}); err != nil { framework.Failf("Error while deleting federation namespace %s: %s", ns.Name, err) } // Verify that it got deleted. err := wait.PollImmediate(5*time.Second, timeout, func() (bool, error) { if _, err := clientset.Core().Namespaces().Get(ns.Name, metav1.GetOptions{}); err != nil { if apierrors.IsNotFound(err) { return true, nil } framework.Logf("Error while waiting for namespace to be terminated: %v", err) return false, nil } return false, nil }) if err != nil { if !apierrors.IsNotFound(err) { framework.Failf("Couldn't delete ns %q: %s", ns.Name, err) } else { framework.Logf("Namespace %v was already deleted", ns.Name) } } }
func (s *statefulSetTester) scale(ss *apps.StatefulSet, count int32) error { name := ss.Name ns := ss.Namespace s.update(ns, name, func(ss *apps.StatefulSet) { *(ss.Spec.Replicas) = count }) var statefulPodList *v1.PodList pollErr := wait.PollImmediate(statefulsetPoll, statefulsetTimeout, func() (bool, error) { statefulPodList = s.getPodList(ss) if int32(len(statefulPodList.Items)) == count { return true, nil } return false, nil }) if pollErr != nil { unhealthy := []string{} for _, statefulPod := range statefulPodList.Items { delTs, phase, readiness := statefulPod.DeletionTimestamp, statefulPod.Status.Phase, v1.IsPodReady(&statefulPod) if delTs != nil || phase != v1.PodRunning || !readiness { unhealthy = append(unhealthy, fmt.Sprintf("%v: deletion %v, phase %v, readiness %v", statefulPod.Name, delTs, phase, readiness)) } } return fmt.Errorf("Failed to scale statefulset to %d in %v. Remaining pods:\n%v", count, statefulsetTimeout, unhealthy) } return nil }
func (o *DrainOptions) waitForDelete(pods []api.Pod, interval, timeout time.Duration, usingEviction bool, getPodFn func(string, string) (*api.Pod, error)) ([]api.Pod, error) { var verbStr string if usingEviction { verbStr = "evicted" } else { verbStr = "deleted" } err := wait.PollImmediate(interval, timeout, func() (bool, error) { pendingPods := []api.Pod{} for i, pod := range pods { p, err := getPodFn(pod.Namespace, pod.Name) if apierrors.IsNotFound(err) || (p != nil && p.ObjectMeta.UID != pod.ObjectMeta.UID) { cmdutil.PrintSuccess(o.mapper, false, o.out, "pod", pod.Name, false, verbStr) continue } else if err != nil { return false, err } else { pendingPods = append(pendingPods, pods[i]) } } pods = pendingPods if len(pendingPods) > 0 { return false, nil } return true, nil }) return pods, err }
func (j *ServiceTestJig) WaitForLoadBalancerDestroyOrFail(namespace, name string, ip string, port int, timeout time.Duration) *v1.Service { // TODO: once support ticket 21807001 is resolved, reduce this timeout back to something reasonable defer func() { if err := EnsureLoadBalancerResourcesDeleted(ip, strconv.Itoa(port)); err != nil { Logf("Failed to delete cloud resources for service: %s %d (%v)", ip, port, err) } }() var service *v1.Service Logf("Waiting up to %v for service %q to have no LoadBalancer", timeout, name) pollFunc := func() (bool, error) { svc, err := j.Client.Core().Services(namespace).Get(name, metav1.GetOptions{}) if err != nil { return false, err } if len(svc.Status.LoadBalancer.Ingress) == 0 { service = svc return true, nil } return false, nil } if err := wait.PollImmediate(Poll, timeout, pollFunc); err != nil { Failf("Timeout waiting for service %q to have no load balancer", name) } return service }
// verifies the cacheWatcher.process goroutine is properly cleaned up even if // the writes to cacheWatcher.result channel is blocked. func TestCacheWatcherCleanupNotBlockedByResult(t *testing.T) { var lock sync.RWMutex count := 0 filter := func(string, labels.Set, fields.Set) bool { return true } forget := func(bool) { lock.Lock() defer lock.Unlock() count++ } initEvents := []*watchCacheEvent{ {Object: &api.Pod{}}, {Object: &api.Pod{}}, } // set the size of the buffer of w.result to 0, so that the writes to // w.result is blocked. w := newCacheWatcher(0, 0, initEvents, filter, forget) w.Stop() if err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) { lock.RLock() defer lock.RUnlock() return count == 2, nil }); err != nil { t.Fatalf("expected forget() to be called twice, because sendWatchCacheEvent should not be blocked by the result channel: %v", err) } }
func cleanupServiceShardLoadBalancer(clusterName string, service *v1.Service, timeout time.Duration) error { provider := framework.TestContext.CloudConfig.Provider if provider == nil { return fmt.Errorf("cloud provider undefined") } internalSvc := &v1.Service{} err := api.Scheme.Convert(service, internalSvc, nil) if err != nil { return fmt.Errorf("failed to convert versioned service object to internal type: %v", err) } err = wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { lbi, supported := provider.LoadBalancer() if !supported { return false, fmt.Errorf("%q doesn't support load balancers", provider.ProviderName()) } err := lbi.EnsureLoadBalancerDeleted(clusterName, internalSvc) if err != nil { // Deletion failed with an error, try again. framework.Logf("Failed to delete cloud provider resources for service %q in namespace %q, in cluster %q", service.Name, service.Namespace, clusterName) return false, nil } By(fmt.Sprintf("Cloud provider resources for Service %q in namespace %q in cluster %q deleted", service.Name, service.Namespace, clusterName)) return true, nil }) return err }
// Ensure a key is in the store before returning (or timeout w/ error) func WaitForStoreUpdate(store util.FederatedReadOnlyStore, clusterName, key string, timeout time.Duration) error { retryInterval := 100 * time.Millisecond err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) { _, found, err := store.GetByKey(clusterName, key) return found, err }) return err }
// WaitForFederationApiserverReady waits for the federation apiserver to be ready. // It tests the readiness by sending a GET request and expecting a non error response. func WaitForFederationApiserverReady(c *federation_clientset.Clientset) error { return wait.PollImmediate(time.Second, 1*time.Minute, func() (bool, error) { _, err := c.Federation().Clusters().List(v1.ListOptions{}) if err != nil { return false, nil } return true, nil }) }
// Scale updates a ReplicationController to a new size, with optional precondition check (if preconditions is not nil), // optional retries (if retry is not nil), and then optionally waits for it's replica count to reach the new value // (if wait is not nil). func (scaler *ReplicationControllerScaler) Scale(namespace, name string, newSize uint, preconditions *ScalePrecondition, retry, waitForReplicas *RetryParams) error { if preconditions == nil { preconditions = &ScalePrecondition{-1, ""} } if retry == nil { // Make it try only once, immediately retry = &RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond} } var updatedResourceVersion string cond := ScaleCondition(scaler, preconditions, namespace, name, newSize, &updatedResourceVersion) if err := wait.PollImmediate(retry.Interval, retry.Timeout, cond); err != nil { return err } if waitForReplicas != nil { checkRC := func(rc *api.ReplicationController) bool { if uint(rc.Spec.Replicas) != newSize { // the size is changed by other party. Don't need to wait for the new change to complete. return true } return rc.Status.ObservedGeneration >= rc.Generation && rc.Status.Replicas == rc.Spec.Replicas } // If number of replicas doesn't change, then the update may not event // be sent to underlying databse (we don't send no-op changes). // In such case, <updatedResourceVersion> will have value of the most // recent update (which may be far in the past) so we may get "too old // RV" error from watch or potentially no ReplicationController events // will be deliver, since it may already be in the expected state. // To protect from these two, we first issue Get() to ensure that we // are not already in the expected state. currentRC, err := scaler.c.ReplicationControllers(namespace).Get(name, metav1.GetOptions{}) if err != nil { return err } if !checkRC(currentRC) { watchOptions := api.ListOptions{ FieldSelector: fields.OneTermEqualSelector("metadata.name", name), ResourceVersion: updatedResourceVersion, } watcher, err := scaler.c.ReplicationControllers(namespace).Watch(watchOptions) if err != nil { return err } _, err = watch.Until(waitForReplicas.Timeout, watcher, func(event watch.Event) (bool, error) { if event.Type != watch.Added && event.Type != watch.Modified { return false, nil } return checkRC(event.Object.(*api.ReplicationController)), nil }) if err == wait.ErrWaitTimeout { return fmt.Errorf("timed out waiting for %q to be synced", name) } return err } } return nil }
// wait for the podInformer to observe the pods. Call this function before // running the RS controller to prevent the rc manager from creating new pods // rather than adopting the existing ones. func waitToObservePods(t *testing.T, podInformer cache.SharedIndexInformer, podNum int) { if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { objects := podInformer.GetIndexer().List() if len(objects) == podNum { return true, nil } return false, nil }); err != nil { t.Fatal(err) } }
// waitForFederatedIngressExists waits for the Ingress object exists. func waitForFederatedIngressExists(c *fedclientset.Clientset, ns, ingName string, timeout time.Duration) error { err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { _, err := c.Extensions().Ingresses(ns).Get(ingName, metav1.GetOptions{}) if err != nil { framework.Logf("Waiting for Ingress %v, error %v", ingName, err) return false, nil } return true, nil }) return err }
// Wait for ingress status to be updated to match the desiredStatus. func WaitForFedStatusUpdate(t *testing.T, store cache.Store, key string, desiredStatus apiv1.LoadBalancerStatus, timeout time.Duration) error { retryInterval := 100 * time.Millisecond err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) { obj, found, err := store.GetByKey(key) if !found || err != nil { return false, err } ingress := obj.(*extensionsv1beta1.Ingress) return reflect.DeepEqual(ingress.Status.LoadBalancer, desiredStatus), nil }) return err }
func cleanupServiceShard(clientset *kubeclientset.Clientset, clusterName, namespace string, service *v1.Service, timeout time.Duration) error { err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { err := clientset.Services(namespace).Delete(service.Name, &v1.DeleteOptions{}) if err != nil && !errors.IsNotFound(err) { // Deletion failed with an error, try again. framework.Logf("Failed to delete service %q in namespace %q, in cluster %q", service.Name, namespace, clusterName) return false, nil } By(fmt.Sprintf("Service %q in namespace %q in cluster %q deleted", service.Name, namespace, clusterName)) return true, nil }) return err }
// WaitForFederatedIngressAddress waits for the Ingress to acquire an address. func waitForFederatedIngressAddress(c *fedclientset.Clientset, ns, ingName string, timeout time.Duration) (string, error) { var address string err := wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { ipOrNameList, err := getFederatedIngressAddress(c, ns, ingName) if err != nil || len(ipOrNameList) == 0 { framework.Logf("Waiting for Ingress %v to acquire IP, error %v", ingName, err) return false, nil } address = ipOrNameList[0] return true, nil }) return address, err }
func waitPDBStable(t *testing.T, clientSet clientset.Interface, podNum int32, ns, pdbName string) { if err := wait.PollImmediate(2*time.Second, 60*time.Second, func() (bool, error) { pdb, err := clientSet.Policy().PodDisruptionBudgets(ns).Get(pdbName, metav1.GetOptions{}) if err != nil { return false, err } if pdb.Status.CurrentHealthy != podNum { return false, nil } return true, nil }); err != nil { t.Fatal(err) } }
// WaitForPodToDisappear polls the API server if the pod has been deleted. func WaitForPodToDisappear(podClient coreclient.PodInterface, podName string, interval, timeout time.Duration) error { return wait.PollImmediate(interval, timeout, func() (bool, error) { _, err := podClient.Get(podName, metav1.GetOptions{}) if err == nil { return false, nil } else { if errors.IsNotFound(err) { return true, nil } else { return false, err } } }) }
// Verify that the cluster is marked ready. func isReady(clusterName string, clientset *federation_clientset.Clientset) error { return wait.PollImmediate(time.Second, 5*time.Minute, func() (bool, error) { c, err := clientset.Federation().Clusters().Get(clusterName, metav1.GetOptions{}) if err != nil { return false, err } for _, condition := range c.Status.Conditions { if condition.Type == federationapi.ClusterReady && condition.Status == v1.ConditionTrue { return true, nil } } return false, nil }) }
// Wait for the cluster ingress to appear in cluster store. func WaitForIngressInClusterStore(store util.FederatedReadOnlyStore, clusterName, key string) error { retryInterval := 100 * time.Millisecond timeout := wait.ForeverTestTimeout err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) { _, found, err := store.GetByKey(clusterName, key) if found && err == nil { return true, nil } if errors.IsNotFound(err) { return false, nil } return false, err }) return err }
// waitForObjectDeletion refreshes the object, waiting until it is deleted, a timeout is reached, or // an error is encountered. It checks once a second. func waitForObjectDeletion(info *resource.Info, timeout time.Duration) error { copied := *info info = &copied // TODO: refactor Reaper so that we can pass the "wait" option into it, and then check for UID change. return wait.PollImmediate(objectDeletionWaitInterval, timeout, func() (bool, error) { switch err := info.Get(); { case err == nil: return false, nil case errors.IsNotFound(err): return true, nil default: return false, err } }) }
func (j *ServiceTestJig) GetHTTPContent(host string, port int, timeout time.Duration, url string) bytes.Buffer { var body bytes.Buffer var err error if pollErr := wait.PollImmediate(Poll, timeout, func() (bool, error) { result, err := TestReachableHTTPWithContent(host, port, url, "", &body) if err != nil { Logf("Error hitting %v:%v%v, retrying: %v", host, port, url, err) return false, nil } return result, nil }); pollErr != nil { Failf("Could not reach HTTP service through %v:%v%v after %v: %v", host, port, url, timeout, err) } return body }
// Ensure a key is in the store before returning (or timeout w/ error) func WaitForStoreUpdateChecking(store util.FederatedReadOnlyStore, clusterName, key string, timeout time.Duration, checkFunction CheckingFunction) error { retryInterval := 500 * time.Millisecond var lastError error err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) { item, found, err := store.GetByKey(clusterName, key) if err != nil || !found { return found, err } runtimeObj := item.(runtime.Object) lastError = checkFunction(runtimeObj) glog.V(2).Infof("Check function failed for %s %v %v", key, runtimeObj, lastError) return lastError == nil, nil }) return err }
func pollReadWithTimeout(statefulPod statefulPodTester, statefulPodNumber int, key, expectedVal string) error { err := wait.PollImmediate(time.Second, readTimeout, func() (bool, error) { val := statefulPod.read(statefulPodNumber, key) if val == "" { return false, nil } else if val != expectedVal { return false, fmt.Errorf("expected value %v, found %v", expectedVal, val) } return true, nil }) if err == wait.ErrWaitTimeout { return fmt.Errorf("timed out when trying to read value for key %v from stateful pod %d", key, statefulPodNumber) } return err }
// Wait till the store is updated with latest secret. func WaitForSecretStoreUpdate(store util.FederatedReadOnlyStore, clusterName, key string, desiredSecret *apiv1.Secret, timeout time.Duration) error { retryInterval := 200 * time.Millisecond err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) { obj, found, err := store.GetByKey(clusterName, key) if !found || err != nil { glog.Infof("%s is not in the store", key) return false, err } equal := secretsEqual(*obj.(*apiv1.Secret), *desiredSecret) if !equal { glog.Infof("wrong content in the store expected:\n%v\nactual:\n%v\n", *desiredSecret, *obj.(*apiv1.Secret)) } return equal, err }) return err }
/* waitForIngressUpdateOrFail waits until a ingress is updated in the specified cluster with same spec of federated ingress. If the condition is not met within timeout, it fails the calling test. */ func waitForIngressUpdateOrFail(clientset *kubeclientset.Clientset, namespace string, ingress *v1beta1.Ingress, timeout time.Duration) { By(fmt.Sprintf("Fetching a federated ingress shard of ingress %q in namespace %q from cluster", ingress.Name, namespace)) err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { clusterIngress, err := clientset.Ingresses(namespace).Get(ingress.Name, metav1.GetOptions{}) if err == nil { // We want it present, and the Get succeeded, so we're all good. if equivalentIngress(*clusterIngress, *ingress) { By(fmt.Sprintf("Success: shard of federated ingress %q in namespace %q in cluster is updated", ingress.Name, namespace)) return true, nil } By(fmt.Sprintf("Ingress %q in namespace %q in cluster, waiting for service being updated, trying again in %s (err=%v)", ingress.Name, namespace, framework.Poll, err)) return false, nil } By(fmt.Sprintf("Ingress %q in namespace %q in cluster, waiting for service being updated, trying again in %s (err=%v)", ingress.Name, namespace, framework.Poll, err)) return false, nil }) framework.ExpectNoError(err, "Failed to verify ingress %q in namespace %q in cluster", ingress.Name, namespace) }
// Wait for finalizers to appear in federation store. func WaitForFinalizersInFederationStore(ingressController *IngressController, store cache.Store, key string) error { retryInterval := 100 * time.Millisecond timeout := wait.ForeverTestTimeout err := wait.PollImmediate(retryInterval, timeout, func() (bool, error) { obj, found, err := store.GetByKey(key) if !found || err != nil { return false, err } ingress := obj.(*extensionsv1beta1.Ingress) if ingressController.hasFinalizerFunc(ingress, apiv1.FinalizerOrphan) && ingressController.hasFinalizerFunc(ingress, deletionhelper.FinalizerDeleteFromUnderlyingClusters) { return true, nil } return false, nil }) return err }
func CheckReachabilityFromPod(expectToBeReachable bool, namespace, pod, target string) { cmd := fmt.Sprintf("wget -T 5 -qO- %q", target) err := wait.PollImmediate(Poll, 2*time.Minute, func() (bool, error) { _, err := RunHostCmd(namespace, pod, cmd) if expectToBeReachable && err != nil { Logf("Expect target to be reachable. But got err: %v. Retry until timeout", err) return false, nil } if !expectToBeReachable && err == nil { Logf("Expect target NOT to be reachable. But it is reachable. Retry until timeout") return false, nil } return true, nil }) Expect(err).NotTo(HaveOccurred()) }
// waitForAllClustersReady wait for all clusters defined in e2e context to be created // return ClusterList until the listed cluster items equals clusterCount func waitForAllClustersReady(f *fedframework.Framework, clusterCount int) *federationapi.ClusterList { var clusterList *federationapi.ClusterList if err := wait.PollImmediate(framework.Poll, FederatedServiceTimeout, func() (bool, error) { var err error clusterList, err = f.FederationClientset.Federation().Clusters().List(v1.ListOptions{}) if err != nil { return false, err } framework.Logf("%d clusters registered, waiting for %d", len(clusterList.Items), clusterCount) if len(clusterList.Items) == clusterCount { return true, nil } return false, nil }); err != nil { framework.Failf("Failed to list registered clusters: %+v", err) } return clusterList }