func checkExistingRCRecovers(f *Framework) {
By("assert that the pre-existing replication controller recovers")
podClient := f.Client.Pods(f.Namespace.Name)
rcSelector := labels.Set{"name": "baz"}.AsSelector()
By("deleting pods from existing replication controller")
expectNoError(wait.Poll(time.Millisecond*500, time.Second*60, func() (bool, error) {
pods, err := podClient.List(rcSelector, fields.Everything())
if err != nil {
Logf("apiserver returned error, as expected before recovery: %v", err)
return false, nil
}
if len(pods.Items) == 0 {
return false, nil
}
for _, pod := range pods.Items {
err = podClient.Delete(pod.Name, api.NewDeleteOptions(0))
Expect(err).NotTo(HaveOccurred())
}
Logf("apiserver has recovered")
return true, nil
}))
By("waiting for replication controller to recover")
expectNoError(wait.Poll(time.Millisecond*500, time.Second*60, func() (bool, error) {
pods, err := podClient.List(rcSelector, fields.Everything())
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods.Items {
if pod.DeletionTimestamp == nil && api.IsPodReady(&pod) {
return true, nil
}
}
return false, nil
}))
}
func CreateAssets(manifestDir string, timeout time.Duration) error {
upFn := func() (bool, error) {
if err := apiTest(); err != nil {
glog.Warningf("Unable to determine api-server version: %v", err)
return false, nil
}
return true, nil
}
createFn := func() (bool, error) {
err := createAssets(manifestDir)
if err != nil {
glog.Warningf("Error creating assets: %v", err)
return !shouldRetry(err), nil
}
return true, nil
}
start := time.Now()
if err := wait.Poll(5*time.Second, timeout, upFn); err != nil {
return fmt.Errorf("API Server unavailable: %v", err)
}
timeout = timeout - time.Since(start)
if err := wait.Poll(5*time.Second, timeout, createFn); err != nil {
return fmt.Errorf("Failed to create assets: %v", err)
}
return nil
}
// Scale updates a replication controller created by the DeploymentConfig with the provided namespace/name,
// to a new size, with optional precondition check (if preconditions is not nil),optional retries (if retry
// is not nil), and then optionally waits for it's replica count to reach the new value (if wait is not nil).
func (scaler *DeploymentConfigScaler) Scale(namespace, name string, newSize uint, preconditions *kubectl.ScalePrecondition, retry, waitForReplicas *kubectl.RetryParams) error {
if preconditions == nil {
preconditions = &kubectl.ScalePrecondition{Size: -1, ResourceVersion: ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &kubectl.RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond}
}
cond := kubectl.ScaleCondition(scaler, preconditions, namespace, name, newSize)
if err := wait.Poll(retry.Interval, retry.Timeout, cond); err != nil {
if scaleErr := err.(kubectl.ControllerScaleError); kerrors.IsNotFound(scaleErr.ActualError) {
glog.Infof("No deployment found for dc/%s. Scaling the deployment configuration template...", name)
dc, err := scaler.dcClient.DeploymentConfigs(namespace).Get(name)
if err != nil {
return err
}
dc.Template.ControllerTemplate.Replicas = int(newSize)
if _, err := scaler.dcClient.DeploymentConfigs(namespace).Update(dc); err != nil {
return err
}
return nil
}
return err
}
if waitForReplicas != nil {
rc, err := scaler.rcClient.ReplicationControllers(namespace).Get(name)
if err != nil {
return err
}
return wait.Poll(waitForReplicas.Interval, waitForReplicas.Timeout, kclient.ControllerHasDesiredReplicas(scaler.clientInterface, rc))
}
return nil
}
// Scale updates a deployment to a new size, with optional precondition check (if preconditions is not nil),
// optional retries (if retry is not nil), and then optionally waits for the status to reach desired count.
func (scaler *DeploymentScaler) Scale(namespace, name string, newSize uint, preconditions *ScalePrecondition, retry, waitForReplicas *RetryParams) error {
if preconditions == nil {
preconditions = &ScalePrecondition{-1, ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond}
}
cond := ScaleCondition(scaler, preconditions, namespace, name, newSize, nil)
if err := wait.Poll(retry.Interval, retry.Timeout, cond); err != nil {
return err
}
if waitForReplicas != nil {
deployment, err := scaler.c.Deployments(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return err
}
err = wait.Poll(waitForReplicas.Interval, waitForReplicas.Timeout, client.DeploymentHasDesiredReplicas(scaler.c, deployment))
if err == wait.ErrWaitTimeout {
return fmt.Errorf("timed out waiting for %q to be synced", name)
}
return err
}
return nil
}
// WaitForABuild waits for a Build object to match either isOK or isFailed conditions.
func WaitForABuild(c client.BuildInterface, name string, isOK, isFailed func(*buildapi.Build) bool) error {
// wait 2 minutes for build to exist
err := wait.Poll(1*time.Second, 2*time.Minute, func() (bool, error) {
if _, err := c.Get(name); err != nil {
return false, nil
}
return true, nil
})
if err == wait.ErrWaitTimeout {
return fmt.Errorf("Timed out waiting for build %q to be created", name)
}
if err != nil {
return err
}
// wait longer for the build to run to completion
err = wait.Poll(5*time.Second, 60*time.Minute, func() (bool, error) {
list, err := c.List(kapi.ListOptions{FieldSelector: fields.Set{"name": name}.AsSelector()})
if err != nil {
return false, err
}
for i := range list.Items {
if name == list.Items[i].Name && isOK(&list.Items[i]) {
return true, nil
}
if name != list.Items[i].Name || isFailed(&list.Items[i]) {
return false, fmt.Errorf("The build %q status is %q", name, list.Items[i].Status.Phase)
}
}
return false, nil
})
if err == wait.ErrWaitTimeout {
return fmt.Errorf("Timed out waiting for build %q to complete", name)
}
return err
}
// Scale updates the DeploymentConfig with the provided namespace/name, to a
// new size, with optional precondition check (if preconditions is not nil),
// optional retries (if retry is not nil), and then optionally waits for its
// deployment replica count to reach the new value (if wait is not nil).
func (scaler *DeploymentConfigScaler) Scale(namespace, name string, newSize uint, preconditions *kubectl.ScalePrecondition, retry, waitForReplicas *kubectl.RetryParams) error {
if preconditions == nil {
preconditions = &kubectl.ScalePrecondition{Size: -1, ResourceVersion: ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &kubectl.RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond}
}
cond := kubectl.ScaleCondition(scaler, preconditions, namespace, name, newSize)
if err := wait.Poll(retry.Interval, retry.Timeout, cond); err != nil {
return err
}
if waitForReplicas != nil {
dc, err := scaler.dcClient.DeploymentConfigs(namespace).Get(name)
if err != nil {
return err
}
rc, err := scaler.rcClient.ReplicationControllers(namespace).Get(util.LatestDeploymentNameForConfig(dc))
if err != nil {
return err
}
return wait.Poll(waitForReplicas.Interval, waitForReplicas.Timeout, controllerHasSpecifiedReplicas(scaler.clientInterface, rc, dc.Template.ControllerTemplate.Replicas))
}
return nil
}
func runSchedulerNoPhantomPodsTest(client *client.Client) {
pod := &api.Pod{
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "c1",
Image: "kubernetes/pause",
Ports: []api.ContainerPort{
{ContainerPort: 1234, HostPort: 9999},
},
ImagePullPolicy: api.PullIfNotPresent,
},
},
},
}
// Assuming we only have two kublets, the third pod here won't schedule
// if the scheduler doesn't correctly handle the delete for the second
// pod.
pod.ObjectMeta.Name = "phantom.foo"
foo, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, longTestTimeout, podRunning(client, foo.Namespace, foo.Name)); err != nil {
glog.Fatalf("FAILED: pod never started running %v", err)
}
pod.ObjectMeta.Name = "phantom.bar"
bar, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, longTestTimeout, podRunning(client, bar.Namespace, bar.Name)); err != nil {
glog.Fatalf("FAILED: pod never started running %v", err)
}
// Delete a pod to free up room.
glog.Infof("Deleting pod %v", bar.Name)
err = client.Pods(api.NamespaceDefault).Delete(bar.Name, api.NewDeleteOptions(0))
if err != nil {
glog.Fatalf("FAILED: couldn't delete pod %q: %v", bar.Name, err)
}
pod.ObjectMeta.Name = "phantom.baz"
baz, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, longTestTimeout, podRunning(client, baz.Namespace, baz.Name)); err != nil {
if pod, perr := client.Pods(api.NamespaceDefault).Get("phantom.bar"); perr == nil {
glog.Fatalf("FAILED: 'phantom.bar' was never deleted: %#v, err: %v", pod, err)
} else {
glog.Fatalf("FAILED: (Scheduler probably didn't process deletion of 'phantom.bar') Pod never started running: err: %v, perr: %v", err, perr)
}
}
glog.Info("Scheduler doesn't make phantom pods: test passed.")
}
// WaitForABuild waits for a Build object to match either isOK or isFailed conditions.
func WaitForABuild(c client.BuildInterface, name string, isOK, isFailed func(*buildapi.Build) bool) error {
// wait 2 minutes for build to exist
err := wait.Poll(1*time.Second, 2*time.Minute, func() (bool, error) {
if _, err := c.Get(name); err != nil {
return false, nil
}
return true, nil
})
if err != nil {
return err
}
// wait longer for the build to run to completion
return wait.Poll(5*time.Second, 20*time.Minute, func() (bool, error) {
list, err := c.List(labels.Everything(), fields.Set{"name": name}.AsSelector())
if err != nil {
return false, err
}
for i := range list.Items {
if name == list.Items[i].Name && isOK(&list.Items[i]) {
return true, nil
}
if name != list.Items[i].Name || isFailed(&list.Items[i]) {
return false, fmt.Errorf("The build %q status is %q", name, &list.Items[i].Status.Phase)
}
}
rv := list.ResourceVersion
w, err := c.Watch(labels.Everything(), fields.Set{"name": name}.AsSelector(), rv)
if err != nil {
return false, err
}
defer w.Stop()
for {
val, ok := <-w.ResultChan()
if !ok {
// reget and re-watch
return false, nil
}
if e, ok := val.Object.(*buildapi.Build); ok {
if name == e.Name && isOK(e) {
return true, nil
}
if name != e.Name || isFailed(e) {
return false, fmt.Errorf("The build %q status is %q", name, e.Status.Phase)
}
}
}
})
}
// MigTemplate (GCE-only) returns the name of the MIG template that the
// nodes of the cluster use.
func MigTemplate() (string, error) {
var errLast error
var templ string
key := "instanceTemplate"
if wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
// TODO(mikedanese): make this hit the compute API directly instead of
// shelling out to gcloud.
// An `instance-groups managed describe` call outputs what we want to stdout.
output, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
"describe",
fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
TestContext.CloudConfig.NodeInstanceGroup)
if err != nil {
errLast = fmt.Errorf("gcloud compute instance-groups managed describe call failed with err: %v", err)
return false, nil
}
// The 'describe' call probably succeeded; parse the output and try to
// find the line that looks like "instanceTemplate: url/to/<templ>" and
// return <templ>.
if val := ParseKVLines(output, key); len(val) > 0 {
url := strings.Split(val, "/")
templ = url[len(url)-1]
Logf("MIG group %s using template: %s", TestContext.CloudConfig.NodeInstanceGroup, templ)
return true, nil
}
errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
return false, nil
}) != nil {
return "", fmt.Errorf("MigTemplate() failed with last error: %v", errLast)
}
return templ, nil
}
func TestMasterService(t *testing.T) {
_, s := framework.RunAMaster(framework.NewIntegrationTestMasterConfig())
defer s.Close()
client := clientset.NewForConfigOrDie(&restclient.Config{Host: s.URL, ContentConfig: restclient.ContentConfig{GroupVersion: ®istered.GroupOrDie(api.GroupName).GroupVersion}})
err := wait.Poll(time.Second, time.Minute, func() (bool, error) {
svcList, err := client.Core().Services(api.NamespaceDefault).List(api.ListOptions{})
if err != nil {
t.Errorf("unexpected error: %v", err)
return false, nil
}
found := false
for i := range svcList.Items {
if svcList.Items[i].Name == "kubernetes" {
found = true
break
}
}
if found {
ep, err := client.Core().Endpoints(api.NamespaceDefault).Get("kubernetes")
if err != nil {
return false, nil
}
if countEndpoints(ep) == 0 {
return false, fmt.Errorf("no endpoints for kubernetes service: %v", ep)
}
return true, nil
}
return false, nil
})
if err != nil {
t.Errorf("unexpected error: %v", err)
}
}
func Rename(c coreclient.ReplicationControllersGetter, rc *api.ReplicationController, newName string) error {
oldName := rc.Name
rc.Name = newName
rc.ResourceVersion = ""
// First delete the oldName RC and orphan its pods.
trueVar := true
err := c.ReplicationControllers(rc.Namespace).Delete(oldName, &api.DeleteOptions{OrphanDependents: &trueVar})
if err != nil && !errors.IsNotFound(err) {
return err
}
err = wait.Poll(5*time.Second, 60*time.Second, func() (bool, error) {
_, err := c.ReplicationControllers(rc.Namespace).Get(oldName)
if err == nil {
return false, nil
} else if errors.IsNotFound(err) {
return true, nil
} else {
return false, err
}
})
if err != nil {
return err
}
// Then create the same RC with the new name.
_, err = c.ReplicationControllers(rc.Namespace).Create(rc)
if err != nil {
return err
}
return nil
}
// UpdatePodWithRetries updates a pod with given applyUpdate function. Note that pod not found error is ignored.
// The returned bool value can be used to tell if the pod is actually updated.
func UpdatePodWithRetries(podClient unversionedcore.PodInterface, pod *api.Pod, applyUpdate updatePodFunc) (*api.Pod, bool, error) {
var err error
var podUpdated bool
oldPod := pod
if err = wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
pod, err = podClient.Get(oldPod.Name)
if err != nil {
return false, err
}
// Apply the update, then attempt to push it to the apiserver.
// TODO: add precondition for update
applyUpdate(pod)
if pod, err = podClient.Update(pod); err == nil {
// Update successful.
return true, nil
}
// TODO: don't retry on perm-failed errors and handle them gracefully
// Update could have failed due to conflict error. Try again.
return false, nil
}); err == nil {
// When there's no error, we've updated this pod.
podUpdated = true
}
if err == wait.ErrWaitTimeout {
err = fmt.Errorf("timed out trying to update pod: %+v", oldPod)
}
if errors.IsNotFound(err) {
glog.V(4).Infof("%s %s/%s is not found, skip updating it.", oldPod.Kind, oldPod.Namespace, oldPod.Name)
err = nil
}
return pod, podUpdated, err
}
func TestCleanUp(t *testing.T) {
m := newTestManager()
for _, probeType := range [...]probeType{liveness, readiness} {
key := probeKey{podUID, containerName, probeType}
w := newTestWorker(m, probeType, api.Probe{})
m.statusManager.SetPodStatus(w.pod, getRunningStatus())
go w.run()
m.workers[key] = w
// Wait for worker to run.
condition := func() (bool, error) {
ready, _ := resultsManager(m, probeType).Get(containerID)
return ready == results.Success, nil
}
if ready, _ := condition(); !ready {
if err := wait.Poll(100*time.Millisecond, util.ForeverTestTimeout, condition); err != nil {
t.Fatalf("[%s] Error waiting for worker ready: %v", probeType, err)
}
}
close(w.stop)
if err := waitForWorkerExit(m, []probeKey{key}); err != nil {
t.Fatalf("[%s] error waiting for worker exit: %v", probeType, err)
}
if _, ok := resultsManager(m, probeType).Get(containerID); ok {
t.Errorf("[%s] Expected result to be cleared.", probeType)
}
if _, ok := m.workers[key]; ok {
t.Errorf("[%s] Expected worker to be cleared.", probeType)
}
}
}
// WaitForAttach runs on the node to detect if the volume (referenced by LUN) is attached. If attached, the device path is returned
func (attacher *azureDiskAttacher) WaitForAttach(spec *volume.Spec, lunStr string, timeout time.Duration) (string, error) {
volumeSource, err := getVolumeSource(spec)
if err != nil {
return "", err
}
if len(lunStr) == 0 {
return "", fmt.Errorf("WaitForAttach failed for Azure disk %q: lun is empty.", volumeSource.DiskName)
}
lun, err := strconv.Atoi(lunStr)
if err != nil {
return "", fmt.Errorf("WaitForAttach: wrong lun %q, err: %v", lunStr, err)
}
scsiHostRescan(&osIOHandler{})
exe := exec.New()
devicePath := ""
err = wait.Poll(checkSleepDuration, timeout, func() (bool, error) {
glog.V(4).Infof("Checking Azure disk %q(lun %s) is attached.", volumeSource.DiskName, lunStr)
if devicePath, err = findDiskByLun(lun, &osIOHandler{}, exe); err == nil {
glog.V(4).Infof("Successfully found attached Azure disk %q(lun %s, device path %s).", volumeSource.DiskName, lunStr, devicePath)
return true, nil
} else {
//Log error, if any, and continue checking periodically
glog.V(4).Infof("Error Stat Azure disk (%q) is attached: %v", volumeSource.DiskName, err)
return false, nil
}
})
return devicePath, err
}
func assertFilesExist(fileNames []string, fileDir string, pod *api.Pod, client *client.Client) {
var failed []string
expectNoError(wait.Poll(time.Second*2, time.Second*60, func() (bool, error) {
failed = []string{}
for _, fileName := range fileNames {
if _, err := client.Get().
Namespace(pod.Namespace).
Resource("pods").
SubResource("proxy").
Name(pod.Name).
Suffix(fileDir, fileName).
Do().Raw(); err != nil {
Logf("Unable to read %s from pod %s: %v", fileName, pod.Name, err)
failed = append(failed, fileName)
}
}
if len(failed) == 0 {
return true, nil
}
Logf("Lookups using %s failed for: %v\n", pod.Name, failed)
return false, nil
}))
Expect(len(failed)).To(Equal(0))
}
func checkTunnelsCorrect(t *testing.T, tunnelList *SSHTunnelList, addresses []string) {
if err := wait.Poll(100*time.Millisecond, 2*time.Second, func() (bool, error) {
return hasCorrectTunnels(tunnelList, addresses), nil
}); err != nil {
t.Errorf("Error waiting for tunnels to reach expected state: %v. Expected %v, had %v", err, addresses, tunnelList)
}
}
// pollForReadyPods polls oldRc and newRc each interval and returns the old
// and new ready counts for their pods. If a pod is observed as being ready,
// it's considered ready even if it later becomes notReady.
func (r *RollingUpdater) pollForReadyPods(interval, timeout time.Duration, oldRc, newRc *api.ReplicationController) (int, int, error) {
controllers := []*api.ReplicationController{oldRc, newRc}
oldReady := 0
newReady := 0
err := wait.Poll(interval, timeout, func() (done bool, err error) {
anyReady := false
for _, controller := range controllers {
selector := labels.Set(controller.Spec.Selector).AsSelector()
pods, err := r.c.Pods(controller.Namespace).List(selector, fields.Everything())
if err != nil {
return false, err
}
for _, pod := range pods.Items {
if api.IsPodReady(&pod) {
switch controller.Name {
case oldRc.Name:
oldReady++
case newRc.Name:
newReady++
}
anyReady = true
}
}
}
if anyReady {
return true, nil
}
return false, nil
})
return oldReady, newReady, err
}
// migRollingUpdatePoll (CKE/GKE-only) polls the progress of the MIG rolling
// update with ID id until it is complete. It returns an error if this takes
// longer than nt times the number of nodes.
func migRollingUpdatePoll(id string, nt time.Duration) error {
// Two keys and a val.
status, progress, done := "status", "statusMessage", "ROLLED_OUT"
start, timeout := time.Now(), nt*time.Duration(framework.TestContext.CloudConfig.NumNodes)
var errLast error
framework.Logf("Waiting up to %v for MIG rolling update to complete.", timeout)
if wait.Poll(restartPoll, timeout, func() (bool, error) {
// A `rolling-updates describe` call outputs what we want to stdout.
output, _, err := retryCmd("gcloud", "alpha", "compute",
"rolling-updates",
fmt.Sprintf("--project=%s", framework.TestContext.CloudConfig.ProjectID),
fmt.Sprintf("--zone=%s", framework.TestContext.CloudConfig.Zone),
"describe",
id)
if err != nil {
errLast = fmt.Errorf("Error calling rolling-updates describe %s: %v", id, err)
framework.Logf("%v", errLast)
return false, nil
}
// The 'describe' call probably succeeded; parse the output and try to
// find the line that looks like "status: <status>" and see whether it's
// done.
framework.Logf("Waiting for MIG rolling update: %s (%v elapsed)",
framework.ParseKVLines(output, progress), time.Since(start))
if st := framework.ParseKVLines(output, status); st == done {
return true, nil
}
return false, nil
}) != nil {
return fmt.Errorf("timeout waiting %v for MIG rolling update to complete. Last error: %v", timeout, errLast)
}
framework.Logf("MIG rolling update complete after %v", time.Since(start))
return nil
}
// waitForNPods tries to list pods using c until it finds expect of them,
// returning their names if it can do so before timeout.
func waitForNPods(ps *testutils.PodStore, expect int, timeout time.Duration) ([]string, error) {
// Loop until we find expect pods or timeout is passed.
var pods []*api.Pod
var errLast error
found := wait.Poll(framework.Poll, timeout, func() (bool, error) {
allPods := ps.List()
pods := filterIrrelevantPods(allPods)
if len(pods) != expect {
errLast = fmt.Errorf("expected to find %d pods but found only %d", expect, len(pods))
framework.Logf("Error getting pods: %v", errLast)
return false, nil
}
return true, nil
}) == nil
// Extract the names of all found pods.
podNames := make([]string, len(pods))
for i, p := range pods {
podNames[i] = p.ObjectMeta.Name
}
if !found {
return podNames, fmt.Errorf("couldn't find %d pods within %v; last error: %v",
expect, timeout, errLast)
}
return podNames, nil
}
func (reaper *DeploymentConfigReaper) updateDeploymentWithRetries(namespace, name string, applyUpdate updateConfigFunc) (*deployapi.DeploymentConfig, error) {
var (
config *deployapi.DeploymentConfig
err error
)
deploymentConfigs := reaper.oc.DeploymentConfigs(namespace)
resultErr := wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
config, err = deploymentConfigs.Get(name)
if err != nil {
return false, err
}
// Apply the update, then attempt to push it to the apiserver.
applyUpdate(config)
config, err = deploymentConfigs.Update(config)
if err != nil {
// Retry only on update conflict
if kerrors.IsConflict(err) {
return false, nil
}
return false, err
}
return true, nil
})
return config, resultErr
}
func waitForDeployment(c *fedclientset.Clientset, namespace string, deploymentName string, clusters map[string]*cluster) error {
err := wait.Poll(10*time.Second, FederatedDeploymentTimeout, func() (bool, error) {
fdep, err := c.Deployments(namespace).Get(deploymentName, metav1.GetOptions{})
if err != nil {
return false, err
}
specReplicas, statusReplicas := int32(0), int32(0)
for _, cluster := range clusters {
dep, err := cluster.Deployments(namespace).Get(deploymentName, metav1.GetOptions{})
if err != nil && !errors.IsNotFound(err) {
By(fmt.Sprintf("Failed getting deployment: %q/%q/%q, err: %v", cluster.name, namespace, deploymentName, err))
return false, err
}
if err == nil {
if !verifyDeployment(fdep, dep) {
By(fmt.Sprintf("Deployment meta or spec not match for cluster %q:\n federation: %v\n cluster: %v", cluster.name, fdep, dep))
return false, nil
}
specReplicas += *dep.Spec.Replicas
statusReplicas += dep.Status.Replicas
}
}
if statusReplicas == fdep.Status.Replicas && specReplicas >= *fdep.Spec.Replicas {
return true, nil
}
By(fmt.Sprintf("Replicas not match, federation replicas: %v/%v, clusters replicas: %v/%v\n", *fdep.Spec.Replicas, fdep.Status.Replicas, specReplicas, statusReplicas))
return false, nil
})
return err
}
func testReachable(ip string, port int) {
url := fmt.Sprintf("http://%s:%d", ip, port)
if ip == "" {
Failf("Got empty IP for reachability check (%s)", url)
}
if port == 0 {
Failf("Got port==0 for reachability check (%s)", url)
}
desc := fmt.Sprintf("the url %s to be reachable", url)
By(fmt.Sprintf("Waiting up to %v for %s", podStartTimeout, desc))
start := time.Now()
err := wait.Poll(poll, podStartTimeout, func() (bool, error) {
resp, err := httpGetNoConnectionPool(url)
if err != nil {
Logf("Got error waiting for reachability of %s: %v (%v)", url, err, time.Since(start))
return false, nil
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Logf("Got error reading response from %s: %v", url, err)
return false, nil
}
if resp.StatusCode != 200 {
return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s", resp.Status, url, string(body))
}
if !strings.Contains(string(body), "test-webserver") {
return false, fmt.Errorf("received response body without expected substring 'test-webserver': %s", string(body))
}
Logf("Successfully reached %v", url)
return true, nil
})
Expect(err).NotTo(HaveOccurred(), "Error waiting for %s", desc)
}
// This test simulates the case where an object is created with an owner that
// doesn't exist. It verifies the GC will delete such an object.
func TestCreateWithNonExisitentOwner(t *testing.T) {
gc, clientSet := setup(t)
oldEnableGarbageCollector := registry.EnableGarbageCollector
registry.EnableGarbageCollector = true
defer func() { registry.EnableGarbageCollector = oldEnableGarbageCollector }()
podClient := clientSet.Core().Pods(framework.TestNS)
pod := newPod(garbageCollectedPodName, []v1.OwnerReference{{UID: "doesn't matter", Name: toBeDeletedRCName}})
_, err := podClient.Create(pod)
if err != nil {
t.Fatalf("Failed to create Pod: %v", err)
}
// set up watch
pods, err := podClient.List(api.ListOptions{})
if err != nil {
t.Fatalf("Failed to list pods: %v", err)
}
if len(pods.Items) != 1 {
t.Fatalf("Expect only 1 pod")
}
stopCh := make(chan struct{})
go gc.Run(5, stopCh)
defer close(stopCh)
// wait for the garbage collector to drain its queue
if err := wait.Poll(10*time.Second, 120*time.Second, func() (bool, error) {
return gc.QueuesDrained(), nil
}); err != nil {
t.Fatal(err)
}
t.Logf("garbage collector queues drained")
if _, err := podClient.Get(garbageCollectedPodName); err == nil || !errors.IsNotFound(err) {
t.Fatalf("expect pod %s to be garbage collected", garbageCollectedPodName)
}
}
func (r *ResourceCollector) Start() {
// Get the cgroup containers for kubelet and docker
kubeletContainer, err := getContainerNameForProcess(kubeletProcessName, "")
dockerContainer, err := getContainerNameForProcess(dockerProcessName, dockerPidFile)
if err == nil {
systemContainers = map[string]string{
stats.SystemContainerKubelet: kubeletContainer,
stats.SystemContainerRuntime: dockerContainer,
}
} else {
framework.Failf("Failed to get docker container name in test-e2e-node resource collector.")
}
wait.Poll(1*time.Second, 1*time.Minute, func() (bool, error) {
var err error
r.client, err = cadvisorclient.NewClient(fmt.Sprintf("http://localhost:%d/", cadvisorPort))
if err == nil {
return true, nil
}
return false, err
})
Expect(r.client).NotTo(BeNil(), "cadvisor client not ready")
r.request = &cadvisorapiv2.RequestOptions{IdType: "name", Count: 1, Recursive: false}
r.stopCh = make(chan struct{})
oldStatsMap := make(map[string]*cadvisorapiv2.ContainerStats)
go wait.Until(func() { r.collectStats(oldStatsMap) }, r.pollingInterval, r.stopCh)
}
func testNotReachable(ip string, port int) {
url := fmt.Sprintf("http://%s:%d", ip, port)
if ip == "" {
Failf("Got empty IP for non-reachability check (%s)", url)
}
if port == 0 {
Failf("Got port==0 for non-reachability check (%s)", url)
}
desc := fmt.Sprintf("the url %s to be *not* reachable", url)
By(fmt.Sprintf("Waiting up to %v for %s", podStartTimeout, desc))
err := wait.Poll(poll, podStartTimeout, func() (bool, error) {
resp, err := httpGetNoConnectionPool(url)
if err != nil {
Logf("Successfully waited for %s", desc)
return true, nil
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Logf("Expecting %s to be unreachable but was reachable and got an error reading response: %v", url, err)
return false, nil
}
Logf("Able to reach service %s when should no longer have been reachable, status:%d and body: %s", url, resp.Status, string(body))
return false, nil
})
Expect(err).NotTo(HaveOccurred(), "Error waiting for %s", desc)
}
// WaitForQueryOutputSatisfies will execute the query multiple times, until the
// specified predicate function is return true.
func WaitForQueryOutputSatisfies(oc *CLI, d Database, timeout time.Duration, admin bool, query string, predicate func(string) bool) error {
err := wait.Poll(5*time.Second, timeout, func() (bool, error) {
var (
out string
err error
)
if admin {
out, err = d.QueryPrivileged(oc, query)
} else {
out, err = d.Query(oc, query)
}
if _, ok := err.(*exec.ExitError); ok {
// Ignore exit errors
return false, nil
}
if err != nil {
return false, err
}
if predicate(out) {
return true, nil
}
return false, nil
})
if err == wait.ErrWaitTimeout {
return fmt.Errorf("timed out waiting for query: %q", query)
}
return err
}
// Wait till the passFunc confirms that the object it expects to see is in the store.
// Used to observe reflected events.
func waitForReflection(t *testing.T, s cache.Store, key string, passFunc func(n interface{}) bool) error {
nodes := []*api.Node{}
err := wait.Poll(time.Millisecond*100, wait.ForeverTestTimeout, func() (bool, error) {
if n, _, err := s.GetByKey(key); err == nil && passFunc(n) {
return true, nil
} else {
if err != nil {
t.Errorf("Unexpected error: %v", err)
} else {
if n == nil {
nodes = append(nodes, nil)
} else {
nodes = append(nodes, n.(*api.Node))
}
}
return false, nil
}
})
if err != nil {
t.Logf("Logging consecutive node versions received from store:")
for i, n := range nodes {
t.Logf("%d: %#v", i, n)
}
}
return err
}
// WaitForQueryOutput will execute the query multiple times, until the
// specified substring is found in the results. This function should be used for
// testing replication, since it might take some time untill the data is propagated
// to slaves.
func WaitForQueryOutput(oc *CLI, d Database, timeout time.Duration, admin bool, query, resultSubstr string) error {
return wait.Poll(5*time.Second, timeout, func() (bool, error) {
var (
out string
err error
)
if admin {
out, err = d.QueryPrivileged(oc, query)
} else {
out, err = d.Query(oc, query)
}
if _, ok := err.(*exec.ExitError); ok {
// Ignore exit errors
return false, nil
}
if err != nil {
return false, err
}
if strings.Contains(out, resultSubstr) {
return true, nil
}
return false, nil
})
}
func (reaper *DaemonSetReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *api.DeleteOptions) error {
ds, err := reaper.client.DaemonSets(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return err
}
// We set the nodeSelector to a random label. This label is nearly guaranteed
// to not be set on any node so the DameonSetController will start deleting
// daemon pods. Once it's done deleting the daemon pods, it's safe to delete
// the DaemonSet.
ds.Spec.Template.Spec.NodeSelector = map[string]string{
string(uuid.NewUUID()): string(uuid.NewUUID()),
}
// force update to avoid version conflict
ds.ResourceVersion = ""
if ds, err = reaper.client.DaemonSets(namespace).Update(ds); err != nil {
return err
}
// Wait for the daemon set controller to kill all the daemon pods.
if err := wait.Poll(reaper.pollInterval, reaper.timeout, func() (bool, error) {
updatedDS, err := reaper.client.DaemonSets(namespace).Get(name, metav1.GetOptions{})
if err != nil {
return false, nil
}
return updatedDS.Status.CurrentNumberScheduled+updatedDS.Status.NumberMisscheduled == 0, nil
}); err != nil {
return err
}
return reaper.client.DaemonSets(namespace).Delete(name, nil)
}
// WaitFor waits for some minimum number of pods to be verified, according to the PodStateVerification
// definition.
func (cl *ClusterVerification) WaitFor(atLeast int, timeout time.Duration) ([]api.Pod, error) {
pods := []api.Pod{}
var returnedErr error
err := wait.Poll(1*time.Second, timeout, func() (bool, error) {
pods, returnedErr = cl.podState.filter(cl.client, cl.namespace)
// Failure
if returnedErr != nil {
Logf("Cutting polling short: We got an error from the pod filtering layer.")
// stop polling if the pod filtering returns an error. that should never happen.
// it indicates, for example, that the client is broken or something non-pod related.
return false, returnedErr
}
Logf("Found %v / %v", len(pods), atLeast)
// Success
if len(pods) >= atLeast {
return true, nil
}
// Keep trying...
return false, nil
})
Logf("WaitFor completed with timeout %v. Pods found = %v out of %v", timeout, len(pods), atLeast)
return pods, err
}