func checkExistingRCRecovers(f Framework) { By("assert that the pre-existing replication controller recovers") podClient := f.Client.Pods(f.Namespace.Name) rcSelector := labels.Set{"name": "baz"}.AsSelector() By("deleting pods from existing replication controller") expectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) { pods, err := podClient.List(rcSelector, fields.Everything()) Expect(err).NotTo(HaveOccurred()) if len(pods.Items) == 0 { return false, nil } for _, pod := range pods.Items { err = podClient.Delete(pod.Name, api.NewDeleteOptions(0)) Expect(err).NotTo(HaveOccurred()) } return true, nil })) By("waiting for replication controller to recover") expectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) { pods, err := podClient.List(rcSelector, fields.Everything()) Expect(err).NotTo(HaveOccurred()) for _, pod := range pods.Items { if api.IsPodReady(&pod) { return true, nil } } return false, nil })) }
func runStaticPodTest(c *client.Client, configFilePath string) { manifest := `version: v1beta2 id: static-pod containers: - name: static-container image: kubernetes/pause` manifestFile, err := ioutil.TempFile(configFilePath, "") defer os.Remove(manifestFile.Name()) ioutil.WriteFile(manifestFile.Name(), []byte(manifest), 0600) // Wait for the mirror pod to be created. hostname, _ := os.Hostname() podName := fmt.Sprintf("static-pod-%s", hostname) namespace := kubelet.NamespaceDefault if err := wait.Poll(time.Second, time.Second*30, podRunning(c, namespace, podName)); err != nil { glog.Fatalf("FAILED: mirror pod has not been created or is not running: %v", err) } // Delete the mirror pod, and wait for it to be recreated. c.Pods(namespace).Delete(podName) if err = wait.Poll(time.Second, time.Second*30, podRunning(c, namespace, podName)); err != nil { glog.Fatalf("FAILED: mirror pod has not been re-created or is not running: %v", err) } // Remove the manifest file, and wait for the mirror pod to be deleted. os.Remove(manifestFile.Name()) if err = wait.Poll(time.Second, time.Second*30, podNotFound(c, namespace, podName)); err != nil { glog.Fatalf("FAILED: mirror pod has not been deleted: %v", err) } }
func runReplicationControllerTest(c *client.Client) { data, err := ioutil.ReadFile("api/examples/controller.json") if err != nil { glog.Fatalf("Unexpected error: %v", err) } var controller api.ReplicationController if err := api.Scheme.DecodeInto(data, &controller); err != nil { glog.Fatalf("Unexpected error: %v", err) } glog.Infof("Creating replication controllers") if _, err := c.ReplicationControllers(api.NamespaceDefault).Create(&controller); err != nil { glog.Fatalf("Unexpected error: %v", err) } glog.Infof("Done creating replication controllers") // Give the controllers some time to actually create the pods if err := wait.Poll(time.Second, time.Second*30, client.ControllerHasDesiredReplicas(c, &controller)); err != nil { glog.Fatalf("FAILED: pods never created %v", err) } // wait for minions to indicate they have info about the desired pods pods, err := c.Pods(api.NamespaceDefault).List(labels.Set(controller.Spec.Selector).AsSelector()) if err != nil { glog.Fatalf("FAILED: unable to get pods to list: %v", err) } if err := wait.Poll(time.Second, time.Second*30, podsOnMinions(c, *pods)); err != nil { glog.Fatalf("FAILED: pods never started running %v", err) } glog.Infof("Pods created") }
func runReplicationControllerTest(c *client.Client) { data, err := ioutil.ReadFile("api/examples/controller.json") if err != nil { glog.Fatalf("Unexpected error: %#v", err) } var controllerRequest api.ReplicationController if err := json.Unmarshal(data, &controllerRequest); err != nil { glog.Fatalf("Unexpected error: %#v", err) } glog.Infof("Creating replication controllers") if _, err := c.CreateReplicationController(controllerRequest); err != nil { glog.Fatalf("Unexpected error: %#v", err) } glog.Infof("Done creating replication controllers") // Give the controllers some time to actually create the pods if err := wait.Poll(time.Second, 10, c.ControllerHasDesiredReplicas(controllerRequest)); err != nil { glog.Fatalf("FAILED: pods never created %v", err) } // wait for minions to indicate they have info about the desired pods pods, err := c.ListPods(labels.Set(controllerRequest.DesiredState.ReplicaSelector).AsSelector()) if err != nil { glog.Fatalf("FAILED: unable to get pods to list: %v", err) } if err := wait.Poll(time.Second, 10, podsOnMinions(c, pods)); err != nil { glog.Fatalf("FAILED: pods never started running %v", err) } glog.Infof("Pods created") }
func runReplicationControllerTest(c *client.Client) { clientAPIVersion := c.APIVersion() data, err := ioutil.ReadFile("cmd/integration/" + clientAPIVersion + "-controller.json") if err != nil { glog.Fatalf("Unexpected error: %v", err) } var controller api.ReplicationController if err := api.Scheme.DecodeInto(data, &controller); err != nil { glog.Fatalf("Unexpected error: %v", err) } glog.Infof("Creating replication controllers") updated, err := c.ReplicationControllers("test").Create(&controller) if err != nil { glog.Fatalf("Unexpected error: %v", err) } glog.Infof("Done creating replication controllers") // Give the controllers some time to actually create the pods if err := wait.Poll(time.Second, time.Second*30, client.ControllerHasDesiredReplicas(c, updated)); err != nil { glog.Fatalf("FAILED: pods never created %v", err) } // Poll till we can retrieve the status of all pods matching the given label selector from their minions. // This involves 3 operations: // - The scheduler must assign all pods to a minion // - The assignment must reflect in a `List` operation against the apiserver, for labels matching the selector // - We need to be able to query the kubelet on that minion for information about the pod if err := wait.Poll( time.Second, time.Second*30, podsOnMinions(c, "test", labels.Set(updated.Spec.Selector).AsSelector())); err != nil { glog.Fatalf("FAILED: pods never started running %v", err) } glog.Infof("Pods created") }
func tour(dockerHelper *docker.Helper) { cmd := os.Args[0] _, kubeClient := clients() // check Docker dockerClient, addr, err := dockerHelper.GetClient() if err != nil { fmt.Printf(tourDockerClientErr, addr, err) os.Exit(1) } if err := dockerClient.Ping(); err != nil { fmt.Printf(tourDockerPingErr, addr, err) //os.Exit(1) continueTour() } fmt.Printf(tourOne) continueTour() // check for server start if serverRunning(kubeClient) { fmt.Printf(tourOneRunning, defaultServerAddr) } else { fmt.Printf(tourOneStart, cmd) if err := wait.Poll(time.Second, maxWait, func() (bool, error) { return serverRunning(kubeClient), nil }); err == wait.ErrWaitTimeout { fmt.Printf(tourHavingTrouble, "The server didn't seem to start in time.") os.Exit(1) } fmt.Printf(tourOneStarted, defaultServerAddr) } continueTour() // create a pod fmt.Printf(tourTwo, cmd) continueTour() fmt.Printf(tourTwoCreate, cmd) if err := wait.Poll(time.Second, maxWait, waitForPod(kubeClient, "hello-openshift")); err == wait.ErrWaitTimeout { fmt.Printf(tourHavingTrouble, "The pod didn't seem to get created in time.") os.Exit(1) } // info about pod creation fmt.Printf(tourTwoCreated, cmd) continueTour() // more to come fmt.Printf(tourThree) }
func runSchedulerNoPhantomPodsTest(client *client.Client) { pod := &api.Pod{ Spec: api.PodSpec{ Containers: []api.Container{ { Name: "c1", Image: "kubernetes/pause", Ports: []api.ContainerPort{ {ContainerPort: 1234, HostPort: 9999}, }, ImagePullPolicy: api.PullIfNotPresent, }, }, }, } // Assuming we only have two kublets, the third pod here won't schedule // if the scheduler doesn't correctly handle the delete for the second // pod. pod.ObjectMeta.Name = "phantom.foo" foo, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, time.Second*30, podRunning(client, foo.Namespace, foo.Name)); err != nil { glog.Fatalf("FAILED: pod never started running %v", err) } pod.ObjectMeta.Name = "phantom.bar" bar, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, time.Second*30, podRunning(client, bar.Namespace, bar.Name)); err != nil { glog.Fatalf("FAILED: pod never started running %v", err) } // Delete a pod to free up room. glog.Infof("Deleting pod %v", bar.Name) err = client.Pods(api.NamespaceDefault).Delete(bar.Name, nil) if err != nil { glog.Fatalf("FAILED: couldn't delete pod %q: %v", bar.Name, err) } pod.ObjectMeta.Name = "phantom.baz" baz, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, time.Second*60, podRunning(client, baz.Namespace, baz.Name)); err != nil { glog.Fatalf("FAILED: (Scheduler probably didn't process deletion of 'phantom.bar') Pod never started running: %v", err) } glog.Info("Scheduler doesn't make phantom pods: test passed.") }
// Update performs a rolling update of a collection of pods. // 'name' points to a replication controller. // 'client' is used for updating pods. // 'updatePeriod' is the time between pod updates. func Update(name string, client client.Interface, updatePeriod time.Duration) error { controller, err := client.GetReplicationController(name) if err != nil { return err } s := labels.Set(controller.DesiredState.ReplicaSelector).AsSelector() podList, err := client.ListPods(s) if err != nil { return err } expected := len(podList.Items) if expected == 0 { return nil } for _, pod := range podList.Items { // We delete the pod here, the controller will recreate it. This will result in pulling // a new Docker image. This isn't a full "update" but it's what we support for now. err = client.DeletePod(pod.ID) if err != nil { return err } time.Sleep(updatePeriod) } return wait.Poll(time.Second*5, time.Second*300, func() (bool, error) { podList, err := client.ListPods(s) if err != nil { return false, err } return len(podList.Items) == expected, nil }) }
// Delete a Replication Controller and all pods it spawned func DeleteRC(c *client.Client, ns, name string) error { rc, err := c.ReplicationControllers(ns).Get(name) if err != nil { return fmt.Errorf("Failed to find replication controller %s in namespace %s: %v", name, ns, err) } rc.Spec.Replicas = 0 if _, err := c.ReplicationControllers(ns).Update(rc); err != nil { return fmt.Errorf("Failed to resize replication controller %s to zero: %v", name, err) } // Wait up to 20 minutes until all replicas are killed. endTime := time.Now().Add(time.Minute * 20) for { if time.Now().After(endTime) { return fmt.Errorf("Timeout while waiting for replication controller %s replicas to 0", name) } remainingTime := endTime.Sub(time.Now()) err := wait.Poll(time.Second, remainingTime, client.ControllerHasDesiredReplicas(c, rc)) if err != nil { Logf("Error while waiting for replication controller %s replicas to read 0: %v", name, err) } else { break } } // Delete the replication controller. if err := c.ReplicationControllers(ns).Delete(name); err != nil { return fmt.Errorf("Failed to delete replication controller %s: %v", name, err) } return nil }
func testNotReachable(ip string, port int) { url := fmt.Sprintf("http://%s:%d", ip, port) if ip == "" { Failf("Got empty IP for non-reachability check (%s)", url) } if port == 0 { Failf("Got port==0 for non-reachability check (%s)", url) } desc := fmt.Sprintf("the url %s to be *not* reachable", url) By(fmt.Sprintf("Waiting up to %v for %s", podStartTimeout, desc)) err := wait.Poll(poll, podStartTimeout, func() (bool, error) { resp, err := httpGetNoConnectionPool(url) if err != nil { Logf("Successfully waited for %s", desc) return true, nil } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { Logf("Expecting %s to be unreachable but was reachable and got an error reading response: %v", url, err) return false, nil } Logf("Able to reach service %s when should no longer have been reachable, status:%d and body: %s", url, resp.Status, string(body)) return false, nil }) Expect(err).NotTo(HaveOccurred(), "Error waiting for %s", desc) }
// migTemlate (GCE/GKE-only) returns the name of the MIG template that the // nodes of the cluster use. func migTemplate() (string, error) { var errLast error var templ string key := "instanceTemplate" // TODO(mbforbes): Refactor this to use cluster_upgrade.go:retryCmd(...) if wait.Poll(poll, singleCallTimeout, func() (bool, error) { // TODO(mbforbes): make this hit the compute API directly instead of // shelling out to gcloud. o, err := exec.Command("gcloud", "preview", "managed-instance-groups", fmt.Sprintf("--project=%s", testContext.CloudConfig.ProjectID), fmt.Sprintf("--zone=%s", testContext.CloudConfig.Zone), "describe", testContext.CloudConfig.NodeInstanceGroup).CombinedOutput() if err != nil { errLast = fmt.Errorf("gcloud preview managed-instance-groups describe call failed with err: %v", err) return false, nil } output := string(o) // The 'describe' call probably succeeded; parse the output and try to // find the line that looks like "instanceTemplate: url/to/<templ>" and // return <templ>. if val := parseKVLines(output, key); len(val) > 0 { url := strings.Split(val, "/") templ = url[len(url)-1] Logf("MIG group %s using template: %s", testContext.CloudConfig.NodeInstanceGroup, templ) return true, nil } errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output) return false, nil }) != nil { return "", fmt.Errorf("migTemplate() failed with last error: %v", errLast) } return templ, nil }
func extinguish(c *client.Client, totalNS int, maxAllowedAfterDel int, maxSeconds int) { var err error for n := 0; n < totalNS; n += 1 { _, err = createTestingNS(fmt.Sprintf("nslifetest-%v", n), c) Expect(err).NotTo(HaveOccurred()) } //Wait 10 seconds, then SEND delete requests for all the namespaces. time.Sleep(time.Duration(10 * time.Second)) nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything()) Expect(err).NotTo(HaveOccurred()) for _, item := range nsList.Items { if strings.Contains(item.Name, "nslifetest") { if err := c.Namespaces().Delete(item.Name); err != nil { Failf("Failed deleting error ::: --- %v ", err) } } Logf("namespace : %v api call to delete is complete ", item) } //Now POLL until all namespaces have been eradicated. expectNoError(wait.Poll(2*time.Second, time.Duration(maxSeconds)*time.Second, func() (bool, error) { if rem, err := countRemaining(c, "nslifetest"); err != nil || rem > maxAllowedAfterDel { Logf("Remaining namespaces : %v", rem) return false, err } else { return true, nil } })) }
// migRollingUpdatePoll (CKE/GKE-only) polls the progress of the MIG rolling // update with ID id until it is complete. It returns an error if this takes // longer than nt times the number of nodes. func migRollingUpdatePoll(id string, nt time.Duration) error { // Two keys and a val. status, progress, done := "status", "statusMessage", "ROLLED_OUT" start, timeout := time.Now(), nt*time.Duration(testContext.CloudConfig.NumNodes) var errLast error Logf("Waiting up to %v for MIG rolling update to complete.", timeout) // TODO(mbforbes): Refactor this to use cluster_upgrade.go:retryCmd(...) if wait.Poll(restartPoll, timeout, func() (bool, error) { o, err := exec.Command("gcloud", "preview", "rolling-updates", fmt.Sprintf("--project=%s", testContext.CloudConfig.ProjectID), fmt.Sprintf("--zone=%s", testContext.CloudConfig.Zone), "describe", id).CombinedOutput() if err != nil { errLast = fmt.Errorf("Error calling rolling-updates describe %s: %v", id, err) Logf("%v", errLast) return false, nil } output := string(o) // The 'describe' call probably succeeded; parse the output and try to // find the line that looks like "status: <status>" and see whether it's // done. Logf("Waiting for MIG rolling update: %s (%v elapsed)", parseKVLines(output, progress), time.Since(start)) if st := parseKVLines(output, status); st == done { return true, nil } return false, nil }) != nil { return fmt.Errorf("timeout waiting %v for MIG rolling update to complete. Last error: %v", timeout, errLast) } Logf("MIG rolling update complete after %v", time.Since(start)) return nil }
func testReachable(ip string, port int) { url := fmt.Sprintf("http://%s:%d", ip, port) if ip == "" { Failf("Got empty IP for reachability check (%s)", url) } if port == 0 { Failf("Got port==0 for reachability check (%s)", url) } desc := fmt.Sprintf("the url %s to be reachable", url) By(fmt.Sprintf("Waiting up to %v for %s", podStartTimeout, desc)) start := time.Now() err := wait.Poll(poll, podStartTimeout, func() (bool, error) { resp, err := httpGetNoConnectionPool(url) if err != nil { Logf("Got error waiting for reachability of %s: %v (%v)", url, err, time.Since(start)) return false, nil } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { Logf("Got error reading response from %s: %v", url, err) return false, nil } if resp.StatusCode != 200 { return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s", resp.Status, url, string(body)) } if !strings.Contains(string(body), "test-webserver") { return false, fmt.Errorf("received response body without expected substring 'test-webserver': %s", string(body)) } Logf("Successfully reached %v", url) return true, nil }) Expect(err).NotTo(HaveOccurred(), "Error waiting for %s", desc) }
func assertFilesExist(fileNames []string, fileDir string, pod *api.Pod, client *client.Client) { var failed []string expectNoError(wait.Poll(time.Second*2, time.Second*60, func() (bool, error) { failed = []string{} for _, fileName := range fileNames { if _, err := client.Get(). Prefix("proxy"). Resource("pods"). Namespace(pod.Namespace). Name(pod.Name). Suffix(fileDir, fileName). Do().Raw(); err != nil { Logf("Unable to read %s from pod %s: %v", fileName, pod.Name, err) failed = append(failed, fileName) } } if len(failed) == 0 { return true, nil } Logf("Lookups using %s failed for: %v\n", pod.Name, failed) return false, nil })) Expect(len(failed)).To(Equal(0)) }
func podsResponding(c *client.Client, ns, name string, wantName bool, pods *api.PodList) error { By("trying to dial each unique pod") retryTimeout := 2 * time.Minute retryInterval := 5 * time.Second label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name})) return wait.Poll(retryInterval, retryTimeout, podResponseChecker{c, ns, label, name, wantName, pods}.checkAllResponses) }
func DoTestUnschedulableNodes(t *testing.T, client *client.Client) { node := &api.Node{ ObjectMeta: api.ObjectMeta{Name: "node"}, Spec: api.NodeSpec{Unschedulable: true}, } if _, err := client.Nodes().Create(node); err != nil { t.Fatalf("Failed to create node: %v", err) } pod := &api.Pod{ ObjectMeta: api.ObjectMeta{Name: "my-pod"}, Spec: api.PodSpec{ Containers: []api.Container{{Name: "container", Image: "kubernetes/pause:go"}}, }, } myPod, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { t.Fatalf("Failed to create pod: %v", err) } // There are no schedulable nodes - the pod shouldn't be scheduled. err = wait.Poll(time.Second, time.Second*10, podScheduled(client, myPod.Namespace, myPod.Name)) if err == nil { t.Errorf("Pod scheduled successfully on unschedulable nodes") } if err != wait.ErrWaitTimeout { t.Errorf("Failed while waiting for scheduled pod: %v", err) } // Make the node schedulable and wait until the pod is scheduled. newNode, err := client.Nodes().Get(node.Name) if err != nil { t.Fatalf("Failed to get node: %v", err) } newNode.Spec.Unschedulable = false if _, err = client.Nodes().Update(newNode); err != nil { t.Fatalf("Failed to update node: %v", err) } err = wait.Poll(time.Second, time.Second*10, podScheduled(client, myPod.Namespace, myPod.Name)) if err != nil { t.Errorf("Failed to schedule a pod: %v", err) } err = client.Pods(api.NamespaceDefault).Delete(myPod.Name) if err != nil { t.Errorf("Failed to delete pod: %v", err) } }
// Wait till the passFunc confirms that the object it expects to see is in the store. // Used to observe reflected events. func waitForReflection(s cache.Store, key string, passFunc func(n interface{}) bool) error { return wait.Poll(time.Millisecond*10, time.Second*20, func() (bool, error) { if n, _, err := s.GetByKey(key); err == nil && passFunc(n) { return true, nil } return false, nil }) }
func (r *RollingUpdater) updateAndWait(rc *api.ReplicationController, interval, timeout time.Duration) (*api.ReplicationController, error) { rc, err := r.c.UpdateReplicationController(r.ns, rc) if err != nil { return nil, err } if err = wait.Poll(interval, timeout, r.c.ControllerHasDesiredReplicas(rc)); err != nil { return nil, err } return r.c.GetReplicationController(r.ns, rc.ObjectMeta.Name) }
// Scale updates a replication controller created by the DeploymentConfig with the provided namespace/name, // to a new size, with optional precondition check (if preconditions is not nil),optional retries (if retry // is not nil), and then optionally waits for it's replica count to reach the new value (if wait is not nil). func (scaler *DeploymentConfigScaler) Scale(namespace, name string, newSize uint, preconditions *kubectl.ScalePrecondition, retry, waitForReplicas *kubectl.RetryParams) error { if preconditions == nil { preconditions = &kubectl.ScalePrecondition{-1, ""} } if retry == nil { // Make it try only once, immediately retry = &kubectl.RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond} } cond := kubectl.ScaleCondition(scaler, preconditions, namespace, name, newSize) if err := wait.Poll(retry.Interval, retry.Timeout, cond); err != nil { return err } if waitForReplicas != nil { rc := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: namespace, Name: rcName}} return wait.Poll(waitForReplicas.Interval, waitForReplicas.Timeout, scaler.c.ControllerHasDesiredReplicas(rc)) } return nil }
// checkNodesReady waits up to nt for expect nodes accessed by c to be ready, // returning an error if this doesn't happen in time. It returns the names of // nodes it finds. func checkNodesReady(c *client.Client, nt time.Duration, expect int) ([]string, error) { // First, keep getting all of the nodes until we get the number we expect. var nodeList *api.NodeList var errLast error start := time.Now() found := wait.Poll(poll, nt, func() (bool, error) { // Even though listNodes(...) has its own retries, a rolling-update // (GCE/GKE implementation of restart) can complete before the apiserver // knows about all of the nodes. Thus, we retry the list nodes call // until we get the expected number of nodes. nodeList, errLast = listNodes(c, labels.Everything(), fields.Everything()) if errLast != nil { return false, nil } if len(nodeList.Items) != expect { errLast = fmt.Errorf("expected to find %d nodes but found only %d (%v elapsed)", expect, len(nodeList.Items), time.Since(start)) Logf("%v", errLast) return false, nil } return true, nil }) == nil nodeNames := make([]string, len(nodeList.Items)) for i, n := range nodeList.Items { nodeNames[i] = n.ObjectMeta.Name } if !found { return nodeNames, fmt.Errorf("couldn't find %d nodes within %v; last error: %v", expect, nt, errLast) } Logf("Successfully found %d nodes", expect) // Next, ensure in parallel that all the nodes are ready. We subtract the // time we spent waiting above. timeout := nt - time.Since(start) result := make(chan bool, len(nodeList.Items)) for _, n := range nodeNames { n := n go func() { result <- waitForNodeToBeReady(c, n, timeout) }() } failed := false // TODO(mbforbes): Change to `for range` syntax once we support only Go // >= 1.4. for i := range nodeList.Items { _ = i if !<-result { failed = true } } if failed { return nodeNames, fmt.Errorf("at least one node failed to be ready") } return nodeNames, nil }
// Resize updates a ReplicationController to a new size, with optional precondition check (if preconditions is not nil), // optional retries (if retry is not nil), and then optionally waits for it's replica count to reach the new value // (if wait is not nil). func (resizer *ReplicationControllerResizer) Resize(namespace, name string, newSize uint, preconditions *ResizePrecondition, retry, waitForReplicas *RetryParams) error { if preconditions == nil { preconditions = &ResizePrecondition{-1, ""} } if retry == nil { // Make it try only once, immediately retry = &RetryParams{interval: time.Millisecond, timeout: time.Millisecond} } cond := ResizeCondition(resizer, preconditions, namespace, name, newSize) if err := wait.Poll(retry.interval, retry.timeout, cond); err != nil { return err } if waitForReplicas != nil { rc := &api.ReplicationController{ObjectMeta: api.ObjectMeta{Namespace: namespace, Name: name}} if err := wait.Poll(waitForReplicas.interval, waitForReplicas.timeout, resizer.c.ControllerHasDesiredReplicas(rc)); err != nil { return err } } return nil }
func getReferencedServiceAccountToken(c *client.Client, ns string, name string, shouldWait bool) (string, string, error) { tokenName := "" token := "" findToken := func() (bool, error) { user, err := c.ServiceAccounts(ns).Get(name) if errors.IsNotFound(err) { return false, nil } if err != nil { return false, err } for _, ref := range user.Secrets { secret, err := c.Secrets(ns).Get(ref.Name) if errors.IsNotFound(err) { continue } if err != nil { return false, err } if secret.Type != api.SecretTypeServiceAccountToken { continue } name := secret.Annotations[api.ServiceAccountNameKey] uid := secret.Annotations[api.ServiceAccountUIDKey] tokenData := secret.Data[api.ServiceAccountTokenKey] if name == user.Name && uid == string(user.UID) && len(tokenData) > 0 { tokenName = secret.Name token = string(tokenData) return true, nil } } return false, nil } if shouldWait { err := wait.Poll(time.Second, 10*time.Second, findToken) if err != nil { return "", "", err } } else { ok, err := findToken() if err != nil { return "", "", err } if !ok { return "", "", fmt.Errorf("No token found for %s/%s", ns, name) } } return tokenName, token, nil }
func RunResize(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string) error { count := cmdutil.GetFlagInt(cmd, "replicas") if len(args) != 2 || count < 0 { return cmdutil.UsageError(cmd, "--replicas=COUNT RESOURCE ID") } cmdNamespace, err := f.DefaultNamespace() if err != nil { return err } mapper, typer := f.Object() r := resource.NewBuilder(mapper, typer, f.ClientMapperForCommand()). ContinueOnError(). NamespaceParam(cmdNamespace).DefaultNamespace(). ResourceTypeOrNameArgs(false, args...). Flatten(). Do() err = r.Err() if err != nil { return err } mapping, err := r.ResourceMapping() if err != nil { return err } infos, err := r.Infos() if err != nil { return err } info := infos[0] resizer, err := f.Resizer(mapping) if err != nil { return err } resourceVersion := cmdutil.GetFlagString(cmd, "resource-version") currentSize := cmdutil.GetFlagInt(cmd, "current-replicas") precondition := &kubectl.ResizePrecondition{currentSize, resourceVersion} cond := kubectl.ResizeCondition(resizer, precondition, info.Namespace, info.Name, uint(count)) msg := "resized" if err = wait.Poll(retryFrequency, retryTimeout, cond); err != nil { msg = fmt.Sprintf("Failed to resize controller in spite of retrying for %s", retryTimeout) if err != nil { return err } } fmt.Fprintf(out, "%s\n", msg) return nil }
// migRollingUpdateStart (GCE/GKE-only) starts a MIG rolling update using templ // as the new template, waiting up to nt per node, and returns the ID of that // update. func migRollingUpdateStart(templ string, nt time.Duration) (string, error) { var errLast error var id string prefix, suffix := "Started [", "]." // TODO(mbforbes): Refactor this to use cluster_upgrade.go:retryCmd(...) if err := wait.Poll(poll, singleCallTimeout, func() (bool, error) { // TODO(mbforbes): make this hit the compute API directly instead of // shelling out to gcloud. // NOTE(mbforbes): If you are changing this gcloud command, update // cluster/gce/upgrade.sh to match this EXACTLY. o, err := exec.Command("gcloud", append(migUdpateCmdBase(), "rolling-updates", fmt.Sprintf("--project=%s", testContext.CloudConfig.ProjectID), fmt.Sprintf("--zone=%s", testContext.CloudConfig.Zone), "start", // Required args. fmt.Sprintf("--group=%s", testContext.CloudConfig.NodeInstanceGroup), fmt.Sprintf("--template=%s", templ), // Optional args to fine-tune behavior. fmt.Sprintf("--instance-startup-timeout=%ds", int(nt.Seconds())), // NOTE: We can speed up this process by increasing // --max-num-concurrent-instances. fmt.Sprintf("--max-num-concurrent-instances=%d", 1), fmt.Sprintf("--max-num-failed-instances=%d", 0), fmt.Sprintf("--min-instance-update-time=%ds", 0))...).CombinedOutput() if err != nil { errLast = fmt.Errorf("rolling-updates call failed with err: %v", err) return false, nil } output := string(o) // The 'start' call probably succeeded; parse the output and try to find // the line that looks like "Started [url/to/<id>]." and return <id>. for _, line := range strings.Split(output, "\n") { // As a sanity check, ensure the line starts with prefix and ends // with suffix. if strings.Index(line, prefix) != 0 || strings.Index(line, suffix) != len(line)-len(suffix) { continue } url := strings.Split(strings.TrimSuffix(strings.TrimPrefix(line, prefix), suffix), "/") id = url[len(url)-1] Logf("Started MIG rolling update; ID: %s", id) return true, nil } errLast = fmt.Errorf("couldn't find line like '%s ... %s' in output to MIG rolling-update start. Output: %s", prefix, suffix, output) return false, nil }); err != nil { return "", fmt.Errorf("migRollingUpdateStart() failed with last error: %v", errLast) } return id, nil }
// Scale updates a ReplicationController to a new size, with optional precondition check (if preconditions is not nil), // optional retries (if retry is not nil), and then optionally waits for it's replica count to reach the new value // (if wait is not nil). func (scaler *ReplicationControllerScaler) Scale(namespace, name string, newSize uint, preconditions *ScalePrecondition, retry, waitForReplicas *RetryParams) error { if preconditions == nil { preconditions = &ScalePrecondition{-1, ""} } if retry == nil { // Make it try only once, immediately retry = &RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond} } cond := ScaleCondition(scaler, preconditions, namespace, name, newSize) if err := wait.Poll(retry.Interval, retry.Timeout, cond); err != nil { return err } if waitForReplicas != nil { rc, err := scaler.c.GetReplicationController(namespace, name) if err != nil { return err } return wait.Poll(waitForReplicas.Interval, waitForReplicas.Timeout, scaler.c.ControllerHasDesiredReplicas(rc)) } return nil }
// retryCmd runs cmd using args and retries it for up to singleCallTimeout if // it returns an error. It returns stdout and stderr. func retryCmd(command string, args ...string) (string, string, error) { var err error stdout, stderr := "", "" wait.Poll(poll, singleCallTimeout, func() (bool, error) { stdout, stderr, err = runCmd(command, args...) if err != nil { Logf("Got %v", err) return false, nil } return true, nil }) return stdout, stderr, err }
func WaitForServiceAccounts(client *kclient.Client, namespace string, accounts []string) error { // Ensure the service accounts needed by build pods exist in the namespace // The extra controllers tend to starve the service account controller serviceAccounts := client.ServiceAccounts(namespace) return wait.Poll(time.Second, ServiceAccountWaitTimeout, func() (bool, error) { for _, account := range accounts { if _, err := serviceAccounts.Get(account); err != nil { return false, nil } } return true, nil }) }
func extinguish(c *client.Client, totalNS int, maxAllowedAfterDel int, maxSeconds int) { var err error By("Creating testing namespaces") wg := &sync.WaitGroup{} for n := 0; n < totalNS; n += 1 { wg.Add(1) go func(n int) { defer wg.Done() defer GinkgoRecover() _, err = createTestingNS(fmt.Sprintf("nslifetest-%v", n), c) Expect(err).NotTo(HaveOccurred()) }(n) } wg.Wait() By("Waiting 10 seconds") //Wait 10 seconds, then SEND delete requests for all the namespaces. time.Sleep(time.Duration(10 * time.Second)) By("Deleting namespaces") nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything()) Expect(err).NotTo(HaveOccurred()) var nsCount = 0 for _, item := range nsList.Items { if strings.Contains(item.Name, "nslifetest") { wg.Add(1) nsCount++ go func(nsName string) { defer wg.Done() defer GinkgoRecover() Expect(c.Namespaces().Delete(nsName)).To(Succeed()) Logf("namespace : %v api call to delete is complete ", nsName) }(item.Name) } } Expect(nsCount).To(Equal(totalNS)) wg.Wait() By("Waiting for namespaces to vanish") //Now POLL until all namespaces have been eradicated. expectNoError(wait.Poll(2*time.Second, time.Duration(maxSeconds)*time.Second, func() (bool, error) { if rem, err := countRemaining(c, "nslifetest"); err != nil || rem > maxAllowedAfterDel { Logf("Remaining namespaces : %v", rem) return false, err } else { return true, nil } })) }
// WaitForPolicyUpdate checks if the given client can perform the named verb and action. // If PolicyCachePollTimeout is reached without the expected condition matching, an error is returned func WaitForPolicyUpdate(c *client.Client, namespace, verb, resource string, allowed bool) error { review := &authorizationapi.SubjectAccessReview{Verb: verb, Resource: resource} err := wait.Poll(PolicyCachePollInterval, PolicyCachePollTimeout, func() (bool, error) { response, err := c.SubjectAccessReviews(namespace).Create(review) if err != nil { return false, err } if response.Allowed != allowed { return false, nil } return true, nil }) return err }