func checkExistingRCRecovers(f Framework) {
By("assert that the pre-existing replication controller recovers")
podClient := f.Client.Pods(f.Namespace.Name)
rcSelector := labels.Set{"name": "baz"}.AsSelector()
By("deleting pods from existing replication controller")
expectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
pods, err := podClient.List(rcSelector, fields.Everything())
Expect(err).NotTo(HaveOccurred())
if len(pods.Items) == 0 {
return false, nil
}
for _, pod := range pods.Items {
err = podClient.Delete(pod.Name, api.NewDeleteOptions(0))
Expect(err).NotTo(HaveOccurred())
}
return true, nil
}))
By("waiting for replication controller to recover")
expectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
pods, err := podClient.List(rcSelector, fields.Everything())
Expect(err).NotTo(HaveOccurred())
for _, pod := range pods.Items {
if api.IsPodReady(&pod) {
return true, nil
}
}
return false, nil
}))
}
func runStaticPodTest(c *client.Client, configFilePath string) {
manifest := `version: v1beta2
id: static-pod
containers:
- name: static-container
image: kubernetes/pause`
manifestFile, err := ioutil.TempFile(configFilePath, "")
defer os.Remove(manifestFile.Name())
ioutil.WriteFile(manifestFile.Name(), []byte(manifest), 0600)
// Wait for the mirror pod to be created.
hostname, _ := os.Hostname()
podName := fmt.Sprintf("static-pod-%s", hostname)
namespace := kubelet.NamespaceDefault
if err := wait.Poll(time.Second, time.Second*30,
podRunning(c, namespace, podName)); err != nil {
glog.Fatalf("FAILED: mirror pod has not been created or is not running: %v", err)
}
// Delete the mirror pod, and wait for it to be recreated.
c.Pods(namespace).Delete(podName)
if err = wait.Poll(time.Second, time.Second*30,
podRunning(c, namespace, podName)); err != nil {
glog.Fatalf("FAILED: mirror pod has not been re-created or is not running: %v", err)
}
// Remove the manifest file, and wait for the mirror pod to be deleted.
os.Remove(manifestFile.Name())
if err = wait.Poll(time.Second, time.Second*30,
podNotFound(c, namespace, podName)); err != nil {
glog.Fatalf("FAILED: mirror pod has not been deleted: %v", err)
}
}
func runReplicationControllerTest(c *client.Client) {
data, err := ioutil.ReadFile("api/examples/controller.json")
if err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
var controller api.ReplicationController
if err := api.Scheme.DecodeInto(data, &controller); err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
glog.Infof("Creating replication controllers")
if _, err := c.ReplicationControllers(api.NamespaceDefault).Create(&controller); err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
glog.Infof("Done creating replication controllers")
// Give the controllers some time to actually create the pods
if err := wait.Poll(time.Second, time.Second*30, client.ControllerHasDesiredReplicas(c, &controller)); err != nil {
glog.Fatalf("FAILED: pods never created %v", err)
}
// wait for minions to indicate they have info about the desired pods
pods, err := c.Pods(api.NamespaceDefault).List(labels.Set(controller.Spec.Selector).AsSelector())
if err != nil {
glog.Fatalf("FAILED: unable to get pods to list: %v", err)
}
if err := wait.Poll(time.Second, time.Second*30, podsOnMinions(c, *pods)); err != nil {
glog.Fatalf("FAILED: pods never started running %v", err)
}
glog.Infof("Pods created")
}
func runReplicationControllerTest(c *client.Client) {
data, err := ioutil.ReadFile("api/examples/controller.json")
if err != nil {
glog.Fatalf("Unexpected error: %#v", err)
}
var controllerRequest api.ReplicationController
if err := json.Unmarshal(data, &controllerRequest); err != nil {
glog.Fatalf("Unexpected error: %#v", err)
}
glog.Infof("Creating replication controllers")
if _, err := c.CreateReplicationController(controllerRequest); err != nil {
glog.Fatalf("Unexpected error: %#v", err)
}
glog.Infof("Done creating replication controllers")
// Give the controllers some time to actually create the pods
if err := wait.Poll(time.Second, 10, c.ControllerHasDesiredReplicas(controllerRequest)); err != nil {
glog.Fatalf("FAILED: pods never created %v", err)
}
// wait for minions to indicate they have info about the desired pods
pods, err := c.ListPods(labels.Set(controllerRequest.DesiredState.ReplicaSelector).AsSelector())
if err != nil {
glog.Fatalf("FAILED: unable to get pods to list: %v", err)
}
if err := wait.Poll(time.Second, 10, podsOnMinions(c, pods)); err != nil {
glog.Fatalf("FAILED: pods never started running %v", err)
}
glog.Infof("Pods created")
}
func runReplicationControllerTest(c *client.Client) {
clientAPIVersion := c.APIVersion()
data, err := ioutil.ReadFile("cmd/integration/" + clientAPIVersion + "-controller.json")
if err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
var controller api.ReplicationController
if err := api.Scheme.DecodeInto(data, &controller); err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
glog.Infof("Creating replication controllers")
updated, err := c.ReplicationControllers("test").Create(&controller)
if err != nil {
glog.Fatalf("Unexpected error: %v", err)
}
glog.Infof("Done creating replication controllers")
// Give the controllers some time to actually create the pods
if err := wait.Poll(time.Second, time.Second*30, client.ControllerHasDesiredReplicas(c, updated)); err != nil {
glog.Fatalf("FAILED: pods never created %v", err)
}
// Poll till we can retrieve the status of all pods matching the given label selector from their minions.
// This involves 3 operations:
// - The scheduler must assign all pods to a minion
// - The assignment must reflect in a `List` operation against the apiserver, for labels matching the selector
// - We need to be able to query the kubelet on that minion for information about the pod
if err := wait.Poll(
time.Second, time.Second*30, podsOnMinions(c, "test", labels.Set(updated.Spec.Selector).AsSelector())); err != nil {
glog.Fatalf("FAILED: pods never started running %v", err)
}
glog.Infof("Pods created")
}
func tour(dockerHelper *docker.Helper) {
cmd := os.Args[0]
_, kubeClient := clients()
// check Docker
dockerClient, addr, err := dockerHelper.GetClient()
if err != nil {
fmt.Printf(tourDockerClientErr, addr, err)
os.Exit(1)
}
if err := dockerClient.Ping(); err != nil {
fmt.Printf(tourDockerPingErr, addr, err)
//os.Exit(1)
continueTour()
}
fmt.Printf(tourOne)
continueTour()
// check for server start
if serverRunning(kubeClient) {
fmt.Printf(tourOneRunning, defaultServerAddr)
} else {
fmt.Printf(tourOneStart, cmd)
if err := wait.Poll(time.Second, maxWait, func() (bool, error) {
return serverRunning(kubeClient), nil
}); err == wait.ErrWaitTimeout {
fmt.Printf(tourHavingTrouble, "The server didn't seem to start in time.")
os.Exit(1)
}
fmt.Printf(tourOneStarted, defaultServerAddr)
}
continueTour()
// create a pod
fmt.Printf(tourTwo, cmd)
continueTour()
fmt.Printf(tourTwoCreate, cmd)
if err := wait.Poll(time.Second, maxWait, waitForPod(kubeClient, "hello-openshift")); err == wait.ErrWaitTimeout {
fmt.Printf(tourHavingTrouble, "The pod didn't seem to get created in time.")
os.Exit(1)
}
// info about pod creation
fmt.Printf(tourTwoCreated, cmd)
continueTour()
// more to come
fmt.Printf(tourThree)
}
func runSchedulerNoPhantomPodsTest(client *client.Client) {
pod := &api.Pod{
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "c1",
Image: "kubernetes/pause",
Ports: []api.ContainerPort{
{ContainerPort: 1234, HostPort: 9999},
},
ImagePullPolicy: api.PullIfNotPresent,
},
},
},
}
// Assuming we only have two kublets, the third pod here won't schedule
// if the scheduler doesn't correctly handle the delete for the second
// pod.
pod.ObjectMeta.Name = "phantom.foo"
foo, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, time.Second*30, podRunning(client, foo.Namespace, foo.Name)); err != nil {
glog.Fatalf("FAILED: pod never started running %v", err)
}
pod.ObjectMeta.Name = "phantom.bar"
bar, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, time.Second*30, podRunning(client, bar.Namespace, bar.Name)); err != nil {
glog.Fatalf("FAILED: pod never started running %v", err)
}
// Delete a pod to free up room.
glog.Infof("Deleting pod %v", bar.Name)
err = client.Pods(api.NamespaceDefault).Delete(bar.Name, nil)
if err != nil {
glog.Fatalf("FAILED: couldn't delete pod %q: %v", bar.Name, err)
}
pod.ObjectMeta.Name = "phantom.baz"
baz, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
glog.Fatalf("Failed to create pod: %v, %v", pod, err)
}
if err := wait.Poll(time.Second, time.Second*60, podRunning(client, baz.Namespace, baz.Name)); err != nil {
glog.Fatalf("FAILED: (Scheduler probably didn't process deletion of 'phantom.bar') Pod never started running: %v", err)
}
glog.Info("Scheduler doesn't make phantom pods: test passed.")
}
// Update performs a rolling update of a collection of pods.
// 'name' points to a replication controller.
// 'client' is used for updating pods.
// 'updatePeriod' is the time between pod updates.
func Update(name string, client client.Interface, updatePeriod time.Duration) error {
controller, err := client.GetReplicationController(name)
if err != nil {
return err
}
s := labels.Set(controller.DesiredState.ReplicaSelector).AsSelector()
podList, err := client.ListPods(s)
if err != nil {
return err
}
expected := len(podList.Items)
if expected == 0 {
return nil
}
for _, pod := range podList.Items {
// We delete the pod here, the controller will recreate it. This will result in pulling
// a new Docker image. This isn't a full "update" but it's what we support for now.
err = client.DeletePod(pod.ID)
if err != nil {
return err
}
time.Sleep(updatePeriod)
}
return wait.Poll(time.Second*5, time.Second*300, func() (bool, error) {
podList, err := client.ListPods(s)
if err != nil {
return false, err
}
return len(podList.Items) == expected, nil
})
}
// Delete a Replication Controller and all pods it spawned
func DeleteRC(c *client.Client, ns, name string) error {
rc, err := c.ReplicationControllers(ns).Get(name)
if err != nil {
return fmt.Errorf("Failed to find replication controller %s in namespace %s: %v", name, ns, err)
}
rc.Spec.Replicas = 0
if _, err := c.ReplicationControllers(ns).Update(rc); err != nil {
return fmt.Errorf("Failed to resize replication controller %s to zero: %v", name, err)
}
// Wait up to 20 minutes until all replicas are killed.
endTime := time.Now().Add(time.Minute * 20)
for {
if time.Now().After(endTime) {
return fmt.Errorf("Timeout while waiting for replication controller %s replicas to 0", name)
}
remainingTime := endTime.Sub(time.Now())
err := wait.Poll(time.Second, remainingTime, client.ControllerHasDesiredReplicas(c, rc))
if err != nil {
Logf("Error while waiting for replication controller %s replicas to read 0: %v", name, err)
} else {
break
}
}
// Delete the replication controller.
if err := c.ReplicationControllers(ns).Delete(name); err != nil {
return fmt.Errorf("Failed to delete replication controller %s: %v", name, err)
}
return nil
}
func testNotReachable(ip string, port int) {
url := fmt.Sprintf("http://%s:%d", ip, port)
if ip == "" {
Failf("Got empty IP for non-reachability check (%s)", url)
}
if port == 0 {
Failf("Got port==0 for non-reachability check (%s)", url)
}
desc := fmt.Sprintf("the url %s to be *not* reachable", url)
By(fmt.Sprintf("Waiting up to %v for %s", podStartTimeout, desc))
err := wait.Poll(poll, podStartTimeout, func() (bool, error) {
resp, err := httpGetNoConnectionPool(url)
if err != nil {
Logf("Successfully waited for %s", desc)
return true, nil
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Logf("Expecting %s to be unreachable but was reachable and got an error reading response: %v", url, err)
return false, nil
}
Logf("Able to reach service %s when should no longer have been reachable, status:%d and body: %s", url, resp.Status, string(body))
return false, nil
})
Expect(err).NotTo(HaveOccurred(), "Error waiting for %s", desc)
}
// migTemlate (GCE/GKE-only) returns the name of the MIG template that the
// nodes of the cluster use.
func migTemplate() (string, error) {
var errLast error
var templ string
key := "instanceTemplate"
// TODO(mbforbes): Refactor this to use cluster_upgrade.go:retryCmd(...)
if wait.Poll(poll, singleCallTimeout, func() (bool, error) {
// TODO(mbforbes): make this hit the compute API directly instead of
// shelling out to gcloud.
o, err := exec.Command("gcloud", "preview", "managed-instance-groups",
fmt.Sprintf("--project=%s", testContext.CloudConfig.ProjectID),
fmt.Sprintf("--zone=%s", testContext.CloudConfig.Zone),
"describe",
testContext.CloudConfig.NodeInstanceGroup).CombinedOutput()
if err != nil {
errLast = fmt.Errorf("gcloud preview managed-instance-groups describe call failed with err: %v", err)
return false, nil
}
output := string(o)
// The 'describe' call probably succeeded; parse the output and try to
// find the line that looks like "instanceTemplate: url/to/<templ>" and
// return <templ>.
if val := parseKVLines(output, key); len(val) > 0 {
url := strings.Split(val, "/")
templ = url[len(url)-1]
Logf("MIG group %s using template: %s", testContext.CloudConfig.NodeInstanceGroup, templ)
return true, nil
}
errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
return false, nil
}) != nil {
return "", fmt.Errorf("migTemplate() failed with last error: %v", errLast)
}
return templ, nil
}
func extinguish(c *client.Client, totalNS int, maxAllowedAfterDel int, maxSeconds int) {
var err error
for n := 0; n < totalNS; n += 1 {
_, err = createTestingNS(fmt.Sprintf("nslifetest-%v", n), c)
Expect(err).NotTo(HaveOccurred())
}
//Wait 10 seconds, then SEND delete requests for all the namespaces.
time.Sleep(time.Duration(10 * time.Second))
nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything())
Expect(err).NotTo(HaveOccurred())
for _, item := range nsList.Items {
if strings.Contains(item.Name, "nslifetest") {
if err := c.Namespaces().Delete(item.Name); err != nil {
Failf("Failed deleting error ::: --- %v ", err)
}
}
Logf("namespace : %v api call to delete is complete ", item)
}
//Now POLL until all namespaces have been eradicated.
expectNoError(wait.Poll(2*time.Second, time.Duration(maxSeconds)*time.Second,
func() (bool, error) {
if rem, err := countRemaining(c, "nslifetest"); err != nil || rem > maxAllowedAfterDel {
Logf("Remaining namespaces : %v", rem)
return false, err
} else {
return true, nil
}
}))
}
// migRollingUpdatePoll (CKE/GKE-only) polls the progress of the MIG rolling
// update with ID id until it is complete. It returns an error if this takes
// longer than nt times the number of nodes.
func migRollingUpdatePoll(id string, nt time.Duration) error {
// Two keys and a val.
status, progress, done := "status", "statusMessage", "ROLLED_OUT"
start, timeout := time.Now(), nt*time.Duration(testContext.CloudConfig.NumNodes)
var errLast error
Logf("Waiting up to %v for MIG rolling update to complete.", timeout)
// TODO(mbforbes): Refactor this to use cluster_upgrade.go:retryCmd(...)
if wait.Poll(restartPoll, timeout, func() (bool, error) {
o, err := exec.Command("gcloud", "preview", "rolling-updates",
fmt.Sprintf("--project=%s", testContext.CloudConfig.ProjectID),
fmt.Sprintf("--zone=%s", testContext.CloudConfig.Zone),
"describe",
id).CombinedOutput()
if err != nil {
errLast = fmt.Errorf("Error calling rolling-updates describe %s: %v", id, err)
Logf("%v", errLast)
return false, nil
}
output := string(o)
// The 'describe' call probably succeeded; parse the output and try to
// find the line that looks like "status: <status>" and see whether it's
// done.
Logf("Waiting for MIG rolling update: %s (%v elapsed)",
parseKVLines(output, progress), time.Since(start))
if st := parseKVLines(output, status); st == done {
return true, nil
}
return false, nil
}) != nil {
return fmt.Errorf("timeout waiting %v for MIG rolling update to complete. Last error: %v", timeout, errLast)
}
Logf("MIG rolling update complete after %v", time.Since(start))
return nil
}
func testReachable(ip string, port int) {
url := fmt.Sprintf("http://%s:%d", ip, port)
if ip == "" {
Failf("Got empty IP for reachability check (%s)", url)
}
if port == 0 {
Failf("Got port==0 for reachability check (%s)", url)
}
desc := fmt.Sprintf("the url %s to be reachable", url)
By(fmt.Sprintf("Waiting up to %v for %s", podStartTimeout, desc))
start := time.Now()
err := wait.Poll(poll, podStartTimeout, func() (bool, error) {
resp, err := httpGetNoConnectionPool(url)
if err != nil {
Logf("Got error waiting for reachability of %s: %v (%v)", url, err, time.Since(start))
return false, nil
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Logf("Got error reading response from %s: %v", url, err)
return false, nil
}
if resp.StatusCode != 200 {
return false, fmt.Errorf("received non-success return status %q trying to access %s; got body: %s", resp.Status, url, string(body))
}
if !strings.Contains(string(body), "test-webserver") {
return false, fmt.Errorf("received response body without expected substring 'test-webserver': %s", string(body))
}
Logf("Successfully reached %v", url)
return true, nil
})
Expect(err).NotTo(HaveOccurred(), "Error waiting for %s", desc)
}
func assertFilesExist(fileNames []string, fileDir string, pod *api.Pod, client *client.Client) {
var failed []string
expectNoError(wait.Poll(time.Second*2, time.Second*60, func() (bool, error) {
failed = []string{}
for _, fileName := range fileNames {
if _, err := client.Get().
Prefix("proxy").
Resource("pods").
Namespace(pod.Namespace).
Name(pod.Name).
Suffix(fileDir, fileName).
Do().Raw(); err != nil {
Logf("Unable to read %s from pod %s: %v", fileName, pod.Name, err)
failed = append(failed, fileName)
}
}
if len(failed) == 0 {
return true, nil
}
Logf("Lookups using %s failed for: %v\n", pod.Name, failed)
return false, nil
}))
Expect(len(failed)).To(Equal(0))
}
func podsResponding(c *client.Client, ns, name string, wantName bool, pods *api.PodList) error {
By("trying to dial each unique pod")
retryTimeout := 2 * time.Minute
retryInterval := 5 * time.Second
label := labels.SelectorFromSet(labels.Set(map[string]string{"name": name}))
return wait.Poll(retryInterval, retryTimeout, podResponseChecker{c, ns, label, name, wantName, pods}.checkAllResponses)
}
func DoTestUnschedulableNodes(t *testing.T, client *client.Client) {
node := &api.Node{
ObjectMeta: api.ObjectMeta{Name: "node"},
Spec: api.NodeSpec{Unschedulable: true},
}
if _, err := client.Nodes().Create(node); err != nil {
t.Fatalf("Failed to create node: %v", err)
}
pod := &api.Pod{
ObjectMeta: api.ObjectMeta{Name: "my-pod"},
Spec: api.PodSpec{
Containers: []api.Container{{Name: "container", Image: "kubernetes/pause:go"}},
},
}
myPod, err := client.Pods(api.NamespaceDefault).Create(pod)
if err != nil {
t.Fatalf("Failed to create pod: %v", err)
}
// There are no schedulable nodes - the pod shouldn't be scheduled.
err = wait.Poll(time.Second, time.Second*10, podScheduled(client, myPod.Namespace, myPod.Name))
if err == nil {
t.Errorf("Pod scheduled successfully on unschedulable nodes")
}
if err != wait.ErrWaitTimeout {
t.Errorf("Failed while waiting for scheduled pod: %v", err)
}
// Make the node schedulable and wait until the pod is scheduled.
newNode, err := client.Nodes().Get(node.Name)
if err != nil {
t.Fatalf("Failed to get node: %v", err)
}
newNode.Spec.Unschedulable = false
if _, err = client.Nodes().Update(newNode); err != nil {
t.Fatalf("Failed to update node: %v", err)
}
err = wait.Poll(time.Second, time.Second*10, podScheduled(client, myPod.Namespace, myPod.Name))
if err != nil {
t.Errorf("Failed to schedule a pod: %v", err)
}
err = client.Pods(api.NamespaceDefault).Delete(myPod.Name)
if err != nil {
t.Errorf("Failed to delete pod: %v", err)
}
}
// Wait till the passFunc confirms that the object it expects to see is in the store.
// Used to observe reflected events.
func waitForReflection(s cache.Store, key string, passFunc func(n interface{}) bool) error {
return wait.Poll(time.Millisecond*10, time.Second*20, func() (bool, error) {
if n, _, err := s.GetByKey(key); err == nil && passFunc(n) {
return true, nil
}
return false, nil
})
}
func (r *RollingUpdater) updateAndWait(rc *api.ReplicationController, interval, timeout time.Duration) (*api.ReplicationController, error) {
rc, err := r.c.UpdateReplicationController(r.ns, rc)
if err != nil {
return nil, err
}
if err = wait.Poll(interval, timeout, r.c.ControllerHasDesiredReplicas(rc)); err != nil {
return nil, err
}
return r.c.GetReplicationController(r.ns, rc.ObjectMeta.Name)
}
// Scale updates a replication controller created by the DeploymentConfig with the provided namespace/name,
// to a new size, with optional precondition check (if preconditions is not nil),optional retries (if retry
// is not nil), and then optionally waits for it's replica count to reach the new value (if wait is not nil).
func (scaler *DeploymentConfigScaler) Scale(namespace, name string, newSize uint, preconditions *kubectl.ScalePrecondition, retry, waitForReplicas *kubectl.RetryParams) error {
if preconditions == nil {
preconditions = &kubectl.ScalePrecondition{-1, ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &kubectl.RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond}
}
cond := kubectl.ScaleCondition(scaler, preconditions, namespace, name, newSize)
if err := wait.Poll(retry.Interval, retry.Timeout, cond); err != nil {
return err
}
if waitForReplicas != nil {
rc := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: namespace, Name: rcName}}
return wait.Poll(waitForReplicas.Interval, waitForReplicas.Timeout,
scaler.c.ControllerHasDesiredReplicas(rc))
}
return nil
}
// checkNodesReady waits up to nt for expect nodes accessed by c to be ready,
// returning an error if this doesn't happen in time. It returns the names of
// nodes it finds.
func checkNodesReady(c *client.Client, nt time.Duration, expect int) ([]string, error) {
// First, keep getting all of the nodes until we get the number we expect.
var nodeList *api.NodeList
var errLast error
start := time.Now()
found := wait.Poll(poll, nt, func() (bool, error) {
// Even though listNodes(...) has its own retries, a rolling-update
// (GCE/GKE implementation of restart) can complete before the apiserver
// knows about all of the nodes. Thus, we retry the list nodes call
// until we get the expected number of nodes.
nodeList, errLast = listNodes(c, labels.Everything(), fields.Everything())
if errLast != nil {
return false, nil
}
if len(nodeList.Items) != expect {
errLast = fmt.Errorf("expected to find %d nodes but found only %d (%v elapsed)",
expect, len(nodeList.Items), time.Since(start))
Logf("%v", errLast)
return false, nil
}
return true, nil
}) == nil
nodeNames := make([]string, len(nodeList.Items))
for i, n := range nodeList.Items {
nodeNames[i] = n.ObjectMeta.Name
}
if !found {
return nodeNames, fmt.Errorf("couldn't find %d nodes within %v; last error: %v",
expect, nt, errLast)
}
Logf("Successfully found %d nodes", expect)
// Next, ensure in parallel that all the nodes are ready. We subtract the
// time we spent waiting above.
timeout := nt - time.Since(start)
result := make(chan bool, len(nodeList.Items))
for _, n := range nodeNames {
n := n
go func() { result <- waitForNodeToBeReady(c, n, timeout) }()
}
failed := false
// TODO(mbforbes): Change to `for range` syntax once we support only Go
// >= 1.4.
for i := range nodeList.Items {
_ = i
if !<-result {
failed = true
}
}
if failed {
return nodeNames, fmt.Errorf("at least one node failed to be ready")
}
return nodeNames, nil
}
// Resize updates a ReplicationController to a new size, with optional precondition check (if preconditions is not nil),
// optional retries (if retry is not nil), and then optionally waits for it's replica count to reach the new value
// (if wait is not nil).
func (resizer *ReplicationControllerResizer) Resize(namespace, name string, newSize uint, preconditions *ResizePrecondition, retry, waitForReplicas *RetryParams) error {
if preconditions == nil {
preconditions = &ResizePrecondition{-1, ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &RetryParams{interval: time.Millisecond, timeout: time.Millisecond}
}
cond := ResizeCondition(resizer, preconditions, namespace, name, newSize)
if err := wait.Poll(retry.interval, retry.timeout, cond); err != nil {
return err
}
if waitForReplicas != nil {
rc := &api.ReplicationController{ObjectMeta: api.ObjectMeta{Namespace: namespace, Name: name}}
if err := wait.Poll(waitForReplicas.interval, waitForReplicas.timeout,
resizer.c.ControllerHasDesiredReplicas(rc)); err != nil {
return err
}
}
return nil
}
func getReferencedServiceAccountToken(c *client.Client, ns string, name string, shouldWait bool) (string, string, error) {
tokenName := ""
token := ""
findToken := func() (bool, error) {
user, err := c.ServiceAccounts(ns).Get(name)
if errors.IsNotFound(err) {
return false, nil
}
if err != nil {
return false, err
}
for _, ref := range user.Secrets {
secret, err := c.Secrets(ns).Get(ref.Name)
if errors.IsNotFound(err) {
continue
}
if err != nil {
return false, err
}
if secret.Type != api.SecretTypeServiceAccountToken {
continue
}
name := secret.Annotations[api.ServiceAccountNameKey]
uid := secret.Annotations[api.ServiceAccountUIDKey]
tokenData := secret.Data[api.ServiceAccountTokenKey]
if name == user.Name && uid == string(user.UID) && len(tokenData) > 0 {
tokenName = secret.Name
token = string(tokenData)
return true, nil
}
}
return false, nil
}
if shouldWait {
err := wait.Poll(time.Second, 10*time.Second, findToken)
if err != nil {
return "", "", err
}
} else {
ok, err := findToken()
if err != nil {
return "", "", err
}
if !ok {
return "", "", fmt.Errorf("No token found for %s/%s", ns, name)
}
}
return tokenName, token, nil
}
func RunResize(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string) error {
count := cmdutil.GetFlagInt(cmd, "replicas")
if len(args) != 2 || count < 0 {
return cmdutil.UsageError(cmd, "--replicas=COUNT RESOURCE ID")
}
cmdNamespace, err := f.DefaultNamespace()
if err != nil {
return err
}
mapper, typer := f.Object()
r := resource.NewBuilder(mapper, typer, f.ClientMapperForCommand()).
ContinueOnError().
NamespaceParam(cmdNamespace).DefaultNamespace().
ResourceTypeOrNameArgs(false, args...).
Flatten().
Do()
err = r.Err()
if err != nil {
return err
}
mapping, err := r.ResourceMapping()
if err != nil {
return err
}
infos, err := r.Infos()
if err != nil {
return err
}
info := infos[0]
resizer, err := f.Resizer(mapping)
if err != nil {
return err
}
resourceVersion := cmdutil.GetFlagString(cmd, "resource-version")
currentSize := cmdutil.GetFlagInt(cmd, "current-replicas")
precondition := &kubectl.ResizePrecondition{currentSize, resourceVersion}
cond := kubectl.ResizeCondition(resizer, precondition, info.Namespace, info.Name, uint(count))
msg := "resized"
if err = wait.Poll(retryFrequency, retryTimeout, cond); err != nil {
msg = fmt.Sprintf("Failed to resize controller in spite of retrying for %s", retryTimeout)
if err != nil {
return err
}
}
fmt.Fprintf(out, "%s\n", msg)
return nil
}
// migRollingUpdateStart (GCE/GKE-only) starts a MIG rolling update using templ
// as the new template, waiting up to nt per node, and returns the ID of that
// update.
func migRollingUpdateStart(templ string, nt time.Duration) (string, error) {
var errLast error
var id string
prefix, suffix := "Started [", "]."
// TODO(mbforbes): Refactor this to use cluster_upgrade.go:retryCmd(...)
if err := wait.Poll(poll, singleCallTimeout, func() (bool, error) {
// TODO(mbforbes): make this hit the compute API directly instead of
// shelling out to gcloud.
// NOTE(mbforbes): If you are changing this gcloud command, update
// cluster/gce/upgrade.sh to match this EXACTLY.
o, err := exec.Command("gcloud", append(migUdpateCmdBase(),
"rolling-updates",
fmt.Sprintf("--project=%s", testContext.CloudConfig.ProjectID),
fmt.Sprintf("--zone=%s", testContext.CloudConfig.Zone),
"start",
// Required args.
fmt.Sprintf("--group=%s", testContext.CloudConfig.NodeInstanceGroup),
fmt.Sprintf("--template=%s", templ),
// Optional args to fine-tune behavior.
fmt.Sprintf("--instance-startup-timeout=%ds", int(nt.Seconds())),
// NOTE: We can speed up this process by increasing
// --max-num-concurrent-instances.
fmt.Sprintf("--max-num-concurrent-instances=%d", 1),
fmt.Sprintf("--max-num-failed-instances=%d", 0),
fmt.Sprintf("--min-instance-update-time=%ds", 0))...).CombinedOutput()
if err != nil {
errLast = fmt.Errorf("rolling-updates call failed with err: %v", err)
return false, nil
}
output := string(o)
// The 'start' call probably succeeded; parse the output and try to find
// the line that looks like "Started [url/to/<id>]." and return <id>.
for _, line := range strings.Split(output, "\n") {
// As a sanity check, ensure the line starts with prefix and ends
// with suffix.
if strings.Index(line, prefix) != 0 || strings.Index(line, suffix) != len(line)-len(suffix) {
continue
}
url := strings.Split(strings.TrimSuffix(strings.TrimPrefix(line, prefix), suffix), "/")
id = url[len(url)-1]
Logf("Started MIG rolling update; ID: %s", id)
return true, nil
}
errLast = fmt.Errorf("couldn't find line like '%s ... %s' in output to MIG rolling-update start. Output: %s",
prefix, suffix, output)
return false, nil
}); err != nil {
return "", fmt.Errorf("migRollingUpdateStart() failed with last error: %v", errLast)
}
return id, nil
}
// Scale updates a ReplicationController to a new size, with optional precondition check (if preconditions is not nil),
// optional retries (if retry is not nil), and then optionally waits for it's replica count to reach the new value
// (if wait is not nil).
func (scaler *ReplicationControllerScaler) Scale(namespace, name string, newSize uint, preconditions *ScalePrecondition, retry, waitForReplicas *RetryParams) error {
if preconditions == nil {
preconditions = &ScalePrecondition{-1, ""}
}
if retry == nil {
// Make it try only once, immediately
retry = &RetryParams{Interval: time.Millisecond, Timeout: time.Millisecond}
}
cond := ScaleCondition(scaler, preconditions, namespace, name, newSize)
if err := wait.Poll(retry.Interval, retry.Timeout, cond); err != nil {
return err
}
if waitForReplicas != nil {
rc, err := scaler.c.GetReplicationController(namespace, name)
if err != nil {
return err
}
return wait.Poll(waitForReplicas.Interval, waitForReplicas.Timeout,
scaler.c.ControllerHasDesiredReplicas(rc))
}
return nil
}
// retryCmd runs cmd using args and retries it for up to singleCallTimeout if
// it returns an error. It returns stdout and stderr.
func retryCmd(command string, args ...string) (string, string, error) {
var err error
stdout, stderr := "", ""
wait.Poll(poll, singleCallTimeout, func() (bool, error) {
stdout, stderr, err = runCmd(command, args...)
if err != nil {
Logf("Got %v", err)
return false, nil
}
return true, nil
})
return stdout, stderr, err
}
func WaitForServiceAccounts(client *kclient.Client, namespace string, accounts []string) error {
// Ensure the service accounts needed by build pods exist in the namespace
// The extra controllers tend to starve the service account controller
serviceAccounts := client.ServiceAccounts(namespace)
return wait.Poll(time.Second, ServiceAccountWaitTimeout, func() (bool, error) {
for _, account := range accounts {
if _, err := serviceAccounts.Get(account); err != nil {
return false, nil
}
}
return true, nil
})
}
func extinguish(c *client.Client, totalNS int, maxAllowedAfterDel int, maxSeconds int) {
var err error
By("Creating testing namespaces")
wg := &sync.WaitGroup{}
for n := 0; n < totalNS; n += 1 {
wg.Add(1)
go func(n int) {
defer wg.Done()
defer GinkgoRecover()
_, err = createTestingNS(fmt.Sprintf("nslifetest-%v", n), c)
Expect(err).NotTo(HaveOccurred())
}(n)
}
wg.Wait()
By("Waiting 10 seconds")
//Wait 10 seconds, then SEND delete requests for all the namespaces.
time.Sleep(time.Duration(10 * time.Second))
By("Deleting namespaces")
nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything())
Expect(err).NotTo(HaveOccurred())
var nsCount = 0
for _, item := range nsList.Items {
if strings.Contains(item.Name, "nslifetest") {
wg.Add(1)
nsCount++
go func(nsName string) {
defer wg.Done()
defer GinkgoRecover()
Expect(c.Namespaces().Delete(nsName)).To(Succeed())
Logf("namespace : %v api call to delete is complete ", nsName)
}(item.Name)
}
}
Expect(nsCount).To(Equal(totalNS))
wg.Wait()
By("Waiting for namespaces to vanish")
//Now POLL until all namespaces have been eradicated.
expectNoError(wait.Poll(2*time.Second, time.Duration(maxSeconds)*time.Second,
func() (bool, error) {
if rem, err := countRemaining(c, "nslifetest"); err != nil || rem > maxAllowedAfterDel {
Logf("Remaining namespaces : %v", rem)
return false, err
} else {
return true, nil
}
}))
}
// WaitForPolicyUpdate checks if the given client can perform the named verb and action.
// If PolicyCachePollTimeout is reached without the expected condition matching, an error is returned
func WaitForPolicyUpdate(c *client.Client, namespace, verb, resource string, allowed bool) error {
review := &authorizationapi.SubjectAccessReview{Verb: verb, Resource: resource}
err := wait.Poll(PolicyCachePollInterval, PolicyCachePollTimeout, func() (bool, error) {
response, err := c.SubjectAccessReviews(namespace).Create(review)
if err != nil {
return false, err
}
if response.Allowed != allowed {
return false, nil
}
return true, nil
})
return err
}
