func waitForAllCaPodsReadyInNamespace(f *framework.Framework, c *client.Client) error {
var notready []string
for start := time.Now(); time.Now().Before(start.Add(scaleUpTimeout)); time.Sleep(20 * time.Second) {
pods, err := c.Pods(f.Namespace.Name).List(api.ListOptions{})
if err != nil {
return fmt.Errorf("failed to get pods: %v", err)
}
notready = make([]string, 0)
for _, pod := range pods.Items {
ready := false
for _, c := range pod.Status.Conditions {
if c.Type == api.PodReady && c.Status == api.ConditionTrue {
ready = true
}
}
if !ready {
notready = append(notready, pod.Name)
}
}
if len(notready) == 0 {
glog.Infof("All pods ready")
return nil
}
glog.Infof("Some pods are not ready yet: %v", notready)
}
glog.Info("Timeout on waiting for pods being ready")
glog.Info(framework.RunKubectlOrDie("get", "pods", "-o", "json", "--all-namespaces"))
glog.Info(framework.RunKubectlOrDie("get", "nodes", "-o", "json"))
// Some pods are still not running.
return fmt.Errorf("Some pods are still not running: %v", notready)
}
// createIngress creates the Ingress and associated service/rc.
// Required: ing.yaml, rc.yaml, svc.yaml must exist in manifestPath
// Optional: secret.yaml, ingAnnotations
// If ingAnnotations is specified it will overwrite any annotations in ing.yaml
func (j *testJig) createIngress(manifestPath, ns string, ingAnnotations map[string]string) {
mkpath := func(file string) string {
return filepath.Join(framework.TestContext.RepoRoot, manifestPath, file)
}
framework.Logf("creating replication controller")
framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", ns))
framework.Logf("creating service")
framework.RunKubectlOrDie("create", "-f", mkpath("svc.yaml"), fmt.Sprintf("--namespace=%v", ns))
if exists(mkpath("secret.yaml")) {
framework.Logf("creating secret")
framework.RunKubectlOrDie("create", "-f", mkpath("secret.yaml"), fmt.Sprintf("--namespace=%v", ns))
}
j.ing = ingFromManifest(mkpath("ing.yaml"))
j.ing.Namespace = ns
j.ing.Annotations = map[string]string{ingressClass: j.class}
for k, v := range ingAnnotations {
j.ing.Annotations[k] = v
}
framework.Logf(fmt.Sprintf("creating" + j.ing.Name + " ingress"))
var err error
j.ing, err = j.client.Extensions().Ingresses(ns).Create(j.ing)
framework.ExpectNoError(err)
}
func (p *statefulSetTester) createStatefulSet(manifestPath, ns string) *apps.StatefulSet {
mkpath := func(file string) string {
return filepath.Join(framework.TestContext.RepoRoot, manifestPath, file)
}
ps := statefulSetFromManifest(mkpath("petset.yaml"), ns)
framework.Logf(fmt.Sprintf("creating " + ps.Name + " service"))
framework.RunKubectlOrDie("create", "-f", mkpath("service.yaml"), fmt.Sprintf("--namespace=%v", ns))
framework.Logf(fmt.Sprintf("creating statefulset %v/%v with %d replicas and selector %+v", ps.Namespace, ps.Name, *(ps.Spec.Replicas), ps.Spec.Selector))
framework.RunKubectlOrDie("create", "-f", mkpath("petset.yaml"), fmt.Sprintf("--namespace=%v", ns))
p.waitForRunningAndReady(*ps.Spec.Replicas, ps)
return ps
}
func (z *zookeeperTester) write(petIndex int, kv map[string]string) {
name := fmt.Sprintf("%v-%d", z.ps.Name, petIndex)
ns := fmt.Sprintf("--namespace=%v", z.ps.Namespace)
for k, v := range kv {
cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh create /%v %v", k, v)
framework.Logf(framework.RunKubectlOrDie("exec", ns, name, "--", "/bin/sh", "-c", cmd))
}
}
func printDebugInfo(c *client.Client) {
set := labels.Set{"k8s-app": "heapster"}
options := api.ListOptions{LabelSelector: set.AsSelector()}
podList, err := c.Pods(api.NamespaceSystem).List(options)
if err != nil {
framework.Logf("Error while listing pods %v", err)
return
}
for _, pod := range podList.Items {
framework.Logf("Kubectl output:\n%v", framework.RunKubectlOrDie("log", pod.Name, "--namespace=kube-system"))
}
}
func waitForAllCaPodsReadyInNamespace(f *framework.Framework, c clientset.Interface) error {
var notready []string
for start := time.Now(); time.Now().Before(start.Add(scaleUpTimeout)); time.Sleep(20 * time.Second) {
pods, err := c.Core().Pods(f.Namespace.Name).List(v1.ListOptions{})
if err != nil {
return fmt.Errorf("failed to get pods: %v", err)
}
notready = make([]string, 0)
for _, pod := range pods.Items {
ready := false
for _, c := range pod.Status.Conditions {
if c.Type == v1.PodReady && c.Status == v1.ConditionTrue {
ready = true
}
}
// Failed pods in this context generally mean that they have been
// double scheduled onto a node, but then failed a constraint check.
if pod.Status.Phase == v1.PodFailed {
glog.Warningf("Pod has failed: %v", pod)
}
if !ready && pod.Status.Phase != v1.PodFailed {
notready = append(notready, pod.Name)
}
}
if len(notready) == 0 {
glog.Infof("All pods ready")
return nil
}
glog.Infof("Some pods are not ready yet: %v", notready)
}
glog.Info("Timeout on waiting for pods being ready")
glog.Info(framework.RunKubectlOrDie("get", "pods", "-o", "json", "--all-namespaces"))
glog.Info(framework.RunKubectlOrDie("get", "nodes", "-o", "json"))
// Some pods are still not running.
return fmt.Errorf("Some pods are still not running: %v", notready)
}
func (cont *NginxIngressController) init() {
mkpath := func(file string) string {
return filepath.Join(framework.TestContext.RepoRoot, ingressManifestPath, "nginx", file)
}
framework.Logf("initializing nginx ingress controller")
framework.RunKubectlOrDie("create", "-f", mkpath("rc.yaml"), fmt.Sprintf("--namespace=%v", cont.ns))
rc, err := cont.c.Core().ReplicationControllers(cont.ns).Get("nginx-ingress-controller", metav1.GetOptions{})
framework.ExpectNoError(err)
cont.rc = rc
framework.Logf("waiting for pods with label %v", rc.Spec.Selector)
sel := labels.SelectorFromSet(labels.Set(rc.Spec.Selector))
framework.ExpectNoError(testutils.WaitForPodsWithLabelRunning(cont.c, cont.ns, sel))
pods, err := cont.c.Core().Pods(cont.ns).List(v1.ListOptions{LabelSelector: sel.String()})
framework.ExpectNoError(err)
if len(pods.Items) == 0 {
framework.Failf("Failed to find nginx ingress controller pods with selector %v", sel)
}
cont.pod = &pods.Items[0]
cont.externalIP, err = framework.GetHostExternalAddress(cont.c, cont.pod)
framework.ExpectNoError(err)
framework.Logf("ingress controller running in pod %v on ip %v", cont.pod.Name, cont.externalIP)
}
CreateNodeSelectorPods(f, "node-selector", minSize+1, labels, false)
By("Waiting for new node to appear and annotating it")
WaitForGroupSize(minMig, int32(minSize+1))
// Verify, that cluster size is increased
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout))
newNodes, err := GetGroupNodes(minMig)
framework.ExpectNoError(err)
newNodesSet := sets.NewString(newNodes...)
newNodesSet.Delete(nodes...)
if len(newNodesSet) > 1 {
By(fmt.Sprintf("Spotted following new nodes in %s: %v", minMig, newNodesSet))
glog.Infof("Usually only 1 new node is expected, investigating")
glog.Infof("Kubectl:%s\n", framework.RunKubectlOrDie("get", "nodes", "-o", "json"))
if output, err := exec.Command("gcloud", "compute", "instances", "list",
"--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil {
glog.Infof("Gcloud compute instances list: %s", output)
} else {
glog.Errorf("Failed to get instances list: %v", err)
}
for newNode := range newNodesSet {
if output, err := exec.Command("gcloud", "compute", "instances", "describe",
newNode,
"--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone).Output(); err == nil {
glog.Infof("Gcloud compute instances describe: %s", output)
} else {
func (c *cockroachDBTester) cockroachDBExec(cmd, ns, podName string) string {
cmd = fmt.Sprintf("/cockroach/cockroach sql --host %s.cockroachdb -e \"%v\"", podName, cmd)
return framework.RunKubectlOrDie(fmt.Sprintf("--namespace=%v", ns), "exec", podName, "--", "/bin/sh", "-c", cmd)
}
func (m *redisTester) redisExec(cmd, ns, podName string) string {
cmd = fmt.Sprintf("/opt/redis/redis-cli -h %v %v", podName, cmd)
return framework.RunKubectlOrDie(fmt.Sprintf("--namespace=%v", ns), "exec", podName, "--", "/bin/sh", "-c", cmd)
}
func (z *zookeeperTester) read(petIndex int, key string) string {
name := fmt.Sprintf("%v-%d", z.ps.Name, petIndex)
ns := fmt.Sprintf("--namespace=%v", z.ps.Namespace)
cmd := fmt.Sprintf("/opt/zookeeper/bin/zkCli.sh get /%v", key)
return lastLine(framework.RunKubectlOrDie("exec", ns, name, "--", "/bin/sh", "-c", cmd))
}
By("Explicitly delete pod here to free the resource it takes.")
err = c.Pods(ns).Delete(podName, api.NewDeleteOptions(0))
framework.ExpectNoError(err)
By("Trying to apply a label with fake az info on the found node.")
k := "kubernetes.io/e2e-az-name"
v := "e2e-az1"
framework.AddOrUpdateLabelOnNode(c, nodeName, k, v)
framework.ExpectNodeHasLabel(c, nodeName, k, v)
defer framework.RemoveLabelOffNode(c, nodeName, k)
By("Trying to launch a pod that with NodeAffinity setting as embedded JSON string in the annotation value.")
labelPodName := "with-labels"
testPodPath := string(framework.ReadOrDie("test/e2e/testing-manifests/node-selection/pod-with-node-affinity.yaml"))
framework.RunKubectlOrDie("create", "-f", testPodPath, fmt.Sprintf("--namespace=%v", ns))
// check that pod got scheduled. We intentionally DO NOT check that the
// pod is running because this will create a race condition with the
// kubelet and the scheduler: the scheduler might have scheduled a pod
// already when the kubelet does not know about its new label yet. The
// kubelet will then refuse to launch the pod.
framework.ExpectNoError(framework.WaitForPodNotPending(c, ns, labelPodName, ""))
labelPod, err := c.Pods(ns).Get(labelPodName)
framework.ExpectNoError(err)
Expect(labelPod.Spec.NodeName).To(Equal(nodeName))
})
// labelSelector Operator is DoesNotExist but values are there in requiredDuringSchedulingIgnoredDuringExecution
// part of podAffinity,so validation fails.
It("validates that a pod with an invalid podAffinity is rejected because of the LabelSelectorRequirement is invalid", func() {
return filepath.Join(framework.TestContext.RepoRoot, "examples/storage/redis", file)
}
bootstrapYaml := mkpath("redis-master.yaml")
sentinelServiceYaml := mkpath("redis-sentinel-service.yaml")
sentinelControllerYaml := mkpath("redis-sentinel-controller.yaml")
controllerYaml := mkpath("redis-controller.yaml")
bootstrapPodName := "redis-master"
redisRC := "redis"
sentinelRC := "redis-sentinel"
nsFlag := fmt.Sprintf("--namespace=%v", ns)
expectedOnServer := "The server is now ready to accept connections"
expectedOnSentinel := "+monitor master"
By("starting redis bootstrap")
framework.RunKubectlOrDie("create", "-f", bootstrapYaml, nsFlag)
err := framework.WaitForPodNameRunningInNamespace(c, bootstrapPodName, ns)
Expect(err).NotTo(HaveOccurred())
_, err = framework.LookForStringInLog(ns, bootstrapPodName, "master", expectedOnServer, serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
_, err = framework.LookForStringInLog(ns, bootstrapPodName, "sentinel", expectedOnSentinel, serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
By("setting up services and controllers")
framework.RunKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag)
framework.RunKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag)
framework.RunKubectlOrDie("create", "-f", controllerYaml, nsFlag)
label := labels.SelectorFromSet(labels.Set(map[string]string{sentinelRC: "true"}))
err = framework.WaitForPodsWithLabelRunning(c, ns, label)
Expect(err).NotTo(HaveOccurred())
frontendPodName := "dns-frontend"
frontendPodContainerName := "dns-frontend"
podOutput := "Hello World!"
// we need two namespaces anyway, so let's forget about
// the one created in BeforeEach and create two new ones.
namespaces := []*api.Namespace{nil, nil}
for i := range namespaces {
var err error
namespaces[i], err = f.CreateNamespace(fmt.Sprintf("dnsexample%d", i), nil)
Expect(err).NotTo(HaveOccurred())
}
for _, ns := range namespaces {
framework.RunKubectlOrDie("create", "-f", backendRcYaml, getNsCmdFlag(ns))
}
for _, ns := range namespaces {
framework.RunKubectlOrDie("create", "-f", backendSvcYaml, getNsCmdFlag(ns))
}
// wait for objects
for _, ns := range namespaces {
framework.WaitForRCPodsRunning(c, ns.Name, backendRcName)
framework.WaitForService(c, ns.Name, backendSvcName, true, framework.Poll, framework.ServiceStartTimeout)
}
// it is not enough that pods are running because they may be set to running, but
// the application itself may have not been initialized. Just query the application.
for _, ns := range namespaces {
label := labels.SelectorFromSet(labels.Set(map[string]string{"name": backendRcName}))
CreateNodeSelectorPods(f, "node-selector", minSize+1, labels, false)
By("Waiting for new node to appear and annotating it")
WaitForGroupSize(minMig, int32(minSize+1))
// Verify, that cluster size is increased
framework.ExpectNoError(WaitForClusterSizeFunc(f.ClientSet,
func(size int) bool { return size >= nodeCount+1 }, scaleUpTimeout))
newNodes, err := GetGroupNodes(minMig)
ExpectNoError(err)
newNodesSet := sets.NewString(newNodes...)
newNodesSet.Delete(nodes...)
if len(newNodesSet) > 1 {
By(fmt.Sprintf("Spotted following new nodes in %s: %v", minMig, newNodesSet))
glog.Infof("Usually only 1 new node is expected, investigating")
glog.Infof("Kubectl:%s\n", framework.RunKubectlOrDie("kubectl get nodes -o yaml"))
if output, err := exec.Command("gcloud", "compute", "instances", "list",
"--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone).Output(); err != nil {
glog.Infof("Gcloud compute instances list: %s", output)
} else {
glog.Errorf("Failed to get instances list: %v", err)
}
for newNode := range newNodesSet {
if output, err := exec.Command("gcloud", "compute", "instances", "describe",
newNode,
"--project="+framework.TestContext.CloudConfig.ProjectID,
"--zone="+framework.TestContext.CloudConfig.Zone).Output(); err != nil {
glog.Infof("Gcloud compute instances describe: %s", output)
} else {
return filepath.Join(framework.TestContext.RepoRoot, "examples/redis", file)
}
bootstrapYaml := mkpath("redis-master.yaml")
sentinelServiceYaml := mkpath("redis-sentinel-service.yaml")
sentinelControllerYaml := mkpath("redis-sentinel-controller.yaml")
controllerYaml := mkpath("redis-controller.yaml")
bootstrapPodName := "redis-master"
redisRC := "redis"
sentinelRC := "redis-sentinel"
nsFlag := fmt.Sprintf("--namespace=%v", ns)
expectedOnServer := "The server is now ready to accept connections"
expectedOnSentinel := "+monitor master"
By("starting redis bootstrap")
framework.RunKubectlOrDie("create", "-f", bootstrapYaml, nsFlag)
err := framework.WaitForPodRunningInNamespace(c, bootstrapPodName, ns)
Expect(err).NotTo(HaveOccurred())
_, err = framework.LookForStringInLog(ns, bootstrapPodName, "master", expectedOnServer, serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
_, err = framework.LookForStringInLog(ns, bootstrapPodName, "sentinel", expectedOnSentinel, serverStartTimeout)
Expect(err).NotTo(HaveOccurred())
By("setting up services and controllers")
framework.RunKubectlOrDie("create", "-f", sentinelServiceYaml, nsFlag)
framework.RunKubectlOrDie("create", "-f", sentinelControllerYaml, nsFlag)
framework.RunKubectlOrDie("create", "-f", controllerYaml, nsFlag)
By("scaling up the deployment")
framework.RunKubectlOrDie("scale", "rc", redisRC, "--replicas=3", nsFlag)