func CreateNewControllerFromCurrentController(c *client.Client, namespace, oldName, newName, image, deploymentKey string) (*api.ReplicationController, error) { // load the old RC into the "new" RC newRc, err := c.ReplicationControllers(namespace).Get(oldName) if err != nil { return nil, err } if len(newRc.Spec.Template.Spec.Containers) > 1 { // TODO: support multi-container image update. return nil, goerrors.New("Image update is not supported for multi-container pods") } if len(newRc.Spec.Template.Spec.Containers) == 0 { return nil, goerrors.New(fmt.Sprintf("Pod has no containers! (%v)", newRc)) } newRc.Spec.Template.Spec.Containers[0].Image = image newHash, err := api.HashObject(newRc, c.Codec) if err != nil { return nil, err } if len(newName) == 0 { newName = fmt.Sprintf("%s-%s", newRc.Name, newHash) } newRc.Name = newName newRc.Spec.Selector[deploymentKey] = newHash newRc.Spec.Template.Labels[deploymentKey] = newHash // Clear resource version after hashing so that identical updates get different hashes. newRc.ResourceVersion = "" return newRc, nil }
func CheckCadvisorHealthOnAllNodes(c *client.Client, timeout time.Duration) { By("getting list of nodes") nodeList, err := c.Nodes().List(labels.Everything(), fields.Everything()) expectNoError(err) var errors []error retries := maxRetries for { errors = []error{} for _, node := range nodeList.Items { // cadvisor is not accessible directly unless its port (4194 by default) is exposed. // Here, we access '/stats/' REST endpoint on the qinglet which polls cadvisor internally. statsResource := fmt.Sprintf("api/v1/proxy/nodes/%s/stats/", node.Name) By(fmt.Sprintf("Querying stats from node %s using url %s", node.Name, statsResource)) _, err = c.Get().AbsPath(statsResource).Timeout(timeout).Do().Raw() if err != nil { errors = append(errors, err) } } if len(errors) == 0 { return } if retries--; retries <= 0 { break } Logf("failed to retrieve qinglet stats -\n %v", errors) time.Sleep(sleepDuration) } Failf("Failed after retrying %d times for cadvisor to be healthy on all nodes. Errors:\n%v", maxRetries, errors) }
// testHostIP tests that a pod gets a host IP func testHostIP(c *client.Client, pod *api.Pod) { ns := "e2e-test-" + string(util.NewUUID()) _, err := createNamespaceIfDoesNotExist(c, ns) expectNoError(err, fmt.Sprintf("creating namespace %s", ns)) podClient := c.Pods(ns) By("creating pod") defer podClient.Delete(pod.Name, nil) _, err = podClient.Create(pod) if err != nil { Fail(fmt.Sprintf("Failed to create pod: %v", err)) } By("ensuring that pod is running and has a hostIP") // Wait for the pods to enter the running state. Waiting loops until the pods // are running so non-running pods cause a timeout for this test. err = waitForPodRunningInNamespace(c, pod.Name, ns) Expect(err).NotTo(HaveOccurred()) // Try to make sure we get a hostIP for each pod. hostIPTimeout := 2 * time.Minute t := time.Now() for { p, err := podClient.Get(pod.Name) Expect(err).NotTo(HaveOccurred()) if p.Status.HostIP != "" { Logf("Pod %s has hostIP: %s", p.Name, p.Status.HostIP) break } if time.Since(t) >= hostIPTimeout { Failf("Gave up waiting for hostIP of pod %s after %v seconds", p.Name, time.Since(t).Seconds()) } Logf("Retrying to get the hostIP of pod %s", p.Name) time.Sleep(5 * time.Second) } }
func verifyExpectedRcsExistAndGetExpectedPods(c *client.Client) ([]string, error) { expectedPods := []string{} // Iterate over the labels that identify the replication controllers that we // want to check. The rcLabels contains the value values for the k8s-app key // that identify the replication controllers that we want to check. Using a label // rather than an explicit name is preferred because the names will typically have // a version suffix e.g. heapster-monitoring-v1 and this will change after a rolling // update e.g. to heapster-monitoring-v2. By using a label query we can check for the // situaiton when a heapster-monitoring-v1 and heapster-monitoring-v2 replication controller // is running (which would be an error except during a rolling update). for _, rcLabel := range rcLabels { rcList, err := c.ReplicationControllers(api.NamespaceDefault).List(labels.Set{"k8s-app": rcLabel}.AsSelector()) if err != nil { return nil, err } if len(rcList.Items) != 1 { return nil, fmt.Errorf("expected to find one replica for RC with label %s but got %d", rcLabel, len(rcList.Items)) } for _, rc := range rcList.Items { podList, err := c.Pods(api.NamespaceDefault).List(labels.Set(rc.Spec.Selector).AsSelector(), fields.Everything()) if err != nil { return nil, err } for _, pod := range podList.Items { expectedPods = append(expectedPods, string(pod.UID)) } } } return expectedPods, nil }
// StartPods check for numPods in TestNS. If they exist, it no-ops, otherwise it starts up // a temp rc, scales it to match numPods, then deletes the rc leaving behind the pods. func StartPods(numPods int, host string, restClient *client.Client) error { start := time.Now() defer func() { glog.Infof("StartPods took %v with numPods %d", time.Since(start), numPods) }() hostField := fields.OneTermEqualSelector(client.PodHost, host) pods, err := restClient.Pods(TestNS).List(labels.Everything(), hostField) if err != nil || len(pods.Items) == numPods { return err } glog.Infof("Found %d pods that match host %v, require %d", len(pods.Items), hostField, numPods) // For the sake of simplicity, assume all pods in TestNS have selectors matching TestRCManifest. controller := RCFromManifest(TestRCManifest) // Make the rc unique to the given host. controller.Spec.Replicas = numPods controller.Spec.Template.Spec.NodeName = host controller.Name = controller.Name + host controller.Spec.Selector["host"] = host controller.Spec.Template.Labels["host"] = host if rc, err := StartRC(controller, restClient); err != nil { return err } else { // Delete the rc, otherwise when we restart master components for the next benchmark // the rc controller will race with the pods controller in the rc manager. return restClient.ReplicationControllers(TestNS).Delete(rc.Name) } }
// createNamespaceIfDoesNotExist ensures that the namespace with specified name exists, or returns an error func createNamespaceIfDoesNotExist(c *client.Client, name string) (*api.Namespace, error) { namespace, err := c.Namespaces().Get(name) if err != nil { namespace, err = c.Namespaces().Create(&api.Namespace{ObjectMeta: api.ObjectMeta{Name: name}}) } return namespace, err }
func runMasterServiceTest(client *client.Client) { time.Sleep(12 * time.Second) svcList, err := client.Services(api.NamespaceDefault).List(labels.Everything()) if err != nil { glog.Fatalf("unexpected error listing services: %v", err) } var foundRW bool found := util.StringSet{} for i := range svcList.Items { found.Insert(svcList.Items[i].Name) if svcList.Items[i].Name == "qingyuan" { foundRW = true } } if foundRW { ep, err := client.Endpoints(api.NamespaceDefault).Get("qingyuan") if err != nil { glog.Fatalf("unexpected error listing endpoints for qingyuan service: %v", err) } if countEndpoints(ep) == 0 { glog.Fatalf("no endpoints for qingyuan service: %v", ep) } } else { glog.Errorf("no RW service found: %v", found) glog.Fatal("QingYuan service test failed") } glog.Infof("Master service test passed.") }
func extinguish(c *client.Client, totalNS int, maxAllowedAfterDel int, maxSeconds int) { var err error for n := 0; n < totalNS; n += 1 { _, err = createTestingNS(fmt.Sprintf("nslifetest-%v", n), c) Expect(err).NotTo(HaveOccurred()) } //Wait 10 seconds, then SEND delete requests for all the namespaces. time.Sleep(time.Duration(10 * time.Second)) nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything()) Expect(err).NotTo(HaveOccurred()) for _, item := range nsList.Items { if strings.Contains(item.Name, "nslifetest") { if err := c.Namespaces().Delete(item.Name); err != nil { Failf("Failed deleting error ::: --- %v ", err) } } Logf("namespace : %v api call to delete is complete ", item) } //Now POLL until all namespaces have been eradicated. expectNoError(wait.Poll(2*time.Second, time.Duration(maxSeconds)*time.Second, func() (bool, error) { if rem, err := countRemaining(c, "nslifetest"); err != nil || rem > maxAllowedAfterDel { Logf("Remaining namespaces : %v", rem) return false, err } else { return true, nil } })) }
func LoadExistingNextReplicationController(c *client.Client, namespace, newName string) (*api.ReplicationController, error) { if len(newName) == 0 { return nil, nil } newRc, err := c.ReplicationControllers(namespace).Get(newName) if err != nil && errors.IsNotFound(err) { return nil, nil } return newRc, err }
func pickNode(c *client.Client) (string, error) { nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) if err != nil { return "", err } if len(nodes.Items) == 0 { return "", fmt.Errorf("no nodes exist, can't test node proxy") } return nodes.Items[0].Name, nil }
func countRemaining(c *client.Client, withName string) (int, error) { var cnt = 0 nsList, err := c.Namespaces().List(labels.Everything(), fields.Everything()) for _, item := range nsList.Items { if strings.Contains(item.Name, "nslifetest") { cnt++ } } return cnt, err }
func runSchedulerNoPhantomPodsTest(client *client.Client) { pod := &api.Pod{ Spec: api.PodSpec{ Containers: []api.Container{ { Name: "c1", Image: "qingyuan/pause", Ports: []api.ContainerPort{ {ContainerPort: 1234, HostPort: 9999}, }, ImagePullPolicy: api.PullIfNotPresent, }, }, }, } // Assuming we only have two kublets, the third pod here won't schedule // if the scheduler doesn't correctly handle the delete for the second // pod. pod.ObjectMeta.Name = "phantom.foo" foo, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, time.Second*30, podRunning(client, foo.Namespace, foo.Name)); err != nil { glog.Fatalf("FAILED: pod never started running %v", err) } pod.ObjectMeta.Name = "phantom.bar" bar, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, time.Second*30, podRunning(client, bar.Namespace, bar.Name)); err != nil { glog.Fatalf("FAILED: pod never started running %v", err) } // Delete a pod to free up room. glog.Infof("Deleting pod %v", bar.Name) err = client.Pods(api.NamespaceDefault).Delete(bar.Name, nil) if err != nil { glog.Fatalf("FAILED: couldn't delete pod %q: %v", bar.Name, err) } pod.ObjectMeta.Name = "phantom.baz" baz, err := client.Pods(api.NamespaceDefault).Create(pod) if err != nil { glog.Fatalf("Failed to create pod: %v, %v", pod, err) } if err := wait.Poll(time.Second, time.Second*60, podRunning(client, baz.Namespace, baz.Name)); err != nil { glog.Fatalf("FAILED: (Scheduler probably didn't process deletion of 'phantom.bar') Pod never started running: %v", err) } glog.Info("Scheduler doesn't make phantom pods: test passed.") }
func getAllNodesInCluster(c *client.Client) ([]string, error) { nodeList, err := c.Nodes().List(labels.Everything(), fields.Everything()) if err != nil { return nil, err } result := []string{} for _, node := range nodeList.Items { result = append(result, node.Name) } return result, nil }
// StartRC creates given rc if it doesn't already exist, then updates it via qingctl's scaler. func StartRC(controller *api.ReplicationController, restClient *client.Client) (*api.ReplicationController, error) { created, err := restClient.ReplicationControllers(controller.Namespace).Get(controller.Name) if err != nil { glog.Infof("Rc %v doesn't exist, creating", controller.Name) created, err = restClient.ReplicationControllers(controller.Namespace).Create(controller) if err != nil { return nil, err } } // If we just created an rc, wait till it creates its replicas. return ScaleRC(created.Name, created.Namespace, controller.Spec.Replicas, restClient) }
func getReferencedServiceAccountToken(c *client.Client, ns string, name string, shouldWait bool) (string, string, error) { tokenName := "" token := "" findToken := func() (bool, error) { user, err := c.ServiceAccounts(ns).Get(name) if errors.IsNotFound(err) { return false, nil } if err != nil { return false, err } for _, ref := range user.Secrets { secret, err := c.Secrets(ns).Get(ref.Name) if errors.IsNotFound(err) { continue } if err != nil { return false, err } if secret.Type != api.SecretTypeServiceAccountToken { continue } name := secret.Annotations[api.ServiceAccountNameKey] uid := secret.Annotations[api.ServiceAccountUIDKey] tokenData := secret.Data[api.ServiceAccountTokenKey] if name == user.Name && uid == string(user.UID) && len(tokenData) > 0 { tokenName = secret.Name token = string(tokenData) return true, nil } } return false, nil } if shouldWait { err := wait.Poll(time.Second, 10*time.Second, findToken) if err != nil { return "", "", err } } else { ok, err := findToken() if err != nil { return "", "", err } if !ok { return "", "", fmt.Errorf("No token found for %s/%s", ns, name) } } return tokenName, token, nil }
func getMinionPublicIps(c *client.Client) ([]string, error) { nodes, err := c.Nodes().List(labels.Everything(), fields.Everything()) if err != nil { return nil, err } ips := collectAddresses(nodes, api.NodeExternalIP) if len(ips) == 0 { ips = collectAddresses(nodes, api.NodeLegacyHostIP) } return ips, nil }
// Retrieve metrics from the qinglet server of the given node. func getQingletMetrics(c *client.Client, node string) (string, error) { metric, err := c.Get(). Prefix("proxy"). Resource("nodes"). Name(fmt.Sprintf("%v:%v", node, ports.QingletPort)). Suffix("metrics"). Do(). Raw() if err != nil { return "", err } return string(metric), nil }
func makeRequestToGuestbook(c *client.Client, cmd, value string, ns string) (string, error) { result, err := c.Get(). Prefix("proxy"). Namespace(ns). Resource("services"). Name("frontend"). Suffix("/index.php"). Param("cmd", cmd). Param("key", "messages"). Param("value", value). Do(). Raw() return string(result), err }
func translatePodNameToIpOrFail(c *client.Client, ns string, expectedEndpoints map[string][]int) map[string][]int { portsByIp := make(map[string][]int) for name, portList := range expectedEndpoints { pod, err := c.Pods(ns).Get(name) if err != nil { Failf("failed to get pod %s, that's pretty weird. validation failed: %s", name, err) } portsByIp[pod.Status.PodIP] = portList By(fmt.Sprintf("")) } By(fmt.Sprintf("successfully translated pod names to ips: %v -> %v on namespace %s", expectedEndpoints, portsByIp, ns)) return portsByIp }
func runAPIVersionsTest(c *client.Client) { v, err := c.ServerAPIVersions() clientVersion := c.APIVersion() if err != nil { glog.Fatalf("failed to get api versions: %v", err) } // Verify that the server supports the API version used by the client. for _, version := range v.Versions { if version == clientVersion { glog.Infof("Version test passed") return } } glog.Fatalf("Server does not support APIVersion used by client. Server supported APIVersions: '%v', client APIVersion: '%v'", v.Versions, clientVersion) }
func podScheduled(c *client.Client, podNamespace, podName string) wait.ConditionFunc { return func() (bool, error) { pod, err := c.Pods(podNamespace).Get(podName) if errors.IsNotFound(err) { return false, nil } if err != nil { // This could be a connection error so we want to retry. return false, nil } if pod.Spec.NodeName == "" { return false, nil } return true, nil } }
func assertFilesExist(fileNames []string, fileDir string, pod *api.Pod, client *client.Client) { var failed []string expectNoError(wait.Poll(time.Second*2, time.Second*60, func() (bool, error) { failed = []string{} for _, fileName := range fileNames { if _, err := client.Get(). Prefix("proxy"). Resource("pods"). Namespace(pod.Namespace). Name(pod.Name). Suffix(fileDir, fileName). Do().Raw(); err != nil { Logf("Unable to read %s from pod %s: %v", fileName, pod.Name, err) failed = append(failed, fileName) } } if len(failed) == 0 { return true, nil } Logf("Lookups using %s failed for: %v\n", pod.Name, failed) return false, nil })) Expect(len(failed)).To(Equal(0)) }
func checkMasterVersion(c *client.Client, want string) error { v, err := c.ServerVersion() if err != nil { return fmt.Errorf("checkMasterVersion() couldn't get the master version: %v", err) } // We do prefix trimming and then matching because: // want looks like: 0.19.3-815-g50e67d4 // got looks like: v0.19.3-815-g50e67d4034e858-dirty got := strings.TrimPrefix(v.GitVersion, "v") if !strings.HasPrefix(got, want) { return fmt.Errorf("master had qing-apiserver version %s which does not start with %s", got, want) } Logf("Master is at version %s", want) return nil }
// ScaleRC scales the given rc to the given replicas. func ScaleRC(name, ns string, replicas int, restClient *client.Client) (*api.ReplicationController, error) { scaler, err := qingctl.ScalerFor("ReplicationController", qingctl.NewScalerClient(restClient)) if err != nil { return nil, err } retry := &qingctl.RetryParams{50 * time.Millisecond, DefaultTimeout} waitForReplicas := &qingctl.RetryParams{50 * time.Millisecond, DefaultTimeout} err = scaler.Scale(ns, name, uint(replicas), nil, retry, waitForReplicas) if err != nil { return nil, err } scaled, err := restClient.ReplicationControllers(ns).Get(name) if err != nil { return nil, err } return scaled, nil }
func podRunning(c *client.Client, podNamespace string, podName string) wait.ConditionFunc { return func() (bool, error) { pod, err := c.Pods(podNamespace).Get(podName) if apierrors.IsNotFound(err) { return false, nil } if err != nil { // This could be a connection error so we want to retry, but log the error. glog.Errorf("Error when reading pod %q: %v", podName, err) return false, nil } if pod.Status.Phase != api.PodRunning { return false, nil } return true, nil } }
func expectedServicesExist(c *client.Client) error { serviceList, err := c.Services(api.NamespaceDefault).List(labels.Everything()) if err != nil { return err } for _, service := range serviceList.Items { if _, ok := expectedServices[service.Name]; ok { expectedServices[service.Name] = true } } for service, found := range expectedServices { if !found { return fmt.Errorf("Service %q not found", service) } } return nil }
// Starts a container specified by config.serverImage and exports all // config.serverPorts from it. The returned pod should be used to get the server // IP address and create appropriate VolumeSource. func startVolumeServer(client *client.Client, config VolumeTestConfig) *api.Pod { podClient := client.Pods(config.namespace) portCount := len(config.serverPorts) serverPodPorts := make([]api.ContainerPort, portCount) for i := 0; i < portCount; i++ { portName := fmt.Sprintf("%s-%d", config.prefix, i) serverPodPorts[i] = api.ContainerPort{ Name: portName, ContainerPort: config.serverPorts[i], Protocol: api.ProtocolTCP, } } By(fmt.Sprint("creating ", config.prefix, " server pod")) privileged := new(bool) *privileged = true serverPod := &api.Pod{ TypeMeta: api.TypeMeta{ Kind: "Pod", APIVersion: "v1", }, ObjectMeta: api.ObjectMeta{ Name: config.prefix + "-server", Labels: map[string]string{ "role": config.prefix + "-server", }, }, Spec: api.PodSpec{ Containers: []api.Container{ { Name: config.prefix + "-server", Image: config.serverImage, SecurityContext: &api.SecurityContext{ Privileged: privileged, }, Ports: serverPodPorts, }, }, }, } _, err := podClient.Create(serverPod) expectNoError(err, "Failed to create %s pod: %v", serverPod.Name, err) expectNoError(waitForPodRunningInNamespace(client, serverPod.Name, config.namespace)) By("locating the server pod") pod, err := podClient.Get(serverPod.Name) expectNoError(err, "Cannot locate the server pod %v: %v", serverPod.Name, err) By("sleeping a bit to give the server time to start") time.Sleep(20 * time.Second) return pod }
func waitForLoadBalancerDestroy(c *client.Client, serviceName, namespace string) (*api.Service, error) { // TODO: once support ticket 21807001 is resolved, reduce this timeout back to something reasonable const timeout = 10 * time.Minute var service *api.Service By(fmt.Sprintf("waiting up to %v for service %s in namespace %s to have no LoadBalancer ingress points", timeout, serviceName, namespace)) for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) { service, err := c.Services(namespace).Get(serviceName) if err != nil { Logf("Get service failed, ignoring for 5s: %v", err) continue } if len(service.Status.LoadBalancer.Ingress) == 0 { return service, nil } Logf("Waiting for service %s in namespace %s to have no LoadBalancer ingress points (%v)", serviceName, namespace, time.Since(start)) } return service, fmt.Errorf("service %s in namespace %s still has LoadBalancer ingress points after %.2f seconds", serviceName, namespace, timeout.Seconds()) }
// NewListWatchFromClient creates a new ListWatch from the specified client, resource, namespace and field selector. func NewListWatchFromClient(c *client.Client, resource string, namespace string, fieldSelector fields.Selector) *ListWatch { listFunc := func() (runtime.Object, error) { return c.Get(). Namespace(namespace). Resource(resource). FieldsSelectorParam(fieldSelector). Do(). Get() } watchFunc := func(resourceVersion string) (watch.Interface, error) { return c.Get(). Prefix("watch"). Namespace(namespace). Resource(resource). FieldsSelectorParam(fieldSelector). Param("resourceVersion", resourceVersion).Watch() } return &ListWatch{ListFunc: listFunc, WatchFunc: watchFunc} }
func getServiceAccount(c *client.Client, ns string, name string, shouldWait bool) (*api.ServiceAccount, error) { if !shouldWait { return c.ServiceAccounts(ns).Get(name) } var user *api.ServiceAccount var err error err = wait.Poll(time.Second, 10*time.Second, func() (bool, error) { user, err = c.ServiceAccounts(ns).Get(name) if errors.IsNotFound(err) { return false, nil } if err != nil { return false, err } return true, nil }) return user, err }