// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node. func recoverDiskSpace(c *client.Client, node *api.Node) { By(fmt.Sprintf("Recovering disk space on node %s", node.Name)) cmd := "rm -f test.img" framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)) ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, false, nodeOODTimeOut) Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut) }
// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk // space on the node is filled in such a way that the available space after filling the disk is just // below the lowDiskSpaceThreshold mark. func fillDiskSpace(c *client.Client, node *api.Node) { avail, err := availSize(c, node) framework.ExpectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err) fillSize := (avail - lowDiskSpaceThreshold + (100 * mb)) framework.Logf("Node %s: disk space available %d bytes", node.Name, avail) By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize)) cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize) framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)) ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, true, nodeOODTimeOut) Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut) avail, err = availSize(c, node) framework.Logf("Node %s: disk space available %d bytes", node.Name, avail) Expect(avail < lowDiskSpaceThreshold).To(BeTrue()) }
// rebootNode takes node name on provider through the following steps using c: // - ensures the node is ready // - ensures all pods on the node are running and ready // - reboots the node (by executing rebootCmd over ssh) // - ensures the node reaches some non-ready state // - ensures the node becomes ready again // - ensures all pods on the node become running and ready again // // It returns true through result only if all of the steps pass; at the first // failed step, it will return false through result and not run the rest. func rebootNode(c *client.Client, provider, name, rebootCmd string) bool { // Setup ns := api.NamespaceSystem ps := framework.NewPodStore(c, ns, labels.Everything(), fields.OneTermEqualSelector(api.PodHostField, name)) defer ps.Stop() // Get the node initially. framework.Logf("Getting %s", name) node, err := c.Nodes().Get(name) if err != nil { framework.Logf("Couldn't get node %s", name) return false } // Node sanity check: ensure it is "ready". if !framework.WaitForNodeToBeReady(c, name, framework.NodeReadyInitialTimeout) { return false } // Get all the pods on the node that don't have liveness probe set. // Liveness probe may cause restart of a pod during node reboot, and the pod may not be running. pods := ps.List() podNames := []string{} for _, p := range pods { probe := false for _, c := range p.Spec.Containers { if c.LivenessProbe != nil { probe = true break } } if !probe { podNames = append(podNames, p.ObjectMeta.Name) } } framework.Logf("Node %s has %d assigned pods with no liveness probes: %v", name, len(podNames), podNames) // For each pod, we do a sanity check to ensure it's running / healthy // or succeeded now, as that's what we'll be checking later. if !framework.CheckPodsRunningReadyOrSucceeded(c, ns, podNames, framework.PodReadyBeforeTimeout) { printStatusAndLogsForNotReadyPods(c, ns, podNames, pods) return false } // Reboot the node. if err = framework.IssueSSHCommand(rebootCmd, provider, node); err != nil { framework.Logf("Error while issuing ssh command: %v", err) return false } // Wait for some kind of "not ready" status. if !framework.WaitForNodeToBeNotReady(c, name, rebootNodeNotReadyTimeout) { return false } // Wait for some kind of "ready" status. if !framework.WaitForNodeToBeReady(c, name, rebootNodeReadyAgainTimeout) { return false } // Ensure all of the pods that we found on this node before the reboot are // running / healthy, or succeeded. if !framework.CheckPodsRunningReadyOrSucceeded(c, ns, podNames, rebootPodReadyAgainTimeout) { newPods := ps.List() printStatusAndLogsForNotReadyPods(c, ns, podNames, newPods) return false } framework.Logf("Reboot successful on node %s", name) return true }
break } } Expect(node).NotTo(BeNil()) By("Generate event list options") selector := fields.Set{ "involvedObject.kind": "Node", "involvedObject.name": node.Name, "involvedObject.namespace": api.NamespaceAll, "source": source, }.AsSelector() eventListOptions = api.ListOptions{FieldSelector: selector} By("Create the test log file") tmpDir = "/tmp/" + name cmd := fmt.Sprintf("mkdir %s; > %s/%s", tmpDir, tmpDir, logFile) Expect(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)).To(Succeed()) By("Create config map for the node problem detector") _, err = c.ConfigMaps(ns).Create(&api.ConfigMap{ ObjectMeta: api.ObjectMeta{ Name: configName, }, Data: map[string]string{configFile: config}, }) Expect(err).NotTo(HaveOccurred()) By("Create the node problem detector") _, err = c.Pods(ns).Create(&api.Pod{ ObjectMeta: api.ObjectMeta{ Name: name, }, Spec: api.PodSpec{ NodeName: node.Name,
break } } Expect(node).NotTo(BeNil()) By("Generate event list options") selector := fields.Set{ "involvedObject.kind": "Node", "involvedObject.name": node.Name, "involvedObject.namespace": v1.NamespaceAll, "source": source, }.AsSelector().String() eventListOptions = v1.ListOptions{FieldSelector: selector} By("Create the test log file") tmpDir = "/tmp/" + name cmd := fmt.Sprintf("mkdir %s; > %s/%s", tmpDir, tmpDir, logFile) Expect(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)).To(Succeed()) By("Create config map for the node problem detector") _, err = c.Core().ConfigMaps(ns).Create(&v1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: configName, }, Data: map[string]string{configFile: config}, }) Expect(err).NotTo(HaveOccurred()) By("Create the node problem detector") _, err = c.Core().Pods(ns).Create(&v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, Spec: v1.PodSpec{ NodeName: node.Name,