// Blocks outgoing network traffic on 'node'. Then verifies that 'podNameToDisappear', // that belongs to replication controller 'rcName', really disappeared. // Finally, it checks that the replication controller recreates the // pods on another node and that now the number of replicas is equal 'replicas'. // At the end (even in case of errors), the network traffic is brought back to normal. // This function executes commands on a node so it will work only for some // environments. func performTemporaryNetworkFailure(c *client.Client, ns, rcName string, replicas int32, podNameToDisappear string, node *api.Node) { host := getNodeExternalIP(node) master := getMaster(c) By(fmt.Sprintf("block network traffic from node %s to the master", node.Name)) defer func() { // This code will execute even if setting the iptables rule failed. // It is on purpose because we may have an error even if the new rule // had been inserted. (yes, we could look at the error code and ssh error // separately, but I prefer to stay on the safe side). By(fmt.Sprintf("Unblock network traffic from node %s to the master", node.Name)) framework.UnblockNetwork(host, master) }() framework.Logf("Waiting %v to ensure node %s is ready before beginning test...", resizeNodeReadyTimeout, node.Name) if !framework.WaitForNodeToBe(c, node.Name, api.NodeReady, true, resizeNodeReadyTimeout) { framework.Failf("Node %s did not become ready within %v", node.Name, resizeNodeReadyTimeout) } framework.BlockNetwork(host, master) framework.Logf("Waiting %v for node %s to be not ready after simulated network failure", resizeNodeNotReadyTimeout, node.Name) if !framework.WaitForNodeToBe(c, node.Name, api.NodeReady, false, resizeNodeNotReadyTimeout) { framework.Failf("Node %s did not become not-ready within %v", node.Name, resizeNodeNotReadyTimeout) } framework.Logf("Waiting for pod %s to be removed", podNameToDisappear) err := framework.WaitForRCPodToDisappear(c, ns, rcName, podNameToDisappear) Expect(err).NotTo(HaveOccurred()) By("verifying whether the pod from the unreachable node is recreated") err = framework.VerifyPods(c, ns, rcName, true, replicas) Expect(err).NotTo(HaveOccurred()) // network traffic is unblocked in a deferred function }
// Blocks outgoing network traffic on 'node'. Then runs testFunc and returns its status. // At the end (even in case of errors), the network traffic is brought back to normal. // This function executes commands on a node so it will work only for some // environments. func testUnderTemporaryNetworkFailure(c clientset.Interface, ns string, node *api.Node, testFunc func()) { host := framework.GetNodeExternalIP(node) master := framework.GetMasterAddress(c) By(fmt.Sprintf("block network traffic from node %s to the master", node.Name)) defer func() { // This code will execute even if setting the iptables rule failed. // It is on purpose because we may have an error even if the new rule // had been inserted. (yes, we could look at the error code and ssh error // separately, but I prefer to stay on the safe side). By(fmt.Sprintf("Unblock network traffic from node %s to the master", node.Name)) framework.UnblockNetwork(host, master) }() framework.Logf("Waiting %v to ensure node %s is ready before beginning test...", resizeNodeReadyTimeout, node.Name) if !framework.WaitForNodeToBe(c, node.Name, api.NodeReady, true, resizeNodeReadyTimeout) { framework.Failf("Node %s did not become ready within %v", node.Name, resizeNodeReadyTimeout) } framework.BlockNetwork(host, master) framework.Logf("Waiting %v for node %s to be not ready after simulated network failure", resizeNodeNotReadyTimeout, node.Name) if !framework.WaitForNodeToBe(c, node.Name, api.NodeReady, false, resizeNodeNotReadyTimeout) { framework.Failf("Node %s did not become not-ready within %v", node.Name, resizeNodeNotReadyTimeout) } testFunc() // network traffic is unblocked in a deferred function }
// recoverDiskSpace recovers disk space, filled by creating a large file, on a given node. func recoverDiskSpace(c *client.Client, node *api.Node) { By(fmt.Sprintf("Recovering disk space on node %s", node.Name)) cmd := "rm -f test.img" framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)) ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, false, nodeOODTimeOut) Expect(ood).To(BeTrue(), "Node %s's out of disk condition status did not change to false within %v", node.Name, nodeOODTimeOut) }
// fillDiskSpace fills the available disk space on a given node by creating a large file. The disk // space on the node is filled in such a way that the available space after filling the disk is just // below the lowDiskSpaceThreshold mark. func fillDiskSpace(c *client.Client, node *api.Node) { avail, err := availSize(c, node) framework.ExpectNoError(err, "Node %s: couldn't obtain available disk size %v", node.Name, err) fillSize := (avail - lowDiskSpaceThreshold + (100 * mb)) framework.Logf("Node %s: disk space available %d bytes", node.Name, avail) By(fmt.Sprintf("Node %s: creating a file of size %d bytes to fill the available disk space", node.Name, fillSize)) cmd := fmt.Sprintf("fallocate -l %d test.img", fillSize) framework.ExpectNoError(framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)) ood := framework.WaitForNodeToBe(c, node.Name, api.NodeOutOfDisk, true, nodeOODTimeOut) Expect(ood).To(BeTrue(), "Node %s did not run out of disk within %v", node.Name, nodeOODTimeOut) avail, err = availSize(c, node) framework.Logf("Node %s: disk space available %d bytes", node.Name, avail) Expect(avail < lowDiskSpaceThreshold).To(BeTrue()) }