func doGetFlowsForNode(oc *testexutil.CLI, nodeName string) ([]string, error) { pod := &kapi.Pod{ TypeMeta: kapiunversioned.TypeMeta{ Kind: "Pod", }, ObjectMeta: kapi.ObjectMeta{ GenerateName: "flow-check", }, Spec: kapi.PodSpec{ Containers: []kapi.Container{ { Name: "flow-check", Image: "openshift/openvswitch", // kubernetes seems to get confused sometimes if the pod exits too quickly Command: []string{"sh", "-c", "ovs-ofctl -O OpenFlow13 dump-flows br0 && sleep 1"}, VolumeMounts: []kapi.VolumeMount{ { Name: "ovs-socket", MountPath: "/var/run/openvswitch/br0.mgmt", }, }, }, }, Volumes: []kapi.Volume{ { Name: "ovs-socket", VolumeSource: kapi.VolumeSource{ HostPath: &kapi.HostPathVolumeSource{ Path: "/var/run/openvswitch/br0.mgmt", }, }, }, }, NodeName: nodeName, RestartPolicy: kapi.RestartPolicyNever, // We don't actually need HostNetwork, we just set it so that deploying this pod won't cause any OVS flows to be added SecurityContext: &kapi.PodSecurityContext{ HostNetwork: true, }, }, } f := oc.KubeFramework() podClient := f.Client.Pods(f.Namespace.Name) pod, err := podClient.Create(pod) if err != nil { return nil, err } defer podClient.Delete(pod.Name, nil) err = waitForPodSuccessInNamespace(f.Client, pod.Name, "flow-check", f.Namespace.Name) if err != nil { return nil, err } logs, err := oc.Run("logs").Args(pod.Name).Output() if err != nil { return nil, err } // For ease of comparison, strip out the parts of the rules that change flows := strings.Split(logs, "\n") strip_re := regexp.MustCompile(`(duration|n_packets|n_bytes)=[^,]*, `) for i := range flows { flows[i] = strip_re.ReplaceAllLiteralString(flows[i], "") } return flows, nil }
func replicationTestFactory(oc *exutil.CLI, template string) func() { return func() { oc.SetOutputDir(exutil.TestContext.OutputDir) defer cleanup(oc) _, err := exutil.SetupHostPathVolumes(oc.AdminKubeREST().PersistentVolumes(), oc.Namespace(), "512Mi", 1) o.Expect(err).NotTo(o.HaveOccurred()) err = testutil.WaitForPolicyUpdate(oc.REST(), oc.Namespace(), "create", "templates", true) o.Expect(err).NotTo(o.HaveOccurred()) err = oc.Run("new-app").Args("-f", template).Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = oc.Run("new-app").Args("-f", helperTemplate, "-p", fmt.Sprintf("DATABASE_SERVICE_NAME=%s", helperName)).Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = oc.KubeFramework().WaitForAnEndpoint(helperName) o.Expect(err).NotTo(o.HaveOccurred()) tableCounter := 0 assertReplicationIsWorking := func(masterDeployment, slaveDeployment string, slaveCount int) (exutil.Database, []exutil.Database, exutil.Database) { tableCounter++ table := fmt.Sprintf("table_%0.2d", tableCounter) master, slaves, helper := CreateMySQLReplicationHelpers(oc.KubeREST().Pods(oc.Namespace()), masterDeployment, slaveDeployment, fmt.Sprintf("%s-1", helperName), slaveCount) o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, []exutil.Database{master, helper})).NotTo(o.HaveOccurred()) o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, slaves)).NotTo(o.HaveOccurred()) // Test if we can query as root oc.KubeFramework().WaitForAnEndpoint("mysql-master") err := helper.TestRemoteLogin(oc, "mysql-master") o.Expect(err).NotTo(o.HaveOccurred()) // Create a new table with random name _, err = master.Query(oc, fmt.Sprintf("CREATE TABLE %s (col1 VARCHAR(20), col2 VARCHAR(20));", table)) o.Expect(err).NotTo(o.HaveOccurred()) // Write new data to the table through master _, err = master.Query(oc, fmt.Sprintf("INSERT INTO %s (col1, col2) VALUES ('val1', 'val2');", table)) o.Expect(err).NotTo(o.HaveOccurred()) // Make sure data is present on master err = exutil.WaitForQueryOutput(oc, master, 10*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2") o.Expect(err).NotTo(o.HaveOccurred()) // Make sure data was replicated to all slaves for _, slave := range slaves { err = exutil.WaitForQueryOutput(oc, slave, 90*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2") o.Expect(err).NotTo(o.HaveOccurred()) } return master, slaves, helper } g.By("after initial deployment") master, _, _ := assertReplicationIsWorking("mysql-master-1", "mysql-slave-1", 1) g.By("after master is restarted by changing the Deployment Config") err = oc.Run("env").Args("dc", "mysql-master", "MYSQL_ROOT_PASSWORD=newpass").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), master.PodName(), 1*time.Minute) master, _, _ = assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1) g.By("after master is restarted by deleting the pod") err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-master-2").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), master.PodName(), 1*time.Minute) o.Expect(err).NotTo(o.HaveOccurred()) _, slaves, _ := assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1) g.By("after slave is restarted by deleting the pod") err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-slave-1").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), slaves[0].PodName(), 1*time.Minute) o.Expect(err).NotTo(o.HaveOccurred()) assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1) pods, err := oc.KubeREST().Pods(oc.Namespace()).List(kapi.ListOptions{LabelSelector: exutil.ParseLabelsOrDie("deployment=mysql-slave-1")}) o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(len(pods.Items)).To(o.Equal(1)) g.By("after slave is scaled to 0 and then back to 4 replicas") err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=0").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), pods.Items[0].Name, 1*time.Minute) o.Expect(err).NotTo(o.HaveOccurred()) err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=4").Execute() o.Expect(err).NotTo(o.HaveOccurred()) assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 4) } }
func replicationTestFactory(oc *exutil.CLI, tc testCase) func() { return func() { oc.SetOutputDir(exutil.TestContext.OutputDir) defer cleanup(oc) _, err := exutil.SetupHostPathVolumes(oc.AdminKubeClient().Core().PersistentVolumes(), oc.Namespace(), "1Gi", 2) o.Expect(err).NotTo(o.HaveOccurred()) err = testutil.WaitForPolicyUpdate(oc.Client(), oc.Namespace(), "create", templateapi.Resource("templates"), true) o.Expect(err).NotTo(o.HaveOccurred()) exutil.CheckOpenShiftNamespaceImageStreams(oc) err = oc.Run("new-app").Args("-f", tc.TemplatePath).Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = oc.Run("new-app").Args("-f", helperTemplate, "-p", fmt.Sprintf("DATABASE_SERVICE_NAME=%s", helperName)).Execute() o.Expect(err).NotTo(o.HaveOccurred()) // oc.KubeFramework().WaitForAnEndpoint currently will wait forever; for now, prefacing with our WaitForADeploymentToComplete, // which does have a timeout, since in most cases a failure in the service coming up stems from a failed deployment g.By("waiting for the deployment to complete") err = exutil.WaitForADeploymentToComplete(oc.KubeClient().Core().ReplicationControllers(oc.Namespace()), helperName, oc) o.Expect(err).NotTo(o.HaveOccurred()) g.By("waiting for an endpoint") err = oc.KubeFramework().WaitForAnEndpoint(helperName) o.Expect(err).NotTo(o.HaveOccurred()) tableCounter := 0 assertReplicationIsWorking := func(masterDeployment, slaveDeployment string, slaveCount int) (exutil.Database, []exutil.Database, exutil.Database) { tableCounter++ table := fmt.Sprintf("table_%0.2d", tableCounter) g.By("creating replication helpers") master, slaves, helper := CreateMySQLReplicationHelpers(oc.KubeClient().Core().Pods(oc.Namespace()), masterDeployment, slaveDeployment, fmt.Sprintf("%s-1", helperName), slaveCount) o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, []exutil.Database{master, helper})).NotTo(o.HaveOccurred()) o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, slaves)).NotTo(o.HaveOccurred()) // Test if we can query as root g.By("wait for mysql-master endpoint") oc.KubeFramework().WaitForAnEndpoint("mysql-master") err := helper.TestRemoteLogin(oc, "mysql-master") o.Expect(err).NotTo(o.HaveOccurred()) // Create a new table with random name g.By("create new table") _, err = master.Query(oc, fmt.Sprintf("CREATE TABLE %s (col1 VARCHAR(20), col2 VARCHAR(20));", table)) o.Expect(err).NotTo(o.HaveOccurred()) // Write new data to the table through master _, err = master.Query(oc, fmt.Sprintf("INSERT INTO %s (col1, col2) VALUES ('val1', 'val2');", table)) o.Expect(err).NotTo(o.HaveOccurred()) // Make sure data is present on master err = exutil.WaitForQueryOutputContains(oc, master, 10*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2") o.Expect(err).NotTo(o.HaveOccurred()) // Make sure data was replicated to all slaves for _, slave := range slaves { err = exutil.WaitForQueryOutputContains(oc, slave, 90*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2") o.Expect(err).NotTo(o.HaveOccurred()) } return master, slaves, helper } g.By("after initial deployment") master, _, _ := assertReplicationIsWorking("mysql-master-1", "mysql-slave-1", 1) if tc.SkipReplication { return } g.By("after master is restarted by changing the Deployment Config") err = oc.Run("env").Args("dc", "mysql-master", "MYSQL_ROOT_PASSWORD=newpass").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), master.PodName(), 1*time.Minute) master, _, _ = assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1) g.By("after master is restarted by deleting the pod") err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-master-2").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), master.PodName(), 1*time.Minute) o.Expect(err).NotTo(o.HaveOccurred()) _, slaves, _ := assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1) g.By("after slave is restarted by deleting the pod") err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-slave-1").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), slaves[0].PodName(), 1*time.Minute) o.Expect(err).NotTo(o.HaveOccurred()) assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1) pods, err := oc.KubeClient().Core().Pods(oc.Namespace()).List(kapi.ListOptions{LabelSelector: exutil.ParseLabelsOrDie("deployment=mysql-slave-1")}) o.Expect(err).NotTo(o.HaveOccurred()) o.Expect(len(pods.Items)).To(o.Equal(1)) // NOTE: Commented out, current template does not support multiple replicas. /* g.By("after slave is scaled to 0 and then back to 4 replicas") err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=0").Execute() o.Expect(err).NotTo(o.HaveOccurred()) err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), pods.Items[0].Name, 1*time.Minute) o.Expect(err).NotTo(o.HaveOccurred()) err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=4").Execute() o.Expect(err).NotTo(o.HaveOccurred()) assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 4) */ } }