// CreateMySQLReplicationHelpers creates a set of MySQL helpers for master,
// slave and an extra helper that is used for remote login test.
func CreateMySQLReplicationHelpers(c kclient.PodInterface, masterDeployment, slaveDeployment, helperDeployment string, slaveCount int) (exutil.Database, []exutil.Database, exutil.Database) {
podNames, err := exutil.WaitForPods(c, exutil.ParseLabelsOrDie(fmt.Sprintf("deployment=%s", masterDeployment)), exutil.CheckPodIsRunningFn, 1, 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
masterPod := podNames[0]
slavePods, err := exutil.WaitForPods(c, exutil.ParseLabelsOrDie(fmt.Sprintf("deployment=%s", slaveDeployment)), exutil.CheckPodIsRunningFn, slaveCount, 2*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
// Create MySQL helper for master
master := db.NewMysql(masterPod, "")
// Create MySQL helpers for slaves
slaves := make([]exutil.Database, len(slavePods))
for i := range slavePods {
slave := db.NewMysql(slavePods[i], masterPod)
slaves[i] = slave
}
helperNames, err := exutil.WaitForPods(c, exutil.ParseLabelsOrDie(fmt.Sprintf("deployment=%s", helperDeployment)), exutil.CheckPodIsRunningFn, 1, 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
helper := db.NewMysql(helperNames[0], masterPod)
return master, slaves, helper
}
func waitForNumberOfPodsWithLabel(oc *exutil.CLI, number int, label string) []string {
g.By(fmt.Sprintf("expecting that there are %d running pods with label name=%s", number, label))
podNames, err := exutil.WaitForPods(
oc.KubeClient().Core().Pods(oc.Namespace()),
exutil.ParseLabelsOrDie("name="+label),
exutil.CheckPodIsRunningFn,
number,
1*time.Minute,
)
o.Expect(err).ShouldNot(o.HaveOccurred())
o.Expect(podNames).Should(o.HaveLen(number))
return podNames
}
g.It(fmt.Sprintf("should process and create the %q template", templatePath), func() {
g.By("creating a new app")
o.Expect(oc.Run("new-app").Args("-f", templatePath).Execute()).Should(o.Succeed())
g.By("waiting for the deployment to complete")
err := exutil.WaitForADeploymentToComplete(oc.KubeREST().ReplicationControllers(oc.Namespace()), "mongodb")
if err != nil {
exutil.DumpDeploymentLogs("mongodb", oc)
}
o.Expect(err).ShouldNot(o.HaveOccurred())
g.By("expecting the mongodb pod is running")
podNames, err := exutil.WaitForPods(
oc.KubeREST().Pods(oc.Namespace()),
exutil.ParseLabelsOrDie("name=mongodb"),
exutil.CheckPodIsRunningFn,
1,
1*time.Minute,
)
o.Expect(err).ShouldNot(o.HaveOccurred())
o.Expect(podNames).Should(o.HaveLen(1))
g.By("expecting the mongodb service is answering for ping")
mongo := db.NewMongoDB(podNames[0])
ok, err := mongo.IsReady(oc)
o.Expect(err).ShouldNot(o.HaveOccurred())
o.Expect(ok).Should(o.BeTrue())
g.By("expecting that we can insert a new record")
result, err := mongo.Query(oc, `db.foo.save({ "status": "passed" })`)
import (
"time"
g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"
exutil "github.com/openshift/origin/test/extended/util"
)
var _ = g.Describe("[builds][Slow] build can have Docker image source", func() {
defer g.GinkgoRecover()
var (
buildFixture = exutil.FixturePath("testdata", "test-imagesource-build.yaml")
oc = exutil.NewCLI("build-image-source", exutil.KubeConfigPath())
imageSourceLabel = exutil.ParseLabelsOrDie("app=imagesourceapp")
imageDockerLabel = exutil.ParseLabelsOrDie("app=imagedockerapp")
)
g.JustBeforeEach(func() {
g.By("waiting for builder service account")
err := exutil.WaitForBuilderAccount(oc.KubeREST().ServiceAccounts(oc.Namespace()))
o.Expect(err).NotTo(o.HaveOccurred())
g.By("waiting for imagestreams to be imported")
err = exutil.WaitForAnImageStream(oc.AdminREST().ImageStreams("openshift"), "jenkins", exutil.CheckImageStreamLatestTagPopulatedFn, exutil.CheckImageStreamTagNotFoundFn)
o.Expect(err).NotTo(o.HaveOccurred())
})
g.Describe("build with image source", func() {
g.It("should complete successfully and contain the expected file", func() {
func replicationTestFactory(oc *exutil.CLI, template string) func() {
return func() {
oc.SetOutputDir(exutil.TestContext.OutputDir)
defer cleanup(oc)
_, err := exutil.SetupHostPathVolumes(oc.AdminKubeREST().PersistentVolumes(), oc.Namespace(), "512Mi", 1)
o.Expect(err).NotTo(o.HaveOccurred())
err = testutil.WaitForPolicyUpdate(oc.REST(), oc.Namespace(), "create", "templates", true)
o.Expect(err).NotTo(o.HaveOccurred())
err = oc.Run("new-app").Args("-f", template).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = oc.Run("new-app").Args("-f", helperTemplate, "-p", fmt.Sprintf("DATABASE_SERVICE_NAME=%s", helperName)).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = oc.KubeFramework().WaitForAnEndpoint(helperName)
o.Expect(err).NotTo(o.HaveOccurred())
tableCounter := 0
assertReplicationIsWorking := func(masterDeployment, slaveDeployment string, slaveCount int) (exutil.Database, []exutil.Database, exutil.Database) {
tableCounter++
table := fmt.Sprintf("table_%0.2d", tableCounter)
master, slaves, helper := CreateMySQLReplicationHelpers(oc.KubeREST().Pods(oc.Namespace()), masterDeployment, slaveDeployment, fmt.Sprintf("%s-1", helperName), slaveCount)
o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, []exutil.Database{master, helper})).NotTo(o.HaveOccurred())
o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, slaves)).NotTo(o.HaveOccurred())
// Test if we can query as root
oc.KubeFramework().WaitForAnEndpoint("mysql-master")
err := helper.TestRemoteLogin(oc, "mysql-master")
o.Expect(err).NotTo(o.HaveOccurred())
// Create a new table with random name
_, err = master.Query(oc, fmt.Sprintf("CREATE TABLE %s (col1 VARCHAR(20), col2 VARCHAR(20));", table))
o.Expect(err).NotTo(o.HaveOccurred())
// Write new data to the table through master
_, err = master.Query(oc, fmt.Sprintf("INSERT INTO %s (col1, col2) VALUES ('val1', 'val2');", table))
o.Expect(err).NotTo(o.HaveOccurred())
// Make sure data is present on master
err = exutil.WaitForQueryOutput(oc, master, 10*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2")
o.Expect(err).NotTo(o.HaveOccurred())
// Make sure data was replicated to all slaves
for _, slave := range slaves {
err = exutil.WaitForQueryOutput(oc, slave, 90*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2")
o.Expect(err).NotTo(o.HaveOccurred())
}
return master, slaves, helper
}
g.By("after initial deployment")
master, _, _ := assertReplicationIsWorking("mysql-master-1", "mysql-slave-1", 1)
g.By("after master is restarted by changing the Deployment Config")
err = oc.Run("env").Args("dc", "mysql-master", "MYSQL_ROOT_PASSWORD=newpass").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), master.PodName(), 1*time.Minute)
master, _, _ = assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1)
g.By("after master is restarted by deleting the pod")
err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-master-2").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), master.PodName(), 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
_, slaves, _ := assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1)
g.By("after slave is restarted by deleting the pod")
err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-slave-1").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), slaves[0].PodName(), 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1)
pods, err := oc.KubeREST().Pods(oc.Namespace()).List(kapi.ListOptions{LabelSelector: exutil.ParseLabelsOrDie("deployment=mysql-slave-1")})
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(pods.Items)).To(o.Equal(1))
g.By("after slave is scaled to 0 and then back to 4 replicas")
err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=0").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeREST().Pods(oc.Namespace()), pods.Items[0].Name, 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=4").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 4)
}
}
g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"
exutil "github.com/openshift/origin/test/extended/util"
)
var _ = g.Describe("[images][php][Slow] hot deploy for openshift php image", func() {
defer g.GinkgoRecover()
var (
cakephpTemplate = "https://raw.githubusercontent.com/openshift/cakephp-ex/master/openshift/templates/cakephp-mysql.json"
oc = exutil.NewCLI("s2i-php", exutil.KubeConfigPath())
hotDeployParam = "OPCACHE_REVALIDATE_FREQ=0"
modifyCommand = []string{"sed", "-ie", `s/\$result\['c'\]/1337/`, "app/View/Layouts/default.ctp"}
pageCountFn = func(count int) string { return fmt.Sprintf(`<span class="code" id="count-value">%d</span>`, count) }
dcName = "cakephp-mysql-example-1"
dcLabel = exutil.ParseLabelsOrDie(fmt.Sprintf("deployment=%s", dcName))
)
g.Describe("CakePHP example", func() {
g.It(fmt.Sprintf("should work with hot deploy"), func() {
oc.SetOutputDir(exutil.TestContext.OutputDir)
g.By(fmt.Sprintf("calling oc new-app -f %q -p %q", cakephpTemplate, hotDeployParam))
err := oc.Run("new-app").Args("-f", cakephpTemplate, "-p", hotDeployParam).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
g.By("waiting for build to finish")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), dcName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
o.Expect(err).NotTo(o.HaveOccurred())
g.By("waiting for endpoint")
err = oc.KubeFramework().WaitForAnEndpoint("cakephp-mysql-example")
g.By("creating a new app")
o.Expect(
oc.Run("new-app").Args(
"-f", templatePath,
"-p", "VOLUME_CAPACITY=256Mi",
"-p", "MEMORY_LIMIT=512Mi",
"-p", "MONGODB_IMAGE=centos/mongodb-32-centos7",
"-p", "MONGODB_SERVICE_NAME=mongodb-replicaset",
).Execute(),
).Should(o.Succeed())
g.By("waiting for pods to running")
podNames, err := exutil.WaitForPods(
oc.KubeREST().Pods(oc.Namespace()),
exutil.ParseLabelsOrDie("name=mongodb-replicaset"),
exutil.CheckPodIsRunningFn,
3,
2*time.Minute,
)
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(podNames).Should(o.HaveLen(3))
g.By("expecting that we can insert a new record on primary node")
mongo := dbutil.NewMongoDB(podNames[0])
replicaSet := mongo.(exutil.ReplicaSet)
_, err = replicaSet.QueryPrimary(oc, `db.test.save({ "status" : "passed" })`)
o.Expect(err).ShouldNot(o.HaveOccurred())
g.By("expecting that we can read a record from all members")
for _, podName := range podNames {
var _ = g.Describe("[job] openshift can execute jobs", func() {
defer g.GinkgoRecover()
var (
configPath = exeutil.FixturePath("fixtures", "job-controller.yaml")
oc = exeutil.NewCLI("job-controller", exeutil.KubeConfigPath())
)
g.Describe("controller", func() {
g.It("should create and run a job in user project", func() {
oc.SetOutputDir(exeutil.TestContext.OutputDir)
g.By(fmt.Sprintf("creating a job from %q", configPath))
err := oc.Run("create").Args("-f", configPath).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
g.By(fmt.Sprintf("Waiting for pod..."))
podNames, err := exeutil.WaitForPods(oc.KubeREST().Pods(oc.Namespace()), exeutil.ParseLabelsOrDie("app=pi"), exeutil.CheckPodIsSucceededFn, 1, 2*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(podNames)).Should(o.Equal(1))
podName := podNames[0]
g.By("retrieving logs from pod " + podName)
logs, err := oc.Run("logs").Args(podName).Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(logs).Should(o.Equal("3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117068"))
g.By("checking job status")
jobs, err := oc.KubeREST().Jobs(oc.Namespace()).List(kapi.ListOptions{LabelSelector: exeutil.ParseLabelsOrDie("app=pi")})
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(jobs.Items)).Should(o.Equal(1))
job := jobs.Items[0]
func replicationTestFactory(oc *exutil.CLI, tc testCase) func() {
return func() {
oc.SetOutputDir(exutil.TestContext.OutputDir)
defer cleanup(oc)
_, err := exutil.SetupHostPathVolumes(oc.AdminKubeClient().Core().PersistentVolumes(), oc.Namespace(), "1Gi", 2)
o.Expect(err).NotTo(o.HaveOccurred())
err = testutil.WaitForPolicyUpdate(oc.Client(), oc.Namespace(), "create", templateapi.Resource("templates"), true)
o.Expect(err).NotTo(o.HaveOccurred())
exutil.CheckOpenShiftNamespaceImageStreams(oc)
err = oc.Run("new-app").Args("-f", tc.TemplatePath).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = oc.Run("new-app").Args("-f", helperTemplate, "-p", fmt.Sprintf("DATABASE_SERVICE_NAME=%s", helperName)).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
// oc.KubeFramework().WaitForAnEndpoint currently will wait forever; for now, prefacing with our WaitForADeploymentToComplete,
// which does have a timeout, since in most cases a failure in the service coming up stems from a failed deployment
g.By("waiting for the deployment to complete")
err = exutil.WaitForADeploymentToComplete(oc.KubeClient().Core().ReplicationControllers(oc.Namespace()), helperName, oc)
o.Expect(err).NotTo(o.HaveOccurred())
g.By("waiting for an endpoint")
err = oc.KubeFramework().WaitForAnEndpoint(helperName)
o.Expect(err).NotTo(o.HaveOccurred())
tableCounter := 0
assertReplicationIsWorking := func(masterDeployment, slaveDeployment string, slaveCount int) (exutil.Database, []exutil.Database, exutil.Database) {
tableCounter++
table := fmt.Sprintf("table_%0.2d", tableCounter)
g.By("creating replication helpers")
master, slaves, helper := CreateMySQLReplicationHelpers(oc.KubeClient().Core().Pods(oc.Namespace()), masterDeployment, slaveDeployment, fmt.Sprintf("%s-1", helperName), slaveCount)
o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, []exutil.Database{master, helper})).NotTo(o.HaveOccurred())
o.Expect(exutil.WaitUntilAllHelpersAreUp(oc, slaves)).NotTo(o.HaveOccurred())
// Test if we can query as root
g.By("wait for mysql-master endpoint")
oc.KubeFramework().WaitForAnEndpoint("mysql-master")
err := helper.TestRemoteLogin(oc, "mysql-master")
o.Expect(err).NotTo(o.HaveOccurred())
// Create a new table with random name
g.By("create new table")
_, err = master.Query(oc, fmt.Sprintf("CREATE TABLE %s (col1 VARCHAR(20), col2 VARCHAR(20));", table))
o.Expect(err).NotTo(o.HaveOccurred())
// Write new data to the table through master
_, err = master.Query(oc, fmt.Sprintf("INSERT INTO %s (col1, col2) VALUES ('val1', 'val2');", table))
o.Expect(err).NotTo(o.HaveOccurred())
// Make sure data is present on master
err = exutil.WaitForQueryOutputContains(oc, master, 10*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2")
o.Expect(err).NotTo(o.HaveOccurred())
// Make sure data was replicated to all slaves
for _, slave := range slaves {
err = exutil.WaitForQueryOutputContains(oc, slave, 90*time.Second, false, fmt.Sprintf("SELECT * FROM %s\\G;", table), "col1: val1\ncol2: val2")
o.Expect(err).NotTo(o.HaveOccurred())
}
return master, slaves, helper
}
g.By("after initial deployment")
master, _, _ := assertReplicationIsWorking("mysql-master-1", "mysql-slave-1", 1)
if tc.SkipReplication {
return
}
g.By("after master is restarted by changing the Deployment Config")
err = oc.Run("env").Args("dc", "mysql-master", "MYSQL_ROOT_PASSWORD=newpass").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), master.PodName(), 1*time.Minute)
master, _, _ = assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1)
g.By("after master is restarted by deleting the pod")
err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-master-2").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), master.PodName(), 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
_, slaves, _ := assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1)
g.By("after slave is restarted by deleting the pod")
err = oc.Run("delete").Args("pod", "-l", "deployment=mysql-slave-1").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), slaves[0].PodName(), 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 1)
pods, err := oc.KubeClient().Core().Pods(oc.Namespace()).List(kapi.ListOptions{LabelSelector: exutil.ParseLabelsOrDie("deployment=mysql-slave-1")})
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(pods.Items)).To(o.Equal(1))
// NOTE: Commented out, current template does not support multiple replicas.
/*
g.By("after slave is scaled to 0 and then back to 4 replicas")
err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=0").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
err = exutil.WaitUntilPodIsGone(oc.KubeClient().Core().Pods(oc.Namespace()), pods.Items[0].Name, 1*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
err = oc.Run("scale").Args("dc", "mysql-slave", "--replicas=4").Execute()
o.Expect(err).NotTo(o.HaveOccurred())
assertReplicationIsWorking("mysql-master-2", "mysql-slave-1", 4)
*/
}
}
oc := exeutil.NewCLI("job-controller", exeutil.KubeConfigPath())
g.Describe("controller", func() {
g.It("should create and run a job in user project", func() {
for _, ver := range []string{"v1beta1", "v1"} {
oc.SetOutputDir(exeutil.TestContext.OutputDir)
configPath := exeutil.FixturePath("testdata", "jobs", fmt.Sprintf("%s.yaml", ver))
name := fmt.Sprintf("simple%s", ver)
labels := fmt.Sprintf("app=%s", name)
g.By(fmt.Sprintf("creating a job from %q...", configPath))
err := oc.Run("create").Args("-f", configPath).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
g.By("waiting for a pod...")
podNames, err := exeutil.WaitForPods(oc.KubeClient().Core().Pods(oc.Namespace()), exeutil.ParseLabelsOrDie(labels), exeutil.CheckPodIsSucceededFn, 1, 2*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(podNames)).Should(o.Equal(1))
g.By("waiting for a job...")
err = exeutil.WaitForAJob(oc.KubeClient().Batch().Jobs(oc.Namespace()), name, 2*time.Minute)
o.Expect(err).NotTo(o.HaveOccurred())
g.By("checking job status...")
jobs, err := oc.KubeClient().Batch().Jobs(oc.Namespace()).List(kapi.ListOptions{LabelSelector: exeutil.ParseLabelsOrDie(labels)})
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(jobs.Items)).Should(o.Equal(1))
job := jobs.Items[0]
o.Expect(len(job.Status.Conditions)).Should(o.Equal(1))
o.Expect(job.Status.Conditions[0].Type).Should(o.Equal(batch.JobComplete))
var _ = g.Describe("Job", func() {
defer g.GinkgoRecover()
var (
configPath = exeutil.FixturePath("fixtures", "job-controller.yaml")
oc = exeutil.NewCLI("job-controller", exeutil.KubeConfigPath())
)
g.Describe("controller", func() {
g.It("should create and run a job in user project", func() {
oc.SetOutputDir(exeutil.TestContext.OutputDir)
g.By(fmt.Sprintf("creating a job from %q", configPath))
err := oc.Run("create").Args("-f", configPath).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
g.By(fmt.Sprintf("Waiting for pod..."))
podNames, err := exeutil.WaitForPods(oc.KubeREST().Pods(oc.Namespace()), exeutil.ParseLabelsOrDie("app=pi"), exeutil.CheckPodIsSucceededFn, 1, 120*time.Second)
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(podNames)).Should(o.Equal(1))
podName := podNames[0]
g.By("retrieving logs from pod " + podName)
logs, err := oc.Run("logs").Args(podName).Output()
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(logs).Should(o.Equal("3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117068"))
g.By("checking job status")
jobs, err := oc.KubeREST().Jobs(oc.Namespace()).List(exeutil.ParseLabelsOrDie("app=pi"), nil)
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(len(jobs.Items)).Should(o.Equal(1))
job := jobs.Items[0]
