}
leader, err := jobIndexOfLeader(kv, boshClient, consulManifest.Name)
Expect(err).ToNot(HaveOccurred())
rand.Seed(time.Now().Unix())
startingIndex := rand.Intn(3)
instances := []int{startingIndex, startingIndex + 1, startingIndex + 2}
if leader < startingIndex || leader > startingIndex+2 {
instances[0] = leader
}
jobIndexToResurrect := startingIndex + 1
err = turbulenceClient.KillIndices(consulManifest.Name, "consul", instances)
Expect(err).NotTo(HaveOccurred())
err = boshClient.SetVMResurrection(consulManifest.Name, "consul", jobIndexToResurrect, true)
Expect(err).NotTo(HaveOccurred())
Eventually(func() error {
return boshClient.ScanAndFix(consulManifest.Name, "consul", []int{jobIndexToResurrect})
}, "10m", "1m").ShouldNot(HaveOccurred())
Eventually(func() ([]bosh.VM, error) {
return helpers.DeploymentVMs(boshClient, consulManifest.Name)
}, "5m", "1m").Should(ContainElement(bosh.VM{JobName: "consul", Index: jobIndexToResurrect, State: "running"}))
})
By("setting and getting a new value", func() {
fakeServer.URL = server.URL
return fakeServer
}
var _ = Describe("Client", func() {
Describe("KillIndices", func() {
var fakeServer *fakeTurbulenceServer
var client turbulence.Client
BeforeEach(func() {
fakeServer = NewFakeTurbulenceServer()
client = turbulence.NewClient(fakeServer.URL, 100*time.Millisecond, 40*time.Millisecond)
})
It("makes a POST request to create an incident and then polls to wait for completion", func() {
errorKillingIndices := client.KillIndices("deployment-name", "job-name", []int{0})
Expect(fakeServer.errorReadingBody).NotTo(HaveOccurred())
Expect(errorKillingIndices).NotTo(HaveOccurred())
Expect(string(fakeServer.receivedPOSTBody)).To(MatchJSON(expectedPOSTRequest))
})
It("returns a timeout error when execution does not complete", func() {
fakeServer.GETResponses = []string{successfulIncompleteGETResponse}
errorKillingIndices := client.KillIndices("deployment-name", "job-name", []int{0})
Expect(errorKillingIndices).NotTo(BeNil())
Expect(errorKillingIndices.Error()).To(ContainSubstring("Did not finish deleting VM in time"))
})
It("returns an error when the turbulence response does not contain any events", func() {
fakeServer.GETResponses = []string{invalidGETResponseWithNoEvents}
By("deleting turbulence", func() {
err := boshClient.DeleteDeployment(turbulenceManifest.Name)
Expect(err).NotTo(HaveOccurred())
})
})
Context("when a consul node is killed", func() {
It("is still able to function on healthy vms", func() {
By("setting a persistent value", func() {
err := kv.Set(testKey, testValue)
Expect(err).NotTo(HaveOccurred())
})
By("killing indices", func() {
spammer.Spam()
err := turbulenceClient.KillIndices(consulManifest.Name, "consul", []int{rand.Intn(3)})
Expect(err).ToNot(HaveOccurred())
yaml, err := consulManifest.ToYAML()
Expect(err).NotTo(HaveOccurred())
Eventually(func() error {
return boshClient.ScanAndFixAll(yaml)
}, "5m", "1m").ShouldNot(HaveOccurred())
Eventually(func() ([]bosh.VM, error) {
return helpers.DeploymentVMs(boshClient, consulManifest.Name)
}, "5m", "10s").Should(ConsistOf(helpers.GetVMsFromManifest(consulManifest)))
spammer.Stop()
})