import (
"github.com/cloudfoundry/hm9000/models"
"github.com/cloudfoundry/hm9000/testhelpers/appfixture"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Evacuation and Shutdown", func() {
var dea appfixture.DeaFixture
var app appfixture.AppFixture
BeforeEach(func() {
dea = appfixture.NewDeaFixture()
app = dea.GetApp(0)
simulator.SetCurrentHeartbeats(dea.HeartbeatWith(app.InstanceAtIndex(0).Heartbeat()))
simulator.SetDesiredState(app.DesiredState(1))
simulator.Tick(simulator.TicksToAttainFreshness)
})
Describe("Shutdown handling by the evacuator component", func() {
Context("when a SHUTDOWN droplet.exited message comes in", func() {
BeforeEach(func() {
cliRunner.StartEvacuator(simulator.currentTimestamp)
coordinator.MessageBus.Publish("droplet.exited", app.InstanceAtIndex(0).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON())
})
AfterEach(func() {
cliRunner.StopEvacuator()
})
},
{
AppGuid: app2.AppGuid,
AppVersion: app2.AppVersion,
InstanceIndex: 0,
CrashCount: 3,
},
{
AppGuid: app4.AppGuid,
AppVersion: app4.AppVersion,
InstanceIndex: 1,
CrashCount: 8,
},
}
store.SyncHeartbeats(dea.HeartbeatWith(actualState...))
store.SyncDesiredState(desiredState...)
store.SaveCrashCounts(crashCount...)
})
Describe("AppKey", func() {
It("should concatenate the app guid and app version appropriately", func() {
key := store.AppKey("abc", "123")
Ω(key).Should(Equal("abc,123"))
})
})
Describe("GetApps", func() {
Context("when all is well", func() {
It("should build and return the set of apps", func() {
apps, err := store.GetApps()
)
duplicateInstance1 = app.InstanceAtIndex(2)
duplicateInstance1.InstanceGuid = models.Guid()
duplicateInstance2 = app.InstanceAtIndex(2)
duplicateInstance2.InstanceGuid = models.Guid()
duplicateInstance3 = app.InstanceAtIndex(2)
duplicateInstance3.InstanceGuid = models.Guid()
})
Context("When there are missing instances on other indices", func() {
It("should not schedule any stops but should start the missing indices", func() {
//[-,-,2|2|2|2]
store.SyncHeartbeats(dea.HeartbeatWith(
app.InstanceAtIndex(2).Heartbeat(),
duplicateInstance1.Heartbeat(),
duplicateInstance2.Heartbeat(),
duplicateInstance3.Heartbeat(),
))
err := analyzer.Analyze()
Ω(err).ShouldNot(HaveOccurred())
Ω(stopMessages()).Should(BeEmpty())
Ω(startMessages()).Should(HaveLen(2))
expectedMessage := models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 0, 2.0/3.0, models.PendingStartMessageReasonMissing)
Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
expectedMessage = models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 1, 2.0/3.0, models.PendingStartMessageReasonMissing)
Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
})
a appfixture.AppFixture
crashingHeartbeat models.Heartbeat
)
BeforeEach(func() {
dea = appfixture.NewDeaFixture()
a = dea.GetApp(0)
})
Describe("when all instances are crashed", func() {
BeforeEach(func() {
simulator.SetDesiredState(a.DesiredState(3))
crashingHeartbeat = dea.HeartbeatWith(
a.CrashedInstanceHeartbeatAtIndex(0),
a.CrashedInstanceHeartbeatAtIndex(1),
a.CrashedInstanceHeartbeatAtIndex(2),
)
simulator.SetCurrentHeartbeats(crashingHeartbeat)
simulator.Tick(simulator.TicksToAttainFreshness)
})
It("should only try to start instance at index 0", func() {
Ω(startStopListener.Starts).Should(HaveLen(1))
Ω(startStopListener.Starts[0].AppVersion).Should(Equal(a.AppVersion))
Ω(startStopListener.Starts[0].InstanceIndex).Should(Equal(0))
})
It("should never try to stop crashes", func() {
Ω(startStopListener.Stops).Should(BeEmpty())
)
var _ = Describe("Expiring Heartbeats Test", func() {
var dea1, dea2 appfixture.DeaFixture
var app1, app2, app3 appfixture.AppFixture
BeforeEach(func() {
dea1 = appfixture.NewDeaFixture()
dea2 = appfixture.NewDeaFixture()
app1 = dea1.GetApp(0)
app2 = dea1.GetApp(1)
app3 = dea2.GetApp(2)
simulator.SetCurrentHeartbeats(
dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat(), app2.InstanceAtIndex(0).Heartbeat()),
dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
)
simulator.SetDesiredState(app1.DesiredState(1), app2.DesiredState(1), app3.DesiredState(1))
simulator.Tick(simulator.TicksToAttainFreshness)
})
Context("when a dea reports than an instance is no longer present", func() {
BeforeEach(func() {
simulator.SetCurrentHeartbeats(
dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat()),
dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
)
})
It("should start the instance after a grace period", func() {
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedIndices", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredApps", Value: 1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredInstances", Value: 3}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredAppsPendingStaging", Value: 0}))
})
})
Context("when a desired app has an instance starting and others running", func() {
BeforeEach(func() {
store.SyncDesiredState(a.DesiredState(3))
startingHB := a.InstanceAtIndex(1).Heartbeat()
startingHB.State = models.InstanceStateStarting
store.SyncHeartbeats(dea.HeartbeatWith(
a.InstanceAtIndex(0).Heartbeat(),
startingHB,
a.InstanceAtIndex(2).Heartbeat(),
))
})
It("should have the correct stats", func() {
context := metricsServer.Emit()
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithAllInstancesReporting", Value: 1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithMissingInstances", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfUndesiredRunningApps", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfRunningInstances", Value: 3}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfMissingIndices", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedInstances", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedIndices", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredApps", Value: 1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredInstances", Value: 3}))
store.SyncDesiredState(app.DesiredState(1))
})
Context("when the index-to-start is within the # of desired instances", func() {
BeforeEach(func() {
indexToStart = 0
})
Context("when there are no running instances at all for that app", func() {
assertMessageWasSent()
})
Context("when there is no running instance reporting at that index", func() {
BeforeEach(func() {
store.SyncHeartbeats(dea.HeartbeatWith(
app.InstanceAtIndex(1).Heartbeat(),
app.InstanceAtIndex(2).Heartbeat(),
))
})
assertMessageWasSent()
})
Context("when there are crashed instances reporting at that index", func() {
BeforeEach(func() {
store.SyncHeartbeats(dea.HeartbeatWith(
app.CrashedInstanceHeartbeatAtIndex(0),
app.CrashedInstanceHeartbeatAtIndex(0),
app.InstanceAtIndex(1).Heartbeat(),
app.InstanceAtIndex(2).Heartbeat(),
))
})
Ω(err).ShouldNot(HaveOccurred())
conf.StoreHeartbeatCacheRefreshIntervalInMilliseconds = 100
store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
dea = appfixture.NewDeaFixture()
otherDea = appfixture.NewDeaFixture()
})
AfterEach(func() {
storeAdapter.Disconnect()
})
Describe("Saving actual state", func() {
BeforeEach(func() {
store.SyncHeartbeats(dea.HeartbeatWith(
dea.GetApp(0).InstanceAtIndex(1).Heartbeat(),
dea.GetApp(1).InstanceAtIndex(3).Heartbeat(),
))
})
It("should save the instance heartbeats for the passed-in heartbeat", func() {
results, err := store.GetInstanceHeartbeats()
Ω(err).ShouldNot(HaveOccurred())
Ω(results).Should(HaveLen(2))
Ω(results).Should(ContainElement(dea.GetApp(0).InstanceAtIndex(1).Heartbeat()))
Ω(results).Should(ContainElement(dea.GetApp(1).InstanceAtIndex(3).Heartbeat()))
})
Context("when there are already instance heartbeats stored for the DEA in question", func() {
var modifiedHeartbeat models.InstanceHeartbeat
BeforeEach(func() {
modifiedHeartbeat = dea.GetApp(1).InstanceAtIndex(3).Heartbeat()
var dea appfixture.DeaFixture
var a appfixture.AppFixture
Context("when there are multiple instances on the same index", func() {
var instance0, instance1, duplicateInstance1 appfixture.Instance
var heartbeat models.Heartbeat
BeforeEach(func() {
dea = appfixture.NewDeaFixture()
a = dea.GetApp(0)
instance0 = a.InstanceAtIndex(0)
instance1 = a.InstanceAtIndex(1)
duplicateInstance1 = a.InstanceAtIndex(1)
duplicateInstance1.InstanceGuid = models.Guid()
heartbeat = dea.HeartbeatWith(instance0.Heartbeat(), instance1.Heartbeat(), duplicateInstance1.Heartbeat())
simulator.SetCurrentHeartbeats(heartbeat)
simulator.SetDesiredState(a.DesiredState(2))
simulator.Tick(simulator.TicksToAttainFreshness)
})
It("should not immediately stop anything", func() {
Ω(startStopListener.Stops).Should(BeEmpty())
})
Context("after four grace periods", func() {
Context("if both instances are still running", func() {
BeforeEach(func() {
simulator.Tick(simulator.GracePeriod * 4)