BeforeEach(func() {
storeAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcdRunner.NodeURLS(), workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests))
err := storeAdapter.Connect()
Ω(err).ShouldNot(HaveOccurred())
store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
dea = appfixture.NewDeaFixture()
app1 = dea.GetApp(0)
app2 = dea.GetApp(1)
app3 = dea.GetApp(2)
app4 = dea.GetApp(3)
actualState := []models.InstanceHeartbeat{
app1.InstanceAtIndex(0).Heartbeat(),
app1.InstanceAtIndex(1).Heartbeat(),
app1.InstanceAtIndex(2).Heartbeat(),
app2.InstanceAtIndex(0).Heartbeat(),
}
desiredState := []models.DesiredAppState{
app1.DesiredState(1),
app3.DesiredState(1),
}
crashCount = []models.CrashCount{
{
AppGuid: app1.AppGuid,
AppVersion: app1.AppVersion,
InstanceIndex: 1,
})
It("should never try to stop crashes", func() {
Ω(startStopListener.Stops).Should(BeEmpty())
simulator.Tick(1)
Ω(startStopListener.Stops).Should(BeEmpty())
})
})
Describe("when at least one instance is running", func() {
BeforeEach(func() {
simulator.SetDesiredState(a.DesiredState(3))
crashingHeartbeat = dea.HeartbeatWith(
a.CrashedInstanceHeartbeatAtIndex(0),
a.InstanceAtIndex(1).Heartbeat(),
a.CrashedInstanceHeartbeatAtIndex(2),
)
simulator.SetCurrentHeartbeats(crashingHeartbeat)
simulator.Tick(simulator.TicksToAttainFreshness)
})
It("should start all the crashed instances", func() {
Ω(startStopListener.Stops).Should(BeEmpty())
Ω(startStopListener.Starts).Should(HaveLen(2))
indicesToStart := []int{
startStopListener.Starts[0].InstanceIndex,
startStopListener.Starts[1].InstanceIndex,
}
import (
"github.com/cloudfoundry/hm9000/models"
"github.com/cloudfoundry/hm9000/testhelpers/appfixture"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Evacuation and Shutdown", func() {
var dea appfixture.DeaFixture
var app appfixture.AppFixture
BeforeEach(func() {
dea = appfixture.NewDeaFixture()
app = dea.GetApp(0)
simulator.SetCurrentHeartbeats(dea.HeartbeatWith(app.InstanceAtIndex(0).Heartbeat()))
simulator.SetDesiredState(app.DesiredState(1))
simulator.Tick(simulator.TicksToAttainFreshness)
})
Describe("Shutdown handling by the evacuator component", func() {
Context("when a SHUTDOWN droplet.exited message comes in", func() {
BeforeEach(func() {
cliRunner.StartEvacuator(simulator.currentTimestamp)
coordinator.MessageBus.Publish("droplet.exited", app.InstanceAtIndex(0).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON())
})
AfterEach(func() {
cliRunner.StopEvacuator()
})
})
})
Describe("Stopping extra instances (index >= numDesired)", func() {
BeforeEach(func() {
store.SyncHeartbeats(app.Heartbeat(3))
})
Context("when there are no desired instances", func() {
It("should return an array of stop messages for the extra instances", func() {
err := analyzer.Analyze()
Ω(err).ShouldNot(HaveOccurred())
Ω(startMessages()).Should(BeEmpty())
Ω(stopMessages()).Should(HaveLen(3))
expectedMessage := models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(1).InstanceGuid, models.PendingStopMessageReasonExtra)
Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(2).InstanceGuid, models.PendingStopMessageReasonExtra)
Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
})
})
Context("when there is an existing stop message", func() {
var existingMessage models.PendingStopMessage
BeforeEach(func() {
existingMessage = models.NewPendingStopMessage(time.Unix(1, 0), 0, 0, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
store.SavePendingStopMessages(
. "github.com/onsi/gomega"
"time"
)
var _ = Describe("App", func() {
var (
fixture appfixture.AppFixture
appGuid string
appVersion string
desired DesiredAppState
instanceHeartbeats []InstanceHeartbeat
crashCounts map[int]CrashCount
)
instance := func(instanceIndex int) appfixture.Instance {
return fixture.InstanceAtIndex(instanceIndex)
}
heartbeat := func(instanceIndex int, state InstanceState) InstanceHeartbeat {
hb := instance(instanceIndex).Heartbeat()
hb.State = state
return hb
}
app := func() *App {
return NewApp(appGuid, appVersion, desired, instanceHeartbeats, crashCounts)
}
BeforeEach(func() {
fixture = appfixture.NewAppFixture()
Context("when the message is malformed", func() {
It("does nothing", func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: []byte("ß"),
})
pendingStarts, err := store.GetPendingStartMessages()
Ω(err).ShouldNot(HaveOccurred())
Ω(pendingStarts).Should(BeEmpty())
})
})
Context("when the reason is DEA_EVACUATION", func() {
BeforeEach(func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
})
})
It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
pendingStarts, err := store.GetPendingStartMessages()
Ω(err).ShouldNot(HaveOccurred())
expectedStartMessage := models.NewPendingStartMessage(timeProvider.Time(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
expectedStartMessage.SkipVerification = true
Ω(pendingStarts).Should(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
})
})
Context("when the reason is DEA_SHUTDOWN", func() {
)
var _ = Describe("Expiring Heartbeats Test", func() {
var dea1, dea2 appfixture.DeaFixture
var app1, app2, app3 appfixture.AppFixture
BeforeEach(func() {
dea1 = appfixture.NewDeaFixture()
dea2 = appfixture.NewDeaFixture()
app1 = dea1.GetApp(0)
app2 = dea1.GetApp(1)
app3 = dea2.GetApp(2)
simulator.SetCurrentHeartbeats(
dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat(), app2.InstanceAtIndex(0).Heartbeat()),
dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
)
simulator.SetDesiredState(app1.DesiredState(1), app2.DesiredState(1), app3.DesiredState(1))
simulator.Tick(simulator.TicksToAttainFreshness)
})
Context("when a dea reports than an instance is no longer present", func() {
BeforeEach(func() {
simulator.SetCurrentHeartbeats(
dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat()),
dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
)
})
It("should start the instance after a grace period", func() {
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfUndesiredRunningApps", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfRunningInstances", Value: 3}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfMissingIndices", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedInstances", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedIndices", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredApps", Value: 1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredInstances", Value: 3}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredAppsPendingStaging", Value: 0}))
})
})
Context("when a desired app has an instance starting and others running", func() {
BeforeEach(func() {
store.SyncDesiredState(a.DesiredState(3))
startingHB := a.InstanceAtIndex(1).Heartbeat()
startingHB.State = models.InstanceStateStarting
store.SyncHeartbeats(dea.HeartbeatWith(
a.InstanceAtIndex(0).Heartbeat(),
startingHB,
a.InstanceAtIndex(2).Heartbeat(),
))
})
It("should have the correct stats", func() {
context := metricsServer.Emit()
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithAllInstancesReporting", Value: 1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithMissingInstances", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfUndesiredRunningApps", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfRunningInstances", Value: 3}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfMissingIndices", Value: 0}))
})
Ω(messageBus.PublishedMessages).Should(BeEmpty())
})
})
Context("when the request contains the droplet and version", func() {
var app appfixture.AppFixture
var expectedApp *models.App
var validRequestPayload string
BeforeEach(func() {
app = appfixture.NewAppFixture()
instanceHeartbeats := []models.InstanceHeartbeat{
app.InstanceAtIndex(0).Heartbeat(),
app.InstanceAtIndex(1).Heartbeat(),
app.InstanceAtIndex(2).Heartbeat(),
}
crashCount := models.CrashCount{
AppGuid: app.AppGuid,
AppVersion: app.AppVersion,
InstanceIndex: 1,
CrashCount: 2,
}
expectedApp = models.NewApp(
app.AppGuid,
app.AppVersion,
app.DesiredState(3),
instanceHeartbeats,
map[int]models.CrashCount{1: crashCount},
})
})
})
})
Context("when there are stop messages", func() {
var keepAliveTime int
var sentOn int64
var err error
var pendingMessage models.PendingStopMessage
var storeSetErrInjector *fakestoreadapter.FakeStoreAdapterErrorInjector
JustBeforeEach(func() {
store.SyncHeartbeats(app.Heartbeat(2))
pendingMessage = models.NewPendingStopMessage(time.Unix(100, 0), 30, keepAliveTime, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonInvalid)
pendingMessage.SentOn = sentOn
store.SavePendingStopMessages(
pendingMessage,
)
storeAdapter.SetErrInjector = storeSetErrInjector
err = sender.Send()
})
BeforeEach(func() {
keepAliveTime = 0
sentOn = 0
err = nil
storeSetErrInjector = nil
})
cliRunner.StopAPIServer()
})
Context("when the store is fresh", func() {
BeforeEach(func() {
simulator.Tick(simulator.TicksToAttainFreshness)
cliRunner.StartAPIServer(simulator.currentTimestamp)
})
It("should return the app", func(done Done) {
replyTo := models.Guid()
_, err := coordinator.MessageBus.Subscribe(replyTo, func(message *yagnats.Message) {
defer GinkgoRecover()
Ω(string(message.Payload)).Should(ContainSubstring(`"droplet":"%s"`, a.AppGuid))
Ω(string(message.Payload)).Should(ContainSubstring(`"instances":2`))
Ω(string(message.Payload)).Should(ContainSubstring(`"instance":"%s"`, a.InstanceAtIndex(0).InstanceGuid))
close(done)
})
Ω(err).ShouldNot(HaveOccurred())
err = coordinator.MessageBus.PublishWithReplyTo("app.state", replyTo, validRequest)
Ω(err).ShouldNot(HaveOccurred())
})
})
Context("when the store is not fresh", func() {
BeforeEach(func() {
simulator.Tick(simulator.TicksToAttainFreshness - 1)
cliRunner.StartAPIServer(simulator.currentTimestamp)
})
Context("when the instance becomes desired", func() {
BeforeEach(func() {
simulator.SetDesiredState(app2.DesiredState(2))
startStopListener.Reset()
simulator.Tick(1)
})
It("should not send a stop message", func() {
Ω(startStopListener.Stops).Should(HaveLen(0))
})
})
Context("when the app is still running", func() {
BeforeEach(func() {
simulator.Tick(1)
})
It("should send a stop message, immediately, for the missing instance", func() {
Ω(startStopListener.Stops).Should(HaveLen(1))
stop := startStopListener.Stops[0]
Ω(stop.AppGuid).Should(Equal(app2.AppGuid))
Ω(stop.AppVersion).Should(Equal(app2.AppVersion))
Ω(stop.InstanceGuid).Should(Equal(app2.InstanceAtIndex(1).InstanceGuid))
Ω(stop.InstanceIndex).Should(Equal(1))
Ω(stop.IsDuplicate).Should(BeFalse())
})
})
})
})
app = appfixture.NewAppFixture()
})
Context("when there is none saved", func() {
It("should come back empty", func() {
results, err := store.GetInstanceHeartbeatsForApp(app.AppGuid, app.AppVersion)
Ω(err).ShouldNot(HaveOccurred())
Ω(results).Should(BeEmpty())
})
})
Context("when there is actual state saved", func() {
var heartbeatA, heartbeatB models.InstanceHeartbeat
BeforeEach(func() {
heartbeatA = app.InstanceAtIndex(0).Heartbeat()
heartbeatA.DeaGuid = "A"
store.SyncHeartbeats(models.Heartbeat{
DeaGuid: "A",
InstanceHeartbeats: []models.InstanceHeartbeat{
heartbeatA,
},
})
heartbeatB = app.InstanceAtIndex(1).Heartbeat()
heartbeatB.DeaGuid = "B"
store.SyncHeartbeats(models.Heartbeat{
DeaGuid: "B",
InstanceHeartbeats: []models.InstanceHeartbeat{
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Stopping Duplicate Instances", func() {
var dea appfixture.DeaFixture
var a appfixture.AppFixture
Context("when there are multiple instances on the same index", func() {
var instance0, instance1, duplicateInstance1 appfixture.Instance
var heartbeat models.Heartbeat
BeforeEach(func() {
dea = appfixture.NewDeaFixture()
a = dea.GetApp(0)
instance0 = a.InstanceAtIndex(0)
instance1 = a.InstanceAtIndex(1)
duplicateInstance1 = a.InstanceAtIndex(1)
duplicateInstance1.InstanceGuid = models.Guid()
heartbeat = dea.HeartbeatWith(instance0.Heartbeat(), instance1.Heartbeat(), duplicateInstance1.Heartbeat())
simulator.SetCurrentHeartbeats(heartbeat)
simulator.SetDesiredState(a.DesiredState(2))
simulator.Tick(simulator.TicksToAttainFreshness)
})
It("should not immediately stop anything", func() {
Ω(startStopListener.Stops).Should(BeEmpty())
})
Describe("ToJson", func() {
It("should, like, totally encode JSON", func() {
jsonHeartbeat, err := NewHeartbeatFromJSON(heartbeat.ToJSON())
Ω(err).ShouldNot(HaveOccurred())
Ω(jsonHeartbeat).Should(Equal(heartbeat))
})
})
Context("With a complex heartbeat", func() {
var heartbeat Heartbeat
var app appfixture.AppFixture
BeforeEach(func() {
app = appfixture.NewAppFixture()
crashedHeartbeat := app.InstanceAtIndex(2).Heartbeat()
crashedHeartbeat.State = InstanceStateCrashed
startingHeartbeat := app.InstanceAtIndex(3).Heartbeat()
startingHeartbeat.State = InstanceStateStarting
evacuatingHeartbeat := app.InstanceAtIndex(4).Heartbeat()
evacuatingHeartbeat.State = InstanceStateEvacuating
heartbeat = Heartbeat{
DeaGuid: "abc",
InstanceHeartbeats: []InstanceHeartbeat{
crashedHeartbeat,
startingHeartbeat,
evacuatingHeartbeat,
app.InstanceAtIndex(0).Heartbeat(),
wpool,
)
Expect(err).NotTo(HaveOccurred())
err = storeAdapter.Connect()
Expect(err).NotTo(HaveOccurred())
store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
dea = appfixture.NewDeaFixture()
app1 = dea.GetApp(0)
app2 = dea.GetApp(1)
app3 = dea.GetApp(2)
app4 = dea.GetApp(3)
actualState := []models.InstanceHeartbeat{
app1.InstanceAtIndex(0).Heartbeat(),
app1.InstanceAtIndex(1).Heartbeat(),
app1.InstanceAtIndex(2).Heartbeat(),
app2.InstanceAtIndex(0).Heartbeat(),
}
desiredState := []models.DesiredAppState{
app1.DesiredState(1),
app3.DesiredState(1),
}
crashCount = []models.CrashCount{
{
AppGuid: app1.AppGuid,
AppVersion: app1.AppVersion,
InstanceIndex: 1,