func (a *appAnalyzer) generatePendingStopsForExtraInstances() {
for _, extraInstance := range a.app.ExtraStartingOrRunningInstances() {
message := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, extraInstance.InstanceGuid, models.PendingStopMessageReasonExtra)
a.appendStopMessageIfNotDuplicate(message, "Identified extra running instance", map[string]string{
"InstanceIndex": strconv.Itoa(extraInstance.InstanceIndex),
"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
})
}
return
}
func (a *appAnalyzer) generatePendingStartsAndStopsForEvacuatingInstances() {
heartbeatsByIndex := a.app.HeartbeatsByIndex()
for index := range heartbeatsByIndex {
evacuatingInstances := a.app.EvacuatingInstancesAtIndex(index)
if len(evacuatingInstances) > 0 {
startMessage := models.NewPendingStartMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, 2.0, models.PendingStartMessageReasonEvacuating)
addStopMessages := func(displayReason string, stopReason models.PendingStopMessageReason) {
for _, evacuatingInstance := range evacuatingInstances {
stopMessage := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, evacuatingInstance.InstanceGuid, stopReason)
a.appendStopMessageIfNotDuplicate(stopMessage, displayReason, map[string]string{})
}
}
if !a.app.IsIndexDesired(index) {
addStopMessages("Identified undesired evacuating instance.", models.PendingStopMessageReasonExtra)
continue
}
if !a.app.IsStaged() {
addStopMessages("Identified evacuating instance that is not staged.", models.PendingStopMessageReasonEvacuationComplete)
}
if a.app.HasRunningInstanceAtIndex(index) {
addStopMessages("Stopping an evacuating instance that has started running elsewhere.", models.PendingStopMessageReasonEvacuationComplete)
continue
}
if a.app.HasStartingInstanceAtIndex(index) {
continue
}
if a.app.CrashCountAtIndex(index, a.currentTime).CrashCount > 0 {
addStopMessages("Stopping an unstable evacuating instance.", models.PendingStopMessageReasonEvacuationComplete)
}
a.appendStartMessageIfNotDuplicate(startMessage, "An instance is evacuating. Starting it elsewhere.", map[string]string{})
}
}
}
func (a *appAnalyzer) generatePendingStopsForDuplicateInstances() {
//stop duplicate instances at indices < numDesired
//this works by scheduling stops for *all* duplicate instances at increasing delays
//the sender will process the stops one at a time and only send stops that don't put
//the system in an invalid state
for index := 0; a.app.IsIndexDesired(index); index++ {
instances := a.app.StartingOrRunningInstancesAtIndex(index)
if len(instances) > 1 {
minimumDuplicateInstanceStopDelay := 4 * a.conf.GracePeriod()
for i, instance := range instances {
delay := i*a.conf.GracePeriod() + minimumDuplicateInstanceStopDelay
message := models.NewPendingStopMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, instance.InstanceGuid, models.PendingStopMessageReasonDuplicate)
a.appendStopMessageIfNotDuplicate(message, "Identified duplicate running instance", map[string]string{
"InstanceIndex": strconv.Itoa(instance.InstanceIndex),
})
}
}
}
return
}
storeAdapter storeadapter.StoreAdapter
conf *config.Config
message1 models.PendingStopMessage
message2 models.PendingStopMessage
message3 models.PendingStopMessage
)
BeforeEach(func() {
var err error
conf, err = config.DefaultConfig()
Ω(err).ShouldNot(HaveOccurred())
storeAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcdRunner.NodeURLS(), workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests))
err = storeAdapter.Connect()
Ω(err).ShouldNot(HaveOccurred())
message1 = models.NewPendingStopMessage(time.Unix(100, 0), 10, 4, "ABC", "123", "XYZ", models.PendingStopMessageReasonInvalid)
message2 = models.NewPendingStopMessage(time.Unix(100, 0), 10, 4, "DEF", "456", "ALPHA", models.PendingStopMessageReasonInvalid)
message3 = models.NewPendingStopMessage(time.Unix(100, 0), 10, 4, "GHI", "789", "BETA", models.PendingStopMessageReasonInvalid)
store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
})
AfterEach(func() {
storeAdapter.Disconnect()
})
Describe("Saving stop messages", func() {
BeforeEach(func() {
err := store.SavePendingStopMessages(
message1,
message2,
})
})
Describe("Stopping extra instances (index >= numDesired)", func() {
BeforeEach(func() {
store.SyncHeartbeats(app.Heartbeat(3))
})
Context("when there are no desired instances", func() {
It("should return an array of stop messages for the extra instances", func() {
err := analyzer.Analyze()
Ω(err).ShouldNot(HaveOccurred())
Ω(startMessages()).Should(BeEmpty())
Ω(stopMessages()).Should(HaveLen(3))
expectedMessage := models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(1).InstanceGuid, models.PendingStopMessageReasonExtra)
Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(2).InstanceGuid, models.PendingStopMessageReasonExtra)
Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
})
})
Context("when there is an existing stop message", func() {
var existingMessage models.PendingStopMessage
BeforeEach(func() {
existingMessage = models.NewPendingStopMessage(time.Unix(1, 0), 0, 0, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
store.SavePendingStopMessages(
})
})
})
})
Context("when there are stop messages", func() {
var keepAliveTime int
var sentOn int64
var err error
var pendingMessage models.PendingStopMessage
var storeSetErrInjector *fakestoreadapter.FakeStoreAdapterErrorInjector
JustBeforeEach(func() {
store.SyncHeartbeats(app.Heartbeat(2))
pendingMessage = models.NewPendingStopMessage(time.Unix(100, 0), 30, keepAliveTime, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonInvalid)
pendingMessage.SentOn = sentOn
store.SavePendingStopMessages(
pendingMessage,
)
storeAdapter.SetErrInjector = storeSetErrInjector
err = sender.Send()
})
BeforeEach(func() {
keepAliveTime = 0
sentOn = 0
err = nil
storeSetErrInjector = nil
})