func (a *appAnalyzer) generatePendingStartsForCrashedInstances(priority float64) (crashCounts []models.CrashCount) {
if !a.app.IsStaged() {
return
}
for index := 0; a.app.IsIndexDesired(index); index++ {
if !a.app.HasStartingOrRunningInstanceAtIndex(index) && a.app.HasCrashedInstanceAtIndex(index) {
if index != 0 && !a.app.HasStartingOrRunningInstances() {
continue
}
crashCount := a.app.CrashCountAtIndex(index, a.currentTime)
delay := a.computeDelayForCrashCount(crashCount)
message := models.NewPendingStartMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, priority, models.PendingStartMessageReasonCrashed)
didAppend := a.appendStartMessageIfNotDuplicate(message, "Identified crashed instance", map[string]string{
"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
"Crash Count": strconv.Itoa(crashCount.CrashCount),
})
if didAppend {
crashCount.CrashCount += 1
a.crashCounts = append(a.crashCounts, crashCount)
}
}
}
return
}
func (a *appAnalyzer) generatePendingStartsForMissingInstances(priority float64) {
if !a.app.IsStaged() {
return
}
for index := 0; a.app.IsIndexDesired(index); index++ {
if !a.app.HasStartingOrRunningInstanceAtIndex(index) && !a.app.HasCrashedInstanceAtIndex(index) {
message := models.NewPendingStartMessage(a.currentTime, a.conf.GracePeriod(), 0, a.app.AppGuid, a.app.AppVersion, index, priority, models.PendingStartMessageReasonMissing)
a.appendStartMessageIfNotDuplicate(message, "Identified missing instance", map[string]string{
"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
})
}
}
return
}
func (e *Evacuator) handleExited(exited models.DropletExited) {
switch exited.Reason {
case models.DropletExitedReasonDEAShutdown, models.DropletExitedReasonDEAEvacuation:
startMessage := models.NewPendingStartMessage(
e.clock.Now(),
0,
e.config.GracePeriod(),
exited.AppGuid,
exited.AppVersion,
exited.InstanceIndex,
2.0,
models.PendingStartMessageReasonEvacuating,
)
startMessage.SkipVerification = true
e.logger.Info("Scheduling start message for droplet.exited message", startMessage.LogDescription(), exited.LogDescription())
e.store.SavePendingStartMessages(startMessage)
}
}
func (a *appAnalyzer) generatePendingStartsAndStopsForEvacuatingInstances() {
heartbeatsByIndex := a.app.HeartbeatsByIndex()
for index := range heartbeatsByIndex {
evacuatingInstances := a.app.EvacuatingInstancesAtIndex(index)
if len(evacuatingInstances) > 0 {
startMessage := models.NewPendingStartMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, 2.0, models.PendingStartMessageReasonEvacuating)
addStopMessages := func(displayReason string, stopReason models.PendingStopMessageReason) {
for _, evacuatingInstance := range evacuatingInstances {
stopMessage := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, evacuatingInstance.InstanceGuid, stopReason)
a.appendStopMessageIfNotDuplicate(stopMessage, displayReason, map[string]string{})
}
}
if !a.app.IsIndexDesired(index) {
addStopMessages("Identified undesired evacuating instance.", models.PendingStopMessageReasonExtra)
continue
}
if !a.app.IsStaged() {
addStopMessages("Identified evacuating instance that is not staged.", models.PendingStopMessageReasonEvacuationComplete)
}
if a.app.HasRunningInstanceAtIndex(index) {
addStopMessages("Stopping an evacuating instance that has started running elsewhere.", models.PendingStopMessageReasonEvacuationComplete)
continue
}
if a.app.HasStartingInstanceAtIndex(index) {
continue
}
if a.app.CrashCountAtIndex(index, a.currentTime).CrashCount > 0 {
addStopMessages("Stopping an unstable evacuating instance.", models.PendingStopMessageReasonEvacuationComplete)
}
a.appendStartMessageIfNotDuplicate(startMessage, "An instance is evacuating. Starting it elsewhere.", map[string]string{})
}
}
}
Describe("Starting missing instances", func() {
Context("where an app has desired instances", func() {
BeforeEach(func() {
store.SyncDesiredState(
app.DesiredState(2),
)
})
Context("and none of the instances are running", func() {
It("should send a start message for each of the missing instances", func() {
err := analyzer.Analyze()
Ω(err).ShouldNot(HaveOccurred())
Ω(stopMessages()).Should(BeEmpty())
Ω(startMessages()).Should(HaveLen(2))
expectedMessage := models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 0, 1, models.PendingStartMessageReasonMissing)
Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
expectedMessage = models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 1, 1, models.PendingStartMessageReasonMissing)
Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
})
It("should set the priority to 1", func() {
analyzer.Analyze()
for _, message := range startMessages() {
Ω(message.Priority).Should(Equal(1.0))
}
})
})
Context("when there is an existing start message", func() {
Ω(pendingStarts).Should(BeEmpty())
})
})
Context("when the reason is DEA_EVACUATION", func() {
BeforeEach(func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
})
})
It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
pendingStarts, err := store.GetPendingStartMessages()
Ω(err).ShouldNot(HaveOccurred())
expectedStartMessage := models.NewPendingStartMessage(timeProvider.Time(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
expectedStartMessage.SkipVerification = true
Ω(pendingStarts).Should(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
})
})
Context("when the reason is DEA_SHUTDOWN", func() {
BeforeEach(func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON(),
})
})
It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
pendingStarts, err := store.GetPendingStartMessages()
storeAdapter storeadapter.StoreAdapter
conf *config.Config
message1 models.PendingStartMessage
message2 models.PendingStartMessage
message3 models.PendingStartMessage
)
BeforeEach(func() {
var err error
conf, err = config.DefaultConfig()
Ω(err).ShouldNot(HaveOccurred())
storeAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcdRunner.NodeURLS(), workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests))
err = storeAdapter.Connect()
Ω(err).ShouldNot(HaveOccurred())
message1 = models.NewPendingStartMessage(time.Unix(100, 0), 10, 4, "ABC", "123", 1, 1.0, models.PendingStartMessageReasonInvalid)
message2 = models.NewPendingStartMessage(time.Unix(100, 0), 10, 4, "DEF", "123", 1, 1.0, models.PendingStartMessageReasonInvalid)
message3 = models.NewPendingStartMessage(time.Unix(100, 0), 10, 4, "ABC", "456", 1, 1.0, models.PendingStartMessageReasonInvalid)
store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
})
AfterEach(func() {
storeAdapter.Disconnect()
})
Describe("Saving start messages", func() {
BeforeEach(func() {
err := store.SavePendingStartMessages(
message1,
message2,
err := sender.Send()
Ω(err).ShouldNot(HaveOccurred())
Ω(messageBus.PublishedMessages).Should(BeEmpty())
})
})
Context("when there are start messages", func() {
var keepAliveTime int
var sentOn int64
var err error
var pendingMessage models.PendingStartMessage
var storeSetErrInjector *fakestoreadapter.FakeStoreAdapterErrorInjector
JustBeforeEach(func() {
store.SyncDesiredState(app.DesiredState(1))
pendingMessage = models.NewPendingStartMessage(time.Unix(100, 0), 30, keepAliveTime, app.AppGuid, app.AppVersion, 0, 1.0, models.PendingStartMessageReasonInvalid)
pendingMessage.SentOn = sentOn
store.SavePendingStartMessages(
pendingMessage,
)
storeAdapter.SetErrInjector = storeSetErrInjector
err = sender.Send()
})
BeforeEach(func() {
keepAliveTime = 0
sentOn = 0
err = nil
storeSetErrInjector = nil
})
Expect(pendingStarts).To(BeEmpty())
})
})
Context("when the reason is DEA_EVACUATION", func() {
BeforeEach(func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
})
})
It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
pendingStarts, err := store.GetPendingStartMessages()
Expect(err).NotTo(HaveOccurred())
expectedStartMessage := models.NewPendingStartMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
expectedStartMessage.SkipVerification = true
Expect(pendingStarts).To(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
})
})
Context("when the reason is DEA_SHUTDOWN", func() {
BeforeEach(func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON(),
})
})
It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
pendingStarts, err := store.GetPendingStartMessages()