func (a *appAnalyzer) generatePendingStartsForCrashedInstances(priority float64) (crashCounts []models.CrashCount) {
	if !a.app.IsStaged() {
		return
	}

	for index := 0; a.app.IsIndexDesired(index); index++ {
		if !a.app.HasStartingOrRunningInstanceAtIndex(index) && a.app.HasCrashedInstanceAtIndex(index) {
			if index != 0 && !a.app.HasStartingOrRunningInstances() {
				continue
			}

			crashCount := a.app.CrashCountAtIndex(index, a.currentTime)
			delay := a.computeDelayForCrashCount(crashCount)
			message := models.NewPendingStartMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, priority, models.PendingStartMessageReasonCrashed)

			didAppend := a.appendStartMessageIfNotDuplicate(message, "Identified crashed instance", map[string]string{
				"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
				"Crash Count":            strconv.Itoa(crashCount.CrashCount),
			})

			if didAppend {
				crashCount.CrashCount += 1
				a.crashCounts = append(a.crashCounts, crashCount)
			}
		}
	}

	return
}
func (a *appAnalyzer) generatePendingStartsForMissingInstances(priority float64) {
	if !a.app.IsStaged() {
		return
	}

	for index := 0; a.app.IsIndexDesired(index); index++ {
		if !a.app.HasStartingOrRunningInstanceAtIndex(index) && !a.app.HasCrashedInstanceAtIndex(index) {
			message := models.NewPendingStartMessage(a.currentTime, a.conf.GracePeriod(), 0, a.app.AppGuid, a.app.AppVersion, index, priority, models.PendingStartMessageReasonMissing)

			a.appendStartMessageIfNotDuplicate(message, "Identified missing instance", map[string]string{
				"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
			})
		}
	}

	return
}
Beispiel #3
0
func (e *Evacuator) handleExited(exited models.DropletExited) {
	switch exited.Reason {
	case models.DropletExitedReasonDEAShutdown, models.DropletExitedReasonDEAEvacuation:
		startMessage := models.NewPendingStartMessage(
			e.clock.Now(),
			0,
			e.config.GracePeriod(),
			exited.AppGuid,
			exited.AppVersion,
			exited.InstanceIndex,
			2.0,
			models.PendingStartMessageReasonEvacuating,
		)
		startMessage.SkipVerification = true

		e.logger.Info("Scheduling start message for droplet.exited message", startMessage.LogDescription(), exited.LogDescription())

		e.store.SavePendingStartMessages(startMessage)
	}
}
func (a *appAnalyzer) generatePendingStartsAndStopsForEvacuatingInstances() {
	heartbeatsByIndex := a.app.HeartbeatsByIndex()

	for index := range heartbeatsByIndex {
		evacuatingInstances := a.app.EvacuatingInstancesAtIndex(index)

		if len(evacuatingInstances) > 0 {
			startMessage := models.NewPendingStartMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, 2.0, models.PendingStartMessageReasonEvacuating)
			addStopMessages := func(displayReason string, stopReason models.PendingStopMessageReason) {
				for _, evacuatingInstance := range evacuatingInstances {
					stopMessage := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, evacuatingInstance.InstanceGuid, stopReason)
					a.appendStopMessageIfNotDuplicate(stopMessage, displayReason, map[string]string{})
				}
			}

			if !a.app.IsIndexDesired(index) {
				addStopMessages("Identified undesired evacuating instance.", models.PendingStopMessageReasonExtra)
				continue
			}

			if !a.app.IsStaged() {
				addStopMessages("Identified evacuating instance that is not staged.", models.PendingStopMessageReasonEvacuationComplete)
			}

			if a.app.HasRunningInstanceAtIndex(index) {
				addStopMessages("Stopping an evacuating instance that has started running elsewhere.", models.PendingStopMessageReasonEvacuationComplete)
				continue
			}

			if a.app.HasStartingInstanceAtIndex(index) {
				continue
			}

			if a.app.CrashCountAtIndex(index, a.currentTime).CrashCount > 0 {
				addStopMessages("Stopping an unstable evacuating instance.", models.PendingStopMessageReasonEvacuationComplete)
			}

			a.appendStartMessageIfNotDuplicate(startMessage, "An instance is evacuating.  Starting it elsewhere.", map[string]string{})
		}
	}
}
Beispiel #5
0
	Describe("Starting missing instances", func() {
		Context("where an app has desired instances", func() {
			BeforeEach(func() {
				store.SyncDesiredState(
					app.DesiredState(2),
				)
			})

			Context("and none of the instances are running", func() {
				It("should send a start message for each of the missing instances", func() {
					err := analyzer.Analyze()
					Ω(err).ShouldNot(HaveOccurred())
					Ω(stopMessages()).Should(BeEmpty())
					Ω(startMessages()).Should(HaveLen(2))

					expectedMessage := models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 0, 1, models.PendingStartMessageReasonMissing)
					Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))

					expectedMessage = models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 1, 1, models.PendingStartMessageReasonMissing)
					Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
				})

				It("should set the priority to 1", func() {
					analyzer.Analyze()
					for _, message := range startMessages() {
						Ω(message.Priority).Should(Equal(1.0))
					}
				})
			})

			Context("when there is an existing start message", func() {
Beispiel #6
0
				Ω(pendingStarts).Should(BeEmpty())
			})
		})

		Context("when the reason is DEA_EVACUATION", func() {
			BeforeEach(func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
				})
			})

			It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
				pendingStarts, err := store.GetPendingStartMessages()
				Ω(err).ShouldNot(HaveOccurred())

				expectedStartMessage := models.NewPendingStartMessage(timeProvider.Time(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
				expectedStartMessage.SkipVerification = true

				Ω(pendingStarts).Should(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
			})
		})

		Context("when the reason is DEA_SHUTDOWN", func() {
			BeforeEach(func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON(),
				})
			})

			It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
				pendingStarts, err := store.GetPendingStartMessages()
		storeAdapter storeadapter.StoreAdapter
		conf         *config.Config
		message1     models.PendingStartMessage
		message2     models.PendingStartMessage
		message3     models.PendingStartMessage
	)

	BeforeEach(func() {
		var err error
		conf, err = config.DefaultConfig()
		Ω(err).ShouldNot(HaveOccurred())
		storeAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcdRunner.NodeURLS(), workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests))
		err = storeAdapter.Connect()
		Ω(err).ShouldNot(HaveOccurred())

		message1 = models.NewPendingStartMessage(time.Unix(100, 0), 10, 4, "ABC", "123", 1, 1.0, models.PendingStartMessageReasonInvalid)
		message2 = models.NewPendingStartMessage(time.Unix(100, 0), 10, 4, "DEF", "123", 1, 1.0, models.PendingStartMessageReasonInvalid)
		message3 = models.NewPendingStartMessage(time.Unix(100, 0), 10, 4, "ABC", "456", 1, 1.0, models.PendingStartMessageReasonInvalid)

		store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
	})

	AfterEach(func() {
		storeAdapter.Disconnect()
	})

	Describe("Saving start messages", func() {
		BeforeEach(func() {
			err := store.SavePendingStartMessages(
				message1,
				message2,
			err := sender.Send()
			Ω(err).ShouldNot(HaveOccurred())
			Ω(messageBus.PublishedMessages).Should(BeEmpty())
		})
	})

	Context("when there are start messages", func() {
		var keepAliveTime int
		var sentOn int64
		var err error
		var pendingMessage models.PendingStartMessage
		var storeSetErrInjector *fakestoreadapter.FakeStoreAdapterErrorInjector

		JustBeforeEach(func() {
			store.SyncDesiredState(app.DesiredState(1))
			pendingMessage = models.NewPendingStartMessage(time.Unix(100, 0), 30, keepAliveTime, app.AppGuid, app.AppVersion, 0, 1.0, models.PendingStartMessageReasonInvalid)
			pendingMessage.SentOn = sentOn
			store.SavePendingStartMessages(
				pendingMessage,
			)
			storeAdapter.SetErrInjector = storeSetErrInjector
			err = sender.Send()
		})

		BeforeEach(func() {
			keepAliveTime = 0
			sentOn = 0
			err = nil
			storeSetErrInjector = nil
		})
Beispiel #9
0
				Expect(pendingStarts).To(BeEmpty())
			})
		})

		Context("when the reason is DEA_EVACUATION", func() {
			BeforeEach(func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
				})
			})

			It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
				pendingStarts, err := store.GetPendingStartMessages()
				Expect(err).NotTo(HaveOccurred())

				expectedStartMessage := models.NewPendingStartMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
				expectedStartMessage.SkipVerification = true

				Expect(pendingStarts).To(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
			})
		})

		Context("when the reason is DEA_SHUTDOWN", func() {
			BeforeEach(func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON(),
				})
			})

			It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
				pendingStarts, err := store.GetPendingStartMessages()