Esempio n. 1
0
func (a *appAnalyzer) generatePendingStopsForExtraInstances() {
	for _, extraInstance := range a.app.ExtraStartingOrRunningInstances() {
		message := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, extraInstance.InstanceGuid, models.PendingStopMessageReasonExtra)

		a.appendStopMessageIfNotDuplicate(message, "Identified extra running instance", map[string]string{
			"InstanceIndex":          strconv.Itoa(extraInstance.InstanceIndex),
			"Desired # of Instances": strconv.Itoa(a.app.NumberOfDesiredInstances()),
		})
	}

	return
}
Esempio n. 2
0
func (a *appAnalyzer) generatePendingStartsAndStopsForEvacuatingInstances() {
	heartbeatsByIndex := a.app.HeartbeatsByIndex()

	for index := range heartbeatsByIndex {
		evacuatingInstances := a.app.EvacuatingInstancesAtIndex(index)

		if len(evacuatingInstances) > 0 {
			startMessage := models.NewPendingStartMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, index, 2.0, models.PendingStartMessageReasonEvacuating)
			addStopMessages := func(displayReason string, stopReason models.PendingStopMessageReason) {
				for _, evacuatingInstance := range evacuatingInstances {
					stopMessage := models.NewPendingStopMessage(a.currentTime, 0, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, evacuatingInstance.InstanceGuid, stopReason)
					a.appendStopMessageIfNotDuplicate(stopMessage, displayReason, map[string]string{})
				}
			}

			if !a.app.IsIndexDesired(index) {
				addStopMessages("Identified undesired evacuating instance.", models.PendingStopMessageReasonExtra)
				continue
			}

			if !a.app.IsStaged() {
				addStopMessages("Identified evacuating instance that is not staged.", models.PendingStopMessageReasonEvacuationComplete)
			}

			if a.app.HasRunningInstanceAtIndex(index) {
				addStopMessages("Stopping an evacuating instance that has started running elsewhere.", models.PendingStopMessageReasonEvacuationComplete)
				continue
			}

			if a.app.HasStartingInstanceAtIndex(index) {
				continue
			}

			if a.app.CrashCountAtIndex(index, a.currentTime).CrashCount > 0 {
				addStopMessages("Stopping an unstable evacuating instance.", models.PendingStopMessageReasonEvacuationComplete)
			}

			a.appendStartMessageIfNotDuplicate(startMessage, "An instance is evacuating.  Starting it elsewhere.", map[string]string{})
		}
	}
}
Esempio n. 3
0
func (a *appAnalyzer) generatePendingStopsForDuplicateInstances() {
	//stop duplicate instances at indices < numDesired
	//this works by scheduling stops for *all* duplicate instances at increasing delays
	//the sender will process the stops one at a time and only send stops that don't put
	//the system in an invalid state
	for index := 0; a.app.IsIndexDesired(index); index++ {
		instances := a.app.StartingOrRunningInstancesAtIndex(index)
		if len(instances) > 1 {
			minimumDuplicateInstanceStopDelay := 4 * a.conf.GracePeriod()

			for i, instance := range instances {
				delay := i*a.conf.GracePeriod() + minimumDuplicateInstanceStopDelay
				message := models.NewPendingStopMessage(a.currentTime, delay, a.conf.GracePeriod(), a.app.AppGuid, a.app.AppVersion, instance.InstanceGuid, models.PendingStopMessageReasonDuplicate)

				a.appendStopMessageIfNotDuplicate(message, "Identified duplicate running instance", map[string]string{
					"InstanceIndex": strconv.Itoa(instance.InstanceIndex),
				})
			}
		}
	}

	return
}
		storeAdapter storeadapter.StoreAdapter
		conf         *config.Config
		message1     models.PendingStopMessage
		message2     models.PendingStopMessage
		message3     models.PendingStopMessage
	)

	BeforeEach(func() {
		var err error
		conf, err = config.DefaultConfig()
		Ω(err).ShouldNot(HaveOccurred())
		storeAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcdRunner.NodeURLS(), workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests))
		err = storeAdapter.Connect()
		Ω(err).ShouldNot(HaveOccurred())

		message1 = models.NewPendingStopMessage(time.Unix(100, 0), 10, 4, "ABC", "123", "XYZ", models.PendingStopMessageReasonInvalid)
		message2 = models.NewPendingStopMessage(time.Unix(100, 0), 10, 4, "DEF", "456", "ALPHA", models.PendingStopMessageReasonInvalid)
		message3 = models.NewPendingStopMessage(time.Unix(100, 0), 10, 4, "GHI", "789", "BETA", models.PendingStopMessageReasonInvalid)

		store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
	})

	AfterEach(func() {
		storeAdapter.Disconnect()
	})

	Describe("Saving stop messages", func() {
		BeforeEach(func() {
			err := store.SavePendingStopMessages(
				message1,
				message2,
Esempio n. 5
0
		})
	})

	Describe("Stopping extra instances (index >= numDesired)", func() {
		BeforeEach(func() {
			store.SyncHeartbeats(app.Heartbeat(3))
		})

		Context("when there are no desired instances", func() {
			It("should return an array of stop messages for the extra instances", func() {
				err := analyzer.Analyze()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(startMessages()).Should(BeEmpty())
				Ω(stopMessages()).Should(HaveLen(3))

				expectedMessage := models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
				Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))

				expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(1).InstanceGuid, models.PendingStopMessageReasonExtra)
				Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))

				expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(2).InstanceGuid, models.PendingStopMessageReasonExtra)
				Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
			})
		})

		Context("when there is an existing stop message", func() {
			var existingMessage models.PendingStopMessage
			BeforeEach(func() {
				existingMessage = models.NewPendingStopMessage(time.Unix(1, 0), 0, 0, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
				store.SavePendingStopMessages(
				})
			})
		})
	})

	Context("when there are stop messages", func() {
		var keepAliveTime int
		var sentOn int64
		var err error
		var pendingMessage models.PendingStopMessage
		var storeSetErrInjector *fakestoreadapter.FakeStoreAdapterErrorInjector

		JustBeforeEach(func() {
			store.SyncHeartbeats(app.Heartbeat(2))

			pendingMessage = models.NewPendingStopMessage(time.Unix(100, 0), 30, keepAliveTime, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonInvalid)
			pendingMessage.SentOn = sentOn
			store.SavePendingStopMessages(
				pendingMessage,
			)

			storeAdapter.SetErrInjector = storeSetErrInjector
			err = sender.Send()
		})

		BeforeEach(func() {
			keepAliveTime = 0
			sentOn = 0
			err = nil
			storeSetErrInjector = nil
		})