예제 #1
0
	BeforeEach(func() {
		storeAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcdRunner.NodeURLS(), workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests))
		err := storeAdapter.Connect()
		Ω(err).ShouldNot(HaveOccurred())

		store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())

		dea = appfixture.NewDeaFixture()
		app1 = dea.GetApp(0)
		app2 = dea.GetApp(1)
		app3 = dea.GetApp(2)
		app4 = dea.GetApp(3)

		actualState := []models.InstanceHeartbeat{
			app1.InstanceAtIndex(0).Heartbeat(),
			app1.InstanceAtIndex(1).Heartbeat(),
			app1.InstanceAtIndex(2).Heartbeat(),
			app2.InstanceAtIndex(0).Heartbeat(),
		}

		desiredState := []models.DesiredAppState{
			app1.DesiredState(1),
			app3.DesiredState(1),
		}

		crashCount = []models.CrashCount{
			{
				AppGuid:       app1.AppGuid,
				AppVersion:    app1.AppVersion,
				InstanceIndex: 1,
예제 #2
0
		})

		It("should never try to stop crashes", func() {
			Ω(startStopListener.Stops).Should(BeEmpty())
			simulator.Tick(1)
			Ω(startStopListener.Stops).Should(BeEmpty())
		})
	})

	Describe("when at least one instance is running", func() {
		BeforeEach(func() {
			simulator.SetDesiredState(a.DesiredState(3))

			crashingHeartbeat = dea.HeartbeatWith(
				a.CrashedInstanceHeartbeatAtIndex(0),
				a.InstanceAtIndex(1).Heartbeat(),
				a.CrashedInstanceHeartbeatAtIndex(2),
			)

			simulator.SetCurrentHeartbeats(crashingHeartbeat)
			simulator.Tick(simulator.TicksToAttainFreshness)
		})

		It("should start all the crashed instances", func() {
			Ω(startStopListener.Stops).Should(BeEmpty())
			Ω(startStopListener.Starts).Should(HaveLen(2))

			indicesToStart := []int{
				startStopListener.Starts[0].InstanceIndex,
				startStopListener.Starts[1].InstanceIndex,
			}
예제 #3
0
import (
	"github.com/cloudfoundry/hm9000/models"
	"github.com/cloudfoundry/hm9000/testhelpers/appfixture"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Evacuation and Shutdown", func() {
	var dea appfixture.DeaFixture
	var app appfixture.AppFixture

	BeforeEach(func() {
		dea = appfixture.NewDeaFixture()
		app = dea.GetApp(0)
		simulator.SetCurrentHeartbeats(dea.HeartbeatWith(app.InstanceAtIndex(0).Heartbeat()))
		simulator.SetDesiredState(app.DesiredState(1))
		simulator.Tick(simulator.TicksToAttainFreshness)
	})

	Describe("Shutdown handling by the evacuator component", func() {
		Context("when a SHUTDOWN droplet.exited message comes in", func() {
			BeforeEach(func() {
				cliRunner.StartEvacuator(simulator.currentTimestamp)
				coordinator.MessageBus.Publish("droplet.exited", app.InstanceAtIndex(0).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON())
			})

			AfterEach(func() {
				cliRunner.StopEvacuator()
			})
예제 #4
0
		})
	})

	Describe("Stopping extra instances (index >= numDesired)", func() {
		BeforeEach(func() {
			store.SyncHeartbeats(app.Heartbeat(3))
		})

		Context("when there are no desired instances", func() {
			It("should return an array of stop messages for the extra instances", func() {
				err := analyzer.Analyze()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(startMessages()).Should(BeEmpty())
				Ω(stopMessages()).Should(HaveLen(3))

				expectedMessage := models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
				Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))

				expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(1).InstanceGuid, models.PendingStopMessageReasonExtra)
				Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))

				expectedMessage = models.NewPendingStopMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, app.InstanceAtIndex(2).InstanceGuid, models.PendingStopMessageReasonExtra)
				Ω(stopMessages()).Should(ContainElement(EqualPendingStopMessage(expectedMessage)))
			})
		})

		Context("when there is an existing stop message", func() {
			var existingMessage models.PendingStopMessage
			BeforeEach(func() {
				existingMessage = models.NewPendingStopMessage(time.Unix(1, 0), 0, 0, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonExtra)
				store.SavePendingStopMessages(
	. "github.com/onsi/gomega"
	"time"
)

var _ = Describe("App", func() {
	var (
		fixture            appfixture.AppFixture
		appGuid            string
		appVersion         string
		desired            DesiredAppState
		instanceHeartbeats []InstanceHeartbeat
		crashCounts        map[int]CrashCount
	)

	instance := func(instanceIndex int) appfixture.Instance {
		return fixture.InstanceAtIndex(instanceIndex)
	}

	heartbeat := func(instanceIndex int, state InstanceState) InstanceHeartbeat {
		hb := instance(instanceIndex).Heartbeat()
		hb.State = state
		return hb
	}

	app := func() *App {
		return NewApp(appGuid, appVersion, desired, instanceHeartbeats, crashCounts)
	}

	BeforeEach(func() {
		fixture = appfixture.NewAppFixture()
예제 #6
0
		Context("when the message is malformed", func() {
			It("does nothing", func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: []byte("ß"),
				})

				pendingStarts, err := store.GetPendingStartMessages()
				Ω(err).ShouldNot(HaveOccurred())
				Ω(pendingStarts).Should(BeEmpty())
			})
		})

		Context("when the reason is DEA_EVACUATION", func() {
			BeforeEach(func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
				})
			})

			It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
				pendingStarts, err := store.GetPendingStartMessages()
				Ω(err).ShouldNot(HaveOccurred())

				expectedStartMessage := models.NewPendingStartMessage(timeProvider.Time(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
				expectedStartMessage.SkipVerification = true

				Ω(pendingStarts).Should(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
			})
		})

		Context("when the reason is DEA_SHUTDOWN", func() {
예제 #7
0
)

var _ = Describe("Expiring Heartbeats Test", func() {
	var dea1, dea2 appfixture.DeaFixture
	var app1, app2, app3 appfixture.AppFixture

	BeforeEach(func() {
		dea1 = appfixture.NewDeaFixture()
		dea2 = appfixture.NewDeaFixture()

		app1 = dea1.GetApp(0)
		app2 = dea1.GetApp(1)
		app3 = dea2.GetApp(2)

		simulator.SetCurrentHeartbeats(
			dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat(), app2.InstanceAtIndex(0).Heartbeat()),
			dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
		)
		simulator.SetDesiredState(app1.DesiredState(1), app2.DesiredState(1), app3.DesiredState(1))
		simulator.Tick(simulator.TicksToAttainFreshness)
	})

	Context("when a dea reports than an instance is no longer present", func() {
		BeforeEach(func() {
			simulator.SetCurrentHeartbeats(
				dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat()),
				dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
			)
		})

		It("should start the instance after a grace period", func() {
예제 #8
0
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfUndesiredRunningApps", Value: 0}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfRunningInstances", Value: 3}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfMissingIndices", Value: 0}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedInstances", Value: 0}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedIndices", Value: 0}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredApps", Value: 1}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredInstances", Value: 3}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredAppsPendingStaging", Value: 0}))
				})
			})

			Context("when a desired app has an instance starting and others running", func() {
				BeforeEach(func() {
					store.SyncDesiredState(a.DesiredState(3))

					startingHB := a.InstanceAtIndex(1).Heartbeat()
					startingHB.State = models.InstanceStateStarting
					store.SyncHeartbeats(dea.HeartbeatWith(
						a.InstanceAtIndex(0).Heartbeat(),
						startingHB,
						a.InstanceAtIndex(2).Heartbeat(),
					))
				})

				It("should have the correct stats", func() {
					context := metricsServer.Emit()
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithAllInstancesReporting", Value: 1}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithMissingInstances", Value: 0}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfUndesiredRunningApps", Value: 0}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfRunningInstances", Value: 3}))
					Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfMissingIndices", Value: 0}))
				})

				Ω(messageBus.PublishedMessages).Should(BeEmpty())
			})
		})

		Context("when the request contains the droplet and version", func() {
			var app appfixture.AppFixture
			var expectedApp *models.App
			var validRequestPayload string

			BeforeEach(func() {
				app = appfixture.NewAppFixture()

				instanceHeartbeats := []models.InstanceHeartbeat{
					app.InstanceAtIndex(0).Heartbeat(),
					app.InstanceAtIndex(1).Heartbeat(),
					app.InstanceAtIndex(2).Heartbeat(),
				}
				crashCount := models.CrashCount{
					AppGuid:       app.AppGuid,
					AppVersion:    app.AppVersion,
					InstanceIndex: 1,
					CrashCount:    2,
				}
				expectedApp = models.NewApp(
					app.AppGuid,
					app.AppVersion,
					app.DesiredState(3),
					instanceHeartbeats,
					map[int]models.CrashCount{1: crashCount},
예제 #10
0
				})
			})
		})
	})

	Context("when there are stop messages", func() {
		var keepAliveTime int
		var sentOn int64
		var err error
		var pendingMessage models.PendingStopMessage
		var storeSetErrInjector *fakestoreadapter.FakeStoreAdapterErrorInjector

		JustBeforeEach(func() {
			store.SyncHeartbeats(app.Heartbeat(2))

			pendingMessage = models.NewPendingStopMessage(time.Unix(100, 0), 30, keepAliveTime, app.AppGuid, app.AppVersion, app.InstanceAtIndex(0).InstanceGuid, models.PendingStopMessageReasonInvalid)
			pendingMessage.SentOn = sentOn
			store.SavePendingStopMessages(
				pendingMessage,
			)

			storeAdapter.SetErrInjector = storeSetErrInjector
			err = sender.Send()
		})

		BeforeEach(func() {
			keepAliveTime = 0
			sentOn = 0
			err = nil
			storeSetErrInjector = nil
		})
예제 #11
0
			cliRunner.StopAPIServer()
		})

		Context("when the store is fresh", func() {
			BeforeEach(func() {
				simulator.Tick(simulator.TicksToAttainFreshness)
				cliRunner.StartAPIServer(simulator.currentTimestamp)
			})

			It("should return the app", func(done Done) {
				replyTo := models.Guid()
				_, err := coordinator.MessageBus.Subscribe(replyTo, func(message *yagnats.Message) {
					defer GinkgoRecover()
					Ω(string(message.Payload)).Should(ContainSubstring(`"droplet":"%s"`, a.AppGuid))
					Ω(string(message.Payload)).Should(ContainSubstring(`"instances":2`))
					Ω(string(message.Payload)).Should(ContainSubstring(`"instance":"%s"`, a.InstanceAtIndex(0).InstanceGuid))

					close(done)
				})
				Ω(err).ShouldNot(HaveOccurred())

				err = coordinator.MessageBus.PublishWithReplyTo("app.state", replyTo, validRequest)
				Ω(err).ShouldNot(HaveOccurred())
			})
		})

		Context("when the store is not fresh", func() {
			BeforeEach(func() {
				simulator.Tick(simulator.TicksToAttainFreshness - 1)
				cliRunner.StartAPIServer(simulator.currentTimestamp)
			})
예제 #12
0
		Context("when the instance becomes desired", func() {
			BeforeEach(func() {
				simulator.SetDesiredState(app2.DesiredState(2))
				startStopListener.Reset()
				simulator.Tick(1)
			})

			It("should not send a stop message", func() {
				Ω(startStopListener.Stops).Should(HaveLen(0))
			})
		})

		Context("when the app is still running", func() {
			BeforeEach(func() {
				simulator.Tick(1)
			})

			It("should send a stop message, immediately, for the missing instance", func() {
				Ω(startStopListener.Stops).Should(HaveLen(1))

				stop := startStopListener.Stops[0]
				Ω(stop.AppGuid).Should(Equal(app2.AppGuid))
				Ω(stop.AppVersion).Should(Equal(app2.AppVersion))
				Ω(stop.InstanceGuid).Should(Equal(app2.InstanceAtIndex(1).InstanceGuid))
				Ω(stop.InstanceIndex).Should(Equal(1))
				Ω(stop.IsDuplicate).Should(BeFalse())
			})
		})
	})
})
예제 #13
0
			app = appfixture.NewAppFixture()
		})

		Context("when there is none saved", func() {
			It("should come back empty", func() {
				results, err := store.GetInstanceHeartbeatsForApp(app.AppGuid, app.AppVersion)
				Ω(err).ShouldNot(HaveOccurred())
				Ω(results).Should(BeEmpty())
			})
		})

		Context("when there is actual state saved", func() {
			var heartbeatA, heartbeatB models.InstanceHeartbeat

			BeforeEach(func() {
				heartbeatA = app.InstanceAtIndex(0).Heartbeat()
				heartbeatA.DeaGuid = "A"

				store.SyncHeartbeats(models.Heartbeat{
					DeaGuid: "A",
					InstanceHeartbeats: []models.InstanceHeartbeat{
						heartbeatA,
					},
				})

				heartbeatB = app.InstanceAtIndex(1).Heartbeat()
				heartbeatB.DeaGuid = "B"

				store.SyncHeartbeats(models.Heartbeat{
					DeaGuid: "B",
					InstanceHeartbeats: []models.InstanceHeartbeat{
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stopping Duplicate Instances", func() {
	var dea appfixture.DeaFixture
	var a appfixture.AppFixture

	Context("when there are multiple instances on the same index", func() {
		var instance0, instance1, duplicateInstance1 appfixture.Instance
		var heartbeat models.Heartbeat
		BeforeEach(func() {
			dea = appfixture.NewDeaFixture()
			a = dea.GetApp(0)

			instance0 = a.InstanceAtIndex(0)
			instance1 = a.InstanceAtIndex(1)
			duplicateInstance1 = a.InstanceAtIndex(1)
			duplicateInstance1.InstanceGuid = models.Guid()

			heartbeat = dea.HeartbeatWith(instance0.Heartbeat(), instance1.Heartbeat(), duplicateInstance1.Heartbeat())
			simulator.SetCurrentHeartbeats(heartbeat)

			simulator.SetDesiredState(a.DesiredState(2))

			simulator.Tick(simulator.TicksToAttainFreshness)
		})

		It("should not immediately stop anything", func() {
			Ω(startStopListener.Stops).Should(BeEmpty())
		})
예제 #15
0
	Describe("ToJson", func() {
		It("should, like, totally encode JSON", func() {
			jsonHeartbeat, err := NewHeartbeatFromJSON(heartbeat.ToJSON())

			Ω(err).ShouldNot(HaveOccurred())
			Ω(jsonHeartbeat).Should(Equal(heartbeat))
		})
	})

	Context("With a complex heartbeat", func() {
		var heartbeat Heartbeat
		var app appfixture.AppFixture
		BeforeEach(func() {
			app = appfixture.NewAppFixture()

			crashedHeartbeat := app.InstanceAtIndex(2).Heartbeat()
			crashedHeartbeat.State = InstanceStateCrashed

			startingHeartbeat := app.InstanceAtIndex(3).Heartbeat()
			startingHeartbeat.State = InstanceStateStarting

			evacuatingHeartbeat := app.InstanceAtIndex(4).Heartbeat()
			evacuatingHeartbeat.State = InstanceStateEvacuating

			heartbeat = Heartbeat{
				DeaGuid: "abc",
				InstanceHeartbeats: []InstanceHeartbeat{
					crashedHeartbeat,
					startingHeartbeat,
					evacuatingHeartbeat,
					app.InstanceAtIndex(0).Heartbeat(),
예제 #16
0
			wpool,
		)
		Expect(err).NotTo(HaveOccurred())
		err = storeAdapter.Connect()
		Expect(err).NotTo(HaveOccurred())

		store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())

		dea = appfixture.NewDeaFixture()
		app1 = dea.GetApp(0)
		app2 = dea.GetApp(1)
		app3 = dea.GetApp(2)
		app4 = dea.GetApp(3)

		actualState := []models.InstanceHeartbeat{
			app1.InstanceAtIndex(0).Heartbeat(),
			app1.InstanceAtIndex(1).Heartbeat(),
			app1.InstanceAtIndex(2).Heartbeat(),
			app2.InstanceAtIndex(0).Heartbeat(),
		}

		desiredState := []models.DesiredAppState{
			app1.DesiredState(1),
			app3.DesiredState(1),
		}

		crashCount = []models.CrashCount{
			{
				AppGuid:       app1.AppGuid,
				AppVersion:    app1.AppVersion,
				InstanceIndex: 1,