import (
"github.com/cloudfoundry/hm9000/models"
"github.com/cloudfoundry/hm9000/testhelpers/appfixture"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Evacuation and Shutdown", func() {
var dea appfixture.DeaFixture
var app appfixture.AppFixture
BeforeEach(func() {
dea = appfixture.NewDeaFixture()
app = dea.GetApp(0)
simulator.SetCurrentHeartbeats(dea.HeartbeatWith(app.InstanceAtIndex(0).Heartbeat()))
simulator.SetDesiredState(app.DesiredState(1))
simulator.Tick(simulator.TicksToAttainFreshness)
})
Describe("Shutdown handling by the evacuator component", func() {
Context("when a SHUTDOWN droplet.exited message comes in", func() {
BeforeEach(func() {
cliRunner.StartEvacuator(simulator.currentTimestamp)
coordinator.MessageBus.Publish("droplet.exited", app.InstanceAtIndex(0).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON())
})
AfterEach(func() {
cliRunner.StopEvacuator()
})
It("should immediately start the app", func() {
var _ = Describe("Crashes", func() {
var (
dea appfixture.DeaFixture
a appfixture.AppFixture
crashingHeartbeat models.Heartbeat
)
BeforeEach(func() {
dea = appfixture.NewDeaFixture()
a = dea.GetApp(0)
})
Describe("when all instances are crashed", func() {
BeforeEach(func() {
simulator.SetDesiredState(a.DesiredState(3))
crashingHeartbeat = dea.HeartbeatWith(
a.CrashedInstanceHeartbeatAtIndex(0),
a.CrashedInstanceHeartbeatAtIndex(1),
a.CrashedInstanceHeartbeatAtIndex(2),
)
simulator.SetCurrentHeartbeats(crashingHeartbeat)
simulator.Tick(simulator.TicksToAttainFreshness)
})
It("should only try to start instance at index 0", func() {
Ω(startStopListener.Starts).Should(HaveLen(1))
Ω(startStopListener.Starts[0].AppVersion).Should(Equal(a.AppVersion))
Ω(startStopListener.Starts[0].InstanceIndex).Should(Equal(0))
app1 = appfixture.NewAppFixture()
app2 = appfixture.NewAppFixture()
app3 = appfixture.NewAppFixture()
store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
})
AfterEach(func() {
storeAdapter.Disconnect()
})
Describe("Syncing desired state", func() {
BeforeEach(func() {
err := store.SyncDesiredState(
app1.DesiredState(1),
app2.DesiredState(1),
)
Expect(err).NotTo(HaveOccurred())
})
It("To store the passed in desired state", func() {
desiredState, err := store.GetDesiredState()
Expect(err).NotTo(HaveOccurred())
Expect(desiredState).To(HaveLen(2))
Expect(desiredState[app1.DesiredState(1).StoreKey()]).To(EqualDesiredState(app1.DesiredState(1)))
Expect(desiredState[app2.DesiredState(1).StoreKey()]).To(EqualDesiredState(app2.DesiredState(1)))
})
Context("When the desired state already exists", func() {
app1 = appfixture.NewAppFixture()
app2 = appfixture.NewAppFixture()
app3 = appfixture.NewAppFixture()
store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
})
AfterEach(func() {
storeAdapter.Disconnect()
})
Describe("Syncing desired state", func() {
BeforeEach(func() {
err := store.SyncDesiredState(
app1.DesiredState(1),
app2.DesiredState(1),
)
Ω(err).ShouldNot(HaveOccurred())
})
It("should store the passed in desired state", func() {
desiredState, err := store.GetDesiredState()
Ω(err).ShouldNot(HaveOccurred())
Ω(desiredState).Should(HaveLen(2))
Ω(desiredState[app1.DesiredState(1).StoreKey()]).Should(EqualDesiredState(app1.DesiredState(1)))
Ω(desiredState[app2.DesiredState(1).StoreKey()]).Should(EqualDesiredState(app2.DesiredState(1)))
})
Context("When the desired state already exists", func() {
Describe("LogDescription", func() {
It("should report the app guid and version", func() {
Ω(app().LogDescription()["AppGuid"]).Should(Equal(appGuid))
Ω(app().LogDescription()["AppVersion"]).Should(Equal(appVersion))
})
Context("when there is no desired state", func() {
It("should report that", func() {
Ω(app().LogDescription()["Desired"]).Should(Equal("None"))
})
})
Context("when there is a desired state", func() {
It("should report on the desired state", func() {
desired = fixture.DesiredState(2)
Ω(app().LogDescription()["Desired"]).Should(ContainSubstring(`"NumberOfInstances":2`))
Ω(app().LogDescription()["Desired"]).Should(ContainSubstring(`"State":"STARTED"`))
Ω(app().LogDescription()["Desired"]).Should(ContainSubstring(`"PackageState":"STAGED"`))
})
})
Context("When there are no heartbeats", func() {
It("should report that", func() {
Ω(app().LogDescription()["InstanceHeartbeats"]).Should(Equal("[]"))
})
})
Context("When there are heartbeats", func() {
It("should report on them", func() {
instanceHeartbeats = []InstanceHeartbeat{
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Simple Cases Test", func() {
var app1, app2 appfixture.AppFixture
BeforeEach(func() {
app1 = appfixture.NewAppFixture()
app2 = appfixture.NewAppFixture()
})
Context("when all running instances are desired", func() {
BeforeEach(func() {
simulator.SetCurrentHeartbeats(app1.Heartbeat(1), app2.Heartbeat(1))
simulator.SetDesiredState(app1.DesiredState(1), app2.DesiredState(1))
simulator.Tick(simulator.TicksToAttainFreshness)
simulator.Tick(1)
})
It("should not send any messages", func() {
Ω(startStopListener.Starts).Should(BeEmpty())
Ω(startStopListener.Stops).Should(BeEmpty())
})
})
Context("when a desired app is pending staging", func() {
Context("and it has a running instance", func() {
BeforeEach(func() {
desired := app1.DesiredState(1)
desired.PackageState = models.AppPackageStatePending
"github.com/nats-io/nats"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/pivotal-golang/localip"
)
var _ = Describe("Serving Metrics", func() {
var (
a appfixture.AppFixture
ip string
)
BeforeEach(func() {
a = appfixture.NewAppFixture()
simulator.SetDesiredState(a.DesiredState(2))
simulator.SetCurrentHeartbeats(a.Heartbeat(1))
var err error
ip, err = localip.LocalIP()
Ω(err).ShouldNot(HaveOccurred())
})
AfterEach(func() {
cliRunner.StopMetricsServer()
})
It("should register with the collector", func(done Done) {
cliRunner.StartMetricsServer(simulator.currentTimestamp)
guid := models.Guid()
instanceHeartbeats := []models.InstanceHeartbeat{
app.InstanceAtIndex(0).Heartbeat(),
app.InstanceAtIndex(1).Heartbeat(),
app.InstanceAtIndex(2).Heartbeat(),
}
crashCount := models.CrashCount{
AppGuid: app.AppGuid,
AppVersion: app.AppVersion,
InstanceIndex: 1,
CrashCount: 2,
}
expectedApp = models.NewApp(
app.AppGuid,
app.AppVersion,
app.DesiredState(3),
instanceHeartbeats,
map[int]models.CrashCount{1: crashCount},
)
store.SyncDesiredState(app.DesiredState(3))
store.SyncHeartbeats(app.Heartbeat(3))
store.SaveCrashCounts(crashCount)
validRequestPayload = fmt.Sprintf(`{"droplet":"%s","version":"%s"}`, app.AppGuid, app.AppVersion)
})
Context("when the store is fresh", func() {
BeforeEach(func() {
store.BumpDesiredFreshness(time.Unix(0, 0))
store.BumpActualFreshness(time.Unix(0, 0))
})
var dea1, dea2 appfixture.DeaFixture
var app1, app2, app3 appfixture.AppFixture
BeforeEach(func() {
dea1 = appfixture.NewDeaFixture()
dea2 = appfixture.NewDeaFixture()
app1 = dea1.GetApp(0)
app2 = dea1.GetApp(1)
app3 = dea2.GetApp(2)
simulator.SetCurrentHeartbeats(
dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat(), app2.InstanceAtIndex(0).Heartbeat()),
dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
)
simulator.SetDesiredState(app1.DesiredState(1), app2.DesiredState(1), app3.DesiredState(1))
simulator.Tick(simulator.TicksToAttainFreshness)
})
Context("when a dea reports than an instance is no longer present", func() {
BeforeEach(func() {
simulator.SetCurrentHeartbeats(
dea1.HeartbeatWith(app1.InstanceAtIndex(0).Heartbeat()),
dea2.HeartbeatWith(app3.InstanceAtIndex(0).Heartbeat()),
)
})
It("should start the instance after a grace period", func() {
simulator.Tick(simulator.GracePeriod)
Ω(startStopListener.StartCount()).Should(Equal(0))
simulator.Tick(1)
a3 appfixture.AppFixture
store storepackage.Store
resultChan chan desiredstatefetcher.DesiredStateFetcherResult
conf *config.Config
storeAdapter *fakestoreadapter.FakeStoreAdapter
)
BeforeEach(func() {
storeAdapter = fakestoreadapter.New()
resultChan = make(chan desiredstatefetcher.DesiredStateFetcherResult, 1)
a1 = appfixture.NewAppFixture()
a2 = appfixture.NewAppFixture()
a3 = appfixture.NewAppFixture()
stateServer.SetDesiredState([]models.DesiredAppState{
a1.DesiredState(1),
a2.DesiredState(1),
a3.DesiredState(1),
})
conf, _ = config.DefaultConfig()
store = storepackage.NewStore(conf, storeAdapter, fakelogger.NewFakeLogger())
fetcher = desiredstatefetcher.New(conf, store, fakemetricsaccountant.New(), httpclient.NewHttpClient(conf.SkipSSLVerification, conf.FetcherNetworkTimeout()), &timeprovider.RealTimeProvider{}, fakelogger.NewFakeLogger())
fetcher.Fetch(resultChan)
})
It("requests for the first set of data from the CC and stores the response", func() {
var desired map[string]models.DesiredAppState
var err error
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithAllInstancesReporting", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithMissingInstances", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfUndesiredRunningApps", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfRunningInstances", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfMissingIndices", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedInstances", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedIndices", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredApps", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredInstances", Value: -1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredAppsPendingStaging", Value: -1}))
})
})
Context("When a desired app is pending staging", func() {
BeforeEach(func() {
desired := a.DesiredState(3)
desired.PackageState = models.AppPackageStatePending
store.SyncDesiredState(desired)
store.SyncHeartbeats(a.Heartbeat(1))
})
It("should have the correct stats", func() {
context := metricsServer.Emit()
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithAllInstancesReporting", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfAppsWithMissingInstances", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfUndesiredRunningApps", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfRunningInstances", Value: 1}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfMissingIndices", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedInstances", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfCrashedIndices", Value: 0}))
Ω(context.Metrics).Should(ContainElement(instrumentation.Metric{Name: "NumberOfDesiredApps", Value: 0}))
Context("when a response with desired state is received", func() {
var (
a1 appfixture.AppFixture
a2 appfixture.AppFixture
stoppedApp appfixture.AppFixture
pendingStagingApp appfixture.AppFixture
failedToStageApp appfixture.AppFixture
deletedApp appfixture.AppFixture
pendingStagingDesiredState models.DesiredAppState
)
BeforeEach(func() {
deletedApp = appfixture.NewAppFixture()
store.SyncDesiredState(deletedApp.DesiredState(1))
a1 = appfixture.NewAppFixture()
a2 = appfixture.NewAppFixture()
stoppedApp = appfixture.NewAppFixture()
stoppedDesiredState := stoppedApp.DesiredState(1)
stoppedDesiredState.State = models.AppStateStopped
pendingStagingApp = appfixture.NewAppFixture()
pendingStagingDesiredState = pendingStagingApp.DesiredState(1)
pendingStagingDesiredState.PackageState = models.AppPackageStatePending
failedToStageApp = appfixture.NewAppFixture()
failedStagingDesiredState := failedToStageApp.DesiredState(1)
failedStagingDesiredState.PackageState = models.AppPackageStateFailed
instanceHeartbeats := []models.InstanceHeartbeat{
app.InstanceAtIndex(0).Heartbeat(),
app.InstanceAtIndex(1).Heartbeat(),
app.InstanceAtIndex(2).Heartbeat(),
}
crashCount := models.CrashCount{
AppGuid: app.AppGuid,
AppVersion: app.AppVersion,
InstanceIndex: 1,
CrashCount: 2,
}
expectedApp = AppResponse{
AppGuid: app.AppGuid,
AppVersion: app.AppVersion,
Desired: app.DesiredState(3),
InstanceHeartbeats: instanceHeartbeats,
CrashCounts: []models.CrashCount{crashCount},
}
store.SyncDesiredState(app.DesiredState(3))
store.SyncHeartbeats(app.Heartbeat(3))
store.SaveCrashCounts(crashCount)
validRequestPayload = fmt.Sprintf(`{"droplet":"%s","version":"%s"}`, app.AppGuid, app.AppVersion)
})
Context("when the store is fresh", func() {
BeforeEach(func() {
store.BumpDesiredFreshness(time.Unix(0, 0))
store.BumpActualFreshness(time.Unix(0, 0))
})
It("should not send any messages", func() {
err := sender.Send()
Ω(err).ShouldNot(HaveOccurred())
Ω(messageBus.PublishedMessages).Should(BeEmpty())
})
})
Context("when there are start messages", func() {
var keepAliveTime int
var sentOn int64
var err error
var pendingMessage models.PendingStartMessage
var storeSetErrInjector *fakestoreadapter.FakeStoreAdapterErrorInjector
JustBeforeEach(func() {
store.SyncDesiredState(app.DesiredState(1))
pendingMessage = models.NewPendingStartMessage(time.Unix(100, 0), 30, keepAliveTime, app.AppGuid, app.AppVersion, 0, 1.0, models.PendingStartMessageReasonInvalid)
pendingMessage.SentOn = sentOn
store.SavePendingStartMessages(
pendingMessage,
)
storeAdapter.SetErrInjector = storeSetErrInjector
err = sender.Send()
})
BeforeEach(func() {
keepAliveTime = 0
sentOn = 0
err = nil
storeSetErrInjector = nil
})
. "github.com/cloudfoundry/hm9000/desiredstatefetcher"
"github.com/cloudfoundry/hm9000/testhelpers/appfixture"
. "github.com/cloudfoundry/hm9000/testhelpers/custommatchers"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Desired State Server Response JSON", func() {
var (
a appfixture.AppFixture
response DesiredStateServerResponse
)
BeforeEach(func() {
a = appfixture.NewAppFixture()
desired, _ := json.Marshal(a.DesiredState(1))
responseJson := fmt.Sprintf(`
{
"results":{"%s":%s},
"bulk_token":{"id":17}
}
`, a.AppGuid, string(desired))
var err error
response, err = NewDesiredStateServerResponse([]byte(responseJson))
Ω(err).ShouldNot(HaveOccurred())
})
It("can parse from JSON", func() {
Ω(response.Results).Should(HaveLen(1))
Ω(response.Results[a.AppGuid]).Should(EqualDesiredState(a.DesiredState(1)))
return messagesArr
}
Describe("The steady state", func() {
Context("When there are no desired or running apps", func() {
It("should not send any start or stop messages", func() {
err := analyzer.Analyze()
Ω(err).ShouldNot(HaveOccurred())
Ω(startMessages()).Should(BeEmpty())
Ω(stopMessages()).Should(BeEmpty())
})
})
Context("When the desired number of instances and the running number of instances match", func() {
BeforeEach(func() {
desired := app.DesiredState(3)
desired.State = models.AppStateStarted
store.SyncDesiredState(
desired,
)
store.SyncHeartbeats(app.Heartbeat(3))
})
It("should not send any start or stop messages", func() {
err := analyzer.Analyze()
Ω(err).ShouldNot(HaveOccurred())
Ω(startMessages()).Should(BeEmpty())
Ω(stopMessages()).Should(BeEmpty())
})
})
})
dea = appfixture.NewDeaFixture()
app1 = dea.GetApp(0)
app2 = dea.GetApp(1)
app3 = dea.GetApp(2)
app4 = dea.GetApp(3)
actualState := []models.InstanceHeartbeat{
app1.InstanceAtIndex(0).Heartbeat(),
app1.InstanceAtIndex(1).Heartbeat(),
app1.InstanceAtIndex(2).Heartbeat(),
app2.InstanceAtIndex(0).Heartbeat(),
}
desiredState := []models.DesiredAppState{
app1.DesiredState(1),
app3.DesiredState(1),
}
crashCount = []models.CrashCount{
{
AppGuid: app1.AppGuid,
AppVersion: app1.AppVersion,
InstanceIndex: 1,
CrashCount: 12,
},
{
AppGuid: app1.AppGuid,
AppVersion: app1.AppVersion,
InstanceIndex: 2,
CrashCount: 17,
dea = appfixture.NewDeaFixture()
app1 = dea.GetApp(0)
app2 = dea.GetApp(1)
app3 = dea.GetApp(2)
app4 = dea.GetApp(3)
actualState := []models.InstanceHeartbeat{
app1.InstanceAtIndex(0).Heartbeat(),
app1.InstanceAtIndex(1).Heartbeat(),
app1.InstanceAtIndex(2).Heartbeat(),
app2.InstanceAtIndex(0).Heartbeat(),
}
desiredState := []models.DesiredAppState{
app1.DesiredState(1),
app3.DesiredState(1),
}
crashCount = []models.CrashCount{
{
AppGuid: app1.AppGuid,
AppVersion: app1.AppVersion,
InstanceIndex: 1,
CrashCount: 12,
},
{
AppGuid: app1.AppGuid,
AppVersion: app1.AppVersion,
InstanceIndex: 2,
CrashCount: 17,