Describe("Starting missing instances", func() {
Context("where an app has desired instances", func() {
BeforeEach(func() {
store.SyncDesiredState(
app.DesiredState(2),
)
})
Context("and none of the instances are running", func() {
It("should send a start message for each of the missing instances", func() {
err := analyzer.Analyze()
Ω(err).ShouldNot(HaveOccurred())
Ω(stopMessages()).Should(BeEmpty())
Ω(startMessages()).Should(HaveLen(2))
expectedMessage := models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 0, 1, models.PendingStartMessageReasonMissing)
Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
expectedMessage = models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 1, 1, models.PendingStartMessageReasonMissing)
Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
})
It("should set the priority to 1", func() {
analyzer.Analyze()
for _, message := range startMessages() {
Ω(message.Priority).Should(Equal(1.0))
}
})
})
Context("when there is an existing start message", func() {
func init() {
Describe("Agent", func() {
var (
logger boshlog.Logger
handler *fakembus.FakeHandler
platform *fakeplatform.FakePlatform
actionDispatcher *fakeagent.FakeActionDispatcher
jobSupervisor *fakejobsuper.FakeJobSupervisor
specService *fakeas.FakeV1Service
syslogServer *fakesyslog.FakeServer
settingsService *fakesettings.FakeSettingsService
uuidGenerator *fakeuuid.FakeGenerator
timeService *fakeclock.FakeClock
agent Agent
)
BeforeEach(func() {
logger = boshlog.NewLogger(boshlog.LevelNone)
handler = &fakembus.FakeHandler{}
platform = fakeplatform.NewFakePlatform()
actionDispatcher = &fakeagent.FakeActionDispatcher{}
jobSupervisor = fakejobsuper.NewFakeJobSupervisor()
specService = fakeas.NewFakeV1Service()
syslogServer = &fakesyslog.FakeServer{}
settingsService = &fakesettings.FakeSettingsService{}
uuidGenerator = &fakeuuid.FakeGenerator{}
timeService = fakeclock.NewFakeClock(time.Now())
agent = New(
logger,
handler,
platform,
actionDispatcher,
jobSupervisor,
specService,
syslogServer,
5*time.Millisecond,
settingsService,
uuidGenerator,
timeService,
)
})
Describe("Run", func() {
It("lets dispatcher handle requests arriving via handler", func() {
err := agent.Run()
Expect(err).ToNot(HaveOccurred())
expectedResp := boshhandler.NewValueResponse("pong")
actionDispatcher.DispatchResp = expectedResp
req := boshhandler.NewRequest("fake-reply", "fake-action", []byte("fake-payload"))
resp := handler.RunFunc(req)
Expect(actionDispatcher.DispatchReq).To(Equal(req))
Expect(resp).To(Equal(expectedResp))
})
It("resumes persistent actions *before* dispatching new requests", func() {
resumedBeforeStartingToDispatch := false
handler.RunCallBack = func() {
resumedBeforeStartingToDispatch = actionDispatcher.ResumedPreviouslyDispatchedTasks
}
err := agent.Run()
Expect(err).ToNot(HaveOccurred())
Expect(resumedBeforeStartingToDispatch).To(BeTrue())
})
Context("when heartbeats can be sent", func() {
BeforeEach(func() {
handler.KeepOnRunning()
})
BeforeEach(func() {
jobName := "fake-job"
nodeID := "node-id"
jobIndex := 1
specService.Spec = boshas.V1ApplySpec{
JobSpec: boshas.JobSpec{Name: &jobName},
Index: &jobIndex,
NodeID: nodeID,
}
jobSupervisor.StatusStatus = "fake-state"
platform.FakeVitalsService.GetVitals = boshvitals.Vitals{
Load: []string{"a", "b", "c"},
}
})
expectedJobName := "fake-job"
expectedJobIndex := 1
expectedNodeID := "node-id"
expectedHb := Heartbeat{
Job: &expectedJobName,
Index: &expectedJobIndex,
JobState: "fake-state",
NodeID: expectedNodeID,
Vitals: boshvitals.Vitals{Load: []string{"a", "b", "c"}},
}
It("sends initial heartbeat", func() {
// Configure periodic heartbeat every 5 hours
// so that we are sure that we will not receive it
agent = New(
logger,
handler,
platform,
actionDispatcher,
jobSupervisor,
specService,
syslogServer,
5*time.Hour,
settingsService,
uuidGenerator,
timeService,
)
// Immediately exit after sending initial heartbeat
handler.SendErr = errors.New("stop")
err := agent.Run()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("stop"))
Expect(handler.SendInputs()).To(Equal([]fakembus.SendInput{
{
Target: boshhandler.HealthMonitor,
Topic: boshhandler.Heartbeat,
Message: expectedHb,
},
}))
})
It("sends periodic heartbeats", func() {
sentRequests := 0
handler.SendCallback = func(_ fakembus.SendInput) {
sentRequests++
if sentRequests == 3 {
handler.SendErr = errors.New("stop")
}
}
err := agent.Run()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("stop"))
Expect(handler.SendInputs()).To(Equal([]fakembus.SendInput{
{
Target: boshhandler.HealthMonitor,
Topic: boshhandler.Heartbeat,
Message: expectedHb,
},
{
Target: boshhandler.HealthMonitor,
Topic: boshhandler.Heartbeat,
Message: expectedHb,
},
{
Target: boshhandler.HealthMonitor,
Topic: boshhandler.Heartbeat,
Message: expectedHb,
},
}))
})
})
Context("when the agent fails to get job spec for a heartbeat", func() {
BeforeEach(func() {
specService.GetErr = errors.New("fake-spec-service-error")
handler.KeepOnRunning()
})
It("returns the error", func() {
err := agent.Run()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("fake-spec-service-error"))
})
})
Context("when the agent fails to get vitals for a heartbeat", func() {
BeforeEach(func() {
platform.FakeVitalsService.GetErr = errors.New("fake-vitals-service-error")
handler.KeepOnRunning()
})
It("returns the error", func() {
err := agent.Run()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("fake-vitals-service-error"))
})
})
It("sends job monitoring alerts to health manager", func() {
handler.KeepOnRunning()
monitAlert := boshalert.MonitAlert{
ID: "fake-monit-alert",
Service: "fake-service",
Event: "fake-event",
Action: "fake-action",
Date: "Sun, 22 May 2011 20:07:41 +0500",
Description: "fake-description",
}
jobSupervisor.JobFailureAlert = &monitAlert
// Fail the first time handler.Send is called for an alert (ignore heartbeats)
handler.SendCallback = func(input fakembus.SendInput) {
if input.Topic == boshhandler.Alert {
handler.SendErr = errors.New("stop")
}
}
err := agent.Run()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("stop"))
expectedAlert := boshalert.Alert{
ID: "fake-monit-alert",
Severity: boshalert.SeverityDefault,
Title: "fake-service - fake-event - fake-action",
Summary: "fake-description",
CreatedAt: int64(1306076861),
}
Expect(handler.SendInputs()).To(ContainElement(fakembus.SendInput{
Target: boshhandler.HealthMonitor,
Topic: boshhandler.Alert,
Message: expectedAlert,
}))
})
It("sends ssh alerts to health manager", func() {
handler.KeepOnRunning()
syslogMsg := boshsyslog.Msg{Content: "disconnected by user"}
syslogServer.StartFirstSyslogMsg = &syslogMsg
uuidGenerator.GeneratedUUID = "fake-uuid"
// Fail the first time handler.Send is called for an alert (ignore heartbeats)
handler.SendCallback = func(input fakembus.SendInput) {
if input.Topic == boshhandler.Alert {
handler.SendErr = errors.New("stop")
}
}
err := agent.Run()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("stop"))
expectedAlert := boshalert.Alert{
ID: "fake-uuid",
Severity: boshalert.SeverityWarning,
Title: "SSH Logout",
Summary: "disconnected by user",
CreatedAt: timeService.Now().Unix(),
}
Expect(handler.SendInputs()).To(ContainElement(fakembus.SendInput{
Target: boshhandler.HealthMonitor,
Topic: boshhandler.Alert,
Message: expectedAlert,
}))
})
})
})
}
monitAlert.Event = event
monitAdapter := NewMonitAdapter(monitAlert, settingsService, timeService)
builtAlert, err := monitAdapter.Alert()
Expect(err).ToNot(HaveOccurred())
Expect(builtAlert.Severity).To(Equal(expectedSeverity))
}
})
It("defaults CreatedAt to time.Now(), when parsing the supplied time fails", func() {
monitAlert := buildMonitAlert()
monitAlert.Date = "Thu, 02 May 2013 20:07:0"
monitAdapter := NewMonitAdapter(monitAlert, settingsService, timeService)
builtAlert, err := monitAdapter.Alert()
Expect(err).ToNot(HaveOccurred())
Expect(builtAlert.CreatedAt).To(Equal(timeService.Now().Unix()))
})
It("sets the title with ips", func() {
monitAlert := buildMonitAlert()
settingsService.Settings.Networks = boshsettings.Networks{
"fake-net1": boshsettings.Network{IP: "192.168.0.1"},
"fake-net2": boshsettings.Network{IP: "10.0.0.1"},
}
monitAdapter := NewMonitAdapter(monitAlert, settingsService, timeService)
builtAlert, err := monitAdapter.Alert()
Expect(err).ToNot(HaveOccurred())
Expect(builtAlert.Title).To(Equal("nats (10.0.0.1, 192.168.0.1) - does not exist - restart"))
})
})
Expect(outputBuffer).To(test_helpers.Say("InstanceGuid"))
Expect(outputBuffer).To(test_helpers.SayLine("a0s9f-u9a8sf-aasdioasdjoi"))
Expect(outputBuffer).To(test_helpers.Say("Cell ID"))
Expect(outputBuffer).To(test_helpers.SayLine("cell-12"))
Expect(outputBuffer).To(test_helpers.Say("Ip"))
Expect(outputBuffer).To(test_helpers.SayLine("10.85.12.100"))
Expect(outputBuffer).To(test_helpers.Say("Port Mapping"))
Expect(outputBuffer).To(test_helpers.Say("1234:3000"))
Expect(outputBuffer).To(test_helpers.Say("5555:6666"))
Expect(outputBuffer).To(test_helpers.SayNewLine())
Expect(outputBuffer).To(test_helpers.Say("Uptime"))
roundedTimeSince := roundTime(fakeClock.Now(), time.Unix(0, epochTime*1e9))
Expect(outputBuffer).To(test_helpers.Say(roundedTimeSince))
Expect(outputBuffer).To(test_helpers.SayNewLine())
Expect(outputBuffer).To(test_helpers.Say("Crash Count"))
Expect(outputBuffer).To(test_helpers.Say("0"))
Expect(outputBuffer).To(test_helpers.SayNewLine())
Expect(outputBuffer).To(test_helpers.Say("CPU"))
Expect(outputBuffer).To(test_helpers.Say("23.46%"))
Expect(outputBuffer).To(test_helpers.SayNewLine())
Expect(outputBuffer).To(test_helpers.Say("Memory"))
Expect(outputBuffer).To(test_helpers.Say("640K"))
Expect(outputBuffer).To(test_helpers.SayNewLine())
}
Context("when a staging task completes", func() {
var taskResponse *models.TaskCallbackResponse
var annotationJson []byte
BeforeEach(func() {
var err error
annotationJson, err = json.Marshal(cc_messages.StagingTaskAnnotation{
Lifecycle: "fake",
})
Expect(err).NotTo(HaveOccurred())
})
JustBeforeEach(func() {
createdAt := fakeClock.Now().UnixNano()
fakeClock.Increment(stagingDurationNano)
taskResponse = &models.TaskCallbackResponse{
TaskGuid: "the-task-guid",
CreatedAt: createdAt,
Result: `{
"buildpack_key":"buildpack-key",
"detected_buildpack":"Some Buildpack",
"execution_metadata":"{\"start_command\":\"./some-start-command\"}",
"detected_start_command":{"web":"./some-start-command"}
}`,
Annotation: string(annotationJson),
}
handler.StagingComplete(responseRecorder, postTask(taskResponse))
Expect(err).NotTo(HaveOccurred())
clients = map[string]*repfakes.FakeSimClient{}
zones = map[string]auctionrunner.Zone{}
logger = lager.NewLogger("fakelogger")
logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG))
})
AfterEach(func() {
workPool.Stop()
})
Context("when there are no cells", func() {
It("immediately returns everything as having failed, incrementing the attempt number", func() {
startAuction := BuildLRPAuction("pg-7", "domain", 0, linuxRootFSURL, 10, 10, clock.Now())
taskAuction := BuildTaskAuction(BuildTask("tg-1", "domain", linuxRootFSURL, 0, 0), clock.Now())
auctionRequest := auctiontypes.AuctionRequest{
LRPs: []auctiontypes.LRPAuction{startAuction},
Tasks: []auctiontypes.TaskAuction{taskAuction},
}
By("no auctions are marked successful")
scheduler := auctionrunner.NewScheduler(workPool, map[string]auctionrunner.Zone{}, clock, logger)
results := scheduler.Schedule(auctionRequest)
Expect(results.SuccessfulLRPs).To(BeEmpty())
Expect(results.SuccessfulTasks).To(BeEmpty())
By("all lrp starts are marked failed, and their attempts are incremented")
Describe("Scanner", func() {
var (
fakeResource *rfakes.FakeResource
times chan time.Time
)
BeforeEach(func() {
fakeResource = new(rfakes.FakeResource)
fakeTracker.InitReturns(fakeResource, nil)
times = make(chan time.Time, 100)
fakeResource.CheckStub = func(atc.Source, atc.Version) ([]atc.Version, error) {
times <- fakeClock.Now()
return nil, nil
}
})
JustBeforeEach(func() {
process = ifrit.Invoke(radar.Scanner(lagertest.NewTestLogger("test"), "some-resource"))
})
AfterEach(func() {
process.Signal(os.Interrupt)
<-process.Wait()
})
Context("when the lease cannot be acquired", func() {
BeforeEach(func() {
Eventually(process.Wait()).Should(Receive(BeNil()))
close(syncChannel)
})
Context("on a specified interval", func() {
It("should sync", func() {
process = ifrit.Invoke(syncerRunner)
var t1 time.Time
var t2 time.Time
clock.Increment(syncInterval + 100*time.Millisecond)
select {
case <-syncChannel:
t1 = clock.Now()
case <-time.After(2 * syncInterval):
Fail("did not receive a sync event")
}
clock.Increment(syncInterval + 100*time.Millisecond)
select {
case <-syncChannel:
t2 = clock.Now()
case <-time.After(2 * syncInterval):
Fail("did not receive a sync event")
}
Expect(t2.Sub(t1)).To(BeNumerically("~", syncInterval, 100*time.Millisecond))
})
Expect(pendingStarts).To(BeEmpty())
})
})
Context("when the reason is DEA_EVACUATION", func() {
BeforeEach(func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
})
})
It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
pendingStarts, err := store.GetPendingStartMessages()
Expect(err).NotTo(HaveOccurred())
expectedStartMessage := models.NewPendingStartMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
expectedStartMessage.SkipVerification = true
Expect(pendingStarts).To(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
})
})
Context("when the reason is DEA_SHUTDOWN", func() {
BeforeEach(func() {
messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON(),
})
})
It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
pendingStarts, err := store.GetPendingStartMessages()
Expect(batch.HasWork).NotTo(Receive())
starts, tasks := batch.DedupeAndDrain()
Expect(starts).To(BeEmpty())
Expect(tasks).To(BeEmpty())
})
Describe("adding work", func() {
Context("when adding start auctions", func() {
BeforeEach(func() {
lrpStart = BuildLRPStartRequest("pg-1", "domain", []int{1}, "linux", 10, 10)
batch.AddLRPStarts([]auctioneer.LRPStartRequest{lrpStart})
})
It("makes the start auction available when drained", func() {
lrpAuctions, _ := batch.DedupeAndDrain()
Expect(lrpAuctions).To(ConsistOf(BuildLRPAuctions(lrpStart, clock.Now())))
})
It("should have work", func() {
Expect(batch.HasWork).To(Receive())
})
})
Context("when adding tasks", func() {
BeforeEach(func() {
task = BuildTaskStartRequest("tg-1", "domain", "linux", 10, 10)
batch.AddTasks([]auctioneer.TaskStartRequest{task})
})
It("makes the stop auction available when drained", func() {
_, taskAuctions := batch.DedupeAndDrain()
})
It("should not error", func() {
Expect(err).NotTo(HaveOccurred())
})
Context("when the message should be kept alive", func() {
BeforeEach(func() {
keepAliveTime = 30
})
It("should update the sent on times", func() {
messages, _ := store.GetPendingStartMessages()
Expect(messages).To(HaveLen(1))
for _, message := range messages {
Expect(message.SentOn).To(Equal(timeProvider.Now().Unix()))
}
})
Context("when saving the start messages fails", func() {
BeforeEach(func() {
storeSetErrInjector = fakestoreadapter.NewFakeStoreAdapterErrorInjector("start", errors.New("oops"))
})
It("should return an error", func() {
Expect(err).To(HaveOccurred())
})
})
})
Context("when the KeepAlive = 0", func() {
})
Context("when the router emits a router.start", func() {
Context("using an interval", func() {
JustBeforeEach(func() {
routerStartMessages <- &nats.Msg{
Data: []byte(`{
"minimumRegisterIntervalInSeconds":1,
"pruneThresholdInSeconds": 3
}`),
}
})
It("should emit routes with the frequency of the passed-in-interval", func() {
Eventually(syncerRunner.Events().Emit, 2).Should(Receive())
t1 := clock.Now()
Eventually(syncerRunner.Events().Emit, 2).Should(Receive())
t2 := clock.Now()
Expect(t2.Sub(t1)).To(BeNumerically("~", 1*time.Second, 200*time.Millisecond))
})
It("should only greet the router once", func() {
Eventually(greetings).Should(Receive())
Consistently(greetings, 1).ShouldNot(Receive())
})
})
})
Context("when the router does not emit a router.start", func() {
bbsClient.DesiredLRPByProcessGuidReturns(&models.DesiredLRP{
LogGuid: logGuid,
ProcessGuid: guid,
}, nil)
actualLRP := &models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey(guid, 5, "some-domain"),
ActualLRPInstanceKey: models.NewActualLRPInstanceKey("instanceId", "some-cell"),
ActualLRPNetInfo: models.NewActualLRPNetInfo(
"host",
models.NewPortMapping(5432, 7890),
models.NewPortMapping(1234, uint32(recipebuilder.DefaultPort)),
),
State: models.ActualLRPStateRunning,
Since: fakeClock.Now().UnixNano(),
}
bbsClient.ActualLRPGroupsByProcessGuidReturns([]*models.ActualLRPGroup{{
Instance: actualLRP},
}, nil)
})
Context("when the LRP has been running for a while", func() {
var expectedSinceTime int64
BeforeEach(func() {
expectedSinceTime = fakeClock.Now().Unix()
fakeClock.Increment(5 * time.Second)
})