Пример #1
0
	Describe("Starting missing instances", func() {
		Context("where an app has desired instances", func() {
			BeforeEach(func() {
				store.SyncDesiredState(
					app.DesiredState(2),
				)
			})

			Context("and none of the instances are running", func() {
				It("should send a start message for each of the missing instances", func() {
					err := analyzer.Analyze()
					Ω(err).ShouldNot(HaveOccurred())
					Ω(stopMessages()).Should(BeEmpty())
					Ω(startMessages()).Should(HaveLen(2))

					expectedMessage := models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 0, 1, models.PendingStartMessageReasonMissing)
					Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))

					expectedMessage = models.NewPendingStartMessage(clock.Now(), conf.GracePeriod(), 0, app.AppGuid, app.AppVersion, 1, 1, models.PendingStartMessageReasonMissing)
					Ω(startMessages()).Should(ContainElement(EqualPendingStartMessage(expectedMessage)))
				})

				It("should set the priority to 1", func() {
					analyzer.Analyze()
					for _, message := range startMessages() {
						Ω(message.Priority).Should(Equal(1.0))
					}
				})
			})

			Context("when there is an existing start message", func() {
Пример #2
0
func init() {
	Describe("Agent", func() {
		var (
			logger           boshlog.Logger
			handler          *fakembus.FakeHandler
			platform         *fakeplatform.FakePlatform
			actionDispatcher *fakeagent.FakeActionDispatcher
			jobSupervisor    *fakejobsuper.FakeJobSupervisor
			specService      *fakeas.FakeV1Service
			syslogServer     *fakesyslog.FakeServer
			settingsService  *fakesettings.FakeSettingsService
			uuidGenerator    *fakeuuid.FakeGenerator
			timeService      *fakeclock.FakeClock
			agent            Agent
		)

		BeforeEach(func() {
			logger = boshlog.NewLogger(boshlog.LevelNone)
			handler = &fakembus.FakeHandler{}
			platform = fakeplatform.NewFakePlatform()
			actionDispatcher = &fakeagent.FakeActionDispatcher{}
			jobSupervisor = fakejobsuper.NewFakeJobSupervisor()
			specService = fakeas.NewFakeV1Service()
			syslogServer = &fakesyslog.FakeServer{}
			settingsService = &fakesettings.FakeSettingsService{}
			uuidGenerator = &fakeuuid.FakeGenerator{}
			timeService = fakeclock.NewFakeClock(time.Now())
			agent = New(
				logger,
				handler,
				platform,
				actionDispatcher,
				jobSupervisor,
				specService,
				syslogServer,
				5*time.Millisecond,
				settingsService,
				uuidGenerator,
				timeService,
			)
		})

		Describe("Run", func() {
			It("lets dispatcher handle requests arriving via handler", func() {
				err := agent.Run()
				Expect(err).ToNot(HaveOccurred())

				expectedResp := boshhandler.NewValueResponse("pong")
				actionDispatcher.DispatchResp = expectedResp

				req := boshhandler.NewRequest("fake-reply", "fake-action", []byte("fake-payload"))
				resp := handler.RunFunc(req)

				Expect(actionDispatcher.DispatchReq).To(Equal(req))
				Expect(resp).To(Equal(expectedResp))
			})

			It("resumes persistent actions *before* dispatching new requests", func() {
				resumedBeforeStartingToDispatch := false
				handler.RunCallBack = func() {
					resumedBeforeStartingToDispatch = actionDispatcher.ResumedPreviouslyDispatchedTasks
				}

				err := agent.Run()
				Expect(err).ToNot(HaveOccurred())
				Expect(resumedBeforeStartingToDispatch).To(BeTrue())
			})

			Context("when heartbeats can be sent", func() {
				BeforeEach(func() {
					handler.KeepOnRunning()
				})

				BeforeEach(func() {
					jobName := "fake-job"
					nodeID := "node-id"
					jobIndex := 1
					specService.Spec = boshas.V1ApplySpec{
						JobSpec: boshas.JobSpec{Name: &jobName},
						Index:   &jobIndex,
						NodeID:  nodeID,
					}

					jobSupervisor.StatusStatus = "fake-state"

					platform.FakeVitalsService.GetVitals = boshvitals.Vitals{
						Load: []string{"a", "b", "c"},
					}
				})

				expectedJobName := "fake-job"
				expectedJobIndex := 1
				expectedNodeID := "node-id"
				expectedHb := Heartbeat{
					Job:      &expectedJobName,
					Index:    &expectedJobIndex,
					JobState: "fake-state",
					NodeID:   expectedNodeID,
					Vitals:   boshvitals.Vitals{Load: []string{"a", "b", "c"}},
				}

				It("sends initial heartbeat", func() {
					// Configure periodic heartbeat every 5 hours
					// so that we are sure that we will not receive it
					agent = New(
						logger,
						handler,
						platform,
						actionDispatcher,
						jobSupervisor,
						specService,
						syslogServer,
						5*time.Hour,
						settingsService,
						uuidGenerator,
						timeService,
					)

					// Immediately exit after sending initial heartbeat
					handler.SendErr = errors.New("stop")

					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("stop"))

					Expect(handler.SendInputs()).To(Equal([]fakembus.SendInput{
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
					}))
				})

				It("sends periodic heartbeats", func() {
					sentRequests := 0
					handler.SendCallback = func(_ fakembus.SendInput) {
						sentRequests++
						if sentRequests == 3 {
							handler.SendErr = errors.New("stop")
						}
					}

					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("stop"))

					Expect(handler.SendInputs()).To(Equal([]fakembus.SendInput{
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
						{
							Target:  boshhandler.HealthMonitor,
							Topic:   boshhandler.Heartbeat,
							Message: expectedHb,
						},
					}))
				})
			})

			Context("when the agent fails to get job spec for a heartbeat", func() {
				BeforeEach(func() {
					specService.GetErr = errors.New("fake-spec-service-error")
					handler.KeepOnRunning()
				})

				It("returns the error", func() {
					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-spec-service-error"))
				})
			})

			Context("when the agent fails to get vitals for a heartbeat", func() {
				BeforeEach(func() {
					platform.FakeVitalsService.GetErr = errors.New("fake-vitals-service-error")
					handler.KeepOnRunning()
				})

				It("returns the error", func() {
					err := agent.Run()
					Expect(err).To(HaveOccurred())
					Expect(err.Error()).To(ContainSubstring("fake-vitals-service-error"))
				})
			})

			It("sends job monitoring alerts to health manager", func() {
				handler.KeepOnRunning()

				monitAlert := boshalert.MonitAlert{
					ID:          "fake-monit-alert",
					Service:     "fake-service",
					Event:       "fake-event",
					Action:      "fake-action",
					Date:        "Sun, 22 May 2011 20:07:41 +0500",
					Description: "fake-description",
				}
				jobSupervisor.JobFailureAlert = &monitAlert

				// Fail the first time handler.Send is called for an alert (ignore heartbeats)
				handler.SendCallback = func(input fakembus.SendInput) {
					if input.Topic == boshhandler.Alert {
						handler.SendErr = errors.New("stop")
					}
				}

				err := agent.Run()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("stop"))

				expectedAlert := boshalert.Alert{
					ID:        "fake-monit-alert",
					Severity:  boshalert.SeverityDefault,
					Title:     "fake-service - fake-event - fake-action",
					Summary:   "fake-description",
					CreatedAt: int64(1306076861),
				}

				Expect(handler.SendInputs()).To(ContainElement(fakembus.SendInput{
					Target:  boshhandler.HealthMonitor,
					Topic:   boshhandler.Alert,
					Message: expectedAlert,
				}))
			})

			It("sends ssh alerts to health manager", func() {
				handler.KeepOnRunning()

				syslogMsg := boshsyslog.Msg{Content: "disconnected by user"}
				syslogServer.StartFirstSyslogMsg = &syslogMsg

				uuidGenerator.GeneratedUUID = "fake-uuid"

				// Fail the first time handler.Send is called for an alert (ignore heartbeats)
				handler.SendCallback = func(input fakembus.SendInput) {
					if input.Topic == boshhandler.Alert {
						handler.SendErr = errors.New("stop")
					}
				}

				err := agent.Run()
				Expect(err).To(HaveOccurred())
				Expect(err.Error()).To(ContainSubstring("stop"))

				expectedAlert := boshalert.Alert{
					ID:        "fake-uuid",
					Severity:  boshalert.SeverityWarning,
					Title:     "SSH Logout",
					Summary:   "disconnected by user",
					CreatedAt: timeService.Now().Unix(),
				}

				Expect(handler.SendInputs()).To(ContainElement(fakembus.SendInput{
					Target:  boshhandler.HealthMonitor,
					Topic:   boshhandler.Alert,
					Message: expectedAlert,
				}))
			})
		})
	})
}
Пример #3
0
				monitAlert.Event = event
				monitAdapter := NewMonitAdapter(monitAlert, settingsService, timeService)
				builtAlert, err := monitAdapter.Alert()
				Expect(err).ToNot(HaveOccurred())
				Expect(builtAlert.Severity).To(Equal(expectedSeverity))
			}
		})

		It("defaults CreatedAt to time.Now(), when parsing the supplied time fails", func() {
			monitAlert := buildMonitAlert()
			monitAlert.Date = "Thu, 02 May 2013 20:07:0"

			monitAdapter := NewMonitAdapter(monitAlert, settingsService, timeService)
			builtAlert, err := monitAdapter.Alert()
			Expect(err).ToNot(HaveOccurred())
			Expect(builtAlert.CreatedAt).To(Equal(timeService.Now().Unix()))
		})

		It("sets the title with ips", func() {
			monitAlert := buildMonitAlert()
			settingsService.Settings.Networks = boshsettings.Networks{
				"fake-net1": boshsettings.Network{IP: "192.168.0.1"},
				"fake-net2": boshsettings.Network{IP: "10.0.0.1"},
			}

			monitAdapter := NewMonitAdapter(monitAlert, settingsService, timeService)
			builtAlert, err := monitAdapter.Alert()
			Expect(err).ToNot(HaveOccurred())
			Expect(builtAlert.Title).To(Equal("nats (10.0.0.1, 192.168.0.1) - does not exist - restart"))
		})
	})
			Expect(outputBuffer).To(test_helpers.Say("InstanceGuid"))
			Expect(outputBuffer).To(test_helpers.SayLine("a0s9f-u9a8sf-aasdioasdjoi"))

			Expect(outputBuffer).To(test_helpers.Say("Cell ID"))
			Expect(outputBuffer).To(test_helpers.SayLine("cell-12"))

			Expect(outputBuffer).To(test_helpers.Say("Ip"))
			Expect(outputBuffer).To(test_helpers.SayLine("10.85.12.100"))

			Expect(outputBuffer).To(test_helpers.Say("Port Mapping"))
			Expect(outputBuffer).To(test_helpers.Say("1234:3000"))
			Expect(outputBuffer).To(test_helpers.Say("5555:6666"))
			Expect(outputBuffer).To(test_helpers.SayNewLine())

			Expect(outputBuffer).To(test_helpers.Say("Uptime"))
			roundedTimeSince := roundTime(fakeClock.Now(), time.Unix(0, epochTime*1e9))
			Expect(outputBuffer).To(test_helpers.Say(roundedTimeSince))
			Expect(outputBuffer).To(test_helpers.SayNewLine())

			Expect(outputBuffer).To(test_helpers.Say("Crash Count"))
			Expect(outputBuffer).To(test_helpers.Say("0"))
			Expect(outputBuffer).To(test_helpers.SayNewLine())

			Expect(outputBuffer).To(test_helpers.Say("CPU"))
			Expect(outputBuffer).To(test_helpers.Say("23.46%"))
			Expect(outputBuffer).To(test_helpers.SayNewLine())

			Expect(outputBuffer).To(test_helpers.Say("Memory"))
			Expect(outputBuffer).To(test_helpers.Say("640K"))
			Expect(outputBuffer).To(test_helpers.SayNewLine())
	}

	Context("when a staging task completes", func() {
		var taskResponse *models.TaskCallbackResponse
		var annotationJson []byte

		BeforeEach(func() {
			var err error
			annotationJson, err = json.Marshal(cc_messages.StagingTaskAnnotation{
				Lifecycle: "fake",
			})
			Expect(err).NotTo(HaveOccurred())
		})

		JustBeforeEach(func() {
			createdAt := fakeClock.Now().UnixNano()
			fakeClock.Increment(stagingDurationNano)

			taskResponse = &models.TaskCallbackResponse{
				TaskGuid:  "the-task-guid",
				CreatedAt: createdAt,
				Result: `{
					"buildpack_key":"buildpack-key",
					"detected_buildpack":"Some Buildpack",
					"execution_metadata":"{\"start_command\":\"./some-start-command\"}",
					"detected_start_command":{"web":"./some-start-command"}
				}`,
				Annotation: string(annotationJson),
			}

			handler.StagingComplete(responseRecorder, postTask(taskResponse))
Пример #6
0
		Expect(err).NotTo(HaveOccurred())

		clients = map[string]*repfakes.FakeSimClient{}
		zones = map[string]auctionrunner.Zone{}

		logger = lager.NewLogger("fakelogger")
		logger.RegisterSink(lager.NewWriterSink(GinkgoWriter, lager.DEBUG))
	})

	AfterEach(func() {
		workPool.Stop()
	})

	Context("when there are no cells", func() {
		It("immediately returns everything as having failed, incrementing the attempt number", func() {
			startAuction := BuildLRPAuction("pg-7", "domain", 0, linuxRootFSURL, 10, 10, clock.Now())

			taskAuction := BuildTaskAuction(BuildTask("tg-1", "domain", linuxRootFSURL, 0, 0), clock.Now())

			auctionRequest := auctiontypes.AuctionRequest{
				LRPs:  []auctiontypes.LRPAuction{startAuction},
				Tasks: []auctiontypes.TaskAuction{taskAuction},
			}

			By("no auctions are marked successful")
			scheduler := auctionrunner.NewScheduler(workPool, map[string]auctionrunner.Zone{}, clock, logger)
			results := scheduler.Schedule(auctionRequest)
			Expect(results.SuccessfulLRPs).To(BeEmpty())
			Expect(results.SuccessfulTasks).To(BeEmpty())

			By("all lrp starts are marked failed, and their attempts are incremented")
Пример #7
0
	Describe("Scanner", func() {
		var (
			fakeResource *rfakes.FakeResource

			times chan time.Time
		)

		BeforeEach(func() {
			fakeResource = new(rfakes.FakeResource)
			fakeTracker.InitReturns(fakeResource, nil)

			times = make(chan time.Time, 100)

			fakeResource.CheckStub = func(atc.Source, atc.Version) ([]atc.Version, error) {
				times <- fakeClock.Now()
				return nil, nil
			}
		})

		JustBeforeEach(func() {
			process = ifrit.Invoke(radar.Scanner(lagertest.NewTestLogger("test"), "some-resource"))
		})

		AfterEach(func() {
			process.Signal(os.Interrupt)
			<-process.Wait()
		})

		Context("when the lease cannot be acquired", func() {
			BeforeEach(func() {
Пример #8
0
		Eventually(process.Wait()).Should(Receive(BeNil()))
		close(syncChannel)
	})

	Context("on a specified interval", func() {

		It("should sync", func() {
			process = ifrit.Invoke(syncerRunner)
			var t1 time.Time
			var t2 time.Time

			clock.Increment(syncInterval + 100*time.Millisecond)

			select {
			case <-syncChannel:
				t1 = clock.Now()
			case <-time.After(2 * syncInterval):
				Fail("did not receive a sync event")
			}

			clock.Increment(syncInterval + 100*time.Millisecond)

			select {
			case <-syncChannel:
				t2 = clock.Now()
			case <-time.After(2 * syncInterval):
				Fail("did not receive a sync event")
			}

			Expect(t2.Sub(t1)).To(BeNumerically("~", syncInterval, 100*time.Millisecond))
		})
Пример #9
0
				Expect(pendingStarts).To(BeEmpty())
			})
		})

		Context("when the reason is DEA_EVACUATION", func() {
			BeforeEach(func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAEvacuation).ToJSON(),
				})
			})

			It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
				pendingStarts, err := store.GetPendingStartMessages()
				Expect(err).NotTo(HaveOccurred())

				expectedStartMessage := models.NewPendingStartMessage(clock.Now(), 0, conf.GracePeriod(), app.AppGuid, app.AppVersion, 1, 2.0, models.PendingStartMessageReasonEvacuating)
				expectedStartMessage.SkipVerification = true

				Expect(pendingStarts).To(ContainElement(EqualPendingStartMessage(expectedStartMessage)))
			})
		})

		Context("when the reason is DEA_SHUTDOWN", func() {
			BeforeEach(func() {
				messageBus.SubjectCallbacks("droplet.exited")[0](&nats.Msg{
					Data: app.InstanceAtIndex(1).DropletExited(models.DropletExitedReasonDEAShutdown).ToJSON(),
				})
			})

			It("should put a high priority pending start message (configured to skip verification) into the queue", func() {
				pendingStarts, err := store.GetPendingStartMessages()
Пример #10
0
		Expect(batch.HasWork).NotTo(Receive())
		starts, tasks := batch.DedupeAndDrain()
		Expect(starts).To(BeEmpty())
		Expect(tasks).To(BeEmpty())
	})

	Describe("adding work", func() {
		Context("when adding start auctions", func() {
			BeforeEach(func() {
				lrpStart = BuildLRPStartRequest("pg-1", "domain", []int{1}, "linux", 10, 10)
				batch.AddLRPStarts([]auctioneer.LRPStartRequest{lrpStart})
			})

			It("makes the start auction available when drained", func() {
				lrpAuctions, _ := batch.DedupeAndDrain()
				Expect(lrpAuctions).To(ConsistOf(BuildLRPAuctions(lrpStart, clock.Now())))
			})

			It("should have work", func() {
				Expect(batch.HasWork).To(Receive())
			})
		})

		Context("when adding tasks", func() {
			BeforeEach(func() {
				task = BuildTaskStartRequest("tg-1", "domain", "linux", 10, 10)
				batch.AddTasks([]auctioneer.TaskStartRequest{task})
			})

			It("makes the stop auction available when drained", func() {
				_, taskAuctions := batch.DedupeAndDrain()
Пример #11
0
			})

			It("should not error", func() {
				Expect(err).NotTo(HaveOccurred())
			})

			Context("when the message should be kept alive", func() {
				BeforeEach(func() {
					keepAliveTime = 30
				})

				It("should update the sent on times", func() {
					messages, _ := store.GetPendingStartMessages()
					Expect(messages).To(HaveLen(1))
					for _, message := range messages {
						Expect(message.SentOn).To(Equal(timeProvider.Now().Unix()))
					}
				})

				Context("when saving the start messages fails", func() {
					BeforeEach(func() {
						storeSetErrInjector = fakestoreadapter.NewFakeStoreAdapterErrorInjector("start", errors.New("oops"))
					})

					It("should return an error", func() {
						Expect(err).To(HaveOccurred())
					})
				})
			})

			Context("when the KeepAlive = 0", func() {
Пример #12
0
		})

		Context("when the router emits a router.start", func() {
			Context("using an interval", func() {
				JustBeforeEach(func() {
					routerStartMessages <- &nats.Msg{
						Data: []byte(`{
						"minimumRegisterIntervalInSeconds":1,
						"pruneThresholdInSeconds": 3
						}`),
					}
				})

				It("should emit routes with the frequency of the passed-in-interval", func() {
					Eventually(syncerRunner.Events().Emit, 2).Should(Receive())
					t1 := clock.Now()

					Eventually(syncerRunner.Events().Emit, 2).Should(Receive())
					t2 := clock.Now()

					Expect(t2.Sub(t1)).To(BeNumerically("~", 1*time.Second, 200*time.Millisecond))
				})

				It("should only greet the router once", func() {
					Eventually(greetings).Should(Receive())
					Consistently(greetings, 1).ShouldNot(Receive())
				})
			})
		})

		Context("when the router does not emit a router.start", func() {
Пример #13
0
			bbsClient.DesiredLRPByProcessGuidReturns(&models.DesiredLRP{
				LogGuid:     logGuid,
				ProcessGuid: guid,
			}, nil)

			actualLRP := &models.ActualLRP{
				ActualLRPKey:         models.NewActualLRPKey(guid, 5, "some-domain"),
				ActualLRPInstanceKey: models.NewActualLRPInstanceKey("instanceId", "some-cell"),
				ActualLRPNetInfo: models.NewActualLRPNetInfo(
					"host",
					models.NewPortMapping(5432, 7890),
					models.NewPortMapping(1234, uint32(recipebuilder.DefaultPort)),
				),
				State: models.ActualLRPStateRunning,
				Since: fakeClock.Now().UnixNano(),
			}

			bbsClient.ActualLRPGroupsByProcessGuidReturns([]*models.ActualLRPGroup{{
				Instance: actualLRP},
			}, nil)
		})

		Context("when the LRP has been running for a while", func() {
			var expectedSinceTime int64

			BeforeEach(func() {
				expectedSinceTime = fakeClock.Now().Unix()
				fakeClock.Increment(5 * time.Second)
			})