func (h *ActualLRPLifecycleHandler) StartActualLRP(logger lager.Logger, w http.ResponseWriter, req *http.Request) {
var err error
logger = logger.Session("start-actual-lrp")
request := &models.StartActualLRPRequest{}
response := &models.ActualLRPLifecycleResponse{}
defer func() { exitIfUnrecoverable(logger, h.exitChan, response.Error) }()
defer writeResponse(w, response)
err = parseRequest(logger, req, request)
if err != nil {
response.Error = models.ConvertError(err)
return
}
before, after, err := h.db.StartActualLRP(logger, request.ActualLrpKey, request.ActualLrpInstanceKey, request.ActualLrpNetInfo)
if err != nil {
response.Error = models.ConvertError(err)
return
}
if before == nil {
go h.actualHub.Emit(models.NewActualLRPCreatedEvent(after))
} else if !before.Equal(after) {
go h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
}
}
func (h *ActualLRPLifecycleHandler) ClaimActualLRP(logger lager.Logger, w http.ResponseWriter, req *http.Request) {
var err error
logger = logger.Session("claim-actual-lrp")
request := &models.ClaimActualLRPRequest{}
response := &models.ActualLRPLifecycleResponse{}
defer func() { exitIfUnrecoverable(logger, h.exitChan, response.Error) }()
defer writeResponse(w, response)
err = parseRequest(logger, req, request)
if err != nil {
response.Error = models.ConvertError(err)
return
}
before, after, err := h.db.ClaimActualLRP(logger, request.ProcessGuid, request.Index, request.ActualLrpInstanceKey)
if err != nil {
response.Error = models.ConvertError(err)
return
}
if !after.Equal(before) {
go h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
}
}
func (h *ActualLRPLifecycleHandler) CrashActualLRP(logger lager.Logger, w http.ResponseWriter, req *http.Request) {
logger = logger.Session("crash-actual-lrp")
request := &models.CrashActualLRPRequest{}
response := &models.ActualLRPLifecycleResponse{}
defer func() { exitIfUnrecoverable(logger, h.exitChan, response.Error) }()
defer writeResponse(w, response)
err := parseRequest(logger, req, request)
if err != nil {
response.Error = models.ConvertError(err)
return
}
actualLRPKey := request.ActualLrpKey
actualLRPInstanceKey := request.ActualLrpInstanceKey
before, after, shouldRestart, err := h.db.CrashActualLRP(logger, actualLRPKey, actualLRPInstanceKey, request.ErrorMessage)
if err != nil {
response.Error = models.ConvertError(err)
return
}
if shouldRestart {
desiredLRP, err := h.desiredLRPDB.DesiredLRPByProcessGuid(logger, actualLRPKey.ProcessGuid)
if err != nil {
logger.Error("failed-fetching-desired-lrp", err)
response.Error = models.ConvertError(err)
return
}
schedInfo := desiredLRP.DesiredLRPSchedulingInfo()
startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(&schedInfo, int(actualLRPKey.Index))
logger.Info("start-lrp-auction-request", lager.Data{"app_guid": schedInfo.ProcessGuid, "index": int(actualLRPKey.Index)})
err = h.auctioneerClient.RequestLRPAuctions(logger, []*auctioneer.LRPStartRequest{&startRequest})
logger.Info("finished-lrp-auction-request", lager.Data{"app_guid": schedInfo.ProcessGuid, "index": int(actualLRPKey.Index)})
if err != nil {
logger.Error("failed-requesting-auction", err)
response.Error = models.ConvertError(err)
return
}
}
actualLRP, _ := after.Resolve()
go h.actualHub.Emit(models.NewActualLRPCrashedEvent(actualLRP))
go h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
}
func (h *EvacuationHandler) unclaimAndRequestAuction(logger lager.Logger, lrpKey *models.ActualLRPKey) error {
before, after, err := h.actualLRPDB.UnclaimActualLRP(logger, lrpKey)
if err != nil {
return err
}
go h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
desiredLRP, err := h.desiredLRPDB.DesiredLRPByProcessGuid(logger, lrpKey.ProcessGuid)
if err != nil {
logger.Error("failed-fetching-desired-lrp", err)
return nil
}
schedInfo := desiredLRP.DesiredLRPSchedulingInfo()
startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(&schedInfo, int(lrpKey.Index))
err = h.auctioneerClient.RequestLRPAuctions([]*auctioneer.LRPStartRequest{&startRequest})
if err != nil {
logger.Error("failed-requesting-auction", err)
}
return nil
}
func (h *ActualLRPLifecycleHandler) FailActualLRP(logger lager.Logger, w http.ResponseWriter, req *http.Request) {
var err error
logger = logger.Session("fail-actual-lrp")
request := &models.FailActualLRPRequest{}
response := &models.ActualLRPLifecycleResponse{}
defer func() { exitIfUnrecoverable(logger, h.exitChan, response.Error) }()
defer writeResponse(w, response)
err = parseRequest(logger, req, request)
if err != nil {
response.Error = models.ConvertError(err)
return
}
before, after, err := h.db.FailActualLRP(logger, request.ActualLrpKey, request.ErrorMessage)
if err != nil {
response.Error = models.ConvertError(err)
return
}
go h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
}
unclaimedKeys = append(unclaimedKeys, key)
}
Expect(unclaimedKeys).To(ContainElement(&unclaimingActualLRP1.ActualLRPKey))
Expect(unclaimedKeys).To(ContainElement(&unclaimingActualLRP2.ActualLRPKey))
Eventually(actualHub.EmitCallCount).Should(Equal(2))
changeEvents := []*models.ActualLRPChangedEvent{}
for i := 0; i < actualHub.EmitCallCount(); i++ {
event := actualHub.EmitArgsForCall(i)
if changeEvent, ok := event.(*models.ActualLRPChangedEvent); ok {
changeEvents = append(changeEvents, changeEvent)
}
}
group1 := &models.ActualLRPGroup{Instance: unclaimingActualLRP1}
group2 := &models.ActualLRPGroup{Instance: unclaimingActualLRP2}
Expect(changeEvents).To(ContainElement(models.NewActualLRPChangedEvent(group1, group1)))
Expect(changeEvents).To(ContainElement(models.NewActualLRPChangedEvent(group2, group2)))
})
Context("when the DB returns an unrecoverable error", func() {
BeforeEach(func() {
fakeLRPDB.UnclaimActualLRPReturns(nil, nil, models.NewUnrecoverableError(nil))
})
It("logs the error", func() {
Eventually(logger).Should(gbytes.Say("unrecoverable-error"))
})
It("returns the error", func() {
Expect(err).To(HaveOccurred())
Expect(err).Should(Equal(models.NewUnrecoverableError(nil)))
It("returns the event", func() {
event, err := eventSource.Next()
Expect(err).NotTo(HaveOccurred())
actualLRPCreatedEvent, ok := event.(*models.ActualLRPCreatedEvent)
Expect(ok).To(BeTrue())
Expect(actualLRPCreatedEvent).To(Equal(expectedEvent))
})
})
Context("when receiving a ActualLRPChangedEvent", func() {
var expectedEvent *models.ActualLRPChangedEvent
BeforeEach(func() {
expectedEvent = models.NewActualLRPChangedEvent(actualLRPGroup, actualLRPGroup)
payload, err := proto.Marshal(expectedEvent)
Expect(err).NotTo(HaveOccurred())
payload = []byte(base64.StdEncoding.EncodeToString(payload))
fakeRawEventSource.NextReturns(
sse.Event{
ID: "sup",
Name: string(expectedEvent.EventType()),
Data: payload,
},
nil,
)
})
It("returns the event", func() {
func (h *LRPConvergenceController) ConvergeLRPs(logger lager.Logger) error {
logger = h.logger.Session("converge-lrps")
var err error
logger.Debug("listing-cells")
var cellSet models.CellSet
cellSet, err = h.serviceClient.Cells(logger)
if err == models.ErrResourceNotFound {
logger.Info("no-cells-found")
cellSet = models.CellSet{}
} else if err != nil {
logger.Error("failed-listing-cells", err)
// convergence should run again later
return nil
}
logger.Debug("succeeded-listing-cells")
startRequests, keysWithMissingCells, keysToRetire := h.db.ConvergeLRPs(logger, cellSet)
retireLogger := logger.WithData(lager.Data{"retiring_lrp_count": len(keysToRetire)})
works := []func(){}
for _, key := range keysToRetire {
key := key
works = append(works, func() {
h.retirer.RetireActualLRP(retireLogger, key.ProcessGuid, key.Index)
})
}
errChan := make(chan *models.Error, 1)
startRequestLock := &sync.Mutex{}
for _, key := range keysWithMissingCells {
key := key
works = append(works, func() {
before, after, err := h.db.UnclaimActualLRP(logger, key.Key)
if err == nil {
h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(key.SchedulingInfo, int(key.Key.Index))
startRequestLock.Lock()
startRequests = append(startRequests, &startRequest)
startRequestLock.Unlock()
} else {
bbsErr := models.ConvertError(err)
if bbsErr.GetType() != models.Error_Unrecoverable {
return
}
logger.Error("unrecoverable-error", bbsErr)
select {
case errChan <- bbsErr:
default:
}
}
})
}
var throttler *workpool.Throttler
throttler, err = workpool.NewThrottler(h.convergenceWorkersSize, works)
if err != nil {
logger.Error("failed-constructing-throttler", err, lager.Data{"max_workers": h.convergenceWorkersSize, "num_works": len(works)})
return nil
}
retireLogger.Debug("retiring-actual-lrps")
throttler.Work()
retireLogger.Debug("done-retiring-actual-lrps")
select {
case err := <-errChan:
return err
default:
}
startLogger := logger.WithData(lager.Data{"start_requests_count": len(startRequests)})
if len(startRequests) > 0 {
startLogger.Debug("requesting-start-auctions")
err = h.auctioneerClient.RequestLRPAuctions(logger, startRequests)
if err != nil {
startLogger.Error("failed-to-request-starts", err, lager.Data{"lrp_start_auctions": startRequests})
}
startLogger.Debug("done-requesting-start-auctions")
}
return nil
}