func (c *convergence) addStartRequestFromSchedulingInfo(logger lager.Logger, schedulingInfo *models.DesiredLRPSchedulingInfo, indices ...int) {
if len(indices) == 0 {
return
}
c.startRequestsMutex.Lock()
defer c.startRequestsMutex.Unlock()
if startRequest, ok := c.guidsToStartRequests[schedulingInfo.ProcessGuid]; ok {
startRequest.Indices = append(startRequest.Indices, indices...)
return
}
startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(schedulingInfo, indices...)
c.guidsToStartRequests[schedulingInfo.ProcessGuid] = &startRequest
}
func (h *ActualLRPLifecycleHandler) CrashActualLRP(logger lager.Logger, w http.ResponseWriter, req *http.Request) {
logger = logger.Session("crash-actual-lrp")
request := &models.CrashActualLRPRequest{}
response := &models.ActualLRPLifecycleResponse{}
defer func() { exitIfUnrecoverable(logger, h.exitChan, response.Error) }()
defer writeResponse(w, response)
err := parseRequest(logger, req, request)
if err != nil {
response.Error = models.ConvertError(err)
return
}
actualLRPKey := request.ActualLrpKey
actualLRPInstanceKey := request.ActualLrpInstanceKey
before, after, shouldRestart, err := h.db.CrashActualLRP(logger, actualLRPKey, actualLRPInstanceKey, request.ErrorMessage)
if err != nil {
response.Error = models.ConvertError(err)
return
}
if shouldRestart {
desiredLRP, err := h.desiredLRPDB.DesiredLRPByProcessGuid(logger, actualLRPKey.ProcessGuid)
if err != nil {
logger.Error("failed-fetching-desired-lrp", err)
response.Error = models.ConvertError(err)
return
}
schedInfo := desiredLRP.DesiredLRPSchedulingInfo()
startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(&schedInfo, int(actualLRPKey.Index))
logger.Info("start-lrp-auction-request", lager.Data{"app_guid": schedInfo.ProcessGuid, "index": int(actualLRPKey.Index)})
err = h.auctioneerClient.RequestLRPAuctions(logger, []*auctioneer.LRPStartRequest{&startRequest})
logger.Info("finished-lrp-auction-request", lager.Data{"app_guid": schedInfo.ProcessGuid, "index": int(actualLRPKey.Index)})
if err != nil {
logger.Error("failed-requesting-auction", err)
response.Error = models.ConvertError(err)
return
}
}
actualLRP, _ := after.Resolve()
go h.actualHub.Emit(models.NewActualLRPCrashedEvent(actualLRP))
go h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
}
func (h *DesiredLRPHandler) startInstanceRange(logger lager.Logger, lower, upper int32, schedulingInfo *models.DesiredLRPSchedulingInfo) {
logger = logger.Session("start-instance-range", lager.Data{"lower": lower, "upper": upper})
logger.Info("starting")
defer logger.Info("complete")
keys := make([]*models.ActualLRPKey, upper-lower)
i := 0
for actualIndex := lower; actualIndex < upper; actualIndex++ {
key := models.NewActualLRPKey(schedulingInfo.ProcessGuid, int32(actualIndex), schedulingInfo.Domain)
keys[i] = &key
i++
}
createdIndices := h.createUnclaimedActualLRPs(logger, keys)
start := auctioneer.NewLRPStartRequestFromSchedulingInfo(schedulingInfo, createdIndices...)
logger.Info("start-lrp-auction-request", lager.Data{"app_guid": schedulingInfo.ProcessGuid, "indices": createdIndices})
err := h.auctioneerClient.RequestLRPAuctions(logger, []*auctioneer.LRPStartRequest{&start})
logger.Info("finished-lrp-auction-request", lager.Data{"app_guid": schedulingInfo.ProcessGuid, "indices": createdIndices})
if err != nil {
logger.Error("failed-to-request-auction", err)
}
}
func (h *EvacuationHandler) unclaimAndRequestAuction(logger lager.Logger, lrpKey *models.ActualLRPKey) error {
before, after, err := h.actualLRPDB.UnclaimActualLRP(logger, lrpKey)
if err != nil {
return err
}
go h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
desiredLRP, err := h.desiredLRPDB.DesiredLRPByProcessGuid(logger, lrpKey.ProcessGuid)
if err != nil {
logger.Error("failed-fetching-desired-lrp", err)
return nil
}
schedInfo := desiredLRP.DesiredLRPSchedulingInfo()
startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(&schedInfo, int(lrpKey.Index))
err = h.auctioneerClient.RequestLRPAuctions([]*auctioneer.LRPStartRequest{&startRequest})
if err != nil {
logger.Error("failed-requesting-auction", err)
}
return nil
}
BeforeEach(func() {
fakeServiceClient.CellsReturns(nil, models.ErrResourceNotFound)
})
It("calls ConvergeLRPs with an empty CellSet", func() {
Expect(err).NotTo(HaveOccurred())
Expect(fakeLRPDB.ConvergeLRPsCallCount()).To(Equal(1))
_, actualCellSet := fakeLRPDB.ConvergeLRPsArgsForCall(0)
Expect(actualCellSet).To(BeEquivalentTo(models.CellSet{}))
})
})
It("auctions off the returned keys", func() {
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))
unclaimedStartRequest1 := auctioneer.NewLRPStartRequestFromSchedulingInfo(&desiredLRP1, 0)
unclaimedStartRequest2 := auctioneer.NewLRPStartRequestFromSchedulingInfo(&desiredLRP2, 1)
expectedStartRequests := append(keysToAuction, &unclaimedStartRequest1)
expectedStartRequests = append(expectedStartRequests, &unclaimedStartRequest2)
_, startAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
Expect(startAuctions).To(HaveLen(4))
Expect(startAuctions).To(ConsistOf(expectedStartRequests))
})
Context("when no lrps to auction", func() {
BeforeEach(func() {
fakeLRPDB.ConvergeLRPsReturns(nil, nil, nil)
})
Expect(responseRecorder.Code).To(Equal(http.StatusOK))
response := &models.ActualLRPLifecycleResponse{}
err := response.Unmarshal(responseRecorder.Body.Bytes())
Expect(err).NotTo(HaveOccurred())
Expect(response.Error).To(BeNil())
Expect(fakeDesiredLRPDB.DesiredLRPByProcessGuidCallCount()).To(Equal(1))
_, processGuid := fakeDesiredLRPDB.DesiredLRPByProcessGuidArgsForCall(0)
Expect(processGuid).To(Equal("process-guid"))
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))
_, startRequests := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
Expect(startRequests).To(HaveLen(1))
schedulingInfo := desiredLRP.DesiredLRPSchedulingInfo()
expectedStartRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(&schedulingInfo, 1)
Expect(startRequests[0]).To(BeEquivalentTo(&expectedStartRequest))
})
})
Context("when the actual lrp should not be restarted (e.g., crashed)", func() {
BeforeEach(func() {
fakeActualLRPDB.CrashActualLRPReturns(&models.ActualLRPGroup{Instance: &actualLRP}, &models.ActualLRPGroup{Instance: &actualLRP}, false, nil)
})
It("does not request an auction", func() {
Expect(responseRecorder.Code).To(Equal(http.StatusOK))
response := &models.ActualLRPLifecycleResponse{}
err := response.Unmarshal(responseRecorder.Body.Bytes())
Expect(err).NotTo(HaveOccurred())
Expect(response.Error).To(BeNil())
Expect(*actualLRPKey).To(Equal(actual.ActualLRPKey))
Expect(*actualLRPInstanceKey).To(Equal(actual.ActualLRPInstanceKey))
Expect(*actualLrpNetInfo).To(Equal(actual.ActualLRPNetInfo))
Expect(ttl).To(BeEquivalentTo(60))
})
It("unclaims the lrp and requests an auction", func() {
Expect(fakeActualLRPDB.UnclaimActualLRPCallCount()).To(Equal(1))
_, actualLRPKey, actualLRPInstanceKey, actualLrpNetInfo, ttl := fakeEvacuationDB.EvacuateActualLRPArgsForCall(0)
Expect(*actualLRPKey).To(Equal(actual.ActualLRPKey))
Expect(*actualLRPInstanceKey).To(Equal(actual.ActualLRPInstanceKey))
Expect(*actualLrpNetInfo).To(Equal(actual.ActualLRPNetInfo))
Expect(ttl).To(BeEquivalentTo(60))
schedulingInfo := desiredLRP.DesiredLRPSchedulingInfo()
expectedStartRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(&schedulingInfo, int(actual.Index))
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))
_, startRequests := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
Expect(startRequests).To(Equal([]*auctioneer.LRPStartRequest{&expectedStartRequest}))
})
It("emits events to the hub", func() {
Eventually(actualHub.EmitCallCount).Should(Equal(2))
for i := 0; i < actualHub.EmitCallCount(); i++ {
switch event := actualHub.EmitArgsForCall(i).(type) {
case *models.ActualLRPCreatedEvent:
Expect(event.ActualLrpGroup).To(Equal(&models.ActualLRPGroup{Evacuating: afterActual}))
case *models.ActualLRPChangedEvent:
Expect(event.Before).To(Equal(&models.ActualLRPGroup{Instance: actual}))
func (h *LRPConvergenceController) ConvergeLRPs(logger lager.Logger) error {
logger = h.logger.Session("converge-lrps")
var err error
logger.Debug("listing-cells")
var cellSet models.CellSet
cellSet, err = h.serviceClient.Cells(logger)
if err == models.ErrResourceNotFound {
logger.Info("no-cells-found")
cellSet = models.CellSet{}
} else if err != nil {
logger.Error("failed-listing-cells", err)
// convergence should run again later
return nil
}
logger.Debug("succeeded-listing-cells")
startRequests, keysWithMissingCells, keysToRetire := h.db.ConvergeLRPs(logger, cellSet)
retireLogger := logger.WithData(lager.Data{"retiring_lrp_count": len(keysToRetire)})
works := []func(){}
for _, key := range keysToRetire {
key := key
works = append(works, func() {
h.retirer.RetireActualLRP(retireLogger, key.ProcessGuid, key.Index)
})
}
errChan := make(chan *models.Error, 1)
startRequestLock := &sync.Mutex{}
for _, key := range keysWithMissingCells {
key := key
works = append(works, func() {
before, after, err := h.db.UnclaimActualLRP(logger, key.Key)
if err == nil {
h.actualHub.Emit(models.NewActualLRPChangedEvent(before, after))
startRequest := auctioneer.NewLRPStartRequestFromSchedulingInfo(key.SchedulingInfo, int(key.Key.Index))
startRequestLock.Lock()
startRequests = append(startRequests, &startRequest)
startRequestLock.Unlock()
} else {
bbsErr := models.ConvertError(err)
if bbsErr.GetType() != models.Error_Unrecoverable {
return
}
logger.Error("unrecoverable-error", bbsErr)
select {
case errChan <- bbsErr:
default:
}
}
})
}
var throttler *workpool.Throttler
throttler, err = workpool.NewThrottler(h.convergenceWorkersSize, works)
if err != nil {
logger.Error("failed-constructing-throttler", err, lager.Data{"max_workers": h.convergenceWorkersSize, "num_works": len(works)})
return nil
}
retireLogger.Debug("retiring-actual-lrps")
throttler.Work()
retireLogger.Debug("done-retiring-actual-lrps")
select {
case err := <-errChan:
return err
default:
}
startLogger := logger.WithData(lager.Data{"start_requests_count": len(startRequests)})
if len(startRequests) > 0 {
startLogger.Debug("requesting-start-auctions")
err = h.auctioneerClient.RequestLRPAuctions(logger, startRequests)
if err != nil {
startLogger.Error("failed-to-request-starts", err, lager.Data{"lrp_start_auctions": startRequests})
}
startLogger.Debug("done-requesting-start-auctions")
}
return nil
}