func (db *ETCDDB) unclaimActualLRPWithIndex(
logger lager.Logger,
lrp *models.ActualLRP,
storeIndex uint64,
actualLRPKey *models.ActualLRPKey,
actualLRPInstanceKey *models.ActualLRPInstanceKey,
) (change stateChange, err error) {
logger = logger.Session("unclaim-actual-lrp-with-index")
defer logger.Debug("complete", lager.Data{"state_change": change, "error": err})
if !lrp.ActualLRPKey.Equal(actualLRPKey) {
logger.Error("failed-actual-lrp-key-differs", models.ErrActualLRPCannotBeUnclaimed)
return stateDidNotChange, models.ErrActualLRPCannotBeUnclaimed
}
if lrp.State == models.ActualLRPStateUnclaimed {
logger.Info("already-unclaimed")
return stateDidNotChange, nil
}
if !lrp.ActualLRPInstanceKey.Equal(actualLRPInstanceKey) {
logger.Error("failed-actual-lrp-instance-key-differs", models.ErrActualLRPCannotBeUnclaimed)
return stateDidNotChange, models.ErrActualLRPCannotBeUnclaimed
}
lrp.Since = db.clock.Now().UnixNano()
lrp.State = models.ActualLRPStateUnclaimed
lrp.ActualLRPInstanceKey = models.ActualLRPInstanceKey{}
lrp.ActualLRPNetInfo = models.EmptyActualLRPNetInfo()
lrp.ModificationTag.Increment()
err = lrp.Validate()
if err != nil {
logger.Error("failed-to-validate-unclaimed-lrp", err)
return stateDidNotChange, models.NewError(models.Error_InvalidRecord, err.Error())
}
lrpData, serialErr := db.serializeModel(logger, lrp)
if serialErr != nil {
logger.Error("failed-to-marshal-unclaimed-lrp", serialErr)
return stateDidNotChange, serialErr
}
_, err = db.client.CompareAndSwap(ActualLRPSchemaPath(actualLRPKey.ProcessGuid, actualLRPKey.Index), lrpData, 0, storeIndex)
if err != nil {
logger.Error("failed-to-compare-and-swap", err)
return stateDidNotChange, models.ErrActualLRPCannotBeUnclaimed
}
logger.Debug("changed-to-unclaimed")
return stateDidChange, nil
}
func itValidatesPresenceOfPlacementError(lrp *models.ActualLRP) {
Context("when placement error is set", func() {
BeforeEach(func() {
lrp.PlacementError = "insufficient capacity"
})
It("validate does not return an error", func() {
Expect(lrp.Validate()).NotTo(HaveOccurred())
})
})
Context("when placement error is not set", func() {
BeforeEach(func() {
lrp.PlacementError = ""
})
It("validate does not return an error", func() {
Expect(lrp.Validate()).NotTo(HaveOccurred())
})
})
}
func itValidatesAbsenceOfPlacementError(lrp *models.ActualLRP) {
Context("when placement error is set", func() {
BeforeEach(func() {
lrp.PlacementError = "insufficient capacity"
})
It("validate returns an error", func() {
err := lrp.Validate()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("placement error"))
})
})
Context("when placement error is not set", func() {
BeforeEach(func() {
lrp.PlacementError = ""
})
It("validate does not return an error", func() {
Expect(lrp.Validate()).NotTo(HaveOccurred())
})
})
}
func itValidatesAbsenceOfNetInfo(lrp *models.ActualLRP) {
Context("when net info is set", func() {
BeforeEach(func() {
lrp.ActualLRPNetInfo = models.NewActualLRPNetInfo("1.2.3.4")
})
It("validate returns an error", func() {
err := lrp.Validate()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("net info"))
})
})
Context("when net info is not set", func() {
BeforeEach(func() {
lrp.ActualLRPNetInfo = models.ActualLRPNetInfo{}
})
It("validate does not return an error", func() {
Expect(lrp.Validate()).NotTo(HaveOccurred())
})
})
}
func itValidatesAbsenceOfTheInstanceKey(lrp *models.ActualLRP) {
Context("when the instance key is set", func() {
BeforeEach(func() {
lrp.ActualLRPInstanceKey = models.NewActualLRPInstanceKey("some-instance", "some-cell")
})
It("validate returns an error", func() {
err := lrp.Validate()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("instance key"))
})
})
Context("when the instance key is not set", func() {
BeforeEach(func() {
lrp.ActualLRPInstanceKey = models.ActualLRPInstanceKey{}
})
It("validate does not return an error", func() {
Expect(lrp.Validate()).NotTo(HaveOccurred())
})
})
}
func itValidatesPresenceOfTheLRPKey(lrp *models.ActualLRP) {
Context("when the lrp key is set", func() {
BeforeEach(func() {
lrp.ActualLRPKey = models.NewActualLRPKey("some-guid", 1, "domain")
})
It("validate does not return an error", func() {
Expect(lrp.Validate()).NotTo(HaveOccurred())
})
})
Context("when the lrp key is not set", func() {
BeforeEach(func() {
lrp.ActualLRPKey = models.ActualLRPKey{}
})
It("validate returns an error", func() {
err := lrp.Validate()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("process_guid"))
})
})
}
func (db *ETCDDB) gatherAndOptionallyPruneActualLRPs(logger lager.Logger, guids map[string]struct{}, doPrune bool, lmc *LRPMetricCounter) (map[string]map[int32]*models.ActualLRP, error) {
response, modelErr := db.fetchRecursiveRaw(logger, ActualLRPSchemaRoot)
if modelErr == models.ErrResourceNotFound {
logger.Info("actual-lrp-schema-root-not-found")
return map[string]map[int32]*models.ActualLRP{}, nil
}
if modelErr != nil {
return nil, modelErr
}
actuals := map[string]map[int32]*models.ActualLRP{}
var guidKeysToDelete, indexKeysToDelete []string
var actualsToDelete []string
var guidsLock, actualsLock, guidKeysToDeleteLock, indexKeysToDeleteLock,
crashingDesiredsLock, actualsToDeleteLock sync.Mutex
logger.Debug("walking-actual-lrp-tree")
works := []func(){}
crashingDesireds := map[string]struct{}{}
for _, guidGroup := range response.Nodes {
guidGroup := guidGroup
works = append(works, func() {
guidGroupWillBeEmpty := true
for _, indexGroup := range guidGroup.Nodes {
indexGroupWillBeEmpty := true
for _, actualNode := range indexGroup.Nodes {
actual := new(models.ActualLRP)
err := db.deserializeModel(logger, actualNode, actual)
if err != nil {
actualsToDeleteLock.Lock()
actualsToDelete = append(actualsToDelete, actualNode.Key)
actualsToDeleteLock.Unlock()
continue
}
err = actual.Validate()
if err != nil {
actualsToDeleteLock.Lock()
actualsToDelete = append(actualsToDelete, actualNode.Key)
actualsToDeleteLock.Unlock()
continue
}
indexGroupWillBeEmpty = false
guidGroupWillBeEmpty = false
switch actual.State {
case models.ActualLRPStateUnclaimed:
atomic.AddInt32(&lmc.unclaimedLRPs, 1)
case models.ActualLRPStateClaimed:
atomic.AddInt32(&lmc.claimedLRPs, 1)
case models.ActualLRPStateRunning:
atomic.AddInt32(&lmc.runningLRPs, 1)
case models.ActualLRPStateCrashed:
crashingDesiredsLock.Lock()
crashingDesireds[actual.ProcessGuid] = struct{}{}
crashingDesiredsLock.Unlock()
atomic.AddInt32(&lmc.crashedActualLRPs, 1)
}
guidsLock.Lock()
guids[actual.ProcessGuid] = struct{}{}
guidsLock.Unlock()
if path.Base(actualNode.Key) == ActualLRPInstanceKey {
actualsLock.Lock()
if actuals[actual.ProcessGuid] == nil {
actuals[actual.ProcessGuid] = map[int32]*models.ActualLRP{}
}
actuals[actual.ProcessGuid][actual.Index] = actual
actualsLock.Unlock()
}
}
if indexGroupWillBeEmpty {
indexKeysToDeleteLock.Lock()
indexKeysToDelete = append(indexKeysToDelete, indexGroup.Key)
indexKeysToDeleteLock.Unlock()
}
}
if guidGroupWillBeEmpty {
guidKeysToDeleteLock.Lock()
guidKeysToDelete = append(guidKeysToDelete, guidGroup.Key)
guidKeysToDeleteLock.Unlock()
}
})
}
logger.Debug("done-walking-actual-lrp-tree")
throttler, err := workpool.NewThrottler(db.convergenceWorkersSize, works)
if err != nil {
logger.Error("failed-to-create-throttler", err)
}
throttler.Work()
if doPrune {
logger.Info("deleting-invalid-actual-lrps", lager.Data{"num_lrps": len(actualsToDelete)})
db.batchDeleteNodes(actualsToDelete, logger)
actualLRPsDeleted.Add(uint64(len(actualsToDelete)))
logger.Info("deleting-empty-actual-indices", lager.Data{"num_indices": len(indexKeysToDelete)})
err = db.deleteLeaves(logger, indexKeysToDelete)
if err != nil {
logger.Error("failed-deleting-empty-actual-indices", err, lager.Data{"num_indices": len(indexKeysToDelete)})
} else {
logger.Info("succeeded-deleting-empty-actual-indices", lager.Data{"num_indices": len(indexKeysToDelete)})
}
logger.Info("deleting-empty-actual-guids", lager.Data{"num_guids": len(guidKeysToDelete)})
err = db.deleteLeaves(logger, guidKeysToDelete)
if err != nil {
logger.Error("failed-deleting-empty-actual-guids", err, lager.Data{"num_guids": len(guidKeysToDelete)})
} else {
logger.Info("succeeded-deleting-empty-actual-guids", lager.Data{"num_guids": len(guidKeysToDelete)})
}
}
lmc.crashingDesiredLRPs = int32(len(crashingDesireds))
return actuals, nil
}
})
})
})
Context("when the existing ActualLRP is Running", func() {
var instanceGuid string
BeforeEach(func() {
instanceGuid = "some-instance-guid"
instanceKey = models.NewActualLRPInstanceKey(instanceGuid, cellID)
actualLRP.State = models.ActualLRPStateRunning
actualLRP.ActualLRPInstanceKey = instanceKey
actualLRP.ActualLRPNetInfo = models.ActualLRPNetInfo{Address: "1"}
Expect(actualLRP.Validate()).NotTo(HaveOccurred())
etcdHelper.SetRawActualLRP(actualLRP)
})
Context("with the same cell and instance guid", func() {
BeforeEach(func() {
instanceKey = models.NewActualLRPInstanceKey(instanceGuid, cellID)
})
It("does not return an error", func() {
Expect(claimErr).NotTo(HaveOccurred())
})
It("reverts the persisted LRP to the CLAIMED state", func() {
lrpGroupInBBS, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, processGuid, index)
itValidatesPresenceOfTheInstanceKey(&lrp)
itValidatesPresenceOfNetInfo(&lrp)
itValidatesAbsenceOfPlacementError(&lrp)
})
Context("when state is not set", func() {
BeforeEach(func() {
lrp = models.ActualLRP{
ActualLRPKey: lrpKey,
State: "",
Since: 1138,
}
})
It("validate returns an error", func() {
err := lrp.Validate()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("state"))
})
})
Context("when since is not set", func() {
BeforeEach(func() {
lrp = models.ActualLRP{
ActualLRPKey: lrpKey,
State: models.ActualLRPStateUnclaimed,
Since: 0,
}
})