func (db *ETCDDB) createActualLRP(logger lager.Logger, desiredLRP *models.DesiredLRP, index int32) error {
logger = logger.Session("create-actual-lrp")
var err error
if index >= desiredLRP.Instances {
err = models.NewError(models.Error_InvalidRecord, "Index too large")
logger.Error("actual-lrp-index-too-large", err, lager.Data{"actual_index": index, "desired_instances": desiredLRP.Instances})
return err
}
guid, err := uuid.NewV4()
if err != nil {
return err
}
actualLRP := &models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey(
desiredLRP.ProcessGuid,
index,
desiredLRP.Domain,
),
State: models.ActualLRPStateUnclaimed,
Since: db.clock.Now().UnixNano(),
ModificationTag: models.ModificationTag{
Epoch: guid.String(),
Index: 0,
},
}
err = db.createRawActualLRP(logger, actualLRP)
if err != nil {
return err
}
return nil
}
func defaultCrashedActual(crashCount int32, lastCrashed int64) *models.ActualLRP {
return &models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey("p-guid", 0, "domain"),
State: models.ActualLRPStateCrashed,
CrashCount: crashCount,
Since: lastCrashed,
}
}
func NewValidActualLRP(guid string, index int32) *models.ActualLRP {
actualLRP := &models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey(guid, index, "some-domain"),
ActualLRPInstanceKey: models.NewActualLRPInstanceKey("some-guid", "some-cell"),
ActualLRPNetInfo: models.NewActualLRPNetInfo("some-address", models.NewPortMapping(2222, 4444)),
CrashCount: 33,
CrashReason: "badness",
State: models.ActualLRPStateRunning,
Since: 1138,
ModificationTag: models.ModificationTag{
Epoch: "some-epoch",
Index: 999,
},
}
err := actualLRP.Validate()
Expect(err).NotTo(HaveOccurred())
return actualLRP
}
// Adds CRASHED Actual LRPs that can be restarted to the list of start requests
// and transitions them to UNCLAIMED.
func (c *convergence) crashedActualLRPs(logger lager.Logger, now time.Time) {
logger = logger.Session("crashed-actual-lrps")
restartCalculator := models.NewDefaultRestartCalculator()
rows, err := c.selectCrashedLRPs(logger, c.db)
if err != nil {
logger.Error("failed-query", err)
return
}
for rows.Next() {
var index int
actual := &models.ActualLRP{}
schedulingInfo, err := c.fetchDesiredLRPSchedulingInfoAndMore(logger, rows, &index, &actual.Since, &actual.CrashCount)
if err != nil {
continue
}
actual.ActualLRPKey = models.NewActualLRPKey(schedulingInfo.ProcessGuid, int32(index), schedulingInfo.Domain)
actual.State = models.ActualLRPStateCrashed
if actual.ShouldRestartCrash(now, restartCalculator) {
c.submit(func() {
_, _, err = c.UnclaimActualLRP(logger, &actual.ActualLRPKey)
if err != nil {
logger.Error("failed-unclaiming-actual-lrp", err)
return
}
c.addStartRequestFromSchedulingInfo(logger, schedulingInfo, index)
})
}
}
if rows.Err() != nil {
logger.Error("failed-getting-next-row", rows.Err())
}
return
}
func lrpForState(state string, timeInState time.Duration) models.ActualLRP {
var actualLRPKey = models.NewActualLRPKey("some-process-guid", 1, "tests")
var instanceKey = models.NewActualLRPInstanceKey("some-instance-guid", "some-cell")
lrp := models.ActualLRP{
ActualLRPKey: actualLRPKey,
State: state,
Since: clock.Now().Add(-timeInState).UnixNano(),
}
switch state {
case models.ActualLRPStateUnclaimed:
case models.ActualLRPStateCrashed:
lrp.CrashReason = "crashed"
case models.ActualLRPStateClaimed:
lrp.ActualLRPInstanceKey = instanceKey
case models.ActualLRPStateRunning:
lrp.ActualLRPInstanceKey = instanceKey
lrp.ActualLRPNetInfo = models.NewActualLRPNetInfo("1.2.3.4", &models.PortMapping{ContainerPort: 1234, HostPort: 5678})
}
return lrp
}
func itValidatesPresenceOfTheLRPKey(lrp *models.ActualLRP) {
Context("when the lrp key is set", func() {
BeforeEach(func() {
lrp.ActualLRPKey = models.NewActualLRPKey("some-guid", 1, "domain")
})
It("validate does not return an error", func() {
Expect(lrp.Validate()).NotTo(HaveOccurred())
})
})
Context("when the lrp key is not set", func() {
BeforeEach(func() {
lrp.ActualLRPKey = models.ActualLRPKey{}
})
It("validate returns an error", func() {
err := lrp.Validate()
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("process_guid"))
})
})
}
func (h *DesiredLRPHandler) startInstanceRange(logger lager.Logger, lower, upper int32, schedulingInfo *models.DesiredLRPSchedulingInfo) {
logger = logger.Session("start-instance-range", lager.Data{"lower": lower, "upper": upper})
logger.Info("starting")
defer logger.Info("complete")
keys := make([]*models.ActualLRPKey, upper-lower)
i := 0
for actualIndex := lower; actualIndex < upper; actualIndex++ {
key := models.NewActualLRPKey(schedulingInfo.ProcessGuid, int32(actualIndex), schedulingInfo.Domain)
keys[i] = &key
i++
}
createdIndices := h.createUnclaimedActualLRPs(logger, keys)
start := auctioneer.NewLRPStartRequestFromSchedulingInfo(schedulingInfo, createdIndices...)
logger.Info("start-lrp-auction-request", lager.Data{"app_guid": schedulingInfo.ProcessGuid, "indices": createdIndices})
err := h.auctioneerClient.RequestLRPAuctions(logger, []*auctioneer.LRPStartRequest{&start})
logger.Info("finished-lrp-auction-request", lager.Data{"app_guid": schedulingInfo.ProcessGuid, "indices": createdIndices})
if err != nil {
logger.Error("failed-to-request-auction", err)
}
}
event, err := eventSource.Next()
Expect(err).NotTo(HaveOccurred())
desiredLRPRemovedEvent, ok := event.(*models.DesiredLRPRemovedEvent)
Expect(ok).To(BeTrue())
Expect(desiredLRPRemovedEvent).To(Equal(expectedEvent))
})
})
})
Describe("Actual LRP Events", func() {
var actualLRPGroup *models.ActualLRPGroup
var actualLRP *models.ActualLRP
BeforeEach(func() {
actualLRP = models.NewUnclaimedActualLRP(models.NewActualLRPKey("some-guid", 0, "some-domain"), 1)
actualLRPGroup = models.NewRunningActualLRPGroup(actualLRP)
})
Context("when receiving a ActualLRPCreatedEvent", func() {
var expectedEvent *models.ActualLRPCreatedEvent
BeforeEach(func() {
expectedEvent = models.NewActualLRPCreatedEvent(actualLRPGroup)
payload, err := proto.Marshal(expectedEvent)
Expect(err).NotTo(HaveOccurred())
payload = []byte(base64.StdEncoding.EncodeToString(payload))
fakeRawEventSource.NextReturns(
sse.Event{
ID: "sup",
BeforeEach(func() {
instanceKey = models.NewActualLRPInstanceKey(
"instance-guid-0",
"cell-id-0",
)
requestBody = &instanceKey
requestBody = &models.ClaimActualLRPRequest{
ProcessGuid: processGuid,
Index: index,
ActualLrpInstanceKey: &instanceKey,
}
actualLRP = models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey(
processGuid,
1,
"domain-0",
),
State: models.ActualLRPStateUnclaimed,
Since: 1138,
}
afterActualLRP = models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey(
processGuid,
1,
"domain-0",
),
State: models.ActualLRPStateClaimed,
Since: 1140,
}
})
func CalculateConvergence(
logger lager.Logger,
clock clock.Clock,
restartCalculator models.RestartCalculator,
input *models.ConvergenceInput,
) *models.ConvergenceChanges {
sess := logger.Session("calculate-convergence")
var extraLRPCount, missingLRPCount int
sess.Info("start")
defer sess.Info("done")
changes := &models.ConvergenceChanges{}
now := clock.Now()
for processGuid, _ := range input.AllProcessGuids {
pLog := sess.WithData(lager.Data{
"process_guid": processGuid,
})
desired, hasDesired := input.DesiredLRPs[processGuid]
actualsByIndex := input.ActualLRPs[processGuid]
if hasDesired {
for i := int32(0); i < desired.Instances; i++ {
if _, hasIndex := actualsByIndex[i]; !hasIndex {
pLog.Info("missing", lager.Data{"index": i})
missingLRPCount++
lrpKey := models.NewActualLRPKey(desired.ProcessGuid, i, desired.Domain)
changes.ActualLRPKeysForMissingIndices = append(
changes.ActualLRPKeysForMissingIndices,
&lrpKey,
)
}
}
for i, actual := range actualsByIndex {
if actual.CellIsMissing(input.Cells) {
pLog.Info("missing-cell", lager.Data{"index": i, "cell_id": actual.CellId})
changes.ActualLRPsWithMissingCells = append(changes.ActualLRPsWithMissingCells, actual)
continue
}
if actual.Index >= desired.Instances && input.Domains.Contains(desired.Domain) {
pLog.Info("extra", lager.Data{"index": i})
extraLRPCount++
changes.ActualLRPsForExtraIndices = append(changes.ActualLRPsForExtraIndices, actual)
continue
}
if actual.ShouldRestartCrash(now, restartCalculator) {
pLog.Info("restart-crash", lager.Data{"index": i})
changes.RestartableCrashedActualLRPs = append(changes.RestartableCrashedActualLRPs, actual)
continue
}
if actual.ShouldStartUnclaimed(now) {
pLog.Info("stale-unclaimed", lager.Data{"index": i})
changes.StaleUnclaimedActualLRPs = append(changes.StaleUnclaimedActualLRPs, actual)
continue
}
}
} else {
for i, actual := range actualsByIndex {
if !input.Domains.Contains(actual.Domain) {
pLog.Info("skipping-unfresh-domain")
continue
}
pLog.Info("no-longer-desired", lager.Data{"index": i})
extraLRPCount++
changes.ActualLRPsForExtraIndices = append(changes.ActualLRPsForExtraIndices, actual)
}
}
}
missingLRPs.Send(missingLRPCount)
extraLRPs.Send(extraLRPCount)
return changes
}
var (
baseLRP *models.ActualLRP
otherIndexLRP *models.ActualLRP
evacuatingLRP *models.ActualLRP
otherDomainLRP *models.ActualLRP
otherCellIdLRP *models.ActualLRP
baseLRPKey models.ActualLRPKey
baseLRPInstanceKey models.ActualLRPInstanceKey
otherLRPInstanceKey models.ActualLRPInstanceKey
netInfo models.ActualLRPNetInfo
)
BeforeEach(func() {
baseLRPKey = models.NewActualLRPKey(baseProcessGuid, baseIndex, baseDomain)
baseLRPInstanceKey = models.NewActualLRPInstanceKey(baseInstanceGuid, cellID)
otherLRPInstanceKey = models.NewActualLRPInstanceKey(otherInstanceGuid, otherCellID)
netInfo = models.NewActualLRPNetInfo("127.0.0.1", models.NewPortMapping(8080, 80))
baseLRP = &models.ActualLRP{
ActualLRPKey: baseLRPKey,
ActualLRPInstanceKey: baseLRPInstanceKey,
ActualLRPNetInfo: netInfo,
State: models.ActualLRPStateRunning,
Since: clock.Now().UnixNano(),
}
evacuatingLRP = &models.ActualLRP{
ActualLRPKey: baseLRPKey,
fakeActualLRPDB = new(dbfakes.FakeActualLRPDB)
logger = lagertest.NewTestLogger("test")
responseRecorder = httptest.NewRecorder()
exitCh = make(chan struct{}, 1)
handler = handlers.NewActualLRPHandler(fakeActualLRPDB, exitCh)
})
Describe("ActualLRPGroups", func() {
var requestBody interface{}
BeforeEach(func() {
requestBody = &models.ActualLRPGroupsRequest{}
actualLRP1 = models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey(
"process-guid-0",
1,
"domain-0",
),
ActualLRPInstanceKey: models.NewActualLRPInstanceKey(
"instance-guid-0",
"cell-id-0",
),
State: models.ActualLRPStateRunning,
Since: 1138,
}
actualLRP2 = models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey(
"process-guid-1",
2,
"domain-1",
if state == models.ActualLRPStateCrashed {
Expect(actual.ShouldRestartCrash(now, calc)).To(BeTrue(), "should restart CRASHED lrp")
} else {
Expect(actual.ShouldRestartCrash(now, calc)).To(BeFalse(), fmt.Sprintf("should not restart %s lrp", state))
}
}
})
})
})
Describe("ActualLRPKey", func() {
Describe("Validate", func() {
var actualLRPKey models.ActualLRPKey
BeforeEach(func() {
actualLRPKey = models.NewActualLRPKey("process-guid", 1, "domain")
})
Context("when valid", func() {
It("returns nil", func() {
Expect(actualLRPKey.Validate()).To(BeNil())
})
})
Context("when the ProcessGuid is blank", func() {
BeforeEach(func() {
actualLRPKey.ProcessGuid = ""
})
It("returns a validation error", func() {
Expect(actualLRPKey.Validate()).To(ConsistOf(models.ErrInvalidField{"process_guid"}))
Describe("RemoveEvacuatingActualLRP", func() {
var (
processGuid = "process-guid"
index = int32(1)
key models.ActualLRPKey
instanceKey models.ActualLRPInstanceKey
actual *models.ActualLRP
requestBody interface{}
)
BeforeEach(func() {
key = models.NewActualLRPKey(
processGuid,
index,
"domain-0",
)
instanceKey = models.NewActualLRPInstanceKey("instance-guid", "cell-id")
requestBody = &models.RemoveEvacuatingActualLRPRequest{
ActualLrpKey: &key,
ActualLrpInstanceKey: &instanceKey,
}
fakeActualLRPDB.ActualLRPGroupByProcessGuidAndIndexReturns(&models.ActualLRPGroup{Evacuating: actual}, nil)
})
JustBeforeEach(func() {
request := newTestRequest(requestBody)
handler.RemoveEvacuatingActualLRP(logger, responseRecorder, request)
})
desiredLRPRemovedEvent, ok := event.(*models.DesiredLRPRemovedEvent)
Expect(ok).To(BeTrue())
Expect(desiredLRPRemovedEvent.DesiredLrp.ProcessGuid).To(Equal(desiredLRP.ProcessGuid))
})
})
Describe("Actual LRPs", func() {
const (
processGuid = "some-process-guid"
domain = "some-domain"
noExpirationTTL = 0
)
BeforeEach(func() {
key = models.NewActualLRPKey(processGuid, 0, domain)
instanceKey = models.NewActualLRPInstanceKey("instance-guid", "cell-id")
newInstanceKey = models.NewActualLRPInstanceKey("other-instance-guid", "other-cell-id")
netInfo = models.NewActualLRPNetInfo("1.1.1.1")
desiredLRP = &models.DesiredLRP{
ProcessGuid: processGuid,
Domain: domain,
RootFs: "some:rootfs",
Instances: 1,
Action: models.WrapAction(&models.RunAction{
Path: "true",
User: "******",
}),
}
})
Expect(err).NotTo(HaveOccurred())
queryStr = `UPDATE actual_lrps SET evacuating = ? WHERE process_guid = ?`
if test_helpers.UsePostgres() {
queryStr = test_helpers.ReplaceQuestionMarks(queryStr)
}
_, err = db.Exec(queryStr, evacuating, processGuid)
Expect(err).NotTo(HaveOccurred())
processGuid = "desired-with-restartable-crashed-actuals" + "-" + domain
desiredLRPWithRestartableCrashedActuals := model_helpers.NewValidDesiredLRP(processGuid)
desiredLRPWithRestartableCrashedActuals.Domain = domain
desiredLRPWithRestartableCrashedActuals.Instances = 2
err = sqlDB.DesireLRP(logger, desiredLRPWithRestartableCrashedActuals)
Expect(err).NotTo(HaveOccurred())
for i := int32(0); i < 2; i++ {
crashedActualLRPKey := models.NewActualLRPKey(processGuid, i, domain)
_, err = sqlDB.CreateUnclaimedActualLRP(logger, &crashedActualLRPKey)
Expect(err).NotTo(HaveOccurred())
instanceGuid = "restartable-crashed-actual" + "-" + domain
_, _, err = sqlDB.ClaimActualLRP(logger, processGuid, i, &models.ActualLRPInstanceKey{InstanceGuid: instanceGuid, CellId: "existing-cell"})
Expect(err).NotTo(HaveOccurred())
actualLRPNetInfo := models.NewActualLRPNetInfo("some-address", models.NewPortMapping(2222, 4444))
_, _, err = sqlDB.StartActualLRP(logger, &crashedActualLRPKey, &models.ActualLRPInstanceKey{InstanceGuid: instanceGuid, CellId: "existing-cell"}, &actualLRPNetInfo)
Expect(err).NotTo(HaveOccurred())
_, _, _, err = sqlDB.CrashActualLRP(logger, &crashedActualLRPKey, &models.ActualLRPInstanceKey{InstanceGuid: instanceGuid, CellId: "existing-cell"}, "because it failed")
Expect(err).NotTo(HaveOccurred())
queryStr = `
UPDATE actual_lrps
SET state = ?
WHERE process_guid = ? AND instance_index = ? AND evacuating = ?
`
