func (s *startRequests) Add(logger lager.Logger, actual *models.ActualLRPKey) {
s.Lock()
defer s.Unlock()
desiredLRP, found := s.desiredMap[actual.ProcessGuid]
if !found {
logger.Info("failed-to-find-desired-lrp-for-stale-unclaimed-actual-lrp", lager.Data{"actual_lrp": actual})
return
}
start, found := s.startMap[desiredLRP.ProcessGuid]
if !found {
startRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actual.Index))
start = &startRequest
} else {
start.Indices = append(start.Indices, int(actual.Index))
}
logger.Info("adding-start-auction", lager.Data{"process_guid": desiredLRP.ProcessGuid, "index": actual.Index})
s.startMap[desiredLRP.ProcessGuid] = start
s.instanceCount++
}
func (t crashTest) Test() {
Context(t.Name, func() {
var (
crashErr error
actualLRPKey *models.ActualLRPKey
instanceKey *models.ActualLRPInstanceKey
initialTimestamp int64
initialModificationIndex uint32
)
BeforeEach(func() {
actualLRP := t.LRP()
actualLRPKey = &actualLRP.ActualLRPKey
instanceKey = &actualLRP.ActualLRPInstanceKey
initialTimestamp = actualLRP.Since
initialModificationIndex = actualLRP.ModificationTag.Index
desiredLRP := models.DesiredLRP{
ProcessGuid: actualLRPKey.ProcessGuid,
Domain: actualLRPKey.Domain,
Instances: actualLRPKey.Index + 1,
RootFs: "foo:bar",
Action: models.WrapAction(&models.RunAction{Path: "true", User: "******"}),
}
etcdHelper.SetRawDesiredLRP(&desiredLRP)
etcdHelper.SetRawActualLRP(&actualLRP)
})
JustBeforeEach(func() {
clock.Increment(600)
crashErr = etcdDB.CrashActualLRP(logger, actualLRPKey, instanceKey, "crashed")
})
if t.Result.ReturnedErr == nil {
It("does not return an error", func() {
Expect(crashErr).NotTo(HaveOccurred())
})
} else {
It(fmt.Sprintf("returned error should be '%s'", t.Result.ReturnedErr.Error()), func() {
Expect(crashErr).To(Equal(t.Result.ReturnedErr))
})
}
It(fmt.Sprintf("has crash count %d", t.Result.CrashCount), func() {
actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
Expect(err).NotTo(HaveOccurred())
Expect(actualLRP.CrashCount).To(Equal(t.Result.CrashCount))
})
It(fmt.Sprintf("has crash reason %s", t.Result.CrashReason), func() {
actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
Expect(err).NotTo(HaveOccurred())
Expect(actualLRP.CrashReason).To(Equal(t.Result.CrashReason))
})
if t.Result.ShouldUpdate {
It("updates the Since", func() {
actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
Expect(err).NotTo(HaveOccurred())
Expect(actualLRP.Since).To(Equal(clock.Now().UnixNano()))
})
It("updates the ModificationIndex", func() {
actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
Expect(err).NotTo(HaveOccurred())
Expect(actualLRP.ModificationTag.Index).To(Equal(initialModificationIndex + 1))
})
} else {
It("does not update the Since", func() {
actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
Expect(err).NotTo(HaveOccurred())
Expect(actualLRP.Since).To(Equal(initialTimestamp))
})
It("does not update the ModificationIndex", func() {
actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
Expect(err).NotTo(HaveOccurred())
Expect(actualLRP.ModificationTag.Index).To(Equal(initialModificationIndex))
})
}
It(fmt.Sprintf("CAS to %s", t.Result.State), func() {
actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
Expect(err).NotTo(HaveOccurred())
Expect(actualLRP.State).To(Equal(t.Result.State))
})
if t.Result.Auction {
It("starts an auction", func() {
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))
requestedAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
Expect(requestedAuctions).To(HaveLen(1))
desiredLRP, err := etcdDB.DesiredLRPByProcessGuid(logger, actualLRPKey.ProcessGuid)
Expect(err).NotTo(HaveOccurred())
expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actualLRPKey.Index))
Expect(*requestedAuctions[0]).To(Equal(expectedStartRequest))
})
Context("when the desired LRP no longer exists", func() {
BeforeEach(func() {
etcdHelper.DeleteDesiredLRP(actualLRPKey.ProcessGuid)
})
It("the actual LRP is also deleted", func() {
Expect(crashErr).NotTo(HaveOccurred())
_, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index)
Expect(err).To(Equal(models.ErrResourceNotFound))
})
})
} else {
It("does not start an auction", func() {
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(0))
})
}
Context("when crashing a different instance key", func() {
var beforeActualGroup *models.ActualLRPGroup
BeforeEach(func() {
var err error
beforeActualGroup, err = etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index)
Expect(err).NotTo(HaveOccurred())
instanceKey.InstanceGuid = "another-guid"
})
It("does not crash", func() {
Expect(crashErr).To(Equal(models.ErrActualLRPCannotBeCrashed))
afterActualGroup, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index)
Expect(err).NotTo(HaveOccurred())
Expect(afterActualGroup).To(Equal(beforeActualGroup))
})
})
})
}
response := models.EvacuationResponse{}
err := response.Unmarshal(responseRecorder.Body.Bytes())
Expect(err).NotTo(HaveOccurred())
Expect(response.KeepContainer).To(BeFalse())
Expect(response.Error).To(BeNil())
Expect(fakeActualLRPDB.UnclaimActualLRPCallCount()).To(Equal(1))
_, lrpKey := fakeActualLRPDB.UnclaimActualLRPArgsForCall(0)
Expect(lrpKey.ProcessGuid).To(Equal("process-guid"))
Expect(lrpKey.Index).To(BeEquivalentTo(1))
Expect(fakeDesiredLRPDB.DesiredLRPByProcessGuidCallCount()).To(Equal(1))
_, guid := fakeDesiredLRPDB.DesiredLRPByProcessGuidArgsForCall(0)
Expect(guid).To(Equal("process-guid"))
expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actual.Index))
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))
startRequests := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
Expect(startRequests).To(Equal([]*auctioneer.LRPStartRequest{&expectedStartRequest}))
})
Context("when the DB returns an unrecoverable error", func() {
BeforeEach(func() {
fakeEvacuationDB.RemoveEvacuatingActualLRPReturns(models.NewUnrecoverableError(nil))
})
It("logs and writes to the exit channel", func() {
Eventually(logger).Should(gbytes.Say("unrecoverable-error"))
Eventually(exitCh).Should(Receive())
})
})
Expect(lrp.ModificationTag.Index).To(BeEquivalentTo(0))
})
Context("when an auctioneer is present", func() {
It("emits start auction requests", func() {
originalAuctionCallCount := fakeAuctioneerClient.RequestLRPAuctionsCallCount()
err := etcdDB.DesireLRP(logger, lrp)
Expect(err).NotTo(HaveOccurred())
desired, err := etcdDB.DesiredLRPByProcessGuid(logger, lrp.ProcessGuid)
Expect(err).NotTo(HaveOccurred())
Consistently(fakeAuctioneerClient.RequestLRPAuctionsCallCount).Should(Equal(originalAuctionCallCount + 1))
expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desired, 0, 1, 2, 3, 4)
startAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(originalAuctionCallCount)
Expect(startAuctions).To(HaveLen(1))
Expect(startAuctions[0].ProcessGuid).To(Equal(desired.ProcessGuid))
Expect(startAuctions[0].Indices).To(ConsistOf(expectedStartRequest.Indices))
})
})
})
Context("when the desired LRP already exists", func() {
var newLRP *models.DesiredLRP
BeforeEach(func() {
err := etcdDB.DesireLRP(logger, lrp)
Expect(err).NotTo(HaveOccurred())
func (t evacuationTest) Test() {
Context(t.Name, func() {
var evacuateErr error
var initialTimestamp int64
var initialInstanceModificationIndex uint32
var initialEvacuatingModificationIndex uint32
var retainContainer bool
BeforeEach(func() {
initialTimestamp = clock.Now().UnixNano()
etcdHelper.SetRawDesiredLRP(&desiredLRP)
if t.InstanceLRP != nil {
actualLRP := t.InstanceLRP()
initialInstanceModificationIndex = actualLRP.ModificationTag.Index
etcdHelper.SetRawActualLRP(&actualLRP)
}
if t.EvacuatingLRP != nil {
evacuatingLRP := t.EvacuatingLRP()
initialEvacuatingModificationIndex = evacuatingLRP.ModificationTag.Index
etcdHelper.SetRawEvacuatingActualLRP(&evacuatingLRP, omegaEvacuationTTL)
}
})
JustBeforeEach(func() {
clock.Increment(timeIncrement)
retainContainer, evacuateErr = t.Subject()
})
if t.Result.ReturnedError == nil {
It("does not return an error", func() {
Expect(evacuateErr).NotTo(HaveOccurred())
})
} else {
It(fmt.Sprintf("returned error should be '%s'", t.Result.ReturnedError.Error()), func() {
Expect(evacuateErr).To(Equal(t.Result.ReturnedError))
})
}
if t.Result.RetainContainer == true {
It("returns true", func() {
Expect(retainContainer).To(Equal(true))
})
} else {
It("returns false", func() {
Expect(retainContainer).To(Equal(false))
})
}
if t.Result.AuctionRequested {
It("starts an auction", func() {
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))
expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(&desiredLRP, int(index))
requestedAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
Expect(requestedAuctions).To(HaveLen(1))
Expect(*requestedAuctions[0]).To(Equal(expectedStartRequest))
})
Context("when starting the auction fails", func() {
BeforeEach(func() {
fakeAuctioneerClient.RequestLRPAuctionsReturns(errors.New("error"))
})
It("returns an UnknownError", func() {
Expect(evacuateErr).To(Equal(models.ErrUnknownError))
})
})
Context("when the desired LRP no longer exists", func() {
BeforeEach(func() {
etcdHelper.DeleteDesiredLRP(desiredLRP.ProcessGuid)
})
It("the actual LRP is also deleted", func() {
Expect(evacuateErr).NotTo(HaveOccurred())
group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, t.InstanceLRP().ProcessGuid, t.InstanceLRP().Index)
if err == nil {
// LRP remaining in one of evacuating or ...not
Expect(group.Instance).To(BeNil())
} else {
// No LRP remaining at all (no group returned)
Expect(err).To(Equal(models.ErrResourceNotFound))
}
})
})
} else {
It("does not start an auction", func() {
Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(0))
})
}
if t.Result.Instance == nil {
It("removes the /instance actualLRP", func() {
_, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).To(Equal(models.ErrResourceNotFound))
})
} else {
if t.Result.Instance.ShouldUpdate {
It("updates the /instance Since", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.Since).To(Equal(clock.Now().UnixNano()))
})
It("updates the /instance ModificationTag", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.ModificationTag.Index).To(Equal(initialInstanceModificationIndex + 1))
})
} else {
It("does not update the /instance Since", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.Since).To(Equal(initialTimestamp))
})
}
It("has the expected /instance state", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.State).To(Equal(t.Result.Instance.State))
})
It("has the expected /instance crash count", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.CrashCount).To(Equal(t.Result.Instance.CrashCount))
})
It("has the expected /instance crash reason", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.CrashReason).To(Equal(t.Result.Instance.CrashReason))
})
It("has the expected /instance instance key", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.ActualLRPInstanceKey).To(Equal(t.Result.Instance.ActualLRPInstanceKey))
})
It("has the expected /instance net info", func() {
lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(lrpInBBS.ActualLRPNetInfo).To(Equal(t.Result.Instance.ActualLRPNetInfo))
})
}
if t.Result.Evacuating == nil {
It("removes the /evacuating actualLRP", func() {
_, err := getEvacuatingActualLRPTTL(lrpKey)
Expect(err).To(Equal(models.ErrResourceNotFound))
})
} else {
if t.Result.Evacuating.ShouldUpdate {
It("updates the /evacuating Since", func() {
group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
Expect(err).NotTo(HaveOccurred())
lrpInBBS := group.Evacuating
Expect(lrpInBBS.Since).To(Equal(clock.Now().UnixNano()))
})
It("updates the /evacuating TTL to the desired value", func() {
ttl, err := getEvacuatingActualLRPTTL(lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(ttl).To(BeNumerically("~", t.Result.Evacuating.TTL, allowedTTLDecay))
})
It("updates the /evacuating ModificationTag", func() {
group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
Expect(err).NotTo(HaveOccurred())
lrpInBBS := group.Evacuating
Expect(lrpInBBS.ModificationTag.Index).To(Equal(initialEvacuatingModificationIndex + 1))
})
} else {
It("does not update the /evacuating Since", func() {
group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
Expect(err).NotTo(HaveOccurred())
lrpInBBS := group.Evacuating
Expect(lrpInBBS.Since).To(Equal(initialTimestamp))
})
It("does not update the /evacuating TTL", func() {
ttl, err := getEvacuatingActualLRPTTL(lrpKey)
Expect(err).NotTo(HaveOccurred())
Expect(ttl).To(BeNumerically("~", omegaEvacuationTTL, allowedTTLDecay))
})
}
It("has the expected /evacuating state", func() {
group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
Expect(err).NotTo(HaveOccurred())
lrpInBBS := group.Evacuating
Expect(lrpInBBS.State).To(Equal(t.Result.Evacuating.State))
})
It("has the expected /evacuating instance key", func() {
group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
Expect(err).NotTo(HaveOccurred())
lrpInBBS := group.Evacuating
Expect(lrpInBBS.ActualLRPInstanceKey).To(Equal(t.Result.Evacuating.ActualLRPInstanceKey))
})
It("has the expected /evacuating net info", func() {
group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
Expect(err).NotTo(HaveOccurred())
lrpInBBS := group.Evacuating
Expect(lrpInBBS.ActualLRPNetInfo).To(Equal(t.Result.Evacuating.ActualLRPNetInfo))
})
}
})
}