func (s *startRequests) Add(logger lager.Logger, actual *models.ActualLRPKey) { s.Lock() defer s.Unlock() desiredLRP, found := s.desiredMap[actual.ProcessGuid] if !found { logger.Info("failed-to-find-desired-lrp-for-stale-unclaimed-actual-lrp", lager.Data{"actual_lrp": actual}) return } start, found := s.startMap[desiredLRP.ProcessGuid] if !found { startRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actual.Index)) start = &startRequest } else { start.Indices = append(start.Indices, int(actual.Index)) } logger.Info("adding-start-auction", lager.Data{"process_guid": desiredLRP.ProcessGuid, "index": actual.Index}) s.startMap[desiredLRP.ProcessGuid] = start s.instanceCount++ }
func (t crashTest) Test() { Context(t.Name, func() { var ( crashErr error actualLRPKey *models.ActualLRPKey instanceKey *models.ActualLRPInstanceKey initialTimestamp int64 initialModificationIndex uint32 ) BeforeEach(func() { actualLRP := t.LRP() actualLRPKey = &actualLRP.ActualLRPKey instanceKey = &actualLRP.ActualLRPInstanceKey initialTimestamp = actualLRP.Since initialModificationIndex = actualLRP.ModificationTag.Index desiredLRP := models.DesiredLRP{ ProcessGuid: actualLRPKey.ProcessGuid, Domain: actualLRPKey.Domain, Instances: actualLRPKey.Index + 1, RootFs: "foo:bar", Action: models.WrapAction(&models.RunAction{Path: "true", User: "******"}), } etcdHelper.SetRawDesiredLRP(&desiredLRP) etcdHelper.SetRawActualLRP(&actualLRP) }) JustBeforeEach(func() { clock.Increment(600) crashErr = etcdDB.CrashActualLRP(logger, actualLRPKey, instanceKey, "crashed") }) if t.Result.ReturnedErr == nil { It("does not return an error", func() { Expect(crashErr).NotTo(HaveOccurred()) }) } else { It(fmt.Sprintf("returned error should be '%s'", t.Result.ReturnedErr.Error()), func() { Expect(crashErr).To(Equal(t.Result.ReturnedErr)) }) } It(fmt.Sprintf("has crash count %d", t.Result.CrashCount), func() { actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey) Expect(err).NotTo(HaveOccurred()) Expect(actualLRP.CrashCount).To(Equal(t.Result.CrashCount)) }) It(fmt.Sprintf("has crash reason %s", t.Result.CrashReason), func() { actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey) Expect(err).NotTo(HaveOccurred()) Expect(actualLRP.CrashReason).To(Equal(t.Result.CrashReason)) }) if t.Result.ShouldUpdate { It("updates the Since", func() { actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey) Expect(err).NotTo(HaveOccurred()) Expect(actualLRP.Since).To(Equal(clock.Now().UnixNano())) }) It("updates the ModificationIndex", func() { actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey) Expect(err).NotTo(HaveOccurred()) Expect(actualLRP.ModificationTag.Index).To(Equal(initialModificationIndex + 1)) }) } else { It("does not update the Since", func() { actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey) Expect(err).NotTo(HaveOccurred()) Expect(actualLRP.Since).To(Equal(initialTimestamp)) }) It("does not update the ModificationIndex", func() { actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey) Expect(err).NotTo(HaveOccurred()) Expect(actualLRP.ModificationTag.Index).To(Equal(initialModificationIndex)) }) } It(fmt.Sprintf("CAS to %s", t.Result.State), func() { actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey) Expect(err).NotTo(HaveOccurred()) Expect(actualLRP.State).To(Equal(t.Result.State)) }) if t.Result.Auction { It("starts an auction", func() { Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1)) requestedAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0) Expect(requestedAuctions).To(HaveLen(1)) desiredLRP, err := etcdDB.DesiredLRPByProcessGuid(logger, actualLRPKey.ProcessGuid) Expect(err).NotTo(HaveOccurred()) expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actualLRPKey.Index)) Expect(*requestedAuctions[0]).To(Equal(expectedStartRequest)) }) Context("when the desired LRP no longer exists", func() { BeforeEach(func() { etcdHelper.DeleteDesiredLRP(actualLRPKey.ProcessGuid) }) It("the actual LRP is also deleted", func() { Expect(crashErr).NotTo(HaveOccurred()) _, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index) Expect(err).To(Equal(models.ErrResourceNotFound)) }) }) } else { It("does not start an auction", func() { Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(0)) }) } Context("when crashing a different instance key", func() { var beforeActualGroup *models.ActualLRPGroup BeforeEach(func() { var err error beforeActualGroup, err = etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index) Expect(err).NotTo(HaveOccurred()) instanceKey.InstanceGuid = "another-guid" }) It("does not crash", func() { Expect(crashErr).To(Equal(models.ErrActualLRPCannotBeCrashed)) afterActualGroup, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index) Expect(err).NotTo(HaveOccurred()) Expect(afterActualGroup).To(Equal(beforeActualGroup)) }) }) }) }
response := models.EvacuationResponse{} err := response.Unmarshal(responseRecorder.Body.Bytes()) Expect(err).NotTo(HaveOccurred()) Expect(response.KeepContainer).To(BeFalse()) Expect(response.Error).To(BeNil()) Expect(fakeActualLRPDB.UnclaimActualLRPCallCount()).To(Equal(1)) _, lrpKey := fakeActualLRPDB.UnclaimActualLRPArgsForCall(0) Expect(lrpKey.ProcessGuid).To(Equal("process-guid")) Expect(lrpKey.Index).To(BeEquivalentTo(1)) Expect(fakeDesiredLRPDB.DesiredLRPByProcessGuidCallCount()).To(Equal(1)) _, guid := fakeDesiredLRPDB.DesiredLRPByProcessGuidArgsForCall(0) Expect(guid).To(Equal("process-guid")) expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actual.Index)) Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1)) startRequests := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0) Expect(startRequests).To(Equal([]*auctioneer.LRPStartRequest{&expectedStartRequest})) }) Context("when the DB returns an unrecoverable error", func() { BeforeEach(func() { fakeEvacuationDB.RemoveEvacuatingActualLRPReturns(models.NewUnrecoverableError(nil)) }) It("logs and writes to the exit channel", func() { Eventually(logger).Should(gbytes.Say("unrecoverable-error")) Eventually(exitCh).Should(Receive()) }) })
Expect(lrp.ModificationTag.Index).To(BeEquivalentTo(0)) }) Context("when an auctioneer is present", func() { It("emits start auction requests", func() { originalAuctionCallCount := fakeAuctioneerClient.RequestLRPAuctionsCallCount() err := etcdDB.DesireLRP(logger, lrp) Expect(err).NotTo(HaveOccurred()) desired, err := etcdDB.DesiredLRPByProcessGuid(logger, lrp.ProcessGuid) Expect(err).NotTo(HaveOccurred()) Consistently(fakeAuctioneerClient.RequestLRPAuctionsCallCount).Should(Equal(originalAuctionCallCount + 1)) expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desired, 0, 1, 2, 3, 4) startAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(originalAuctionCallCount) Expect(startAuctions).To(HaveLen(1)) Expect(startAuctions[0].ProcessGuid).To(Equal(desired.ProcessGuid)) Expect(startAuctions[0].Indices).To(ConsistOf(expectedStartRequest.Indices)) }) }) }) Context("when the desired LRP already exists", func() { var newLRP *models.DesiredLRP BeforeEach(func() { err := etcdDB.DesireLRP(logger, lrp) Expect(err).NotTo(HaveOccurred())
func (t evacuationTest) Test() { Context(t.Name, func() { var evacuateErr error var initialTimestamp int64 var initialInstanceModificationIndex uint32 var initialEvacuatingModificationIndex uint32 var retainContainer bool BeforeEach(func() { initialTimestamp = clock.Now().UnixNano() etcdHelper.SetRawDesiredLRP(&desiredLRP) if t.InstanceLRP != nil { actualLRP := t.InstanceLRP() initialInstanceModificationIndex = actualLRP.ModificationTag.Index etcdHelper.SetRawActualLRP(&actualLRP) } if t.EvacuatingLRP != nil { evacuatingLRP := t.EvacuatingLRP() initialEvacuatingModificationIndex = evacuatingLRP.ModificationTag.Index etcdHelper.SetRawEvacuatingActualLRP(&evacuatingLRP, omegaEvacuationTTL) } }) JustBeforeEach(func() { clock.Increment(timeIncrement) retainContainer, evacuateErr = t.Subject() }) if t.Result.ReturnedError == nil { It("does not return an error", func() { Expect(evacuateErr).NotTo(HaveOccurred()) }) } else { It(fmt.Sprintf("returned error should be '%s'", t.Result.ReturnedError.Error()), func() { Expect(evacuateErr).To(Equal(t.Result.ReturnedError)) }) } if t.Result.RetainContainer == true { It("returns true", func() { Expect(retainContainer).To(Equal(true)) }) } else { It("returns false", func() { Expect(retainContainer).To(Equal(false)) }) } if t.Result.AuctionRequested { It("starts an auction", func() { Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1)) expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(&desiredLRP, int(index)) requestedAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0) Expect(requestedAuctions).To(HaveLen(1)) Expect(*requestedAuctions[0]).To(Equal(expectedStartRequest)) }) Context("when starting the auction fails", func() { BeforeEach(func() { fakeAuctioneerClient.RequestLRPAuctionsReturns(errors.New("error")) }) It("returns an UnknownError", func() { Expect(evacuateErr).To(Equal(models.ErrUnknownError)) }) }) Context("when the desired LRP no longer exists", func() { BeforeEach(func() { etcdHelper.DeleteDesiredLRP(desiredLRP.ProcessGuid) }) It("the actual LRP is also deleted", func() { Expect(evacuateErr).NotTo(HaveOccurred()) group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, t.InstanceLRP().ProcessGuid, t.InstanceLRP().Index) if err == nil { // LRP remaining in one of evacuating or ...not Expect(group.Instance).To(BeNil()) } else { // No LRP remaining at all (no group returned) Expect(err).To(Equal(models.ErrResourceNotFound)) } }) }) } else { It("does not start an auction", func() { Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(0)) }) } if t.Result.Instance == nil { It("removes the /instance actualLRP", func() { _, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).To(Equal(models.ErrResourceNotFound)) }) } else { if t.Result.Instance.ShouldUpdate { It("updates the /instance Since", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.Since).To(Equal(clock.Now().UnixNano())) }) It("updates the /instance ModificationTag", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.ModificationTag.Index).To(Equal(initialInstanceModificationIndex + 1)) }) } else { It("does not update the /instance Since", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.Since).To(Equal(initialTimestamp)) }) } It("has the expected /instance state", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.State).To(Equal(t.Result.Instance.State)) }) It("has the expected /instance crash count", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.CrashCount).To(Equal(t.Result.Instance.CrashCount)) }) It("has the expected /instance crash reason", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.CrashReason).To(Equal(t.Result.Instance.CrashReason)) }) It("has the expected /instance instance key", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.ActualLRPInstanceKey).To(Equal(t.Result.Instance.ActualLRPInstanceKey)) }) It("has the expected /instance net info", func() { lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(lrpInBBS.ActualLRPNetInfo).To(Equal(t.Result.Instance.ActualLRPNetInfo)) }) } if t.Result.Evacuating == nil { It("removes the /evacuating actualLRP", func() { _, err := getEvacuatingActualLRPTTL(lrpKey) Expect(err).To(Equal(models.ErrResourceNotFound)) }) } else { if t.Result.Evacuating.ShouldUpdate { It("updates the /evacuating Since", func() { group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index) Expect(err).NotTo(HaveOccurred()) lrpInBBS := group.Evacuating Expect(lrpInBBS.Since).To(Equal(clock.Now().UnixNano())) }) It("updates the /evacuating TTL to the desired value", func() { ttl, err := getEvacuatingActualLRPTTL(lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(ttl).To(BeNumerically("~", t.Result.Evacuating.TTL, allowedTTLDecay)) }) It("updates the /evacuating ModificationTag", func() { group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index) Expect(err).NotTo(HaveOccurred()) lrpInBBS := group.Evacuating Expect(lrpInBBS.ModificationTag.Index).To(Equal(initialEvacuatingModificationIndex + 1)) }) } else { It("does not update the /evacuating Since", func() { group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index) Expect(err).NotTo(HaveOccurred()) lrpInBBS := group.Evacuating Expect(lrpInBBS.Since).To(Equal(initialTimestamp)) }) It("does not update the /evacuating TTL", func() { ttl, err := getEvacuatingActualLRPTTL(lrpKey) Expect(err).NotTo(HaveOccurred()) Expect(ttl).To(BeNumerically("~", omegaEvacuationTTL, allowedTTLDecay)) }) } It("has the expected /evacuating state", func() { group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index) Expect(err).NotTo(HaveOccurred()) lrpInBBS := group.Evacuating Expect(lrpInBBS.State).To(Equal(t.Result.Evacuating.State)) }) It("has the expected /evacuating instance key", func() { group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index) Expect(err).NotTo(HaveOccurred()) lrpInBBS := group.Evacuating Expect(lrpInBBS.ActualLRPInstanceKey).To(Equal(t.Result.Evacuating.ActualLRPInstanceKey)) }) It("has the expected /evacuating net info", func() { group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index) Expect(err).NotTo(HaveOccurred()) lrpInBBS := group.Evacuating Expect(lrpInBBS.ActualLRPNetInfo).To(Equal(t.Result.Evacuating.ActualLRPNetInfo)) }) } }) }