Exemple #1
0
func (s *startRequests) Add(logger lager.Logger, actual *models.ActualLRPKey) {
	s.Lock()
	defer s.Unlock()

	desiredLRP, found := s.desiredMap[actual.ProcessGuid]
	if !found {
		logger.Info("failed-to-find-desired-lrp-for-stale-unclaimed-actual-lrp", lager.Data{"actual_lrp": actual})
		return
	}

	start, found := s.startMap[desiredLRP.ProcessGuid]
	if !found {
		startRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actual.Index))
		start = &startRequest
	} else {
		start.Indices = append(start.Indices, int(actual.Index))
	}

	logger.Info("adding-start-auction", lager.Data{"process_guid": desiredLRP.ProcessGuid, "index": actual.Index})
	s.startMap[desiredLRP.ProcessGuid] = start
	s.instanceCount++
}
func (t crashTest) Test() {
	Context(t.Name, func() {
		var (
			crashErr                 error
			actualLRPKey             *models.ActualLRPKey
			instanceKey              *models.ActualLRPInstanceKey
			initialTimestamp         int64
			initialModificationIndex uint32
		)

		BeforeEach(func() {
			actualLRP := t.LRP()
			actualLRPKey = &actualLRP.ActualLRPKey
			instanceKey = &actualLRP.ActualLRPInstanceKey

			initialTimestamp = actualLRP.Since
			initialModificationIndex = actualLRP.ModificationTag.Index

			desiredLRP := models.DesiredLRP{
				ProcessGuid: actualLRPKey.ProcessGuid,
				Domain:      actualLRPKey.Domain,
				Instances:   actualLRPKey.Index + 1,
				RootFs:      "foo:bar",
				Action:      models.WrapAction(&models.RunAction{Path: "true", User: "******"}),
			}

			etcdHelper.SetRawDesiredLRP(&desiredLRP)
			etcdHelper.SetRawActualLRP(&actualLRP)
		})

		JustBeforeEach(func() {
			clock.Increment(600)
			crashErr = etcdDB.CrashActualLRP(logger, actualLRPKey, instanceKey, "crashed")
		})

		if t.Result.ReturnedErr == nil {
			It("does not return an error", func() {
				Expect(crashErr).NotTo(HaveOccurred())
			})
		} else {
			It(fmt.Sprintf("returned error should be '%s'", t.Result.ReturnedErr.Error()), func() {
				Expect(crashErr).To(Equal(t.Result.ReturnedErr))
			})
		}

		It(fmt.Sprintf("has crash count %d", t.Result.CrashCount), func() {
			actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRP.CrashCount).To(Equal(t.Result.CrashCount))
		})

		It(fmt.Sprintf("has crash reason %s", t.Result.CrashReason), func() {
			actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRP.CrashReason).To(Equal(t.Result.CrashReason))
		})

		if t.Result.ShouldUpdate {
			It("updates the Since", func() {
				actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
				Expect(err).NotTo(HaveOccurred())
				Expect(actualLRP.Since).To(Equal(clock.Now().UnixNano()))
			})

			It("updates the ModificationIndex", func() {
				actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
				Expect(err).NotTo(HaveOccurred())
				Expect(actualLRP.ModificationTag.Index).To(Equal(initialModificationIndex + 1))
			})
		} else {
			It("does not update the Since", func() {
				actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
				Expect(err).NotTo(HaveOccurred())
				Expect(actualLRP.Since).To(Equal(initialTimestamp))
			})

			It("does not update the ModificationIndex", func() {
				actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
				Expect(err).NotTo(HaveOccurred())
				Expect(actualLRP.ModificationTag.Index).To(Equal(initialModificationIndex))
			})
		}

		It(fmt.Sprintf("CAS to %s", t.Result.State), func() {
			actualLRP, err := etcdHelper.GetInstanceActualLRP(actualLRPKey)
			Expect(err).NotTo(HaveOccurred())
			Expect(actualLRP.State).To(Equal(t.Result.State))
		})

		if t.Result.Auction {
			It("starts an auction", func() {
				Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))

				requestedAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
				Expect(requestedAuctions).To(HaveLen(1))

				desiredLRP, err := etcdDB.DesiredLRPByProcessGuid(logger, actualLRPKey.ProcessGuid)
				Expect(err).NotTo(HaveOccurred())
				expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actualLRPKey.Index))
				Expect(*requestedAuctions[0]).To(Equal(expectedStartRequest))
			})

			Context("when the desired LRP no longer exists", func() {
				BeforeEach(func() {
					etcdHelper.DeleteDesiredLRP(actualLRPKey.ProcessGuid)
				})

				It("the actual LRP is also deleted", func() {
					Expect(crashErr).NotTo(HaveOccurred())

					_, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index)
					Expect(err).To(Equal(models.ErrResourceNotFound))
				})
			})
		} else {
			It("does not start an auction", func() {
				Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(0))
			})
		}

		Context("when crashing a different instance key", func() {
			var beforeActualGroup *models.ActualLRPGroup

			BeforeEach(func() {
				var err error
				beforeActualGroup, err = etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index)
				Expect(err).NotTo(HaveOccurred())
				instanceKey.InstanceGuid = "another-guid"
			})

			It("does not crash", func() {
				Expect(crashErr).To(Equal(models.ErrActualLRPCannotBeCrashed))

				afterActualGroup, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, actualLRPKey.ProcessGuid, actualLRPKey.Index)
				Expect(err).NotTo(HaveOccurred())
				Expect(afterActualGroup).To(Equal(beforeActualGroup))
			})
		})
	})
}
			response := models.EvacuationResponse{}
			err := response.Unmarshal(responseRecorder.Body.Bytes())
			Expect(err).NotTo(HaveOccurred())
			Expect(response.KeepContainer).To(BeFalse())
			Expect(response.Error).To(BeNil())

			Expect(fakeActualLRPDB.UnclaimActualLRPCallCount()).To(Equal(1))
			_, lrpKey := fakeActualLRPDB.UnclaimActualLRPArgsForCall(0)
			Expect(lrpKey.ProcessGuid).To(Equal("process-guid"))
			Expect(lrpKey.Index).To(BeEquivalentTo(1))

			Expect(fakeDesiredLRPDB.DesiredLRPByProcessGuidCallCount()).To(Equal(1))
			_, guid := fakeDesiredLRPDB.DesiredLRPByProcessGuidArgsForCall(0)
			Expect(guid).To(Equal("process-guid"))

			expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desiredLRP, int(actual.Index))
			Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))
			startRequests := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
			Expect(startRequests).To(Equal([]*auctioneer.LRPStartRequest{&expectedStartRequest}))
		})

		Context("when the DB returns an unrecoverable error", func() {
			BeforeEach(func() {
				fakeEvacuationDB.RemoveEvacuatingActualLRPReturns(models.NewUnrecoverableError(nil))
			})

			It("logs and writes to the exit channel", func() {
				Eventually(logger).Should(gbytes.Say("unrecoverable-error"))
				Eventually(exitCh).Should(Receive())
			})
		})
				Expect(lrp.ModificationTag.Index).To(BeEquivalentTo(0))
			})

			Context("when an auctioneer is present", func() {
				It("emits start auction requests", func() {
					originalAuctionCallCount := fakeAuctioneerClient.RequestLRPAuctionsCallCount()

					err := etcdDB.DesireLRP(logger, lrp)
					Expect(err).NotTo(HaveOccurred())

					desired, err := etcdDB.DesiredLRPByProcessGuid(logger, lrp.ProcessGuid)
					Expect(err).NotTo(HaveOccurred())

					Consistently(fakeAuctioneerClient.RequestLRPAuctionsCallCount).Should(Equal(originalAuctionCallCount + 1))

					expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(desired, 0, 1, 2, 3, 4)

					startAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(originalAuctionCallCount)
					Expect(startAuctions).To(HaveLen(1))
					Expect(startAuctions[0].ProcessGuid).To(Equal(desired.ProcessGuid))
					Expect(startAuctions[0].Indices).To(ConsistOf(expectedStartRequest.Indices))
				})
			})
		})

		Context("when the desired LRP already exists", func() {
			var newLRP *models.DesiredLRP

			BeforeEach(func() {
				err := etcdDB.DesireLRP(logger, lrp)
				Expect(err).NotTo(HaveOccurred())
func (t evacuationTest) Test() {
	Context(t.Name, func() {
		var evacuateErr error
		var initialTimestamp int64
		var initialInstanceModificationIndex uint32
		var initialEvacuatingModificationIndex uint32
		var retainContainer bool

		BeforeEach(func() {
			initialTimestamp = clock.Now().UnixNano()

			etcdHelper.SetRawDesiredLRP(&desiredLRP)
			if t.InstanceLRP != nil {
				actualLRP := t.InstanceLRP()
				initialInstanceModificationIndex = actualLRP.ModificationTag.Index
				etcdHelper.SetRawActualLRP(&actualLRP)
			}
			if t.EvacuatingLRP != nil {
				evacuatingLRP := t.EvacuatingLRP()
				initialEvacuatingModificationIndex = evacuatingLRP.ModificationTag.Index
				etcdHelper.SetRawEvacuatingActualLRP(&evacuatingLRP, omegaEvacuationTTL)
			}
		})

		JustBeforeEach(func() {
			clock.Increment(timeIncrement)
			retainContainer, evacuateErr = t.Subject()
		})

		if t.Result.ReturnedError == nil {
			It("does not return an error", func() {
				Expect(evacuateErr).NotTo(HaveOccurred())
			})
		} else {
			It(fmt.Sprintf("returned error should be '%s'", t.Result.ReturnedError.Error()), func() {
				Expect(evacuateErr).To(Equal(t.Result.ReturnedError))
			})
		}

		if t.Result.RetainContainer == true {
			It("returns true", func() {
				Expect(retainContainer).To(Equal(true))
			})

		} else {
			It("returns false", func() {
				Expect(retainContainer).To(Equal(false))
			})

		}

		if t.Result.AuctionRequested {
			It("starts an auction", func() {
				Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(1))

				expectedStartRequest := auctioneer.NewLRPStartRequestFromModel(&desiredLRP, int(index))
				requestedAuctions := fakeAuctioneerClient.RequestLRPAuctionsArgsForCall(0)
				Expect(requestedAuctions).To(HaveLen(1))

				Expect(*requestedAuctions[0]).To(Equal(expectedStartRequest))
			})

			Context("when starting the auction fails", func() {
				BeforeEach(func() {
					fakeAuctioneerClient.RequestLRPAuctionsReturns(errors.New("error"))
				})

				It("returns an UnknownError", func() {
					Expect(evacuateErr).To(Equal(models.ErrUnknownError))
				})
			})

			Context("when the desired LRP no longer exists", func() {
				BeforeEach(func() {
					etcdHelper.DeleteDesiredLRP(desiredLRP.ProcessGuid)
				})

				It("the actual LRP is also deleted", func() {
					Expect(evacuateErr).NotTo(HaveOccurred())

					group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, t.InstanceLRP().ProcessGuid, t.InstanceLRP().Index)
					if err == nil {
						// LRP remaining in one of evacuating or ...not
						Expect(group.Instance).To(BeNil())
					} else {
						// No LRP remaining at all (no group returned)
						Expect(err).To(Equal(models.ErrResourceNotFound))
					}
				})
			})
		} else {
			It("does not start an auction", func() {
				Expect(fakeAuctioneerClient.RequestLRPAuctionsCallCount()).To(Equal(0))
			})
		}

		if t.Result.Instance == nil {
			It("removes the /instance actualLRP", func() {
				_, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
				Expect(err).To(Equal(models.ErrResourceNotFound))
			})
		} else {
			if t.Result.Instance.ShouldUpdate {
				It("updates the /instance Since", func() {
					lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
					Expect(err).NotTo(HaveOccurred())

					Expect(lrpInBBS.Since).To(Equal(clock.Now().UnixNano()))
				})

				It("updates the /instance ModificationTag", func() {
					lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
					Expect(err).NotTo(HaveOccurred())

					Expect(lrpInBBS.ModificationTag.Index).To(Equal(initialInstanceModificationIndex + 1))
				})
			} else {
				It("does not update the /instance Since", func() {
					lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
					Expect(err).NotTo(HaveOccurred())

					Expect(lrpInBBS.Since).To(Equal(initialTimestamp))
				})
			}

			It("has the expected /instance state", func() {
				lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
				Expect(err).NotTo(HaveOccurred())

				Expect(lrpInBBS.State).To(Equal(t.Result.Instance.State))
			})

			It("has the expected /instance crash count", func() {
				lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
				Expect(err).NotTo(HaveOccurred())

				Expect(lrpInBBS.CrashCount).To(Equal(t.Result.Instance.CrashCount))
			})

			It("has the expected /instance crash reason", func() {
				lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
				Expect(err).NotTo(HaveOccurred())

				Expect(lrpInBBS.CrashReason).To(Equal(t.Result.Instance.CrashReason))
			})

			It("has the expected /instance instance key", func() {
				lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
				Expect(err).NotTo(HaveOccurred())

				Expect(lrpInBBS.ActualLRPInstanceKey).To(Equal(t.Result.Instance.ActualLRPInstanceKey))
			})

			It("has the expected /instance net info", func() {
				lrpInBBS, err := etcdHelper.GetInstanceActualLRP(&lrpKey)
				Expect(err).NotTo(HaveOccurred())

				Expect(lrpInBBS.ActualLRPNetInfo).To(Equal(t.Result.Instance.ActualLRPNetInfo))
			})
		}

		if t.Result.Evacuating == nil {
			It("removes the /evacuating actualLRP", func() {
				_, err := getEvacuatingActualLRPTTL(lrpKey)
				Expect(err).To(Equal(models.ErrResourceNotFound))
			})
		} else {
			if t.Result.Evacuating.ShouldUpdate {
				It("updates the /evacuating Since", func() {
					group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
					Expect(err).NotTo(HaveOccurred())
					lrpInBBS := group.Evacuating

					Expect(lrpInBBS.Since).To(Equal(clock.Now().UnixNano()))
				})

				It("updates the /evacuating TTL to the desired value", func() {
					ttl, err := getEvacuatingActualLRPTTL(lrpKey)
					Expect(err).NotTo(HaveOccurred())

					Expect(ttl).To(BeNumerically("~", t.Result.Evacuating.TTL, allowedTTLDecay))
				})

				It("updates the /evacuating ModificationTag", func() {
					group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
					Expect(err).NotTo(HaveOccurred())
					lrpInBBS := group.Evacuating

					Expect(lrpInBBS.ModificationTag.Index).To(Equal(initialEvacuatingModificationIndex + 1))
				})
			} else {
				It("does not update the /evacuating Since", func() {
					group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
					Expect(err).NotTo(HaveOccurred())
					lrpInBBS := group.Evacuating

					Expect(lrpInBBS.Since).To(Equal(initialTimestamp))
				})

				It("does not update the /evacuating TTL", func() {
					ttl, err := getEvacuatingActualLRPTTL(lrpKey)
					Expect(err).NotTo(HaveOccurred())

					Expect(ttl).To(BeNumerically("~", omegaEvacuationTTL, allowedTTLDecay))
				})
			}

			It("has the expected /evacuating state", func() {
				group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
				Expect(err).NotTo(HaveOccurred())
				lrpInBBS := group.Evacuating

				Expect(lrpInBBS.State).To(Equal(t.Result.Evacuating.State))
			})

			It("has the expected /evacuating instance key", func() {
				group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
				Expect(err).NotTo(HaveOccurred())
				lrpInBBS := group.Evacuating

				Expect(lrpInBBS.ActualLRPInstanceKey).To(Equal(t.Result.Evacuating.ActualLRPInstanceKey))
			})

			It("has the expected /evacuating net info", func() {
				group, err := etcdDB.ActualLRPGroupByProcessGuidAndIndex(logger, lrpKey.ProcessGuid, lrpKey.Index)
				Expect(err).NotTo(HaveOccurred())
				lrpInBBS := group.Evacuating

				Expect(lrpInBBS.ActualLRPNetInfo).To(Equal(t.Result.Evacuating.ActualLRPNetInfo))
			})
		}
	})
}