Exemplo n.º 1
0
// Describe returns the description of a DeploymentConfig
func (d *DeploymentConfigDescriber) Describe(namespace, name string) (string, error) {
	deploymentConfig, err := d.client.getDeploymentConfig(namespace, name)
	if err != nil {
		return "", err
	}
	events, err := d.client.listEvents(deploymentConfig)
	if err != nil {
		return "", err
	}

	return tabbedString(func(out *tabwriter.Writer) error {
		formatMeta(out, deploymentConfig.ObjectMeta)

		if deploymentConfig.LatestVersion == 0 {
			formatString(out, "Latest Version", "Not deployed")
		} else {
			formatString(out, "Latest Version", strconv.Itoa(deploymentConfig.LatestVersion))
		}

		printTriggers(deploymentConfig.Triggers, out)

		formatString(out, "Strategy", deploymentConfig.Template.Strategy.Type)
		printStrategy(deploymentConfig.Template.Strategy, out)
		printReplicationControllerSpec(deploymentConfig.Template.ControllerTemplate, out)

		deploymentName := deployutil.LatestDeploymentNameForConfig(deploymentConfig)
		deployment, err := d.client.getDeployment(namespace, deploymentName)
		if err != nil {
			if kerrors.IsNotFound(err) {
				formatString(out, "Latest Deployment", "<none>")
			} else {
				formatString(out, "Latest Deployment", fmt.Sprintf("error: %v", err))
			}
		} else {
			header := fmt.Sprintf("Deployment #%d (latest)", deployutil.DeploymentVersionFor(deployment))
			printDeploymentRc(deployment, d.client, out, header, true)
		}
		deploymentsHistory, err := d.client.listDeployments(namespace, labels.Everything())
		if err == nil {
			sorted := rcSorter{}
			sorted = append(sorted, deploymentsHistory.Items...)
			sort.Sort(sorted)
			for _, item := range sorted {
				if item.Name != deploymentName && deploymentConfig.Name == deployutil.DeploymentConfigNameFor(&item) {
					header := fmt.Sprintf("Deployment #%d", deployutil.DeploymentVersionFor(&item))
					printDeploymentRc(&item, d.client, out, header, false)
				}
			}
		}

		if events != nil {
			kctl.DescribeEvents(events, out)
		}
		return nil
	})
}
func TestTriggers_manual(t *testing.T) {
	testutil.DeleteAllEtcdKeys()
	openshift := NewTestDeployOpenshift(t)
	defer openshift.Close()

	config := deploytest.OkDeploymentConfig(0)
	config.Namespace = testutil.Namespace()
	config.Triggers = []deployapi.DeploymentTriggerPolicy{
		{
			Type: deployapi.DeploymentTriggerManual,
		},
	}

	var err error

	dc, err := openshift.Client.DeploymentConfigs(testutil.Namespace()).Create(config)
	if err != nil {
		t.Fatalf("Couldn't create DeploymentConfig: %v %#v", err, config)
	}

	watch, err := openshift.KubeClient.ReplicationControllers(testutil.Namespace()).Watch(labels.Everything(), fields.Everything(), dc.ResourceVersion)
	if err != nil {
		t.Fatalf("Couldn't subscribe to Deployments: %v", err)
	}
	defer watch.Stop()

	config, err = openshift.Client.DeploymentConfigs(testutil.Namespace()).Generate(config.Name)
	if err != nil {
		t.Fatalf("Error generating config: %v", err)
	}
	if config.LatestVersion != 1 {
		t.Fatalf("Generated deployment should have version 1: %#v", config)
	}
	glog.Infof("config(1): %#v", config)

	new, err := openshift.Client.DeploymentConfigs(testutil.Namespace()).Update(config)
	if err != nil {
		t.Fatalf("Couldn't create updated DeploymentConfig: %v %#v", err, config)
	}
	glog.Infof("config(2): %#v", new)

	event := <-watch.ResultChan()
	if e, a := watchapi.Added, event.Type; e != a {
		t.Fatalf("expected watch event type %s, got %s", e, a)
	}
	deployment := event.Object.(*kapi.ReplicationController)

	if e, a := config.Name, deployutil.DeploymentConfigNameFor(deployment); e != a {
		t.Fatalf("Expected deployment annotated with deploymentConfig '%s', got '%s'", e, a)
	}
	if e, a := 1, deployutil.DeploymentVersionFor(deployment); e != a {
		t.Fatalf("Deployment annotation version does not match: %#v", deployment)
	}
}
Exemplo n.º 3
0
// cancel cancels any deployment process in progress for config.
func (c *cancelDeploymentCommand) cancel(config *deployapi.DeploymentConfig, out io.Writer) error {
	deployments, err := c.client.ListDeploymentsForConfig(config.Namespace, config.Name)
	if err != nil {
		return err
	}
	if len(deployments.Items) == 0 {
		fmt.Fprintln(out, "no deployments found to cancel")
		return nil
	}
	failedCancellations := []string{}
	for _, deployment := range deployments.Items {
		status := deployutil.DeploymentStatusFor(&deployment)

		switch status {
		case deployapi.DeploymentStatusNew,
			deployapi.DeploymentStatusPending,
			deployapi.DeploymentStatusRunning:

			if deployutil.IsDeploymentCancelled(&deployment) {
				continue
			}

			deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
			deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser
			_, err := c.client.UpdateDeployment(&deployment)
			if err == nil {
				fmt.Fprintf(out, "cancelled #%d\n", config.LatestVersion)
			} else {
				fmt.Fprintf(out, "couldn't cancel deployment %d (status: %s): %v\n", deployutil.DeploymentVersionFor(&deployment), status, err)
				failedCancellations = append(failedCancellations, strconv.Itoa(deployutil.DeploymentVersionFor(&deployment)))
			}
		default:
			fmt.Fprintln(out, "no active deployments to cancel")
		}
	}
	if len(failedCancellations) == 0 {
		return nil
	} else {
		return fmt.Errorf("couldn't cancel deployment %s", strings.Join(failedCancellations, ", "))
	}
}
Exemplo n.º 4
0
func JoinDeployments(node *DeploymentConfigNode, deploys []kapi.ReplicationController) {
	matches := []*kapi.ReplicationController{}
	for i := range deploys {
		if belongsToDeploymentConfig(node.DeploymentConfig, &deploys[i]) {
			matches = append(matches, &deploys[i])
		}
	}
	if len(matches) == 0 {
		return
	}
	sort.Sort(RecentDeploymentReferences(matches))
	if node.DeploymentConfig.LatestVersion == deployutil.DeploymentVersionFor(matches[0]) {
		node.ActiveDeployment = matches[0]
		node.Deployments = matches[1:]
		return
	}
	node.Deployments = matches
}
func describeDeploymentStatus(deploy *kapi.ReplicationController, first bool) string {
	timeAt := strings.ToLower(formatRelativeTime(deploy.CreationTimestamp.Time))
	status := deployutil.DeploymentStatusFor(deploy)
	version := deployutil.DeploymentVersionFor(deploy)
	switch status {
	case deployapi.DeploymentStatusFailed:
		reason := deployutil.DeploymentStatusReasonFor(deploy)
		if len(reason) > 0 {
			reason = fmt.Sprintf(": %s", reason)
		}
		// TODO: encode fail time in the rc
		return fmt.Sprintf("#%d deployment failed %s ago%s%s", version, timeAt, reason, describeDeploymentPodSummaryInline(deploy, false))
	case deployapi.DeploymentStatusComplete:
		// TODO: pod status output
		return fmt.Sprintf("#%d deployed %s ago%s", version, timeAt, describeDeploymentPodSummaryInline(deploy, first))
	case deployapi.DeploymentStatusRunning:
		return fmt.Sprintf("#%d deployment running for %s%s", version, timeAt, describeDeploymentPodSummaryInline(deploy, false))
	default:
		return fmt.Sprintf("#%d deployment %s %s ago%s", version, strings.ToLower(string(status)), timeAt, describeDeploymentPodSummaryInline(deploy, false))
	}
}
Exemplo n.º 6
0
func (m RecentDeploymentReferences) Less(i, j int) bool {
	return deployutil.DeploymentVersionFor(m[i]) > deployutil.DeploymentVersionFor(m[j])
}
func TestDeployer_deployScenarios(t *testing.T) {
	mkd := func(version int, status deployapi.DeploymentStatus, replicas int, desired int) *kapi.ReplicationController {
		deployment := mkdeployment(version, status)
		deployment.Spec.Replicas = replicas
		if desired > 0 {
			deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desired)
		}
		return deployment
	}
	type scaleEvent struct {
		version int
		size    int
	}
	scenarios := []struct {
		name        string
		deployments []*kapi.ReplicationController
		fromVersion int
		toVersion   int
		scaleEvents []scaleEvent
	}{
		{
			"initial deployment",
			// existing deployments
			[]*kapi.ReplicationController{
				mkd(1, deployapi.DeploymentStatusNew, 0, 3),
			},
			// from and to version
			0, 1,
			// expected scale events
			[]scaleEvent{},
		},
		{
			"last deploy failed",
			// existing deployments
			[]*kapi.ReplicationController{
				mkd(1, deployapi.DeploymentStatusComplete, 3, 0),
				mkd(2, deployapi.DeploymentStatusFailed, 1, 3),
				mkd(3, deployapi.DeploymentStatusNew, 0, 3),
			},
			// from and to version
			1, 3,
			// expected scale events
			[]scaleEvent{
				{2, 0},
			},
		},
		{
			"sequential complete",
			// existing deployments
			[]*kapi.ReplicationController{
				mkd(1, deployapi.DeploymentStatusComplete, 0, 0),
				mkd(2, deployapi.DeploymentStatusComplete, 3, 0),
				mkd(3, deployapi.DeploymentStatusNew, 0, 3),
			},
			// from and to version
			2, 3,
			// expected scale events
			[]scaleEvent{},
		},
		{
			"sequential failure",
			// existing deployments
			[]*kapi.ReplicationController{
				mkd(1, deployapi.DeploymentStatusFailed, 1, 3),
				mkd(2, deployapi.DeploymentStatusFailed, 1, 3),
				mkd(3, deployapi.DeploymentStatusNew, 0, 3),
			},
			// from and to version
			0, 3,
			// expected scale events
			[]scaleEvent{
				{1, 0},
				{2, 0},
			},
		},
	}

	for _, s := range scenarios {
		t.Logf("executing scenario %s", s.name)
		findDeployment := func(version int) *kapi.ReplicationController {
			for _, d := range s.deployments {
				if deployutil.DeploymentVersionFor(d) == version {
					return d
				}
			}
			return nil
		}

		var actualFrom, actualTo *kapi.ReplicationController
		var actualDesired int
		to := findDeployment(s.toVersion)
		scaler := &scalertest.FakeScaler{}

		deployer := &Deployer{
			strategyFor: func(config *deployapi.DeploymentConfig) (strategy.DeploymentStrategy, error) {
				return &testStrategy{
					deployFunc: func(from *kapi.ReplicationController, to *kapi.ReplicationController, desiredReplicas int) error {
						actualFrom = from
						actualTo = to
						actualDesired = desiredReplicas
						return nil
					},
				}, nil
			},
			getDeployment: func(namespace, name string) (*kapi.ReplicationController, error) {
				return to, nil
			},
			getDeployments: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
				list := &kapi.ReplicationControllerList{}
				for _, d := range s.deployments {
					list.Items = append(list.Items, *d)
				}
				return list, nil
			},
			scaler: scaler,
		}

		err := deployer.Deploy(to.Namespace, to.Name)
		if err != nil {
			t.Fatalf("unexpcted error: %v", err)
		}

		if s.fromVersion > 0 {
			if e, a := s.fromVersion, deployutil.DeploymentVersionFor(actualFrom); e != a {
				t.Fatalf("expected from.latestVersion %d, got %d", e, a)
			}
		}
		if e, a := s.toVersion, deployutil.DeploymentVersionFor(actualTo); e != a {
			t.Fatalf("expected to.latestVersion %d, got %d", e, a)
		}
		if e, a := len(s.scaleEvents), len(scaler.Events); e != a {
			t.Fatalf("expected %d scale events, got %d", e, a)
		}
		for _, expected := range s.scaleEvents {
			expectedTo := findDeployment(expected.version)
			expectedWasScaled := false
			for _, actual := range scaler.Events {
				if actual.Name != expectedTo.Name {
					continue
				}
				if e, a := uint(expected.size), actual.Size; e != a {
					t.Fatalf("expected version %d to be scaled to %d, got %d", expected.version, e, a)
				}
				expectedWasScaled = true
			}
			if !expectedWasScaled {
				t.Fatalf("expected version %d to be scaled to %d, but it wasn't scaled at all", expected.version, expected.size)
			}
		}
	}
}
Exemplo n.º 8
0
// Handle processes config and creates a new deployment if necessary.
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
	// Only deploy when the version has advanced past 0.
	if config.LatestVersion == 0 {
		glog.V(5).Infof("Waiting for first version of %s", deployutil.LabelForDeploymentConfig(config))
		return nil
	}

	// Check if any existing inflight deployments (any non-terminal state).
	existingDeployments, err := c.deploymentClient.listDeploymentsForConfig(config.Namespace, config.Name)
	if err != nil {
		return fmt.Errorf("couldn't list Deployments for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
	}
	var inflightDeployment *kapi.ReplicationController
	latestDeploymentExists := false
	for _, deployment := range existingDeployments.Items {
		// check if this is the latest deployment
		// we'll return after we've dealt with the multiple-active-deployments case
		if deployutil.DeploymentVersionFor(&deployment) == config.LatestVersion {
			latestDeploymentExists = true
		}

		deploymentStatus := deployutil.DeploymentStatusFor(&deployment)
		switch deploymentStatus {
		case deployapi.DeploymentStatusFailed,
			deployapi.DeploymentStatusComplete:
			// Previous deployment in terminal state - can ignore
			// Ignoring specific deployment states so that any newly introduced
			// deployment state will not be ignored
		default:
			if inflightDeployment == nil {
				inflightDeployment = &deployment
				continue
			}
			var deploymentForCancellation *kapi.ReplicationController
			if deployutil.DeploymentVersionFor(inflightDeployment) < deployutil.DeploymentVersionFor(&deployment) {
				deploymentForCancellation, inflightDeployment = inflightDeployment, &deployment
			} else {
				deploymentForCancellation = &deployment
			}

			deploymentForCancellation.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
			deploymentForCancellation.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
			if _, err := c.deploymentClient.updateDeployment(deploymentForCancellation.Namespace, deploymentForCancellation); err != nil {
				util.HandleError(fmt.Errorf("couldn't cancel Deployment %s: %v", deployutil.LabelForDeployment(deploymentForCancellation), err))
			}
			glog.V(4).Infof("Cancelled Deployment %s for DeploymentConfig %s", deployutil.LabelForDeployment(deploymentForCancellation), deployutil.LabelForDeploymentConfig(config))
		}
	}

	// if the latest deployment exists then nothing else needs to be done
	if latestDeploymentExists {
		return nil
	}

	// check to see if there are inflight deployments
	if inflightDeployment != nil {
		// raise a transientError so that the deployment config can be re-queued
		glog.V(4).Infof("Found previous inflight Deployment for %s - will requeue", deployutil.LabelForDeploymentConfig(config))
		return transientError(fmt.Sprintf("found previous inflight Deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)))
	}

	// Try and build a deployment for the config.
	deployment, err := c.makeDeployment(config)
	if err != nil {
		return fatalError(fmt.Sprintf("couldn't make Deployment from (potentially invalid) DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err))
	}

	// Compute the desired replicas for the deployment. Use the last completed
	// deployment's current replica count, or the config template if there is no
	// prior completed deployment available.
	desiredReplicas := config.Template.ControllerTemplate.Replicas
	if len(existingDeployments.Items) > 0 {
		sort.Sort(deployutil.DeploymentsByLatestVersionDesc(existingDeployments.Items))
		for _, existing := range existingDeployments.Items {
			if deployutil.DeploymentStatusFor(&existing) == deployapi.DeploymentStatusComplete {
				desiredReplicas = existing.Spec.Replicas
				glog.V(4).Infof("Desired replicas for %s set to %d based on prior completed deployment %s", deployutil.LabelForDeploymentConfig(config), desiredReplicas, existing.Name)
				break
			}
		}
	}
	deployment.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(desiredReplicas)

	// Create the deployment.
	if _, err := c.deploymentClient.createDeployment(config.Namespace, deployment); err == nil {
		glog.V(4).Infof("Created Deployment for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config))
		return nil
	} else {
		// If the deployment was already created, just move on. The cache could be stale, or another
		// process could have already handled this update.
		if errors.IsAlreadyExists(err) {
			glog.V(4).Infof("Deployment already exists for DeploymentConfig %s", deployutil.LabelForDeploymentConfig(config))
			return nil
		}

		// log an event if the deployment could not be created that the user can discover
		c.recorder.Eventf(config, "failedCreate", "Error creating: %v", err)
		return fmt.Errorf("couldn't create Deployment for DeploymentConfig %s: %v", deployutil.LabelForDeploymentConfig(config), err)
	}
}
Exemplo n.º 9
0
// TestCmdDeploy_cancelOk ensures that attempts to cancel deployments
// for a config result in cancelling all in-progress deployments
// and none of the completed/faild ones.
func TestCmdDeploy_cancelOk(t *testing.T) {
	var (
		config              *deployapi.DeploymentConfig
		existingDeployments *kapi.ReplicationControllerList
		updatedDeployments  []kapi.ReplicationController
	)

	commandClient := &deployCommandClientImpl{
		GetDeploymentFn: func(namespace, name string) (*kapi.ReplicationController, error) {
			t.Fatalf("unexpected call to GetDeployment: %s", name)
			return nil, nil
		},
		ListDeploymentsForConfigFn: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
			return existingDeployments, nil
		},
		UpdateDeploymentConfigFn: func(config *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) {
			t.Fatalf("unexpected call to UpdateDeploymentConfig")
			return nil, nil
		},
		UpdateDeploymentFn: func(deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
			updatedDeployments = append(updatedDeployments, *deployment)
			return deployment, nil
		},
	}

	type existing struct {
		version      int
		status       deployapi.DeploymentStatus
		shouldCancel bool
	}
	type scenario struct {
		version  int
		existing []existing
	}

	scenarios := []scenario{
		// No existing deployments
		{1, []existing{{1, deployapi.DeploymentStatusComplete, false}}},
		// A single existing failed deployment
		{1, []existing{{1, deployapi.DeploymentStatusFailed, false}}},
		// Multiple existing completed/failed deployments
		{2, []existing{{2, deployapi.DeploymentStatusFailed, false}, {1, deployapi.DeploymentStatusComplete, false}}},
		// A single existing new deployment
		{1, []existing{{1, deployapi.DeploymentStatusNew, true}}},
		// A single existing pending deployment
		{1, []existing{{1, deployapi.DeploymentStatusPending, true}}},
		// A single existing running deployment
		{1, []existing{{1, deployapi.DeploymentStatusRunning, true}}},
		// Multiple existing deployments with one in new/pending/running
		{3, []existing{{3, deployapi.DeploymentStatusRunning, true}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusFailed, false}}},
		// Multiple existing deployments with more than one in new/pending/running
		{3, []existing{{3, deployapi.DeploymentStatusNew, true}, {2, deployapi.DeploymentStatusRunning, true}, {1, deployapi.DeploymentStatusFailed, false}}},
	}

	c := &cancelDeploymentCommand{client: commandClient}
	for _, scenario := range scenarios {
		updatedDeployments = []kapi.ReplicationController{}
		config = deploytest.OkDeploymentConfig(scenario.version)
		existingDeployments = &kapi.ReplicationControllerList{}
		for _, e := range scenario.existing {
			d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), api.Codec)
			d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status)
			existingDeployments.Items = append(existingDeployments.Items, *d)
		}

		err := c.cancel(config, ioutil.Discard)
		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}

		expectedCancellations := []int{}
		actualCancellations := []int{}
		for _, e := range scenario.existing {
			if e.shouldCancel {
				expectedCancellations = append(expectedCancellations, e.version)
			}
		}
		for _, d := range updatedDeployments {
			actualCancellations = append(actualCancellations, deployutil.DeploymentVersionFor(&d))
		}

		sort.Ints(actualCancellations)
		sort.Ints(expectedCancellations)
		if !reflect.DeepEqual(actualCancellations, expectedCancellations) {
			t.Fatalf("expected cancellations: %v, actual: %v", expectedCancellations, actualCancellations)
		}
	}
}
// TestHandle_existingDeployments ensures that an attempt to create a
// new deployment for a config that has existing deployments succeeds of fails
// depending upon the state of the existing deployments
func TestHandle_existingDeployments(t *testing.T) {
	var updatedDeployments []kapi.ReplicationController
	var (
		config              *deployapi.DeploymentConfig
		deployed            *kapi.ReplicationController
		existingDeployments *kapi.ReplicationControllerList
	)

	controller := &DeploymentConfigController{
		makeDeployment: func(config *deployapi.DeploymentConfig) (*kapi.ReplicationController, error) {
			return deployutil.MakeDeployment(config, api.Codec)
		},
		deploymentClient: &deploymentClientImpl{
			createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
				deployed = deployment
				return deployment, nil
			},
			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
				return existingDeployments, nil
			},
			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
				updatedDeployments = append(updatedDeployments, *deployment)
				//t.Fatalf("unexpected update call with deployment %v", deployment)
				return deployment, nil
			},
		},
		recorder: &record.FakeRecorder{},
	}

	type existing struct {
		version      int
		status       deployapi.DeploymentStatus
		shouldCancel bool
	}

	type scenario struct {
		version          int
		existing         []existing
		errorType        reflect.Type
		expectDeployment bool
	}

	transientErrorType := reflect.TypeOf(transientError(""))
	scenarios := []scenario{
		// No existing deployments
		{1, []existing{}, nil, true},
		// A single existing completed deployment
		{2, []existing{{1, deployapi.DeploymentStatusComplete, false}}, nil, true},
		// A single existing failed deployment
		{2, []existing{{1, deployapi.DeploymentStatusFailed, false}}, nil, true},
		// Multiple existing completed/failed deployments
		{3, []existing{{2, deployapi.DeploymentStatusFailed, false}, {1, deployapi.DeploymentStatusComplete, false}}, nil, true},

		// A single existing deployment in the default state
		{2, []existing{{1, "", false}}, transientErrorType, false},
		// A single existing new deployment
		{2, []existing{{1, deployapi.DeploymentStatusNew, false}}, transientErrorType, false},
		// A single existing pending deployment
		{2, []existing{{1, deployapi.DeploymentStatusPending, false}}, transientErrorType, false},
		// A single existing running deployment
		{2, []existing{{1, deployapi.DeploymentStatusRunning, false}}, transientErrorType, false},
		// Multiple existing deployments with one in new/pending/running
		{4, []existing{{3, deployapi.DeploymentStatusRunning, false}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusFailed, false}}, transientErrorType, false},

		// Latest deployment exists and has already failed/completed
		{2, []existing{{2, deployapi.DeploymentStatusFailed, false}, {1, deployapi.DeploymentStatusComplete, false}}, nil, false},
		// Latest deployment exists and is in new/pending/running state
		{2, []existing{{2, deployapi.DeploymentStatusRunning, false}, {1, deployapi.DeploymentStatusComplete, false}}, nil, false},

		// Multiple existing deployments with more than one in new/pending/running
		{4, []existing{{3, deployapi.DeploymentStatusNew, false}, {2, deployapi.DeploymentStatusRunning, true}, {1, deployapi.DeploymentStatusFailed, false}}, transientErrorType, false},
		// Multiple existing deployments with more than one in new/pending/running
		// Latest deployment has already failed
		{6, []existing{{5, deployapi.DeploymentStatusFailed, false}, {4, deployapi.DeploymentStatusRunning, false}, {3, deployapi.DeploymentStatusNew, true}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusNew, true}}, transientErrorType, false},
	}

	for _, scenario := range scenarios {
		updatedDeployments = []kapi.ReplicationController{}
		deployed = nil
		config = deploytest.OkDeploymentConfig(scenario.version)
		existingDeployments = &kapi.ReplicationControllerList{}
		for _, e := range scenario.existing {
			d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), api.Codec)
			if e.status != "" {
				d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status)
			}
			existingDeployments.Items = append(existingDeployments.Items, *d)
		}
		err := controller.Handle(config)

		if scenario.expectDeployment && deployed == nil {
			t.Fatalf("expected a deployment")
		}

		if scenario.errorType == nil {
			if err != nil {
				t.Fatalf("unexpected error: %v", err)
			}
		} else {
			if err == nil {
				t.Fatalf("expected error")
			}
			if reflect.TypeOf(err) != scenario.errorType {
				t.Fatalf("error expected: %s, got: %s", scenario.errorType, reflect.TypeOf(err))
			}
		}

		expectedCancellations := []int{}
		actualCancellations := []int{}
		for _, e := range scenario.existing {
			if e.shouldCancel {
				expectedCancellations = append(expectedCancellations, e.version)
			}
		}
		for _, d := range updatedDeployments {
			actualCancellations = append(actualCancellations, deployutil.DeploymentVersionFor(&d))
		}

		sort.Ints(actualCancellations)
		sort.Ints(expectedCancellations)
		if !reflect.DeepEqual(actualCancellations, expectedCancellations) {
			t.Fatalf("expected cancellations: %v, actual: %v", expectedCancellations, actualCancellations)
		}
	}
}
// TestHandle_cleanupDeploymentFailure ensures that clean up happens
// for the deployment if the deployer pod fails.
//  - failed deployment is scaled down
//  - the last completed deployment is scaled back up
func TestHandle_cleanupDeploymentFailure(t *testing.T) {
	var existingDeployments *kapi.ReplicationControllerList
	var failedDeployment *kapi.ReplicationController
	// map of deployment-version to updated replicas
	var updatedDeployments map[int]*kapi.ReplicationController

	controller := &DeployerPodController{
		deploymentClient: &deploymentClientImpl{
			getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
				return failedDeployment, nil
			},
			updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
				if _, found := updatedDeployments[deployutil.DeploymentVersionFor(deployment)]; found {
					t.Fatalf("unexpected multiple updates for deployment #%d", deployutil.DeploymentVersionFor(deployment))
				}
				updatedDeployments[deployutil.DeploymentVersionFor(deployment)] = deployment
				return deployment, nil
			},
			listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
				return existingDeployments, nil
			},
		},
	}

	type existing struct {
		version         int
		status          deployapi.DeploymentStatus
		initialReplicas int
		updatedReplicas int
	}

	type scenario struct {
		name string
		// this is the deployment that is passed to Handle
		version int
		// this is the target replicas for the deployment that failed
		desiredReplicas int
		// existing deployments also include the one being handled currently
		existing []existing
	}

	// existing deployments intentionally placed un-ordered
	// in order to verify sorting
	scenarios := []scenario{
		{"No previous deployments",
			1, 3, []existing{
				{1, deployapi.DeploymentStatusRunning, 3, 0},
			}},
		{"Multiple existing deployments - none in complete state",
			3, 2, []existing{
				{1, deployapi.DeploymentStatusFailed, 2, 2},
				{2, deployapi.DeploymentStatusFailed, 0, 0},
				{3, deployapi.DeploymentStatusRunning, 2, 0},
			}},
		{"Failed deployment is already at 0 replicas",
			3, 2, []existing{
				{1, deployapi.DeploymentStatusFailed, 2, 2},
				{2, deployapi.DeploymentStatusFailed, 0, 0},
				{3, deployapi.DeploymentStatusRunning, 0, 0},
			}},
		{"Multiple existing completed deployments",
			4, 2, []existing{
				{3, deployapi.DeploymentStatusComplete, 0, 2},
				{2, deployapi.DeploymentStatusComplete, 0, 0},
				{4, deployapi.DeploymentStatusRunning, 1, 0},
				{1, deployapi.DeploymentStatusFailed, 0, 0},
			}},
		// A deployment already exists after the current failed deployment
		// only the current deployment is marked as failed
		// the completed deployment is not scaled up
		{"Deployment exists after current failed",
			4, 2, []existing{
				{3, deployapi.DeploymentStatusComplete, 1, 1},
				{2, deployapi.DeploymentStatusComplete, 0, 0},
				{4, deployapi.DeploymentStatusRunning, 2, 0},
				{5, deployapi.DeploymentStatusNew, 0, 0},
				{1, deployapi.DeploymentStatusFailed, 0, 0},
			}},
	}

	for _, scenario := range scenarios {
		t.Logf("running scenario: %s", scenario.name)
		updatedDeployments = make(map[int]*kapi.ReplicationController)
		failedDeployment = nil
		existingDeployments = &kapi.ReplicationControllerList{}

		for _, e := range scenario.existing {
			d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), kapi.Codec)
			d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status)
			d.Spec.Replicas = e.initialReplicas
			// if this is the deployment passed to Handle, set the desired replica annotation
			if e.version == scenario.version {
				d.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(scenario.desiredReplicas)
				failedDeployment = d
			}
			existingDeployments.Items = append(existingDeployments.Items, *d)
		}
		associatedDeployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(scenario.version), kapi.Codec)
		err := controller.Handle(terminatedPod(associatedDeployment))

		if err != nil {
			t.Fatalf("unexpected error: %v", err)
		}

		// only the failed and the last completed deployment should be updated
		if len(updatedDeployments) > 2 {
			t.Fatalf("expected to update only the failed and last completed deployment")
		}

		for _, existing := range scenario.existing {
			updatedDeployment, ok := updatedDeployments[existing.version]
			if existing.initialReplicas != existing.updatedReplicas {
				if !ok {
					t.Fatalf("expected deployment #%d to be updated", existing.version)
				}
				if e, a := existing.updatedReplicas, updatedDeployment.Spec.Replicas; e != a {
					t.Fatalf("expected deployment #%d to be scaled to %d, got %d", existing.version, e, a)
				}
			} else if ok && existing.version != scenario.version {
				t.Fatalf("unexpected update for deployment #%d; replicas %d; status: %s", existing.version, updatedDeployment.Spec.Replicas, deployutil.DeploymentStatusFor(updatedDeployment))
			}
		}
		if deployutil.DeploymentStatusFor(updatedDeployments[scenario.version]) != deployapi.DeploymentStatusFailed {
			t.Fatalf("status for deployment #%d expected to be updated to failed; got %s", scenario.version, deployutil.DeploymentStatusFor(updatedDeployments[scenario.version]))
		}
		if updatedDeployments[scenario.version].Spec.Replicas != 0 {
			t.Fatalf("deployment #%d expected to be scaled down to 0; got %d", scenario.version, updatedDeployments[scenario.version].Spec.Replicas)
		}
	}
}