// TestHandle_cleanupDeploymentFailure ensures that clean up happens
// for the deployment if the deployer pod fails.
// - failed deployment is scaled down
// - the last completed deployment is scaled back up
func TestHandle_cleanupDeploymentFailure(t *testing.T) {
var existingDeployments *kapi.ReplicationControllerList
var failedDeployment *kapi.ReplicationController
// map of deployment-version to updated replicas
var updatedDeployments map[int]*kapi.ReplicationController
controller := &DeployerPodController{
deploymentClient: &deploymentClientImpl{
getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
return failedDeployment, nil
},
updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
if _, found := updatedDeployments[deployutil.DeploymentVersionFor(deployment)]; found {
t.Fatalf("unexpected multiple updates for deployment #%d", deployutil.DeploymentVersionFor(deployment))
}
updatedDeployments[deployutil.DeploymentVersionFor(deployment)] = deployment
return deployment, nil
},
listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return existingDeployments, nil
},
},
}
type existing struct {
version int
status deployapi.DeploymentStatus
initialReplicas int
updatedReplicas int
}
type scenario struct {
name string
// this is the deployment that is passed to Handle
version int
// this is the target replicas for the deployment that failed
desiredReplicas int
// existing deployments also include the one being handled currently
existing []existing
}
// existing deployments intentionally placed un-ordered
// in order to verify sorting
scenarios := []scenario{
{"No previous deployments",
1, 3, []existing{
{1, deployapi.DeploymentStatusRunning, 3, 0},
}},
{"Multiple existing deployments - none in complete state",
3, 2, []existing{
{1, deployapi.DeploymentStatusFailed, 2, 2},
{2, deployapi.DeploymentStatusFailed, 0, 0},
{3, deployapi.DeploymentStatusRunning, 2, 0},
}},
{"Failed deployment is already at 0 replicas",
3, 2, []existing{
{1, deployapi.DeploymentStatusFailed, 2, 2},
{2, deployapi.DeploymentStatusFailed, 0, 0},
{3, deployapi.DeploymentStatusRunning, 0, 0},
}},
{"Multiple existing completed deployments",
4, 2, []existing{
{3, deployapi.DeploymentStatusComplete, 0, 2},
{2, deployapi.DeploymentStatusComplete, 0, 0},
{4, deployapi.DeploymentStatusRunning, 1, 0},
{1, deployapi.DeploymentStatusFailed, 0, 0},
}},
// A deployment already exists after the current failed deployment
// only the current deployment is marked as failed
// the completed deployment is not scaled up
{"Deployment exists after current failed",
4, 2, []existing{
{3, deployapi.DeploymentStatusComplete, 1, 1},
{2, deployapi.DeploymentStatusComplete, 0, 0},
{4, deployapi.DeploymentStatusRunning, 2, 0},
{5, deployapi.DeploymentStatusNew, 0, 0},
{1, deployapi.DeploymentStatusFailed, 0, 0},
}},
}
for _, scenario := range scenarios {
t.Logf("running scenario: %s", scenario.name)
updatedDeployments = make(map[int]*kapi.ReplicationController)
failedDeployment = nil
existingDeployments = &kapi.ReplicationControllerList{}
for _, e := range scenario.existing {
d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), kapi.Codec)
d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status)
d.Spec.Replicas = e.initialReplicas
// if this is the deployment passed to Handle, set the desired replica annotation
if e.version == scenario.version {
d.Annotations[deployapi.DesiredReplicasAnnotation] = strconv.Itoa(scenario.desiredReplicas)
failedDeployment = d
}
existingDeployments.Items = append(existingDeployments.Items, *d)
}
associatedDeployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(scenario.version), kapi.Codec)
err := controller.Handle(terminatedPod(associatedDeployment))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// only the failed and the last completed deployment should be updated
if len(updatedDeployments) > 2 {
t.Fatalf("expected to update only the failed and last completed deployment")
}
for _, existing := range scenario.existing {
updatedDeployment, ok := updatedDeployments[existing.version]
if existing.initialReplicas != existing.updatedReplicas {
if !ok {
t.Fatalf("expected deployment #%d to be updated", existing.version)
}
if e, a := existing.updatedReplicas, updatedDeployment.Spec.Replicas; e != a {
t.Fatalf("expected deployment #%d to be scaled to %d, got %d", existing.version, e, a)
}
} else if ok && existing.version != scenario.version {
t.Fatalf("unexpected update for deployment #%d; replicas %d; status: %s", existing.version, updatedDeployment.Spec.Replicas, deployutil.DeploymentStatusFor(updatedDeployment))
}
}
if deployutil.DeploymentStatusFor(updatedDeployments[scenario.version]) != deployapi.DeploymentStatusFailed {
t.Fatalf("status for deployment #%d expected to be updated to failed; got %s", scenario.version, deployutil.DeploymentStatusFor(updatedDeployments[scenario.version]))
}
if updatedDeployments[scenario.version].Spec.Replicas != 0 {
t.Fatalf("deployment #%d expected to be scaled down to 0; got %d", scenario.version, updatedDeployments[scenario.version].Spec.Replicas)
}
}
}
// TestHandle_existingDeployments ensures that an attempt to create a
// new deployment for a config that has existing deployments succeeds of fails
// depending upon the state of the existing deployments
func TestHandle_existingDeployments(t *testing.T) {
var (
config *deployapi.DeploymentConfig
deployed *kapi.ReplicationController
existingDeployments *kapi.ReplicationControllerList
updatedDeployments []kapi.ReplicationController
)
controller := &DeploymentConfigController{
makeDeployment: func(cfg *deployapi.DeploymentConfig) (*kapi.ReplicationController, error) {
return deployutil.MakeDeployment(cfg, api.Codec)
},
deploymentClient: &deploymentClientImpl{
createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
deployed = deployment
return deployment, nil
},
listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return existingDeployments, nil
},
updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
updatedDeployments = append(updatedDeployments, *deployment)
return deployment, nil
},
},
osClient: testclient.NewSimpleFake(),
}
type existing struct {
version int
status deployapi.DeploymentStatus
shouldCancel bool
}
type scenario struct {
version int
existing []existing
errorType reflect.Type
expectDeployment bool
}
transientErrorType := reflect.TypeOf(transientError(""))
scenarios := []scenario{
// No existing deployments
{1, []existing{}, nil, true},
// A single existing completed deployment
{2, []existing{{1, deployapi.DeploymentStatusComplete, false}}, nil, true},
// A single existing failed deployment
{2, []existing{{1, deployapi.DeploymentStatusFailed, false}}, nil, true},
// Multiple existing completed/failed deployments
{3, []existing{{2, deployapi.DeploymentStatusFailed, false}, {1, deployapi.DeploymentStatusComplete, false}}, nil, true},
// A single existing deployment in the default state
{2, []existing{{1, "", false}}, transientErrorType, false},
// A single existing new deployment
{2, []existing{{1, deployapi.DeploymentStatusNew, false}}, transientErrorType, false},
// A single existing pending deployment
{2, []existing{{1, deployapi.DeploymentStatusPending, false}}, transientErrorType, false},
// A single existing running deployment
{2, []existing{{1, deployapi.DeploymentStatusRunning, false}}, transientErrorType, false},
// Multiple existing deployments with one in new/pending/running
{4, []existing{{3, deployapi.DeploymentStatusRunning, false}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusFailed, false}}, transientErrorType, false},
// Latest deployment exists and has already failed/completed
{2, []existing{{2, deployapi.DeploymentStatusFailed, false}, {1, deployapi.DeploymentStatusComplete, false}}, nil, false},
// Latest deployment exists and is in new/pending/running state
{2, []existing{{2, deployapi.DeploymentStatusRunning, false}, {1, deployapi.DeploymentStatusComplete, false}}, nil, false},
// Multiple existing deployments with more than one in new/pending/running
{4, []existing{{3, deployapi.DeploymentStatusNew, false}, {2, deployapi.DeploymentStatusRunning, true}, {1, deployapi.DeploymentStatusFailed, false}}, transientErrorType, false},
// Multiple existing deployments with more than one in new/pending/running
// Latest deployment has already failed
{6, []existing{{5, deployapi.DeploymentStatusFailed, false}, {4, deployapi.DeploymentStatusRunning, false}, {3, deployapi.DeploymentStatusNew, true}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusNew, true}}, transientErrorType, false},
}
for _, scenario := range scenarios {
updatedDeployments = []kapi.ReplicationController{}
deployed = nil
config = deploytest.OkDeploymentConfig(scenario.version)
existingDeployments = &kapi.ReplicationControllerList{}
for _, e := range scenario.existing {
d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), api.Codec)
if e.status != "" {
d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status)
}
existingDeployments.Items = append(existingDeployments.Items, *d)
}
controller.osClient = testclient.NewSimpleFake(config)
err := controller.Handle(config)
if scenario.expectDeployment && deployed == nil {
t.Fatalf("expected a deployment")
}
if scenario.errorType == nil {
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
} else {
if err == nil {
t.Fatalf("expected error")
}
if reflect.TypeOf(err) != scenario.errorType {
t.Fatalf("error expected: %s, got: %s", scenario.errorType, reflect.TypeOf(err))
}
}
expectedCancellations := []int{}
actualCancellations := []int{}
for _, e := range scenario.existing {
if e.shouldCancel {
expectedCancellations = append(expectedCancellations, e.version)
}
}
for _, d := range updatedDeployments {
actualCancellations = append(actualCancellations, deployutil.DeploymentVersionFor(&d))
}
sort.Ints(actualCancellations)
sort.Ints(expectedCancellations)
if !reflect.DeepEqual(actualCancellations, expectedCancellations) {
t.Fatalf("expected cancellations: %v, actual: %v", expectedCancellations, actualCancellations)
}
}
}
// TestHandle_updateOk ensures that an updated config (version >0) results in
// a new deployment with the appropriate replica count based on a variety of
// existing prior deployments.
func TestHandle_updateOk(t *testing.T) {
var (
config *deployapi.DeploymentConfig
deployed *kapi.ReplicationController
existingDeployments *kapi.ReplicationControllerList
)
controller := &DeploymentConfigController{
makeDeployment: func(config *deployapi.DeploymentConfig) (*kapi.ReplicationController, error) {
return deployutil.MakeDeployment(config, api.Codec)
},
deploymentClient: &deploymentClientImpl{
createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
deployed = deployment
return deployment, nil
},
listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return existingDeployments, nil
},
updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
t.Fatalf("unexpected update call with deployment %v", deployment)
return nil, nil
},
},
osClient: testclient.NewSimpleFake(),
}
type existing struct {
version int
replicas int
status deployapi.DeploymentStatus
}
type scenario struct {
version int
expectedReplicas int
existing []existing
}
scenarios := []scenario{
{1, 1, []existing{}},
{2, 1, []existing{
{1, 1, deployapi.DeploymentStatusComplete},
}},
{3, 4, []existing{
{1, 0, deployapi.DeploymentStatusComplete},
{2, 4, deployapi.DeploymentStatusComplete},
}},
{3, 4, []existing{
{1, 4, deployapi.DeploymentStatusComplete},
{2, 1, deployapi.DeploymentStatusFailed},
}},
{4, 2, []existing{
{1, 0, deployapi.DeploymentStatusComplete},
{2, 0, deployapi.DeploymentStatusFailed},
{3, 2, deployapi.DeploymentStatusComplete},
}},
// Scramble the order of the previous to ensure we still get it right.
{4, 2, []existing{
{2, 0, deployapi.DeploymentStatusFailed},
{3, 2, deployapi.DeploymentStatusComplete},
{1, 0, deployapi.DeploymentStatusComplete},
}},
}
for _, scenario := range scenarios {
deployed = nil
config = deploytest.OkDeploymentConfig(scenario.version)
config.Triggers = []deployapi.DeploymentTriggerPolicy{}
existingDeployments = &kapi.ReplicationControllerList{}
for _, e := range scenario.existing {
d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), api.Codec)
d.Spec.Replicas = e.replicas
d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status)
existingDeployments.Items = append(existingDeployments.Items, *d)
}
err := controller.Handle(config)
if deployed == nil {
t.Fatalf("expected a deployment")
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
desired, hasDesired := deployutil.DeploymentDesiredReplicas(deployed)
if !hasDesired {
t.Fatalf("expected desired replicas")
}
if e, a := scenario.expectedReplicas, desired; e != a {
t.Errorf("expected desired replicas %d, got %d", e, a)
}
}
}
// TestCmdDeploy_cancelOk ensures that attempts to cancel deployments
// for a config result in cancelling all in-progress deployments
// and none of the completed/faild ones.
func TestCmdDeploy_cancelOk(t *testing.T) {
var (
config *deployapi.DeploymentConfig
existingDeployments *kapi.ReplicationControllerList
updatedDeployments []kapi.ReplicationController
)
commandClient := &deployCommandClientImpl{
GetDeploymentFn: func(namespace, name string) (*kapi.ReplicationController, error) {
t.Fatalf("unexpected call to GetDeployment: %s", name)
return nil, nil
},
ListDeploymentsForConfigFn: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return existingDeployments, nil
},
UpdateDeploymentConfigFn: func(config *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) {
t.Fatalf("unexpected call to UpdateDeploymentConfig")
return nil, nil
},
UpdateDeploymentFn: func(deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
updatedDeployments = append(updatedDeployments, *deployment)
return deployment, nil
},
}
type existing struct {
version int
status deployapi.DeploymentStatus
shouldCancel bool
}
type scenario struct {
version int
existing []existing
}
scenarios := []scenario{
// No existing deployments
{1, []existing{{1, deployapi.DeploymentStatusComplete, false}}},
// A single existing failed deployment
{1, []existing{{1, deployapi.DeploymentStatusFailed, false}}},
// Multiple existing completed/failed deployments
{2, []existing{{2, deployapi.DeploymentStatusFailed, false}, {1, deployapi.DeploymentStatusComplete, false}}},
// A single existing new deployment
{1, []existing{{1, deployapi.DeploymentStatusNew, true}}},
// A single existing pending deployment
{1, []existing{{1, deployapi.DeploymentStatusPending, true}}},
// A single existing running deployment
{1, []existing{{1, deployapi.DeploymentStatusRunning, true}}},
// Multiple existing deployments with one in new/pending/running
{3, []existing{{3, deployapi.DeploymentStatusRunning, true}, {2, deployapi.DeploymentStatusComplete, false}, {1, deployapi.DeploymentStatusFailed, false}}},
// Multiple existing deployments with more than one in new/pending/running
{3, []existing{{3, deployapi.DeploymentStatusNew, true}, {2, deployapi.DeploymentStatusRunning, true}, {1, deployapi.DeploymentStatusFailed, false}}},
}
c := &cancelDeploymentCommand{client: commandClient}
for _, scenario := range scenarios {
updatedDeployments = []kapi.ReplicationController{}
config = deploytest.OkDeploymentConfig(scenario.version)
existingDeployments = &kapi.ReplicationControllerList{}
for _, e := range scenario.existing {
d, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(e.version), api.Codec)
d.Annotations[deployapi.DeploymentStatusAnnotation] = string(e.status)
existingDeployments.Items = append(existingDeployments.Items, *d)
}
err := c.cancel(config, ioutil.Discard)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectedCancellations := []int{}
actualCancellations := []int{}
for _, e := range scenario.existing {
if e.shouldCancel {
expectedCancellations = append(expectedCancellations, e.version)
}
}
for _, d := range updatedDeployments {
actualCancellations = append(actualCancellations, deployutil.DeploymentVersionFor(&d))
}
sort.Ints(actualCancellations)
sort.Ints(expectedCancellations)
if !reflect.DeepEqual(actualCancellations, expectedCancellations) {
t.Fatalf("expected cancellations: %v, actual: %v", expectedCancellations, actualCancellations)
}
}
}