func TestRecreate_initialDeployment(t *testing.T) {
var deployment *kapi.ReplicationController
scaler := &cmdtest.FakeScaler{}
strategy := &RecreateDeploymentStrategy{
out: &bytes.Buffer{},
errOut: &bytes.Buffer{},
decoder: kapi.Codecs.UniversalDecoder(),
retryPeriod: 1 * time.Millisecond,
getUpdateAcceptor: getUpdateAcceptor,
scaler: scaler,
eventClient: fake.NewSimpleClientset().Core(),
}
config := deploytest.OkDeploymentConfig(1)
config.Spec.Strategy = recreateParams(30, "", "", "")
deployment, _ = deployutil.MakeDeployment(config, kapi.Codecs.LegacyCodec(registered.GroupOrDie(kapi.GroupName).GroupVersions[0]))
strategy.rcClient = &fakeControllerClient{deployment: deployment}
strategy.podClient = &fakePodClient{deployerName: deployutil.DeployerPodNameForDeployment(deployment.Name)}
err := strategy.Deploy(nil, deployment, 3)
if err != nil {
t.Fatalf("unexpected deploy error: %#v", err)
}
if e, a := 1, len(scaler.Events); e != a {
t.Fatalf("expected %d scale calls, got %d", e, a)
}
if e, a := uint(3), scaler.Events[0].Size; e != a {
t.Errorf("expected scale up to %d, got %d", e, a)
}
}
// makeDeployerPod creates a pod which implements deployment behavior. The pod is correlated to
// the deployment with an annotation.
func (c *DeploymentController) makeDeployerPod(deployment *kapi.ReplicationController) (*kapi.Pod, error) {
deploymentConfig, err := c.decodeConfig(deployment)
if err != nil {
return nil, err
}
container, err := c.makeContainer(&deploymentConfig.Spec.Strategy)
if err != nil {
return nil, err
}
// Add deployment environment variables to the container.
envVars := []kapi.EnvVar{}
for _, env := range container.Env {
envVars = append(envVars, env)
}
envVars = append(envVars, kapi.EnvVar{Name: "OPENSHIFT_DEPLOYMENT_NAME", Value: deployment.Name})
envVars = append(envVars, kapi.EnvVar{Name: "OPENSHIFT_DEPLOYMENT_NAMESPACE", Value: deployment.Namespace})
// Assigning to a variable since its address is required
maxDeploymentDurationSeconds := deployapi.MaxDeploymentDurationSeconds
pod := &kapi.Pod{
ObjectMeta: kapi.ObjectMeta{
Name: deployutil.DeployerPodNameForDeployment(deployment.Name),
Annotations: map[string]string{
deployapi.DeploymentAnnotation: deployment.Name,
},
Labels: map[string]string{
deployapi.DeployerPodForDeploymentLabel: deployment.Name,
},
},
Spec: kapi.PodSpec{
Containers: []kapi.Container{
{
Name: "deployment",
Command: container.Command,
Args: container.Args,
Image: container.Image,
Env: envVars,
Resources: deploymentConfig.Spec.Strategy.Resources,
},
},
ActiveDeadlineSeconds: &maxDeploymentDurationSeconds,
// Setting the node selector on the deployer pod so that it is created
// on the same set of nodes as the pods.
NodeSelector: deployment.Spec.Template.Spec.NodeSelector,
RestartPolicy: kapi.RestartPolicyNever,
ServiceAccountName: c.serviceAccount,
},
}
// MergeInfo will not overwrite values unless the flag OverwriteExistingDstKey is set.
util.MergeInto(pod.Labels, deploymentConfig.Spec.Strategy.Labels, 0)
util.MergeInto(pod.Annotations, deploymentConfig.Spec.Strategy.Annotations, 0)
pod.Spec.Containers[0].ImagePullPolicy = kapi.PullIfNotPresent
return pod, nil
}
func TestRecreate_deploymentMidHookSuccess(t *testing.T) {
config := deploytest.OkDeploymentConfig(1)
config.Spec.Strategy = recreateParams(30, "", deployapi.LifecycleHookFailurePolicyAbort, "")
deployment, _ := deployutil.MakeDeployment(config, kapi.Codecs.LegacyCodec(deployv1.SchemeGroupVersion))
scaler := &cmdtest.FakeScaler{}
strategy := &RecreateDeploymentStrategy{
out: &bytes.Buffer{},
errOut: &bytes.Buffer{},
decoder: kapi.Codecs.UniversalDecoder(),
retryPeriod: 1 * time.Millisecond,
rcClient: &fakeControllerClient{deployment: deployment},
eventClient: fake.NewSimpleClientset().Core(),
getUpdateAcceptor: getUpdateAcceptor,
hookExecutor: &hookExecutorImpl{
executeFunc: func(hook *deployapi.LifecycleHook, deployment *kapi.ReplicationController, suffix, label string) error {
return fmt.Errorf("hook execution failure")
},
},
scaler: scaler,
}
strategy.podClient = &fakePodClient{deployerName: deployutil.DeployerPodNameForDeployment(deployment.Name)}
err := strategy.Deploy(nil, deployment, 2)
if err == nil {
t.Fatalf("expected a deploy error")
}
if len(scaler.Events) > 0 {
t.Fatalf("unexpected scaling events: %v", scaler.Events)
}
}
// TestHandle_deployerPodAlreadyExists ensures that attempts to create a
// deployer pod which was already created don't result in an error
// (effectively skipping the handling as redundant).
func TestHandle_deployerPodAlreadyExists(t *testing.T) {
var updatedDeployment *kapi.ReplicationController
config := deploytest.OkDeploymentConfig(1)
deployment, _ := deployutil.MakeDeployment(config, codec)
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name)
fake := &ktestclient.Fake{}
fake.AddReactor("create", "pods", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
name := action.(ktestclient.CreateAction).GetObject().(*kapi.Pod).Name
return true, nil, kerrors.NewAlreadyExists(kapi.Resource("Pod"), name)
})
fake.AddReactor("update", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
rc := action.(ktestclient.UpdateAction).GetObject().(*kapi.ReplicationController)
updatedDeployment = rc
return true, rc, nil
})
controller := okDeploymentController(fake, deployment, nil, true)
if err := controller.Handle(deployment); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if updatedDeployment.Annotations[deployapi.DeploymentPodAnnotation] != deployerPodName {
t.Fatalf("deployment not updated with pod name annotation")
}
if updatedDeployment.Annotations[deployapi.DeploymentStatusAnnotation] != string(deployapi.DeploymentStatusPending) {
t.Fatalf("deployment status not updated to pending")
}
}
func deployerPod(deployment *kapi.ReplicationController, alternateName string, related bool) *kapi.Pod {
deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name)
if len(alternateName) > 0 {
deployerPodName = alternateName
}
deployment.Namespace = "test"
pod := &kapi.Pod{
ObjectMeta: kapi.ObjectMeta{
Name: deployerPodName,
Namespace: deployment.Namespace,
Labels: map[string]string{
deployapi.DeployerPodForDeploymentLabel: deployment.Name,
},
Annotations: map[string]string{
deployapi.DeploymentAnnotation: deployment.Name,
},
},
}
if !related {
delete(pod.Annotations, deployapi.DeploymentAnnotation)
}
return pod
}
func TestRecreate_deploymentPreHookSuccess(t *testing.T) {
config := deploytest.OkDeploymentConfig(1)
config.Spec.Strategy = recreateParams(30, deployapi.LifecycleHookFailurePolicyAbort, "", "")
deployment, _ := deployutil.MakeDeployment(config, kapi.Codecs.LegacyCodec(registered.GroupOrDie(kapi.GroupName).GroupVersions[0]))
scaler := &cmdtest.FakeScaler{}
hookExecuted := false
strategy := &RecreateDeploymentStrategy{
out: &bytes.Buffer{},
errOut: &bytes.Buffer{},
decoder: kapi.Codecs.UniversalDecoder(),
retryPeriod: 1 * time.Millisecond,
getUpdateAcceptor: getUpdateAcceptor,
eventClient: fake.NewSimpleClientset().Core(),
rcClient: &fakeControllerClient{deployment: deployment},
hookExecutor: &hookExecutorImpl{
executeFunc: func(hook *deployapi.LifecycleHook, deployment *kapi.ReplicationController, suffix, label string) error {
hookExecuted = true
return nil
},
},
scaler: scaler,
}
strategy.podClient = &fakePodClient{deployerName: deployutil.DeployerPodNameForDeployment(deployment.Name)}
err := strategy.Deploy(nil, deployment, 2)
if err != nil {
t.Fatalf("unexpected deploy error: %#v", err)
}
if !hookExecuted {
t.Fatalf("expected hook execution")
}
}
func okPod(deployment *kapi.ReplicationController) *kapi.Pod {
return &kapi.Pod{
ObjectMeta: kapi.ObjectMeta{
Name: deployutil.DeployerPodNameForDeployment(deployment.Name),
Annotations: map[string]string{
deployapi.DeploymentAnnotation: deployment.Name,
},
},
Status: kapi.PodStatus{
ContainerStatuses: []kapi.ContainerStatus{
{},
},
},
}
}
// TestHandle_orphanedPod ensures that deployer pods associated with a non-
// existent deployment results in all deployer pods being deleted.
func TestHandle_orphanedPod(t *testing.T) {
deleted := kutil.NewStringSet()
controller := &DeployerPodController{
deploymentClient: &deploymentClientImpl{
updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
t.Fatalf("Unexpected deployment update")
return nil, nil
},
getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
return nil, kerrors.NewNotFound("ReplicationController", name)
},
},
deployerPodsFor: func(namespace, name string) (*kapi.PodList, error) {
mkpod := func(suffix string) kapi.Pod {
deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec)
p := okPod(deployment)
p.Name = p.Name + suffix
return *p
}
return &kapi.PodList{
Items: []kapi.Pod{
mkpod(""),
mkpod("-prehook"),
mkpod("-posthook"),
},
}, nil
},
deletePod: func(namespace, name string) error {
deleted.Insert(name)
return nil
},
}
deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codec)
err := controller.Handle(runningPod(deployment))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
deployerName := deployutil.DeployerPodNameForDeployment(deployment.Name)
if !deleted.HasAll(deployerName, deployerName+"-prehook", deployerName+"-posthook") {
t.Fatalf("unexpected deleted names: %v", deleted.List())
}
}
func TestRecreate_acceptorSuccess(t *testing.T) {
var deployment *kapi.ReplicationController
scaler := &cmdtest.FakeScaler{}
strategy := &RecreateDeploymentStrategy{
out: &bytes.Buffer{},
errOut: &bytes.Buffer{},
eventClient: fake.NewSimpleClientset().Core(),
decoder: kapi.Codecs.UniversalDecoder(),
retryPeriod: 1 * time.Millisecond,
scaler: scaler,
}
acceptorCalled := false
acceptor := &testAcceptor{
acceptFn: func(deployment *kapi.ReplicationController) error {
acceptorCalled = true
return nil
},
}
oldDeployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codecs.LegacyCodec(registered.GroupOrDie(kapi.GroupName).GroupVersions[0]))
deployment, _ = deployutil.MakeDeployment(deploytest.OkDeploymentConfig(2), kapi.Codecs.LegacyCodec(registered.GroupOrDie(kapi.GroupName).GroupVersions[0]))
strategy.rcClient = &fakeControllerClient{deployment: deployment}
strategy.podClient = &fakePodClient{deployerName: deployutil.DeployerPodNameForDeployment(deployment.Name)}
err := strategy.DeployWithAcceptor(oldDeployment, deployment, 2, acceptor)
if err != nil {
t.Fatalf("unexpected deploy error: %#v", err)
}
if !acceptorCalled {
t.Fatalf("expected acceptor to be called")
}
if e, a := 2, len(scaler.Events); e != a {
t.Fatalf("expected %d scale calls, got %d", e, a)
}
if e, a := uint(1), scaler.Events[0].Size; e != a {
t.Errorf("expected scale down to %d, got %d", e, a)
}
if e, a := uint(2), scaler.Events[1].Size; e != a {
t.Errorf("expected scale up to %d, got %d", e, a)
}
}
// TestHandle_orphanedPod ensures that deployer pods associated with a non-
// existent deployment results in all deployer pods being deleted.
func TestHandle_orphanedPod(t *testing.T) {
deleted := sets.NewString()
kFake := &ktestclient.Fake{}
kFake.PrependReactor("get", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
name := action.(ktestclient.GetAction).GetName()
return true, nil, kerrors.NewNotFound(kapi.Resource("ReplicationController"), name)
})
kFake.PrependReactor("update", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
t.Fatalf("Unexpected deployment update")
return true, nil, nil
})
kFake.PrependReactor("list", "pods", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
mkpod := func(suffix string) kapi.Pod {
deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codecs.LegacyCodec(deployapi.SchemeGroupVersion))
p := okPod(deployment)
p.Name = p.Name + suffix
return *p
}
return true, &kapi.PodList{Items: []kapi.Pod{mkpod(""), mkpod("-prehook"), mkpod("-posthook")}}, nil
})
kFake.PrependReactor("delete", "pods", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
name := action.(ktestclient.DeleteAction).GetName()
deleted.Insert(name)
return true, nil, nil
})
controller := &DeployerPodController{
store: cache.NewStore(cache.MetaNamespaceKeyFunc),
kClient: kFake,
}
deployment, _ := deployutil.MakeDeployment(deploytest.OkDeploymentConfig(1), kapi.Codecs.LegacyCodec(deployapi.SchemeGroupVersion))
err := controller.Handle(runningPod(deployment))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
deployerName := deployutil.DeployerPodNameForDeployment(deployment.Name)
if !deleted.HasAll(deployerName, deployerName+"-prehook", deployerName+"-posthook") {
t.Fatalf("unexpected deleted names: %v", deleted.List())
}
}
// Handle processes deployment and either creates a deployer pod or responds
// to a terminal deployment status. Since this controller started using caches,
// the provided rc MUST be deep-copied beforehand (see work() in factory.go).
func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error {
currentStatus := deployutil.DeploymentStatusFor(deployment)
nextStatus := currentStatus
deploymentScaled := false
deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name)
deployer, deployerErr := c.getPod(deployment.Namespace, deployerPodName)
switch currentStatus {
case deployapi.DeploymentStatusNew:
// If the deployment has been cancelled, don't create a deployer pod.
// Instead try to delete any deployer pods found and transition the
// deployment to Pending so that the deployment config controller
// continues to see the deployment as in-flight. Eventually the deletion
// of the deployer pod should cause a requeue of this deployment and
// then it can be transitioned to Failed by this controller.
if deployutil.IsDeploymentCancelled(deployment) {
nextStatus = deployapi.DeploymentStatusPending
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
break
}
// If the pod already exists, it's possible that a previous CreatePod
// succeeded but the deployment state update failed and now we're re-
// entering. Ensure that the pod is the one we created by verifying the
// annotation on it, and throw a retryable error.
if deployerErr != nil && !kerrors.IsNotFound(deployerErr) {
return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), deployerErr)
}
if deployerErr == nil && deployer != nil {
// Do a stronger check to validate that the existing deployer pod is
// actually for this deployment, and if not, fail this deployment.
//
// TODO: Investigate checking the container image of the running pod and
// comparing with the intended deployer pod image. If we do so, we'll need
// to ensure that changes to 'unrelated' pods don't result in updates to
// the deployment. So, the image check will have to be done in other areas
// of the code as well.
if deployutil.DeploymentNameFor(deployer) != deployment.Name {
nextStatus = deployapi.DeploymentStatusFailed
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists
c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "FailedCreate", fmt.Sprintf("Error creating deployer pod since another pod with the same name (%q) exists", deployer.Name))
glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), deployer.Name)
} else {
// Update to pending relative to the existing validated deployer pod.
deployment.Annotations[deployapi.DeploymentPodAnnotation] = deployer.Name
nextStatus = deployapi.DeploymentStatusPending
glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", deployer.Name, deployutil.LabelForDeployment(deployment))
}
// Don't try and re-create the deployer pod.
break
}
if _, ok := deployment.Annotations[deployapi.DeploymentIgnorePodAnnotation]; ok {
return nil
}
// Generate a deployer pod spec.
deployerPod, err := c.makeDeployerPod(deployment)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err))
}
// Create the deployer pod.
deploymentPod, err := c.pn.Pods(deployment.Namespace).Create(deployerPod)
// Retry on error.
if err != nil {
return actionableError(fmt.Sprintf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err))
}
deployment.Annotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name
nextStatus = deployapi.DeploymentStatusPending
glog.V(4).Infof("Created deployer pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment))
case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning:
switch {
case kerrors.IsNotFound(deployerErr):
nextStatus = deployapi.DeploymentStatusFailed
// If the deployment is cancelled here then we deleted the deployer in a previous
// resync of the deployment.
if !deployutil.IsDeploymentCancelled(deployment) {
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists
c.emitDeploymentEvent(deployment, kapi.EventTypeWarning, "Failed", fmt.Sprintf("Deployer pod %q has gone missing", deployerPodName))
glog.V(4).Infof("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName)
}
case deployerErr != nil:
// We'll try again later on resync. Continue to process cancellations.
glog.V(4).Infof("Error getting deployer pod %s for deployment %s: %v", deployerPodName, deployutil.LabelForDeployment(deployment), deployerErr)
default: /* err == nil */
// If the deployment has been cancelled, delete any deployer pods
// found and transition the deployment to Pending so that the
// deployment config controller continues to see the deployment
// as in-flight. Eventually the deletion of the deployer pod should
// cause a requeue of this deployment and then it can be transitioned
// to Failed by this controller.
if deployutil.IsDeploymentCancelled(deployment) {
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
}
}
case deployapi.DeploymentStatusFailed:
// Check for test deployment and ensure the deployment scale matches
if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test {
deploymentScaled = deployment.Spec.Replicas != 0
deployment.Spec.Replicas = 0
}
// Try to cleanup once more a cancelled deployment in case hook pods
// were created just after we issued the first cleanup request.
if deployutil.IsDeploymentCancelled(deployment) {
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
}
case deployapi.DeploymentStatusComplete:
// Check for test deployment and ensure the deployment scale matches
if config, err := deployutil.DecodeDeploymentConfig(deployment, c.codec); err == nil && config.Spec.Test {
deploymentScaled = deployment.Spec.Replicas != 0
deployment.Spec.Replicas = 0
}
if err := c.cleanupDeployerPods(deployment); err != nil {
return err
}
}
if deployutil.CanTransitionPhase(currentStatus, nextStatus) || deploymentScaled {
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
if _, err := c.rn.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil {
return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
}
glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas)
}
return nil
}
// Get returns a streamer resource with the contents of the deployment log
func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) {
// Ensure we have a namespace in the context
namespace, ok := kapi.NamespaceFrom(ctx)
if !ok {
return nil, errors.NewBadRequest("namespace parameter required.")
}
// Validate DeploymentLogOptions
deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions)
if !ok {
return nil, errors.NewBadRequest("did not get an expected options.")
}
if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 {
return nil, errors.NewInvalid(deployapi.Kind("DeploymentLogOptions"), "", errs)
}
// Fetch deploymentConfig and check latest version; if 0, there are no deployments
// for this config
config, err := r.dn.DeploymentConfigs(namespace).Get(name)
if err != nil {
return nil, errors.NewNotFound(deployapi.Resource("deploymentconfig"), name)
}
desiredVersion := config.Status.LatestVersion
if desiredVersion == 0 {
return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name))
}
// Support retrieving logs for older deployments
switch {
case deployLogOpts.Version == nil:
// Latest or previous
if deployLogOpts.Previous {
desiredVersion--
if desiredVersion < 1 {
return nil, errors.NewBadRequest(fmt.Sprintf("no previous deployment exists for deploymentConfig %q", config.Name))
}
}
case *deployLogOpts.Version <= 0 || *deployLogOpts.Version > config.Status.LatestVersion:
// Invalid version
return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version))
default:
desiredVersion = *deployLogOpts.Version
}
// Get desired deployment
targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion)
target, err := r.waitForExistingDeployment(namespace, targetName)
if err != nil {
return nil, err
}
podName := deployutil.DeployerPodNameForDeployment(target.Name)
// Check for deployment status; if it is new or pending, we will wait for it. If it is complete,
// the deployment completed successfully and the deployer pod will be deleted so we will return a
// success message. If it is running or failed, retrieve the log from the deployer pod.
status := deployutil.DeploymentStatusFor(target)
switch status {
case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending:
if deployLogOpts.NoWait {
glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status)
return &genericrest.LocationStreamer{}, nil
}
glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status)
if err := deployutil.WaitForRunningDeployerPod(r.pn, target, r.timeout); err != nil {
return nil, errors.NewBadRequest(fmt.Sprintf("failed to run deployer pod %s: %v", podName, err))
}
latest, ok, err := registry.WaitForRunningDeployment(r.rn, target, r.timeout)
if err != nil {
return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err))
}
if !ok {
return nil, errors.NewServerTimeout(kapi.Resource("ReplicationController"), "get", 2)
}
if deployutil.DeploymentStatusFor(latest) == deployapi.DeploymentStatusComplete {
podName, err = r.returnApplicationPodName(target)
if err != nil {
return nil, err
}
}
case deployapi.DeploymentStatusComplete:
podName, err = r.returnApplicationPodName(target)
if err != nil {
return nil, err
}
}
logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts)
location, transport, err := pod.LogLocation(&podGetter{r.pn}, r.connInfo, ctx, podName, logOpts)
if err != nil {
return nil, errors.NewBadRequest(err.Error())
}
return &genericrest.LocationStreamer{
Location: location,
Transport: transport,
ContentType: "text/plain",
Flush: deployLogOpts.Follow,
ResponseChecker: genericrest.NewGenericHttpResponseChecker(kapi.Resource("pod"), podName),
}, nil
}
// TestHandle_deployerPodAlreadyExists ensures that attempts to create a
// deployer pod which was already created don't result in an error
// (effectively skipping the handling as redundant).
func TestHandle_deployerPodAlreadyExists(t *testing.T) {
tests := []struct {
name string
podPhase kapi.PodPhase
expected deployapi.DeploymentStatus
}{
{
name: "pending",
podPhase: kapi.PodPending,
expected: deployapi.DeploymentStatusPending,
},
{
name: "running",
podPhase: kapi.PodRunning,
expected: deployapi.DeploymentStatusRunning,
},
{
name: "complete",
podPhase: kapi.PodFailed,
expected: deployapi.DeploymentStatusFailed,
},
{
name: "failed",
podPhase: kapi.PodSucceeded,
expected: deployapi.DeploymentStatusComplete,
},
}
for _, test := range tests {
var updatedDeployment *kapi.ReplicationController
config := deploytest.OkDeploymentConfig(1)
deployment, _ := deployutil.MakeDeployment(config, codec)
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(deployapi.DeploymentStatusNew)
deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name)
fake := &ktestclient.Fake{}
fake.AddReactor("create", "pods", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
name := action.(ktestclient.CreateAction).GetObject().(*kapi.Pod).Name
return true, nil, kerrors.NewAlreadyExists(kapi.Resource("Pod"), name)
})
fake.AddReactor("update", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
rc := action.(ktestclient.UpdateAction).GetObject().(*kapi.ReplicationController)
updatedDeployment = rc
return true, rc, nil
})
controller := okDeploymentController(fake, deployment, nil, true, test.podPhase)
if err := controller.Handle(deployment); err != nil {
t.Errorf("%s: unexpected error: %v", test.name, err)
continue
}
if updatedDeployment.Annotations[deployapi.DeploymentPodAnnotation] != deployerPodName {
t.Errorf("%s: deployment not updated with pod name annotation", test.name)
continue
}
if e, a := string(test.expected), updatedDeployment.Annotations[deployapi.DeploymentStatusAnnotation]; e != a {
t.Errorf("%s: deployment status not updated. Expected %q, got %q", test.name, e, a)
}
}
}
// Handle processes deployment and either creates a deployer pod or responds
// to a terminal deployment status.
func (c *DeploymentController) Handle(deployment *kapi.ReplicationController) error {
currentStatus := deployutil.DeploymentStatusFor(deployment)
nextStatus := currentStatus
deploymentScaled := false
switch currentStatus {
case deployapi.DeploymentStatusNew:
// If the deployment has been cancelled, don't create a deployer pod, and
// transition to failed immediately.
if deployutil.IsDeploymentCancelled(deployment) {
nextStatus = deployapi.DeploymentStatusFailed
break
}
// Generate a deployer pod spec.
podTemplate, err := c.makeDeployerPod(deployment)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err))
}
// Create the deployer pod.
deploymentPod, err := c.podClient.createPod(deployment.Namespace, podTemplate)
if err == nil {
deployment.Annotations[deployapi.DeploymentPodAnnotation] = deploymentPod.Name
nextStatus = deployapi.DeploymentStatusPending
glog.V(4).Infof("Created pod %s for deployment %s", deploymentPod.Name, deployutil.LabelForDeployment(deployment))
break
}
// Retry on error.
if !kerrors.IsAlreadyExists(err) {
c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error creating deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)
return fmt.Errorf("couldn't create deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)
}
// If the pod already exists, it's possible that a previous CreatePod
// succeeded but the deployment state update failed and now we're re-
// entering. Ensure that the pod is the one we created by verifying the
// annotation on it, and throw a retryable error.
existingPod, err := c.podClient.getPod(deployment.Namespace, deployutil.DeployerPodNameForDeployment(deployment.Name))
if err != nil {
c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error getting existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)
return fmt.Errorf("couldn't fetch existing deployer pod for %s: %v", deployutil.LabelForDeployment(deployment), err)
}
// Do a stronger check to validate that the existing deployer pod is
// actually for this deployment, and if not, fail this deployment.
//
// TODO: Investigate checking the container image of the running pod and
// comparing with the intended deployer pod image. If we do so, we'll need
// to ensure that changes to 'unrelated' pods don't result in updates to
// the deployment. So, the image check will have to be done in other areas
// of the code as well.
if deployutil.DeploymentNameFor(existingPod) != deployment.Name {
nextStatus = deployapi.DeploymentStatusFailed
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedUnrelatedDeploymentExists
c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedCreate", "Error creating deployer pod for %s since another pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), existingPod.Name)
glog.V(2).Infof("Couldn't create deployer pod for %s since an unrelated pod with the same name (%q) exists", deployutil.LabelForDeployment(deployment), existingPod.Name)
break
}
// Update to pending relative to the existing validated deployer pod.
deployment.Annotations[deployapi.DeploymentPodAnnotation] = existingPod.Name
nextStatus = deployapi.DeploymentStatusPending
glog.V(4).Infof("Detected existing deployer pod %s for deployment %s", existingPod.Name, deployutil.LabelForDeployment(deployment))
case deployapi.DeploymentStatusPending, deployapi.DeploymentStatusRunning:
// If the deployer pod has vanished, consider the deployment a failure.
deployerPodName := deployutil.DeployerPodNameForDeployment(deployment.Name)
if _, err := c.podClient.getPod(deployment.Namespace, deployerPodName); err != nil {
if kerrors.IsNotFound(err) {
nextStatus = deployapi.DeploymentStatusFailed
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentFailedDeployerPodNoLongerExists
c.recorder.Eventf(deployment, kapi.EventTypeWarning, "failed", "Deployer pod %q has gone missing", deployerPodName)
glog.V(4).Infof("Failing deployment %q because its deployer pod %q disappeared", deployutil.LabelForDeployment(deployment), deployerPodName)
break
} else {
// We'll try again later on resync. Continue to process cancellations.
glog.V(2).Infof("Error getting deployer pod %s for deployment %s: %#v", deployerPodName, deployutil.LabelForDeployment(deployment), err)
}
}
// If the deployment is cancelled, terminate any deployer/hook pods.
// NOTE: Do not mark the deployment as Failed just yet.
// The deployment will be marked as Failed by the deployer pod controller
// when the deployer pod failure state is picked up
// Also, it will scale down the failed deployment and scale back up
// the last successful completed deployment
if deployutil.IsDeploymentCancelled(deployment) {
deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name)
if err != nil {
return fmt.Errorf("couldn't fetch deployer pods for %s while trying to cancel deployment: %v", deployutil.LabelForDeployment(deployment), err)
}
glog.V(4).Infof("Cancelling %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment))
zeroDelay := int64(1)
for _, deployerPod := range deployerPods {
// Set the ActiveDeadlineSeconds on the pod so it's terminated very soon.
if deployerPod.Spec.ActiveDeadlineSeconds == nil || *deployerPod.Spec.ActiveDeadlineSeconds != zeroDelay {
deployerPod.Spec.ActiveDeadlineSeconds = &zeroDelay
if _, err := c.podClient.updatePod(deployerPod.Namespace, &deployerPod); err != nil {
c.recorder.Eventf(deployment, kapi.EventTypeWarning, "failedCancellation", "Error cancelling deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err)
return fmt.Errorf("couldn't cancel deployer pod %s for deployment %s: %v", deployerPod.Name, deployutil.LabelForDeployment(deployment), err)
}
glog.V(4).Infof("Cancelled deployer pod %s for deployment %s", deployerPod.Name, deployutil.LabelForDeployment(deployment))
}
}
c.recorder.Eventf(deployment, kapi.EventTypeNormal, "Cancelled", "Cancelled deployment")
}
case deployapi.DeploymentStatusFailed:
// Check for test deployment and ensure the deployment scale matches
if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test {
deploymentScaled = deployment.Spec.Replicas != 0
deployment.Spec.Replicas = 0
}
case deployapi.DeploymentStatusComplete:
// Check for test deployment and ensure the deployment scale matches
if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test {
deploymentScaled = deployment.Spec.Replicas != 0
deployment.Spec.Replicas = 0
}
// now list any pods in the namespace that have the specified label
deployerPods, err := c.podClient.getDeployerPodsFor(deployment.Namespace, deployment.Name)
if err != nil {
return fmt.Errorf("couldn't fetch deployer pods for %s after successful completion: %v", deployutil.LabelForDeployment(deployment), err)
}
if len(deployerPods) > 0 {
glog.V(4).Infof("Deleting %d deployer pods for deployment %s", len(deployerPods), deployutil.LabelForDeployment(deployment))
}
cleanedAll := true
for _, deployerPod := range deployerPods {
if err := c.podClient.deletePod(deployerPod.Namespace, deployerPod.Name); err != nil {
if !kerrors.IsNotFound(err) {
// if the pod deletion failed, then log the error and continue
// we will try to delete any remaining deployer pods and return an error later
utilruntime.HandleError(fmt.Errorf("couldn't delete completed deployer pod %s/%s for deployment %s: %v", deployment.Namespace, deployerPod.Name, deployutil.LabelForDeployment(deployment), err))
cleanedAll = false
}
// Already deleted
} else {
glog.V(4).Infof("Deleted completed deployer pod %s/%s for deployment %s", deployment.Namespace, deployerPod.Name, deployutil.LabelForDeployment(deployment))
}
}
if !cleanedAll {
return fmt.Errorf("couldn't clean up all deployer pods for %s", deployutil.LabelForDeployment(deployment))
}
}
if currentStatus != nextStatus || deploymentScaled {
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil {
c.recorder.Eventf(deployment, kapi.EventTypeWarning, "FailedUpdate", "Error updating deployment %s status to %s", deployutil.LabelForDeployment(deployment), nextStatus)
return fmt.Errorf("couldn't update deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
}
glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas)
}
return nil
}
// mockREST mocks a DeploymentLog REST
func mockREST(version, desired int64, status api.DeploymentStatus) *REST {
connectionInfo := &kubeletclient.HTTPKubeletClient{Config: &kubeletclient.KubeletClientConfig{EnableHttps: true, Port: 12345}, Client: &http.Client{}}
// Fake deploymentConfig
config := deploytest.OkDeploymentConfig(version)
fakeDn := testclient.NewSimpleFake(config)
fakeDn.PrependReactor("get", "deploymentconfigs", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
return true, config, nil
})
// Used for testing validation errors prior to getting replication controllers.
if desired > version {
return &REST{
dn: fakeDn,
connInfo: connectionInfo,
timeout: defaultTimeout,
}
}
// Fake deployments
fakeDeployments := makeDeploymentList(version)
fakeRn := ktestclient.NewSimpleFake(fakeDeployments)
fakeRn.PrependReactor("get", "replicationcontrollers", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
return true, &fakeDeployments.Items[desired-1], nil
})
// Fake watcher for deployments
fakeWatch := watch.NewFake()
fakeRn.PrependWatchReactor("replicationcontrollers", ktestclient.DefaultWatchReactor(fakeWatch, nil))
obj := &fakeDeployments.Items[desired-1]
obj.Annotations[api.DeploymentStatusAnnotation] = string(status)
go fakeWatch.Add(obj)
fakePn := ktestclient.NewSimpleFake()
if status == api.DeploymentStatusComplete {
// If the deployment is complete, we will try to get the logs from the oldest
// application pod...
fakePn.PrependReactor("list", "pods", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
return true, fakePodList, nil
})
fakePn.PrependReactor("get", "pods", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
return true, &fakePodList.Items[0], nil
})
} else {
// ...otherwise try to get the logs from the deployer pod.
fakeDeployer := &kapi.Pod{
ObjectMeta: kapi.ObjectMeta{
Name: deployutil.DeployerPodNameForDeployment(obj.Name),
Namespace: kapi.NamespaceDefault,
},
Spec: kapi.PodSpec{
Containers: []kapi.Container{
{
Name: deployutil.DeployerPodNameForDeployment(obj.Name) + "-container",
},
},
NodeName: "some-host",
},
}
fakePn.PrependReactor("get", "pods", func(action ktestclient.Action) (handled bool, ret runtime.Object, err error) {
return true, fakeDeployer, nil
})
}
return &REST{
dn: fakeDn,
rn: fakeRn,
pn: fakePn,
connInfo: connectionInfo,
timeout: defaultTimeout,
}
}
// Handle syncs pod's status with any associated deployment.
func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
// Find the deployment associated with the deployer pod.
deploymentName := deployutil.DeploymentNameFor(pod)
if len(deploymentName) == 0 {
return nil
}
// Reject updates to anything but the main deployer pod
// TODO: Find a way to filter this on the watch side.
if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) {
return nil
}
deployment, err := c.deploymentClient.getDeployment(pod.Namespace, deploymentName)
// If the deployment for this pod has disappeared, we should clean up this
// and any other deployer pods, then bail out.
if err != nil {
// Some retrieval error occured. Retry.
if !kerrors.IsNotFound(err) {
return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace)
}
// Find all the deployer pods for the deployment (including this one).
deployers, err := c.deployerPodsFor(pod.Namespace, deploymentName)
if err != nil {
// Retry.
return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err)
}
// Delete all deployers.
for _, deployer := range deployers.Items {
err := c.deletePod(deployer.Namespace, deployer.Name)
if err != nil {
if !kerrors.IsNotFound(err) {
// TODO: Should this fire an event?
glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err)
}
} else {
// TODO: Should this fire an event?
glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name)
}
}
return nil
}
currentStatus := deployutil.DeploymentStatusFor(deployment)
nextStatus := currentStatus
switch pod.Status.Phase {
case kapi.PodRunning:
nextStatus = deployapi.DeploymentStatusRunning
case kapi.PodSucceeded:
// Detect failure based on the container state
nextStatus = deployapi.DeploymentStatusComplete
for _, info := range pod.Status.ContainerStatuses {
if info.State.Termination != nil && info.State.Termination.ExitCode != 0 {
nextStatus = deployapi.DeploymentStatusFailed
}
}
case kapi.PodFailed:
// if the deployment is already marked Failed, do not attempt clean up again
if currentStatus != deployapi.DeploymentStatusFailed {
// clean up will also update the deployment status to Failed
// failure to clean up will result in retries and
// the deployment will not be marked Failed
// Note: this will prevent new deployments from being created for this config
err := c.cleanupFailedDeployment(deployment)
if err != nil {
return transientError(fmt.Sprintf("couldn't clean up failed deployment: %v", err))
}
}
}
if currentStatus != nextStatus {
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil {
if kerrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
}
glog.V(4).Infof("Updated Deployment %s status from %s to %s", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus)
}
return nil
}
// Get returns a streamer resource with the contents of the deployment log
func (r *REST) Get(ctx kapi.Context, name string, opts runtime.Object) (runtime.Object, error) {
// Ensure we have a namespace in the context
namespace, ok := kapi.NamespaceFrom(ctx)
if !ok {
return nil, errors.NewBadRequest("namespace parameter required.")
}
// Validate DeploymentLogOptions
deployLogOpts, ok := opts.(*deployapi.DeploymentLogOptions)
if !ok {
return nil, errors.NewBadRequest("did not get an expected options.")
}
if errs := validation.ValidateDeploymentLogOptions(deployLogOpts); len(errs) > 0 {
return nil, errors.NewInvalid("deploymentLogOptions", "", errs)
}
// Fetch deploymentConfig and check latest version; if 0, there are no deployments
// for this config
config, err := r.ConfigGetter.DeploymentConfigs(namespace).Get(name)
if err != nil {
return nil, errors.NewNotFound("deploymentConfig", name)
}
desiredVersion := config.Status.LatestVersion
if desiredVersion == 0 {
return nil, errors.NewBadRequest(fmt.Sprintf("no deployment exists for deploymentConfig %q", config.Name))
}
// Support retrieving logs for older deployments
switch {
case deployLogOpts.Version == nil:
// Latest
case *deployLogOpts.Version <= 0 || int(*deployLogOpts.Version) > config.Status.LatestVersion:
// Invalid version
return nil, errors.NewBadRequest(fmt.Sprintf("invalid version for deploymentConfig %q: %d", config.Name, *deployLogOpts.Version))
default:
desiredVersion = int(*deployLogOpts.Version)
}
// Get desired deployment
targetName := deployutil.DeploymentNameForConfigVersion(config.Name, desiredVersion)
target, err := r.DeploymentGetter.ReplicationControllers(namespace).Get(targetName)
if err != nil {
return nil, err
}
// Check for deployment status; if it is new or pending, we will wait for it. If it is complete,
// the deployment completed successfully and the deployer pod will be deleted so we will return a
// success message. If it is running or failed, retrieve the log from the deployer pod.
status := deployutil.DeploymentStatusFor(target)
switch status {
case deployapi.DeploymentStatusNew, deployapi.DeploymentStatusPending:
if deployLogOpts.NoWait {
glog.V(4).Infof("Deployment %s is in %s state. No logs to retrieve yet.", deployutil.LabelForDeployment(target), status)
return &genericrest.LocationStreamer{}, nil
}
glog.V(4).Infof("Deployment %s is in %s state, waiting for it to start...", deployutil.LabelForDeployment(target), status)
latest, ok, err := registry.WaitForRunningDeployment(r.DeploymentGetter, target, r.Timeout)
if err != nil {
return nil, errors.NewBadRequest(fmt.Sprintf("unable to wait for deployment %s to run: %v", deployutil.LabelForDeployment(target), err))
}
if !ok {
return nil, errors.NewTimeoutError(fmt.Sprintf("timed out waiting for deployment %s to start after %s", deployutil.LabelForDeployment(target), r.Timeout), 1)
}
if deployutil.DeploymentStatusFor(latest) == deployapi.DeploymentStatusComplete {
// Deployer pod has been deleted, no logs to retrieve
glog.V(4).Infof("Deployment %s was successful so the deployer pod is deleted. No logs to retrieve.", deployutil.LabelForDeployment(target))
return &genericrest.LocationStreamer{}, nil
}
case deployapi.DeploymentStatusComplete:
// Deployer pod has been deleted, no logs to retrieve
glog.V(4).Infof("Deployment %s was successful so the deployer pod is deleted. No logs to retrieve.", deployutil.LabelForDeployment(target))
return &genericrest.LocationStreamer{}, nil
}
// Setup url of the deployer pod
deployPodName := deployutil.DeployerPodNameForDeployment(target.Name)
logOpts := deployapi.DeploymentToPodLogOptions(deployLogOpts)
location, transport, err := pod.LogLocation(r.PodGetter, r.ConnectionInfo, ctx, deployPodName, logOpts)
if err != nil {
return nil, errors.NewBadRequest(err.Error())
}
return &genericrest.LocationStreamer{
Location: location,
Transport: transport,
ContentType: "text/plain",
Flush: deployLogOpts.Follow,
ResponseChecker: genericrest.NewGenericHttpResponseChecker("Pod", deployPodName),
}, nil
}
// Handle syncs pod's status with any associated deployment.
func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
// Find the deployment associated with the deployer pod.
deploymentName := deployutil.DeploymentNameFor(pod)
if len(deploymentName) == 0 {
return nil
}
// Reject updates to anything but the main deployer pod
// TODO: Find a way to filter this on the watch side.
if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) {
return nil
}
deployment := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: pod.Namespace, Name: deploymentName}}
cached, exists, err := c.store.Get(deployment)
if err == nil && exists {
// Try to use the cache first. Trust hits and return them.
deployment = cached.(*kapi.ReplicationController)
} else {
// Double-check with the master for cache misses/errors, since those
// are rare and API calls are expensive but more reliable.
deployment, err = c.kClient.ReplicationControllers(pod.Namespace).Get(deploymentName)
}
// If the deployment for this pod has disappeared, we should clean up this
// and any other deployer pods, then bail out.
if err != nil {
// Some retrieval error occurred. Retry.
if !kerrors.IsNotFound(err) {
return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace)
}
// Find all the deployer pods for the deployment (including this one).
opts := kapi.ListOptions{LabelSelector: deployutil.DeployerPodSelector(deploymentName)}
deployers, err := c.kClient.Pods(pod.Namespace).List(opts)
if err != nil {
// Retry.
return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err)
}
// Delete all deployers.
for _, deployer := range deployers.Items {
err := c.kClient.Pods(deployer.Namespace).Delete(deployer.Name, kapi.NewDeleteOptions(0))
if err != nil {
if !kerrors.IsNotFound(err) {
// TODO: Should this fire an event?
glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err)
}
} else {
// TODO: Should this fire an event?
glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name)
}
}
return nil
}
currentStatus := deployutil.DeploymentStatusFor(deployment)
nextStatus := currentStatus
switch pod.Status.Phase {
case kapi.PodRunning:
nextStatus = deployapi.DeploymentStatusRunning
case kapi.PodSucceeded:
nextStatus = deployapi.DeploymentStatusComplete
config, decodeErr := c.decodeConfig(deployment)
// If the deployment was cancelled just prior to the deployer pod succeeding
// then we need to remove the cancel annotations from the complete deployment
// and emit an event letting users know their cancellation failed.
if deployutil.IsDeploymentCancelled(deployment) {
delete(deployment.Annotations, deployapi.DeploymentCancelledAnnotation)
delete(deployment.Annotations, deployapi.DeploymentStatusReasonAnnotation)
if decodeErr == nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "FailedCancellation", "Deployment %q succeeded before cancel recorded", deployutil.LabelForDeployment(deployment))
} else {
c.recorder.Event(deployment, kapi.EventTypeWarning, "FailedCancellation", "Succeeded before cancel recorded")
}
}
// Sync the internal replica annotation with the target so that we can
// distinguish deployer updates from other scaling events.
deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation]
if nextStatus == deployapi.DeploymentStatusComplete {
delete(deployment.Annotations, deployapi.DesiredReplicasAnnotation)
}
// reset the size of any test container, since we are the ones updating the RC
if decodeErr == nil && config.Spec.Test {
deployment.Spec.Replicas = 0
}
case kapi.PodFailed:
nextStatus = deployapi.DeploymentStatusFailed
// reset the size of any test container, since we are the ones updating the RC
if config, err := c.decodeConfig(deployment); err == nil && config.Spec.Test {
deployment.Spec.Replicas = 0
}
}
if deployutil.CanTransitionPhase(currentStatus, nextStatus) {
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
if _, err := c.kClient.ReplicationControllers(deployment.Namespace).Update(deployment); err != nil {
if kerrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
}
glog.V(4).Infof("Updated deployment %s status from %s to %s (scale: %d)", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus, deployment.Spec.Replicas)
// If the deployment was canceled, trigger a reconcilation of its deployment config
// so that the latest complete deployment can immediately rollback in place of the
// canceled deployment.
if nextStatus == deployapi.DeploymentStatusFailed && deployutil.IsDeploymentCancelled(deployment) {
// If we are unable to get the deployment config, then the deploymentconfig controller will
// perform its duties once the resync interval forces the deploymentconfig to be reconciled.
name := deployutil.DeploymentConfigNameFor(deployment)
kclient.RetryOnConflict(kclient.DefaultRetry, func() error {
config, err := c.client.DeploymentConfigs(deployment.Namespace).Get(name)
if err != nil {
return err
}
if config.Annotations == nil {
config.Annotations = make(map[string]string)
}
config.Annotations[deployapi.DeploymentCancelledAnnotation] = strconv.Itoa(config.Status.LatestVersion)
_, err = c.client.DeploymentConfigs(config.Namespace).Update(config)
return err
})
}
}
return nil
}
// Handle syncs pod's status with any associated deployment.
func (c *DeployerPodController) Handle(pod *kapi.Pod) error {
// Find the deployment associated with the deployer pod.
deploymentName := deployutil.DeploymentNameFor(pod)
if len(deploymentName) == 0 {
return nil
}
// Reject updates to anything but the main deployer pod
// TODO: Find a way to filter this on the watch side.
if pod.Name != deployutil.DeployerPodNameForDeployment(deploymentName) {
return nil
}
deployment, err := c.deploymentClient.getDeployment(pod.Namespace, deploymentName)
// If the deployment for this pod has disappeared, we should clean up this
// and any other deployer pods, then bail out.
if err != nil {
// Some retrieval error occurred. Retry.
if !kerrors.IsNotFound(err) {
return fmt.Errorf("couldn't get deployment %s/%s which owns deployer pod %s/%s", pod.Namespace, deploymentName, pod.Name, pod.Namespace)
}
// Find all the deployer pods for the deployment (including this one).
deployers, err := c.deployerPodsFor(pod.Namespace, deploymentName)
if err != nil {
// Retry.
return fmt.Errorf("couldn't get deployer pods for %s: %v", deployutil.LabelForDeployment(deployment), err)
}
// Delete all deployers.
for _, deployer := range deployers.Items {
err := c.deletePod(deployer.Namespace, deployer.Name)
if err != nil {
if !kerrors.IsNotFound(err) {
// TODO: Should this fire an event?
glog.V(2).Infof("Couldn't delete orphaned deployer pod %s/%s: %v", deployer.Namespace, deployer.Name, err)
}
} else {
// TODO: Should this fire an event?
glog.V(2).Infof("Deleted orphaned deployer pod %s/%s", deployer.Namespace, deployer.Name)
}
}
return nil
}
currentStatus := deployutil.DeploymentStatusFor(deployment)
nextStatus := currentStatus
switch pod.Status.Phase {
case kapi.PodRunning:
if !deployutil.IsTerminatedDeployment(deployment) {
nextStatus = deployapi.DeploymentStatusRunning
}
case kapi.PodSucceeded:
// Detect failure based on the container state
nextStatus = deployapi.DeploymentStatusComplete
for _, info := range pod.Status.ContainerStatuses {
if info.State.Terminated != nil && info.State.Terminated.ExitCode != 0 {
nextStatus = deployapi.DeploymentStatusFailed
break
}
}
// Sync the internal replica annotation with the target so that we can
// distinguish deployer updates from other scaling events.
deployment.Annotations[deployapi.DeploymentReplicasAnnotation] = deployment.Annotations[deployapi.DesiredReplicasAnnotation]
if nextStatus == deployapi.DeploymentStatusComplete {
delete(deployment.Annotations, deployapi.DesiredReplicasAnnotation)
}
case kapi.PodFailed:
nextStatus = deployapi.DeploymentStatusFailed
}
if currentStatus != nextStatus {
deployment.Annotations[deployapi.DeploymentStatusAnnotation] = string(nextStatus)
if _, err := c.deploymentClient.updateDeployment(deployment.Namespace, deployment); err != nil {
if kerrors.IsNotFound(err) {
return nil
}
return fmt.Errorf("couldn't update Deployment %s to status %s: %v", deployutil.LabelForDeployment(deployment), nextStatus, err)
}
glog.V(4).Infof("Updated Deployment %s status from %s to %s", deployutil.LabelForDeployment(deployment), currentStatus, nextStatus)
}
return nil
}
// executeExecNewPod executes a ExecNewPod hook by creating a new pod based on
// the hook parameters and deployment. The pod is then synchronously watched
// until the pod completes, and if the pod failed, an error is returned.
//
// The hook pod inherits the following from the container the hook refers to:
//
// * Environment (hook keys take precedence)
// * Working directory
// * Resources
func (e *HookExecutor) executeExecNewPod(hook *deployapi.LifecycleHook, deployment *kapi.ReplicationController, suffix, label string) error {
config, err := deployutil.DecodeDeploymentConfig(deployment, e.decoder)
if err != nil {
return err
}
deployerPod, err := e.pods.Pods(deployment.Namespace).Get(deployutil.DeployerPodNameForDeployment(deployment.Name))
if err != nil {
return err
}
// Build a pod spec from the hook config and deployment
podSpec, err := makeHookPod(hook, deployment, deployerPod, &config.Spec.Strategy, suffix)
if err != nil {
return err
}
// Track whether the pod has already run to completion and avoid showing logs
// or the Success message twice.
completed, created := false, false
// Try to create the pod.
pod, err := e.pods.Pods(deployment.Namespace).Create(podSpec)
if err != nil {
if !kerrors.IsAlreadyExists(err) {
return fmt.Errorf("couldn't create lifecycle pod for %s: %v", deployment.Name, err)
}
completed = true
pod = podSpec
pod.Namespace = deployment.Namespace
} else {
created = true
fmt.Fprintf(e.out, "--> %s: Running hook pod ...\n", label)
}
stopChannel := make(chan struct{})
defer close(stopChannel)
nextPod := NewPodWatch(e.pods.Pods(pod.Namespace), pod.Namespace, pod.Name, pod.ResourceVersion, stopChannel)
// Wait for the hook pod to reach a terminal phase. Start reading logs as
// soon as the pod enters a usable phase.
var updatedPod *kapi.Pod
wg := &sync.WaitGroup{}
wg.Add(1)
restarts := int32(0)
alreadyRead := false
waitLoop:
for {
updatedPod = nextPod()
switch updatedPod.Status.Phase {
case kapi.PodRunning:
completed = false
// We should read only the first time or in any container restart when we want to retry.
canRetry, restartCount := canRetryReading(updatedPod, restarts)
if alreadyRead && !canRetry {
break
}
// The hook container has restarted; we need to notify that we are retrying in the logs.
// TODO: Maybe log the container id
if restarts != restartCount {
wg.Add(1)
restarts = restartCount
fmt.Fprintf(e.out, "--> %s: Retrying hook pod (retry #%d)\n", label, restartCount)
}
alreadyRead = true
go e.readPodLogs(pod, wg)
case kapi.PodSucceeded, kapi.PodFailed:
if completed {
if updatedPod.Status.Phase == kapi.PodSucceeded {
fmt.Fprintf(e.out, "--> %s: Hook pod already succeeded\n", label)
}
wg.Done()
break waitLoop
}
if !created {
fmt.Fprintf(e.out, "--> %s: Hook pod is already running ...\n", label)
}
if !alreadyRead {
go e.readPodLogs(pod, wg)
}
break waitLoop
default:
completed = false
}
}
// The pod is finished, wait for all logs to be consumed before returning.
wg.Wait()
if updatedPod.Status.Phase == kapi.PodFailed {
fmt.Fprintf(e.out, "--> %s: Failed\n", label)
return fmt.Errorf(updatedPod.Status.Message)
}
// Only show this message if we created the pod ourselves, or we saw
// the pod in a running or pending state.
if !completed {
fmt.Fprintf(e.out, "--> %s: Success\n", label)
}
return nil
}
func updateConditions(config deployapi.DeploymentConfig, newStatus *deployapi.DeploymentConfigStatus, latestRC *kapi.ReplicationController) {
// Availability condition.
if newStatus.AvailableReplicas >= config.Spec.Replicas-deployutil.MaxUnavailable(config) && newStatus.AvailableReplicas > 0 {
minAvailability := deployutil.NewDeploymentCondition(deployapi.DeploymentAvailable, kapi.ConditionTrue, "", "Deployment config has minimum availability.")
deployutil.SetDeploymentCondition(newStatus, *minAvailability)
} else {
noMinAvailability := deployutil.NewDeploymentCondition(deployapi.DeploymentAvailable, kapi.ConditionFalse, "", "Deployment config does not have minimum availability.")
deployutil.SetDeploymentCondition(newStatus, *noMinAvailability)
}
// Condition about progress.
if latestRC != nil {
switch deployutil.DeploymentStatusFor(latestRC) {
case deployapi.DeploymentStatusPending:
msg := fmt.Sprintf("Replication controller %q is waiting for pod %q to run", latestRC.Name, deployutil.DeployerPodNameForDeployment(latestRC.Name))
condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionUnknown, "", msg)
deployutil.SetDeploymentCondition(newStatus, *condition)
case deployapi.DeploymentStatusRunning:
if deployutil.IsProgressing(config, *newStatus) {
deployutil.RemoveDeploymentCondition(newStatus, deployapi.DeploymentProgressing)
msg := fmt.Sprintf("Replication controller %q is progressing", latestRC.Name)
condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionTrue, deployutil.ReplicationControllerUpdatedReason, msg)
// TODO: Right now, we use lastTransitionTime for storing the last time we had any progress instead
// of the last time the condition transitioned to a new status. We should probably change that.
deployutil.SetDeploymentCondition(newStatus, *condition)
}
case deployapi.DeploymentStatusFailed:
msg := fmt.Sprintf("Replication controller %q has failed progressing", latestRC.Name)
condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionFalse, deployutil.TimedOutReason, msg)
deployutil.SetDeploymentCondition(newStatus, *condition)
case deployapi.DeploymentStatusComplete:
msg := fmt.Sprintf("Replication controller %q has completed progressing", latestRC.Name)
condition := deployutil.NewDeploymentCondition(deployapi.DeploymentProgressing, kapi.ConditionTrue, deployutil.NewRcAvailableReason, msg)
deployutil.SetDeploymentCondition(newStatus, *condition)
}
}
}