// NewDeployer makes a new Deployer from a kube client.
func NewDeployer(client kclient.Interface, oclient client.Interface, out, errOut io.Writer, until string) *Deployer {
scaler, _ := kubectl.ScalerFor(kapi.Kind("ReplicationController"), client)
return &Deployer{
out: out,
errOut: errOut,
until: until,
getDeployment: func(namespace, name string) (*kapi.ReplicationController, error) {
return client.ReplicationControllers(namespace).Get(name)
},
getDeployments: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return client.ReplicationControllers(namespace).List(kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(configName)})
},
scaler: scaler,
strategyFor: func(config *deployapi.DeploymentConfig) (strategy.DeploymentStrategy, error) {
switch config.Spec.Strategy.Type {
case deployapi.DeploymentStrategyTypeRecreate:
return recreate.NewRecreateDeploymentStrategy(client, oclient, client.Events(""), kapi.Codecs.UniversalDecoder(), out, errOut, until), nil
case deployapi.DeploymentStrategyTypeRolling:
recreate := recreate.NewRecreateDeploymentStrategy(client, oclient, client.Events(""), kapi.Codecs.UniversalDecoder(), out, errOut, until)
return rolling.NewRollingDeploymentStrategy(config.Namespace, client, oclient, client.Events(""), kapi.Codecs.UniversalDecoder(), recreate, out, errOut, until), nil
default:
return nil, fmt.Errorf("unsupported strategy type: %s", config.Spec.Strategy.Type)
}
},
}
}
// findTargetDeployment finds the deployment which is the rollback target by
// searching for deployments associated with config. If desiredVersion is >0,
// the deployment matching desiredVersion will be returned. If desiredVersion
// is <=0, the last completed deployment which is older than the config's
// version will be returned.
func (o *RollbackOptions) findTargetDeployment(config *deployapi.DeploymentConfig, desiredVersion int) (*kapi.ReplicationController, error) {
// Find deployments for the config sorted by version descending.
deployments, err := o.kc.ReplicationControllers(config.Namespace).List(deployutil.ConfigSelector(config.Name), fields.Everything())
if err != nil {
return nil, err
}
sort.Sort(deployutil.ByLatestVersionDesc(deployments.Items))
// Find the target deployment for rollback. If a version was specified,
// use the version for a search. Otherwise, use the last completed
// deployment.
var target *kapi.ReplicationController
for _, deployment := range deployments.Items {
version := deployutil.DeploymentVersionFor(&deployment)
if desiredVersion > 0 {
if version == desiredVersion {
target = &deployment
break
}
} else {
if version < config.LatestVersion && deployutil.DeploymentStatusFor(&deployment) == deployapi.DeploymentStatusComplete {
target = &deployment
break
}
}
}
if target == nil {
return nil, fmt.Errorf("couldn't find deployment for rollback")
}
return target, nil
}
// ViewHistory returns a description of all the history it can find for a deployment config.
func (h *DeploymentConfigHistoryViewer) ViewHistory(namespace, name string, revision int64) (string, error) {
opts := kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(name)}
deploymentList, err := h.rn.ReplicationControllers(namespace).List(opts)
if err != nil {
return "", err
}
if len(deploymentList.Items) == 0 {
return "No rollout history found.", nil
}
items := deploymentList.Items
history := make([]*kapi.ReplicationController, 0, len(items))
for i := range items {
history = append(history, &items[i])
}
// Print details of a specific revision
if revision > 0 {
var desired *kapi.PodTemplateSpec
// We could use a binary search here but brute-force is always faster to write
for i := range history {
rc := history[i]
if deployutil.DeploymentVersionFor(rc) == revision {
desired = rc.Spec.Template
break
}
}
if desired == nil {
return "", fmt.Errorf("unable to find the specified revision")
}
buf := bytes.NewBuffer([]byte{})
kubectl.DescribePodTemplate(desired, buf)
return buf.String(), nil
}
sort.Sort(deployutil.ByLatestVersionAsc(history))
return tabbedString(func(out *tabwriter.Writer) error {
fmt.Fprintf(out, "REVISION\tSTATUS\tCAUSE\n")
for i := range history {
rc := history[i]
rev := deployutil.DeploymentVersionFor(rc)
status := deployutil.DeploymentStatusFor(rc)
cause := rc.Annotations[deployapi.DeploymentStatusReasonAnnotation]
if len(cause) == 0 {
cause = "<unknown>"
}
fmt.Fprintf(out, "%d\t%s\t%s\n", rev, status, cause)
}
return nil
})
}
func (r *ScaleREST) replicasForDeploymentConfig(namespace, configName string) (int32, error) {
options := kapi.ListOptions{LabelSelector: util.ConfigSelector(configName)}
rcList, err := r.rcNamespacer.ReplicationControllers(namespace).List(options)
if err != nil {
return 0, err
}
replicas := int32(0)
for _, rc := range rcList.Items {
replicas += rc.Spec.Replicas
}
return replicas, nil
}
func (r *ScaleREST) replicasForDeploymentConfig(namespace, configName string) (int, error) {
selector := util.ConfigSelector(configName)
rcList, err := r.rcNamespacer.ReplicationControllers(namespace).List(selector, fields.Everything())
if err != nil {
return 0, err
}
replicas := 0
for _, rc := range rcList.Items {
replicas += rc.Spec.Replicas
}
return replicas, nil
}
// cancel cancels any deployment process in progress for config.
func (o DeployOptions) cancel(config *deployapi.DeploymentConfig, out io.Writer) error {
deployments, err := o.kubeClient.ReplicationControllers(config.Namespace).List(deployutil.ConfigSelector(config.Name), fields.Everything())
if err != nil {
return err
}
if len(deployments.Items) == 0 {
fmt.Fprintf(out, "There have been no deployments for %s/%s\n", config.Namespace, config.Name)
return nil
}
sort.Sort(deployutil.ByLatestVersionDesc(deployments.Items))
failedCancellations := []string{}
anyCancelled := false
for _, deployment := range deployments.Items {
status := deployutil.DeploymentStatusFor(&deployment)
switch status {
case deployapi.DeploymentStatusNew,
deployapi.DeploymentStatusPending,
deployapi.DeploymentStatusRunning:
if deployutil.IsDeploymentCancelled(&deployment) {
continue
}
deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser
_, err := o.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment)
if err == nil {
fmt.Fprintf(out, "Cancelled deployment #%d\n", config.LatestVersion)
anyCancelled = true
} else {
fmt.Fprintf(out, "Couldn't cancel deployment #%d (status: %s): %v\n", deployutil.DeploymentVersionFor(&deployment), status, err)
failedCancellations = append(failedCancellations, strconv.Itoa(deployutil.DeploymentVersionFor(&deployment)))
}
}
}
if len(failedCancellations) > 0 {
return fmt.Errorf("couldn't cancel deployment %s", strings.Join(failedCancellations, ", "))
}
if !anyCancelled {
latest := &deployments.Items[0]
timeAt := strings.ToLower(units.HumanDuration(time.Now().Sub(latest.CreationTimestamp.Time)))
fmt.Fprintf(out, "No deployments are in progress (latest deployment #%d %s %s ago)\n",
deployutil.DeploymentVersionFor(latest),
strings.ToLower(string(deployutil.DeploymentStatusFor(latest))),
timeAt)
}
return nil
}
// cancel cancels any deployment process in progress for config.
func (o *DeployOptions) cancel(config *deployapi.DeploymentConfig, out io.Writer) error {
deployments, err := o.kubeClient.ReplicationControllers(config.Namespace).List(deployutil.ConfigSelector(config.Name))
if err != nil {
return err
}
if len(deployments.Items) == 0 {
fmt.Fprintln(out, "no deployments found to cancel")
return nil
}
failedCancellations := []string{}
anyCancelled := false
for _, deployment := range deployments.Items {
status := deployutil.DeploymentStatusFor(&deployment)
switch status {
case deployapi.DeploymentStatusNew,
deployapi.DeploymentStatusPending,
deployapi.DeploymentStatusRunning:
if deployutil.IsDeploymentCancelled(&deployment) {
continue
}
deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledByUser
_, err := o.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment)
if err == nil {
fmt.Fprintf(out, "cancelled deployment #%d\n", config.LatestVersion)
anyCancelled = true
} else {
fmt.Fprintf(out, "couldn't cancel deployment #%d (status: %s): %v\n", deployutil.DeploymentVersionFor(&deployment), status, err)
failedCancellations = append(failedCancellations, strconv.Itoa(deployutil.DeploymentVersionFor(&deployment)))
}
}
}
if len(failedCancellations) > 0 {
return fmt.Errorf("couldn't cancel deployment %s", strings.Join(failedCancellations, ", "))
}
if !anyCancelled {
fmt.Fprintln(out, "no active deployments to cancel")
}
return nil
}
// Stop scales a replication controller via its deployment configuration down to
// zero replicas, waits for all of them to get deleted and then deletes both the
// replication controller and its deployment configuration.
func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error {
// If the config is already deleted, it may still have associated
// deployments which didn't get cleaned up during prior calls to Stop. If
// the config can't be found, still make an attempt to clean up the
// deployments.
//
// It's important to delete the config first to avoid an undesirable side
// effect which can cause the deployment to be re-triggered upon the
// config's deletion. See https://github.com/openshift/origin/issues/2721
// for more details.
err := reaper.oc.DeploymentConfigs(namespace).Delete(name)
configNotFound := kerrors.IsNotFound(err)
if err != nil && !configNotFound {
return err
}
// Clean up deployments related to the config.
options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)}
rcList, err := reaper.kc.ReplicationControllers(namespace).List(options)
if err != nil {
return err
}
rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc)
if err != nil {
return err
}
// If there is neither a config nor any deployments, we can return NotFound.
deployments := rcList.Items
if configNotFound && len(deployments) == 0 {
return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name)
}
for _, rc := range deployments {
if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
// Better not error out here...
glog.Infof("Cannot delete ReplicationController %s/%s: %v", rc.Namespace, rc.Name, err)
}
}
return nil
}
func deploymentInfo(oc *exutil.CLI, name string) (*deployapi.DeploymentConfig, []kapi.ReplicationController, []kapi.Pod, error) {
dc, err := oc.REST().DeploymentConfigs(oc.Namespace()).Get(name)
if err != nil {
return nil, nil, nil, err
}
// get pods before RCs, so we see more RCs than pods.
pods, err := oc.KubeREST().Pods(oc.Namespace()).List(kapi.ListOptions{})
if err != nil {
return nil, nil, nil, err
}
rcs, err := oc.KubeREST().ReplicationControllers(oc.Namespace()).List(kapi.ListOptions{
LabelSelector: deployutil.ConfigSelector(name),
})
if err != nil {
return nil, nil, nil, err
}
sort.Sort(deployutil.ByLatestVersionAsc(rcs.Items))
return dc, rcs.Items, pods.Items, nil
}
// NewDeployer makes a new Deployer from a kube client.
func NewDeployer(client kclient.Interface) *Deployer {
scaler, _ := kubectl.ScalerFor("ReplicationController", kubectl.NewScalerClient(client))
return &Deployer{
getDeployment: func(namespace, name string) (*kapi.ReplicationController, error) {
return client.ReplicationControllers(namespace).Get(name)
},
getDeployments: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return client.ReplicationControllers(namespace).List(deployutil.ConfigSelector(configName))
},
scaler: scaler,
strategyFor: func(config *deployapi.DeploymentConfig) (strategy.DeploymentStrategy, error) {
switch config.Template.Strategy.Type {
case deployapi.DeploymentStrategyTypeRecreate:
return recreate.NewRecreateDeploymentStrategy(client, latest.Codec), nil
case deployapi.DeploymentStrategyTypeRolling:
recreate := recreate.NewRecreateDeploymentStrategy(client, latest.Codec)
return rolling.NewRollingDeploymentStrategy(config.Namespace, client, latest.Codec, recreate), nil
default:
return nil, fmt.Errorf("unsupported strategy type: %s", config.Template.Strategy.Type)
}
},
}
}
// Handle implements the loop that processes deployment configs. Since this controller started
// using caches, the provided config MUST be deep-copied beforehand (see work() in factory.go).
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
// There's nothing to reconcile until the version is nonzero.
if config.Status.LatestVersion == 0 {
return c.updateStatus(config, []kapi.ReplicationController{})
}
// Find all deployments owned by the deployment config.
selector := deployutil.ConfigSelector(config.Name)
existingDeployments, err := c.rcStore.ReplicationControllers(config.Namespace).List(selector)
if err != nil {
return err
}
// In case the deployment config has been marked for deletion, merely update its status with
// the latest available information. Some deletions make take some time to complete so there
// is value in doing this.
if config.DeletionTimestamp != nil {
return c.updateStatus(config, existingDeployments)
}
latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
// If the latest deployment doesn't exist yet, cancel any running
// deployments to allow them to be superceded by the new config version.
awaitingCancellations := false
if !latestIsDeployed {
for i := range existingDeployments {
deployment := existingDeployments[i]
// Skip deployments with an outcome.
if deployutil.IsTerminatedDeployment(&deployment) {
continue
}
// Cancel running deployments.
awaitingCancellations = true
if !deployutil.IsDeploymentCancelled(&deployment) {
// Retry faster on conflicts
var updatedDeployment *kapi.ReplicationController
if err := kclient.RetryOnConflict(kclient.DefaultBackoff, func() error {
rc, err := c.rcStore.ReplicationControllers(deployment.Namespace).Get(deployment.Name)
if kapierrors.IsNotFound(err) {
return nil
}
if err != nil {
return err
}
copied, err := deployutil.DeploymentDeepCopy(rc)
if err != nil {
return err
}
copied.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
copied.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
updatedDeployment, err = c.rn.ReplicationControllers(copied.Namespace).Update(copied)
return err
}); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err)
} else {
if updatedDeployment != nil {
// replace the current deployment with the updated copy so that a future update has a chance at working
existingDeployments[i] = *updatedDeployment
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion)
}
}
}
}
}
// Wait for deployment cancellations before reconciling or creating a new
// deployment to avoid competing with existing deployment processes.
if awaitingCancellations {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion)
return fmt.Errorf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config))
}
// If the latest deployment already exists, reconcile existing deployments
// and return early.
if latestIsDeployed {
// If the latest deployment is still running, try again later. We don't
// want to compete with the deployer.
if !deployutil.IsTerminatedDeployment(latestDeployment) {
return c.updateStatus(config, existingDeployments)
}
return c.reconcileDeployments(existingDeployments, config)
}
// If the config is paused we shouldn't create new deployments for it.
if config.Spec.Paused {
// in order for revision history limit cleanup to work for paused
// deployments, we need to trigger it here
if err := c.cleanupOldDeployments(existingDeployments, config); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err)
}
return c.updateStatus(config, existingDeployments)
}
// No deployments are running and the latest deployment doesn't exist, so
// create the new deployment.
deployment, err := deployutil.MakeDeployment(config, c.codec)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err))
}
created, err := c.rn.ReplicationControllers(config.Namespace).Create(deployment)
if err != nil {
// If the deployment was already created, just move on. The cache could be
// stale, or another process could have already handled this update.
if kapierrors.IsAlreadyExists(err) {
return c.updateStatus(config, existingDeployments)
}
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err)
return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion)
// As we've just created a new deployment, we need to make sure to clean
// up old deployments if we have reached our deployment history quota
existingDeployments = append(existingDeployments, *created)
if err := c.cleanupOldDeployments(existingDeployments, config); err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCleanupFailed", "Couldn't clean up deployments: %v", err)
}
return c.updateStatus(config, existingDeployments)
}
// Describe returns the description of the latest deployments for a config
func (d *LatestDeploymentsDescriber) Describe(namespace, name string) (string, error) {
var f formatter
config, err := d.osClient.DeploymentConfigs(namespace).Get(name)
if err != nil {
return "", err
}
var deployments []kapi.ReplicationController
if d.count == -1 || d.count > 1 {
list, err := d.kubeClient.ReplicationControllers(namespace).List(kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(name)})
if err != nil && !kerrors.IsNotFound(err) {
return "", err
}
deployments = list.Items
} else {
deploymentName := deployutil.LatestDeploymentNameForConfig(config)
deployment, err := d.kubeClient.ReplicationControllers(config.Namespace).Get(deploymentName)
if err != nil && !kerrors.IsNotFound(err) {
return "", err
}
if deployment != nil {
deployments = []kapi.ReplicationController{*deployment}
}
}
g := graph.New()
dcNode := deploygraph.EnsureDeploymentConfigNode(g, config)
for i := range deployments {
kubegraph.EnsureReplicationControllerNode(g, &deployments[i])
}
deployedges.AddTriggerEdges(g, dcNode)
deployedges.AddDeploymentEdges(g, dcNode)
activeDeployment, inactiveDeployments := deployedges.RelevantDeployments(g, dcNode)
return tabbedString(func(out *tabwriter.Writer) error {
descriptions := describeDeployments(f, dcNode, activeDeployment, inactiveDeployments, d.count)
for i, description := range descriptions {
descriptions[i] = fmt.Sprintf("%v %v", name, description)
}
printLines(out, "", 0, descriptions...)
return nil
})
}
func (c *ApplicationController) handleDeploymentConfigLabel(app *api.Application, itemIndex int) error {
labelSelectorStr := fmt.Sprintf("%s.application.%s", app.Namespace, app.Name)
client := c.Client.DeploymentConfigs(app.Namespace)
resource, err := client.Get(app.Spec.Items[itemIndex].Name)
if err != nil {
if kerrors.IsNotFound(err) {
c.Recorder.Eventf(app, kapi.EventTypeWarning, getItemErrEventReason(app.Status, app.Spec.Items[itemIndex]), "get deploymentconfig has error: %s", err.Error())
c.deleteApplicationItem(app, itemIndex)
return nil
}
return err
}
switch app.Status.Phase {
case api.ApplicationActiveUpdate:
if _, exists := resource.Labels[labelSelectorStr]; exists {
//Active正常状态,当有新的更新时,如果这个label不存在,则新建
return nil
}
fallthrough
case api.ApplicationNew:
if resource.Labels == nil {
resource.Labels = make(map[string]string)
}
resource.Labels[labelSelectorStr] = app.Name
for _, trigger := range resource.Spec.Triggers {
if trigger.Type == deployapi.DeploymentTriggerOnConfigChange {
trigger.Type = deployapi.DeploymentTriggerManual
}
}
if _, err := client.Update(resource); err != nil {
c.Recorder.Eventf(app, kapi.EventTypeWarning, addItemEvent(app.Spec.Items[itemIndex]), "error: %s", err.Error())
return err
}
c.Recorder.Event(app, kapi.EventTypeWarning, "Application", addItemEvent(app.Spec.Items[itemIndex])+"success")
case api.ApplicationTerminating:
if !labelExistsOtherApplicationKey(resource.Labels, labelSelectorStr) {
if err := client.Delete(app.Spec.Items[itemIndex].Name); err != nil {
c.Recorder.Eventf(app, kapi.EventTypeWarning, getItemErrEventReason(app.Status, app.Spec.Items[itemIndex]), "delete deploymentconfig has error: %s", err.Error())
return err
}
sel := deployutil.ConfigSelector(app.Spec.Items[itemIndex].Name)
existingDeployments, err := c.KubeClient.ReplicationControllers(app.Namespace).List(kapi.ListOptions{LabelSelector: sel, FieldSelector: fields.Everything()})
if err != nil && !kerrors.IsNotFound(err) {
fmt.Printf("delete application dc %s err ", app.Spec.Items[itemIndex].Name, err)
}
for _, v := range existingDeployments.Items {
if err := c.KubeClient.ReplicationControllers(app.Namespace).Delete(v.Name); err != nil {
c.Recorder.Eventf(app, kapi.EventTypeWarning, getItemErrEventReason(app.Status, app.Spec.Items[itemIndex]), "delete deploymentconfig-replicationcontroller %s has error: %v", v.Name, err)
}
c.Recorder.Eventf(app, kapi.EventTypeWarning, getItemErrEventReason(app.Status, app.Spec.Items[itemIndex]), "delete deploymentconfig-replicationcontroller %s has error", v.Name)
}
existingPods, err := c.KubeClient.Pods(app.Namespace).List(kapi.ListOptions{LabelSelector: labels.Set{deployapi.DeploymentConfigLabel: app.Spec.Items[itemIndex].Name}.AsSelector(), FieldSelector: fields.Everything()})
if err != nil && !kerrors.IsNotFound(err) {
fmt.Printf("delete application dc %s err ", app.Spec.Items[itemIndex].Name, err)
}
for _, v := range existingPods.Items {
if err := c.KubeClient.Pods(app.Namespace).Delete(v.Name, nil); err != nil {
c.Recorder.Eventf(app, kapi.EventTypeWarning, getItemErrEventReason(app.Status, app.Spec.Items[itemIndex]), "delete deploymentconfig-pod %s has error: %v", v.Name, err)
}
c.Recorder.Eventf(app, kapi.EventTypeWarning, getItemErrEventReason(app.Status, app.Spec.Items[itemIndex]), "delete deploymentconfig-pod %s has error", v.Name)
}
} else {
delete(resource.Labels, labelSelectorStr)
if _, err := client.Update(resource); err != nil {
c.Recorder.Eventf(app, kapi.EventTypeWarning, getItemErrEventReason(app.Status, app.Spec.Items[itemIndex]), "update deploymentconfig has error: %s", err.Error())
return err
}
}
app.Spec.Items = append(app.Spec.Items[:itemIndex], app.Spec.Items[itemIndex+1:]...)
if len(app.Spec.Items) == 0 {
if err := c.Client.Applications(app.Namespace).Delete(app.Name); err != nil {
c.Recorder.Eventf(app, kapi.EventTypeWarning, "Clean Application", "delete application has error: %s", err.Error())
}
}
case api.ApplicationTerminatingLabel:
delete(resource.Labels, labelSelectorStr)
if _, err := client.Update(resource); err != nil {
return err
}
app.Spec.Items = append(app.Spec.Items[:itemIndex], app.Spec.Items[itemIndex+1:]...)
if len(app.Spec.Items) == 0 {
if err := c.Client.Applications(app.Namespace).Delete(app.Name); err != nil {
c.Recorder.Eventf(app, kapi.EventTypeWarning, "Clean Application", "delete application has error: %s", err.Error())
}
}
}
return nil
}
// registers all deployment related steps
func init() {
RegisterSteps(func(c *Context) {
c.Then(`^I should not have a deploymentconfig "(.+?)"$`, func(dcName string) {
found, err := c.DeploymentConfigExists(dcName)
if err != nil {
c.Fail("Failed to check for Deployment Config '%s' existance: %v", dcName, err)
return
}
if found {
c.Fail("Deployment Config %s should not exists", dcName)
return
}
})
c.Then(`^I should have a deploymentconfig "(.+?)"$`, func(dcName string) {
dc, err := c.GetDeploymentConfig(dcName)
if err != nil {
c.Fail("Failed to get Deployment Config '%s': %v", dcName, err)
return
}
assert.Equal(c.T, dcName, dc.Name)
})
c.Given(`^I have a deploymentconfig "(.+?)"$`, func(dcName string) {
dc, err := c.GetDeploymentConfig(dcName)
if err != nil {
c.Fail("Failed to get Deployment Config '%s': %v", dcName, err)
return
}
assert.Equal(c.T, dcName, dc.Name)
})
c.When(`^the deploymentconfig "(.+?)" has at least (\d+) deployments?$`, func(dcName string, requiredDeployments int) {
dc, err := c.GetDeploymentConfig(dcName)
if err != nil {
c.Fail("Failed to get Deployment Config '%s': %v", dcName, err)
return
}
if dc.Status.LatestVersion < requiredDeployments {
c.Fail("The Deployment Config '%s' has only %v deployments, instead of the %v deployments required", dcName, dc.Status.LatestVersion, requiredDeployments)
return
}
})
c.Then(`^the latest deployment of "(.+?)" should succeed in less than "(.+?)"$`, func(dcName string, timeout string) {
timeoutDuration, err := time.ParseDuration(timeout)
if err != nil {
c.Fail("Failed to parse duration '%s': %v", timeout, err)
return
}
dc, err := c.GetDeploymentConfig(dcName)
if err != nil {
c.Fail("Failed to get Deployment Config '%s': %v", dcName, err)
return
}
latestDeploymentName := fmt.Sprintf("%s-%d", dc.Name, dc.Status.LatestVersion)
success, err := c.IsDeploymentComplete(latestDeploymentName, timeoutDuration)
if err != nil {
c.Fail("Failed to check status of the deployment '%s': %v", latestDeploymentName, err)
return
}
if !success {
logs, err := c.GetDeploymentLogs(latestDeploymentName)
if err != nil {
fmt.Printf("Failed to get deployment logs '%v'\n", err)
} else {
fmt.Printf("Deployment logs '%v'\n", logs)
}
c.Fail("Deployment '%s' was not successful!", latestDeploymentName)
return
}
})
c.When(`^I have a successful deployment of "(.+?)"$`, func(dcName string) {
rcList, err := c.GetReplicationControllers(deployutil.ConfigSelector(dcName))
if err != nil {
c.Fail("Failed to get Deployment Config '%s': %v", dcName, err)
return
}
var successfulDeployment bool
for _, rc := range rcList.Items {
if status, ok := rc.Annotations[deployapi.DeploymentStatusAnnotation]; ok {
switch status {
case string(deployapi.DeploymentStatusComplete):
successfulDeployment = true
default:
}
}
}
if !successfulDeployment {
logs, err := c.GetDeploymentLogs(dcName)
if err != nil {
fmt.Printf("Failed to get deployment logs '%v'\n", err)
} else {
fmt.Printf("Deployment logs '%v'\n", logs)
}
c.Fail("No successful deployment for '%s'", dcName)
return
}
})
c.When(`^I delete the deploymentconfig "(.+?)"$`, func(dcName string) {
if err := c.DeleteDeploymentConfig(dcName); err != nil {
c.Fail("Failed to delete deployment config %s", dcName)
}
})
})
}
// Create creates a DeploymentConfigController.
func (factory *DeploymentConfigControllerFactory) Create() controller.RunnableController {
deploymentConfigLW := &deployutil.ListWatcherImpl{
ListFunc: func() (runtime.Object, error) {
return factory.Client.DeploymentConfigs(kapi.NamespaceAll).List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return factory.Client.DeploymentConfigs(kapi.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
queue := cache.NewFIFO(cache.MetaNamespaceKeyFunc)
cache.NewReflector(deploymentConfigLW, &deployapi.DeploymentConfig{}, queue, 2*time.Minute).Run()
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartRecordingToSink(factory.KubeClient.Events(""))
configController := &DeploymentConfigController{
deploymentClient: &deploymentClientImpl{
createDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
return factory.KubeClient.ReplicationControllers(namespace).Create(deployment)
},
listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
sel := deployutil.ConfigSelector(configName)
list, err := factory.KubeClient.ReplicationControllers(namespace).List(sel)
if err != nil {
return nil, err
}
return list, nil
},
updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
return factory.KubeClient.ReplicationControllers(namespace).Update(deployment)
},
},
makeDeployment: func(config *deployapi.DeploymentConfig) (*kapi.ReplicationController, error) {
return deployutil.MakeDeployment(config, factory.Codec)
},
recorder: eventBroadcaster.NewRecorder(kapi.EventSource{Component: "deployer"}),
}
return &controller.RetryController{
Queue: queue,
RetryManager: controller.NewQueueRetryManager(
queue,
cache.MetaNamespaceKeyFunc,
func(obj interface{}, err error, retries controller.Retry) bool {
config := obj.(*deployapi.DeploymentConfig)
// no retries for a fatal error
if _, isFatal := err.(fatalError); isFatal {
glog.V(4).Infof("Will not retry fatal error for deploymentConfig %s/%s: %v", config.Namespace, config.Name, err)
kutil.HandleError(err)
return false
}
// infinite retries for a transient error
if _, isTransient := err.(transientError); isTransient {
glog.V(4).Infof("Retrying deploymentConfig %s/%s with error: %v", config.Namespace, config.Name, err)
return true
}
kutil.HandleError(err)
// no retries for anything else
if retries.Count > 0 {
return false
}
return true
},
kutil.NewTokenBucketRateLimiter(1, 10),
),
Handle: func(obj interface{}) error {
config := obj.(*deployapi.DeploymentConfig)
return configController.Handle(config)
},
}
}
// Create creates a DeployerPodController.
func (factory *DeployerPodControllerFactory) Create() controller.RunnableController {
deploymentLW := &deployutil.ListWatcherImpl{
ListFunc: func() (runtime.Object, error) {
return factory.KubeClient.ReplicationControllers(kapi.NamespaceAll).List(labels.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return factory.KubeClient.ReplicationControllers(kapi.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
deploymentStore := cache.NewStore(cache.MetaNamespaceKeyFunc)
cache.NewReflector(deploymentLW, &kapi.ReplicationController{}, deploymentStore, 2*time.Minute).Run()
// TODO: These should be filtered somehow to include only the primary
// deployer pod. For now, the controller is filtering.
// TODO: Even with the label selector, this is inefficient on the backend
// and we should work to consolidate namespace-spanning pod watches. For
// example, the build infra is also watching pods across namespaces.
podLW := &deployutil.ListWatcherImpl{
ListFunc: func() (runtime.Object, error) {
return factory.KubeClient.Pods(kapi.NamespaceAll).List(deployutil.AnyDeployerPodSelector(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return factory.KubeClient.Pods(kapi.NamespaceAll).Watch(deployutil.AnyDeployerPodSelector(), fields.Everything(), resourceVersion)
},
}
podQueue := cache.NewFIFO(cache.MetaNamespaceKeyFunc)
cache.NewReflector(podLW, &kapi.Pod{}, podQueue, 2*time.Minute).Run()
podController := &DeployerPodController{
deploymentClient: &deploymentClientImpl{
getDeploymentFunc: func(namespace, name string) (*kapi.ReplicationController, error) {
// Try to use the cache first. Trust hits and return them.
example := &kapi.ReplicationController{ObjectMeta: kapi.ObjectMeta{Namespace: namespace, Name: name}}
cached, exists, err := deploymentStore.Get(example)
if err == nil && exists {
return cached.(*kapi.ReplicationController), nil
}
// Double-check with the master for cache misses/errors, since those
// are rare and API calls are expensive but more reliable.
return factory.KubeClient.ReplicationControllers(namespace).Get(name)
},
updateDeploymentFunc: func(namespace string, deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
return factory.KubeClient.ReplicationControllers(namespace).Update(deployment)
},
listDeploymentsForConfigFunc: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
return factory.KubeClient.ReplicationControllers(namespace).List(deployutil.ConfigSelector(configName))
},
},
deployerPodsFor: func(namespace, name string) (*kapi.PodList, error) {
return factory.KubeClient.Pods(namespace).List(deployutil.DeployerPodSelector(name), fields.Everything())
},
deletePod: func(namespace, name string) error {
return factory.KubeClient.Pods(namespace).Delete(name, kapi.NewDeleteOptions(0))
},
}
return &controller.RetryController{
Queue: podQueue,
RetryManager: controller.NewQueueRetryManager(
podQueue,
cache.MetaNamespaceKeyFunc,
func(obj interface{}, err error, retries controller.Retry) bool {
kutil.HandleError(err)
// infinite retries for a transient error
if _, isTransient := err.(transientError); isTransient {
return true
}
// no retries for anything else
if retries.Count > 0 {
return false
}
return true
},
kutil.NewTokenBucketRateLimiter(1, 10),
),
Handle: func(obj interface{}) error {
pod := obj.(*kapi.Pod)
return podController.Handle(pod)
},
}
}
// Describe returns the description of a DeploymentConfig
func (d *DeploymentConfigDescriber) Describe(namespace, name string, settings kctl.DescriberSettings) (string, error) {
var deploymentConfig *deployapi.DeploymentConfig
if d.config != nil {
// If a deployment config is already provided use that.
// This is used by `oc rollback --dry-run`.
deploymentConfig = d.config
} else {
var err error
deploymentConfig, err = d.osClient.DeploymentConfigs(namespace).Get(name)
if err != nil {
return "", err
}
}
return tabbedString(func(out *tabwriter.Writer) error {
formatMeta(out, deploymentConfig.ObjectMeta)
var (
deploymentsHistory []kapi.ReplicationController
activeDeploymentName string
)
if d.config == nil {
if rcs, err := d.kubeClient.ReplicationControllers(namespace).List(kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(deploymentConfig.Name)}); err == nil {
deploymentsHistory = rcs.Items
}
}
if deploymentConfig.Status.LatestVersion == 0 {
formatString(out, "Latest Version", "Not deployed")
} else {
formatString(out, "Latest Version", strconv.FormatInt(deploymentConfig.Status.LatestVersion, 10))
}
printDeploymentConfigSpec(d.kubeClient, *deploymentConfig, out)
fmt.Fprintln(out)
latestDeploymentName := deployutil.LatestDeploymentNameForConfig(deploymentConfig)
if activeDeployment := deployutil.ActiveDeployment(deploymentConfig, deploymentsHistory); activeDeployment != nil {
activeDeploymentName = activeDeployment.Name
}
var deployment *kapi.ReplicationController
isNotDeployed := len(deploymentsHistory) == 0
for _, item := range deploymentsHistory {
if item.Name == latestDeploymentName {
deployment = &item
}
}
if isNotDeployed {
formatString(out, "Latest Deployment", "<none>")
} else {
header := fmt.Sprintf("Deployment #%d (latest)", deployutil.DeploymentVersionFor(deployment))
// Show details if the current deployment is the active one or it is the
// initial deployment.
printDeploymentRc(deployment, d.kubeClient, out, header, (deployment.Name == activeDeploymentName) || len(deploymentsHistory) == 1)
}
// We don't show the deployment history when running `oc rollback --dry-run`.
if d.config == nil && !isNotDeployed {
sorted := deploymentsHistory
sort.Sort(sort.Reverse(rcutils.OverlappingControllers(sorted)))
counter := 1
for _, item := range sorted {
if item.Name != latestDeploymentName && deploymentConfig.Name == deployutil.DeploymentConfigNameFor(&item) {
header := fmt.Sprintf("Deployment #%d", deployutil.DeploymentVersionFor(&item))
printDeploymentRc(&item, d.kubeClient, out, header, item.Name == activeDeploymentName)
counter++
}
if counter == maxDisplayDeployments {
break
}
}
}
if settings.ShowEvents {
// Events
if events, err := d.kubeClient.Events(deploymentConfig.Namespace).Search(deploymentConfig); err == nil && events != nil {
latestDeploymentEvents := &kapi.EventList{Items: []kapi.Event{}}
for i := len(events.Items); i != 0 && i > len(events.Items)-maxDisplayDeploymentsEvents; i-- {
latestDeploymentEvents.Items = append(latestDeploymentEvents.Items, events.Items[i-1])
}
fmt.Fprintln(out)
kctl.DescribeEvents(latestDeploymentEvents, out)
}
}
return nil
})
}
func (c *DeploymentConfigController) Handle(config *deployapi.DeploymentConfig) error {
// There's nothing to reconcile until the version is nonzero.
if config.Status.LatestVersion == 0 {
glog.V(5).Infof("Waiting for first version of %q", deployutil.LabelForDeploymentConfig(config))
return c.updateStatus(config)
}
// Find all deployments owned by the deploymentConfig.
sel := deployutil.ConfigSelector(config.Name)
existingDeployments, err := c.kubeClient.ReplicationControllers(config.Namespace).List(kapi.ListOptions{LabelSelector: sel})
if err != nil {
return err
}
latestIsDeployed, latestDeployment := deployutil.LatestDeploymentInfo(config, existingDeployments)
// If the latest deployment doesn't exist yet, cancel any running
// deployments to allow them to be superceded by the new config version.
awaitingCancellations := false
if !latestIsDeployed {
for _, deployment := range existingDeployments.Items {
// Skip deployments with an outcome.
if deployutil.IsTerminatedDeployment(&deployment) {
continue
}
// Cancel running deployments.
awaitingCancellations = true
if !deployutil.IsDeploymentCancelled(&deployment) {
deployment.Annotations[deployapi.DeploymentCancelledAnnotation] = deployapi.DeploymentCancelledAnnotationValue
deployment.Annotations[deployapi.DeploymentStatusReasonAnnotation] = deployapi.DeploymentCancelledNewerDeploymentExists
_, err := c.kubeClient.ReplicationControllers(deployment.Namespace).Update(&deployment)
if err != nil {
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCancellationFailed", "Failed to cancel deployment %q superceded by version %d: %s", deployment.Name, config.Status.LatestVersion, err)
} else {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCancelled", "Cancelled deployment %q superceded by version %d", deployment.Name, config.Status.LatestVersion)
}
}
}
}
// Wait for deployment cancellations before reconciling or creating a new
// deployment to avoid competing with existing deployment processes.
if awaitingCancellations {
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentAwaitingCancellation", "Deployment of version %d awaiting cancellation of older running deployments", config.Status.LatestVersion)
// raise a transientError so that the deployment config can be re-queued
return transientError(fmt.Sprintf("found previous inflight deployment for %s - requeuing", deployutil.LabelForDeploymentConfig(config)))
}
// If the latest deployment already exists, reconcile existing deployments
// and return early.
if latestIsDeployed {
// If the latest deployment is still running, try again later. We don't
// want to compete with the deployer.
if !deployutil.IsTerminatedDeployment(latestDeployment) {
return c.updateStatus(config)
}
return c.reconcileDeployments(existingDeployments, config)
}
// No deployments are running and the latest deployment doesn't exist, so
// create the new deployment.
deployment, err := deployutil.MakeDeployment(config, c.codec)
if err != nil {
return fatalError(fmt.Sprintf("couldn't make deployment from (potentially invalid) deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err))
}
created, err := c.kubeClient.ReplicationControllers(config.Namespace).Create(deployment)
if err != nil {
// If the deployment was already created, just move on. The cache could be
// stale, or another process could have already handled this update.
if errors.IsAlreadyExists(err) {
return c.updateStatus(config)
}
c.recorder.Eventf(config, kapi.EventTypeWarning, "DeploymentCreationFailed", "Couldn't deploy version %d: %s", config.Status.LatestVersion, err)
return fmt.Errorf("couldn't create deployment for deployment config %s: %v", deployutil.LabelForDeploymentConfig(config), err)
}
c.recorder.Eventf(config, kapi.EventTypeNormal, "DeploymentCreated", "Created new deployment %q for version %d", created.Name, config.Status.LatestVersion)
return c.updateStatus(config)
}
// Stop scales a replication controller via its deployment configuration down to
// zero replicas, waits for all of them to get deleted and then deletes both the
// replication controller and its deployment configuration.
func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error {
// Pause the deployment configuration to prevent the new deployments from
// being triggered.
config, err := reaper.pause(namespace, name)
configNotFound := kerrors.IsNotFound(err)
if err != nil && !configNotFound {
return err
}
var (
isPaused bool
legacy bool
)
// Determine if the deployment config controller noticed the pause.
if !configNotFound {
if err := wait.Poll(1*time.Second, 1*time.Minute, func() (bool, error) {
dc, err := reaper.oc.DeploymentConfigs(namespace).Get(name)
if err != nil {
return false, err
}
isPaused = dc.Spec.Paused
return dc.Status.ObservedGeneration >= config.Generation, nil
}); err != nil {
return err
}
// If we failed to pause the deployment config, it means we are talking to
// old API that does not support pausing. In that case, we delete the
// deployment config to stay backward compatible.
if !isPaused {
if err := reaper.oc.DeploymentConfigs(namespace).Delete(name); err != nil {
return err
}
// Setting this to true avoid deleting the config at the end.
legacy = true
}
}
// Clean up deployments related to the config. Even if the deployment
// configuration has been deleted, we want to sweep the existing replication
// controllers and clean them up.
options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)}
rcList, err := reaper.kc.ReplicationControllers(namespace).List(options)
if err != nil {
return err
}
rcReaper, err := kubectl.ReaperFor(kapi.Kind("ReplicationController"), reaper.kc)
if err != nil {
return err
}
// If there is neither a config nor any deployments, nor any deployer pods, we can return NotFound.
deployments := rcList.Items
if configNotFound && len(deployments) == 0 {
return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name)
}
for _, rc := range deployments {
if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
// Better not error out here...
glog.Infof("Cannot delete ReplicationController %s/%s for deployment config %s/%s: %v", rc.Namespace, rc.Name, namespace, name, err)
}
// Only remove deployer pods when the deployment was failed. For completed
// deployment the pods should be already deleted.
if !util.IsFailedDeployment(&rc) {
continue
}
// Delete all deployer and hook pods
options = kapi.ListOptions{LabelSelector: util.DeployerPodSelector(rc.Name)}
podList, err := reaper.kc.Pods(rc.Namespace).List(options)
if err != nil {
return err
}
for _, pod := range podList.Items {
err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod)
if err != nil {
// Better not error out here...
glog.Infof("Cannot delete lifecycle Pod %s/%s for deployment config %s/%s: %v", pod.Namespace, pod.Name, namespace, name, err)
}
}
}
// Nothing to delete or we already deleted the deployment config because we
// failed to pause.
if configNotFound || legacy {
return nil
}
return reaper.oc.DeploymentConfigs(namespace).Delete(name)
}
func (o DeployOptions) RunDeploy() error {
config, err := o.osClient.DeploymentConfigs(o.namespace).Get(o.deploymentConfigName)
if err != nil {
return err
}
commandClient := &deployCommandClientImpl{
GetDeploymentFn: func(namespace, name string) (*kapi.ReplicationController, error) {
return o.kubeClient.ReplicationControllers(namespace).Get(name)
},
ListDeploymentsForConfigFn: func(namespace, configName string) (*kapi.ReplicationControllerList, error) {
list, err := o.kubeClient.ReplicationControllers(namespace).List(deployutil.ConfigSelector(configName))
if err != nil {
return nil, err
}
return list, nil
},
UpdateDeploymentConfigFn: func(config *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) {
return o.osClient.DeploymentConfigs(config.Namespace).Update(config)
},
UpdateDeploymentFn: func(deployment *kapi.ReplicationController) (*kapi.ReplicationController, error) {
return o.kubeClient.ReplicationControllers(deployment.Namespace).Update(deployment)
},
ListDeployerPodsForFn: func(namespace, deploymentName string) (*kapi.PodList, error) {
selector, err := labels.Parse(fmt.Sprintf("%s=%s", deployapi.DeployerPodForDeploymentLabel, deploymentName))
if err != nil {
return nil, err
}
return o.kubeClient.Pods(namespace).List(selector, fields.Everything())
},
DeletePodFn: func(pod *kapi.Pod) error {
return o.kubeClient.Pods(pod.Namespace).Delete(pod.Name, nil)
},
}
switch {
case o.deployLatest:
c := &deployLatestCommand{client: commandClient}
err = c.deploy(config, o.out)
case o.retryDeploy:
c := &retryDeploymentCommand{client: commandClient}
err = c.retry(config, o.out)
case o.cancelDeploy:
c := &cancelDeploymentCommand{client: commandClient}
err = c.cancel(config, o.out)
case o.enableTriggers:
t := &triggerEnabler{
updateConfig: func(namespace string, config *deployapi.DeploymentConfig) (*deployapi.DeploymentConfig, error) {
return o.osClient.DeploymentConfigs(namespace).Update(config)
},
}
err = t.enableTriggers(config, o.out)
default:
describer := describe.NewLatestDeploymentsDescriber(o.osClient, o.kubeClient, -1)
desc, err := describer.Describe(config.Namespace, config.Name)
if err != nil {
return err
}
fmt.Fprint(o.out, desc)
}
return err
}
func (o CancelOptions) forEachControllerInConfig(namespace, name string, mutateFunc func(*kapi.ReplicationController) bool) ([]kapi.ReplicationController, bool, error) {
deployments, err := o.Clientset.ReplicationControllers(namespace).List(kapi.ListOptions{LabelSelector: deployutil.ConfigSelector(name)})
if err != nil {
return nil, false, err
}
if len(deployments.Items) == 0 {
return nil, false, fmt.Errorf("there have been no replication controllers for %s/%s\n", namespace, name)
}
sort.Sort(deployutil.ByLatestVersionDesc(deployments.Items))
allErrs := []error{}
cancelled := false
for _, deployment := range deployments.Items {
status := deployutil.DeploymentStatusFor(&deployment)
switch status {
case deployapi.DeploymentStatusNew,
deployapi.DeploymentStatusPending,
deployapi.DeploymentStatusRunning:
cancelled = mutateFunc(&deployment)
}
}
return deployments.Items, cancelled, utilerrors.NewAggregate(allErrs)
}