func MetaAndSpecCheckingFunction(expected runtime.Object) CheckingFunction { return func(obj runtime.Object) error { if util.ObjectMetaAndSpecEquivalent(obj, expected) { return nil } return fmt.Errorf("Object different expected=%#v received=%#v", expected, obj) } }
func waitForDaemonSetUpdateOrFail(clientset *kubeclientset.Clientset, namespace string, daemonset *v1beta1.DaemonSet, timeout time.Duration) { By(fmt.Sprintf("Fetching a federated daemonset shard of daemonset %q in namespace %q from cluster", daemonset.Name, namespace)) err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { clusterDaemonSet, err := clientset.Extensions().DaemonSets(namespace).Get(daemonset.Name) if err == nil { // We want it present, and the Get succeeded, so we're all good. if util.ObjectMetaAndSpecEquivalent(clusterDaemonSet, daemonset) { By(fmt.Sprintf("Success: shard of federated daemonset %q in namespace %q in cluster is updated", daemonset.Name, namespace)) return true, nil } else { By(fmt.Sprintf("Expected equal daemonsets. expected: %+v\nactual: %+v", *daemonset, *clusterDaemonSet)) } By(fmt.Sprintf("DaemonSet %q in namespace %q in cluster, waiting for daemonset being updated, trying again in %s (err=%v)", daemonset.Name, namespace, framework.Poll, err)) return false, nil } By(fmt.Sprintf("DaemonSet %q in namespace %q in cluster, waiting for being updated, trying again in %s (err=%v)", daemonset.Name, namespace, framework.Poll, err)) return false, nil }) framework.ExpectNoError(err, "Failed to verify daemonset %q in namespace %q in cluster", daemonset.Name, namespace) }
func waitForDaemonSetOrFail(clientset *kubeclientset.Clientset, namespace string, daemonset *v1beta1.DaemonSet, present bool, timeout time.Duration) { By(fmt.Sprintf("Fetching a federated daemonset shard of daemonset %q in namespace %q from cluster", daemonset.Name, namespace)) var clusterDaemonSet *v1beta1.DaemonSet err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) { clusterDaemonSet, err := clientset.Extensions().DaemonSets(namespace).Get(daemonset.Name) if (!present) && errors.IsNotFound(err) { // We want it gone, and it's gone. By(fmt.Sprintf("Success: shard of federated daemonset %q in namespace %q in cluster is absent", daemonset.Name, namespace)) return true, nil // Success } if present && err == nil { // We want it present, and the Get succeeded, so we're all good. By(fmt.Sprintf("Success: shard of federated daemonset %q in namespace %q in cluster is present", daemonset.Name, namespace)) return true, nil // Success } By(fmt.Sprintf("DaemonSet %q in namespace %q in cluster. Found: %v, waiting for Found: %v, trying again in %s (err=%v)", daemonset.Name, namespace, clusterDaemonSet != nil && err == nil, present, framework.Poll, err)) return false, nil }) framework.ExpectNoError(err, "Failed to verify daemonset %q in namespace %q in cluster: Present=%v", daemonset.Name, namespace, present) if present && clusterDaemonSet != nil { Expect(util.ObjectMetaAndSpecEquivalent(clusterDaemonSet, daemonset)) } }
func (nc *NamespaceController) reconcileNamespace(namespace string) { if !nc.isSynced() { nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false) return } baseNamespaceObj, exist, err := nc.namespaceInformerStore.GetByKey(namespace) if err != nil { glog.Errorf("Failed to query main namespace store for %v: %v", namespace, err) nc.deliverNamespace(namespace, 0, true) return } if !exist { // Not federated namespace, ignoring. return } baseNamespace := baseNamespaceObj.(*api_v1.Namespace) if baseNamespace.DeletionTimestamp != nil { if err := nc.delete(baseNamespace); err != nil { glog.Errorf("Failed to delete %s: %v", namespace, err) nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "DeleteFailed", "Namespace delete failed: %v", err) nc.deliverNamespace(namespace, 0, true) } return } glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for namespace: %s", baseNamespace.Name) // Add the DeleteFromUnderlyingClusters finalizer before creating a namespace in // underlying clusters. // This ensures that the dependent namespaces are deleted in underlying // clusters when the federated namespace is deleted. updatedNamespaceObj, err := nc.deletionHelper.EnsureDeleteFromUnderlyingClustersFinalizer(baseNamespace) if err != nil { glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in namespace %s: %v", baseNamespace.Name, err) nc.deliverNamespace(namespace, 0, false) return } baseNamespace = updatedNamespaceObj.(*api_v1.Namespace) glog.V(3).Infof("Syncing namespace %s in underlying clusters", baseNamespace.Name) // Sync the namespace in all underlying clusters. clusters, err := nc.namespaceFederatedInformer.GetReadyClusters() if err != nil { glog.Errorf("Failed to get cluster list: %v", err) nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false) return } operations := make([]util.FederatedOperation, 0) for _, cluster := range clusters { clusterNamespaceObj, found, err := nc.namespaceFederatedInformer.GetTargetStore().GetByKey(cluster.Name, namespace) if err != nil { glog.Errorf("Failed to get %s from %s: %v", namespace, cluster.Name, err) nc.deliverNamespace(namespace, 0, true) return } desiredNamespace := &api_v1.Namespace{ ObjectMeta: util.CopyObjectMeta(baseNamespace.ObjectMeta), Spec: baseNamespace.Spec, } glog.V(5).Infof("Desired namespace in underlying clusters: %+v", desiredNamespace) if !found { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "CreateInCluster", "Creating namespace in cluster %s", cluster.Name) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeAdd, Obj: desiredNamespace, ClusterName: cluster.Name, }) } else { clusterNamespace := clusterNamespaceObj.(*api_v1.Namespace) // Update existing namespace, if needed. if !util.ObjectMetaAndSpecEquivalent(desiredNamespace, clusterNamespace) { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInCluster", "Updating namespace in cluster %s. Desired: %+v\n Actual: %+v\n", cluster.Name, desiredNamespace, clusterNamespace) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeUpdate, Obj: desiredNamespace, ClusterName: cluster.Name, }) } } } if len(operations) == 0 { // Everything is in order return } glog.V(2).Infof("Updating namespace %s in underlying clusters. Operations: %d", baseNamespace.Name, len(operations)) err = nc.federatedUpdater.UpdateWithOnError(operations, nc.updateTimeout, func(op util.FederatedOperation, operror error) { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInClusterFailed", "Namespace update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v", namespace, err) nc.deliverNamespace(namespace, 0, true) return } // Evertyhing is in order but lets be double sure nc.deliverNamespace(namespace, nc.namespaceReviewDelay, false) }
func (ic *IngressController) reconcileIngress(ingress types.NamespacedName) { glog.V(4).Infof("Reconciling ingress %q for all clusters", ingress) if !ic.isSynced() { ic.deliverIngress(ingress, ic.clusterAvailableDelay, false) return } key := ingress.String() baseIngressObjFromStore, exist, err := ic.ingressInformerStore.GetByKey(key) if err != nil { glog.Errorf("Failed to query main ingress store for %v: %v", ingress, err) ic.deliverIngress(ingress, 0, true) return } if !exist { // Not federated ingress, ignoring. glog.V(4).Infof("Ingress %q is not federated. Ignoring.", ingress) return } baseIngressObj, err := conversion.NewCloner().DeepCopy(baseIngressObjFromStore) baseIngress, ok := baseIngressObj.(*extensionsv1beta1.Ingress) if err != nil || !ok { glog.Errorf("Internal Error %v : Object retrieved from ingressInformerStore with key %q is not of correct type *extensionsv1beta1.Ingress: %v", err, key, baseIngressObj) } else { glog.V(4).Infof("Base (federated) ingress: %v", baseIngress) } if baseIngress.DeletionTimestamp != nil { if err := ic.delete(baseIngress); err != nil { glog.Errorf("Failed to delete %s: %v", ingress, err) ic.eventRecorder.Eventf(baseIngress, api.EventTypeNormal, "DeleteFailed", "Ingress delete failed: %v", err) ic.deliverIngress(ingress, 0, true) } return } glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for ingress: %s", baseIngress.Name) // Add the required finalizers before creating a ingress in underlying clusters. updatedIngressObj, err := ic.deletionHelper.EnsureFinalizers(baseIngress) if err != nil { glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in ingress %s: %v", baseIngress.Name, err) ic.deliverIngress(ingress, 0, true) return } baseIngress = updatedIngressObj.(*extensionsv1beta1.Ingress) glog.V(3).Infof("Syncing ingress %s in underlying clusters", baseIngress.Name) clusters, err := ic.ingressFederatedInformer.GetReadyClusters() if err != nil { glog.Errorf("Failed to get cluster list: %v", err) ic.deliverIngress(ingress, ic.clusterAvailableDelay, false) return } else { glog.V(4).Infof("Found %d ready clusters across which to reconcile ingress %q", len(clusters), ingress) } operations := make([]util.FederatedOperation, 0) for _, cluster := range clusters { baseIPName, baseIPAnnotationExists := baseIngress.ObjectMeta.Annotations[staticIPNameKeyWritable] firstClusterName, firstClusterExists := baseIngress.ObjectMeta.Annotations[firstClusterAnnotation] clusterIngressObj, clusterIngressFound, err := ic.ingressFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key) if err != nil { glog.Errorf("Failed to get cached ingress %s for cluster %s, will retry: %v", ingress, cluster.Name, err) ic.deliverIngress(ingress, 0, true) return } desiredIngress := &extensionsv1beta1.Ingress{} objMeta, err := conversion.NewCloner().DeepCopy(baseIngress.ObjectMeta) if err != nil { glog.Errorf("Error deep copying ObjectMeta: %v", err) } objSpec, err := conversion.NewCloner().DeepCopy(baseIngress.Spec) if err != nil { glog.Errorf("Error deep copying Spec: %v", err) } desiredIngress.ObjectMeta, ok = objMeta.(v1.ObjectMeta) if !ok { glog.Errorf("Internal error: Failed to cast to v1.ObjectMeta: %v", objMeta) } desiredIngress.Spec = objSpec.(extensionsv1beta1.IngressSpec) if !ok { glog.Errorf("Internal error: Failed to cast to extensionsv1beta1.Ingressespec: %v", objSpec) } glog.V(4).Infof("Desired Ingress: %v", desiredIngress) if !clusterIngressFound { glog.V(4).Infof("No existing Ingress %s in cluster %s - checking if appropriate to queue a create operation", ingress, cluster.Name) // We can't supply server-created fields when creating a new object. desiredIngress.ObjectMeta = util.DeepCopyRelevantObjectMeta(baseIngress.ObjectMeta) ic.eventRecorder.Eventf(baseIngress, api.EventTypeNormal, "CreateInCluster", "Creating ingress in cluster %s", cluster.Name) // We always first create an ingress in the first available cluster. Once that ingress // has been created and allocated a global IP (visible via an annotation), // we record that annotation on the federated ingress, and create all other cluster // ingresses with that same global IP. // Note: If the first cluster becomes (e.g. temporarily) unavailable, the // second cluster will become the first cluster, but eventually all ingresses // will share the single global IP recorded in the annotation of the // federated ingress. haveFirstCluster := firstClusterExists && firstClusterName != "" && ic.isClusterReady(firstClusterName) if !haveFirstCluster { glog.V(4).Infof("No cluster has been chosen as the first cluster. Electing cluster %s as the first cluster to create ingress in", cluster.Name) ic.updateAnnotationOnIngress(baseIngress, firstClusterAnnotation, cluster.Name) return } if baseIPAnnotationExists || firstClusterName == cluster.Name { if baseIPAnnotationExists { glog.V(4).Infof("No existing Ingress %s in cluster %s and static IP annotation (%q) exists on base ingress - queuing a create operation", ingress, cluster.Name, staticIPNameKeyWritable) } else { glog.V(4).Infof("No existing Ingress %s in cluster %s and no static IP annotation (%q) on base ingress - queuing a create operation in first cluster", ingress, cluster.Name, staticIPNameKeyWritable) } operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeAdd, Obj: desiredIngress, ClusterName: cluster.Name, }) } else { glog.V(4).Infof("No annotation %q exists on ingress %q in federation and waiting for ingress in cluster %s. Not queueing create operation for ingress until annotation exists", staticIPNameKeyWritable, ingress, firstClusterName) } } else { clusterIngress := clusterIngressObj.(*extensionsv1beta1.Ingress) glog.V(4).Infof("Found existing Ingress %s in cluster %s - checking if update is required (in either direction)", ingress, cluster.Name) clusterIPName, clusterIPNameExists := clusterIngress.ObjectMeta.Annotations[staticIPNameKeyReadonly] baseLBStatusExists := len(baseIngress.Status.LoadBalancer.Ingress) > 0 clusterLBStatusExists := len(clusterIngress.Status.LoadBalancer.Ingress) > 0 logStr := fmt.Sprintf("Cluster ingress %q has annotation %q=%q, loadbalancer status exists? [%v], federated ingress has annotation %q=%q, loadbalancer status exists? [%v]. %%s annotation and/or loadbalancer status from cluster ingress to federated ingress.", ingress, staticIPNameKeyReadonly, clusterIPName, clusterLBStatusExists, staticIPNameKeyWritable, baseIPName, baseLBStatusExists) if (!baseIPAnnotationExists && clusterIPNameExists) || (!baseLBStatusExists && clusterLBStatusExists) { // copy the IP name from the readonly annotation on the cluster ingress, to the writable annotation on the federated ingress glog.V(4).Infof(logStr, "Transferring") if !baseIPAnnotationExists && clusterIPNameExists { ic.updateAnnotationOnIngress(baseIngress, staticIPNameKeyWritable, clusterIPName) return } if !baseLBStatusExists && clusterLBStatusExists { lbstatusObj, lbErr := conversion.NewCloner().DeepCopy(&clusterIngress.Status.LoadBalancer) lbstatus, ok := lbstatusObj.(*v1.LoadBalancerStatus) if lbErr != nil || !ok { glog.Errorf("Internal error: Failed to clone LoadBalancerStatus of %q in cluster %q while attempting to update master loadbalancer ingress status, will try again later. error: %v, Object to be cloned: %v", ingress, cluster.Name, lbErr, lbstatusObj) ic.deliverIngress(ingress, ic.ingressReviewDelay, true) return } baseIngress.Status.LoadBalancer = *lbstatus glog.V(4).Infof("Attempting to update base federated ingress status: %v", baseIngress) if updatedFedIngress, err := ic.federatedApiClient.Extensions().Ingresses(baseIngress.Namespace).UpdateStatus(baseIngress); err != nil { glog.Errorf("Failed to update federated ingress status of %q (loadbalancer status), will try again later: %v", ingress, err) ic.deliverIngress(ingress, ic.ingressReviewDelay, true) return } else { glog.V(4).Infof("Successfully updated federated ingress status of %q (added loadbalancer status), after update: %q", ingress, updatedFedIngress) ic.deliverIngress(ingress, ic.smallDelay, false) return } } } else { glog.V(4).Infof(logStr, "Not transferring") } // Update existing cluster ingress, if needed. if util.ObjectMetaAndSpecEquivalent(baseIngress, clusterIngress) { glog.V(4).Infof("Ingress %q in cluster %q does not need an update: cluster ingress is equivalent to federated ingress", ingress, cluster.Name) } else { glog.V(4).Infof("Ingress %s in cluster %s needs an update: cluster ingress %v is not equivalent to federated ingress %v", ingress, cluster.Name, clusterIngress, desiredIngress) objMeta, err := conversion.NewCloner().DeepCopy(clusterIngress.ObjectMeta) if err != nil { glog.Errorf("Error deep copying ObjectMeta: %v", err) ic.deliverIngress(ingress, ic.ingressReviewDelay, true) } desiredIngress.ObjectMeta, ok = objMeta.(v1.ObjectMeta) if !ok { glog.Errorf("Internal error: Failed to cast to v1.ObjectMeta: %v", objMeta) ic.deliverIngress(ingress, ic.ingressReviewDelay, true) } // Merge any annotations and labels on the federated ingress onto the underlying cluster ingress, // overwriting duplicates. if desiredIngress.ObjectMeta.Annotations == nil { desiredIngress.ObjectMeta.Annotations = make(map[string]string) } for key, val := range baseIngress.ObjectMeta.Annotations { desiredIngress.ObjectMeta.Annotations[key] = val } if desiredIngress.ObjectMeta.Labels == nil { desiredIngress.ObjectMeta.Labels = make(map[string]string) } for key, val := range baseIngress.ObjectMeta.Labels { desiredIngress.ObjectMeta.Labels[key] = val } ic.eventRecorder.Eventf(baseIngress, api.EventTypeNormal, "UpdateInCluster", "Updating ingress in cluster %s", cluster.Name) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeUpdate, Obj: desiredIngress, ClusterName: cluster.Name, }) // TODO: Transfer any readonly (target-proxy, url-map etc) annotations from the master cluster to the federation, if this is the master cluster. // This is only for consistency, so that the federation ingress metadata matches the underlying clusters. It's not actually required } } } } if len(operations) == 0 { // Everything is in order glog.V(4).Infof("Ingress %q is up-to-date in all clusters - no propagation to clusters required.", ingress) return } glog.V(4).Infof("Calling federatedUpdater.Update() - operations: %v", operations) err = ic.federatedIngressUpdater.UpdateWithOnError(operations, ic.updateTimeout, func(op util.FederatedOperation, operror error) { ic.eventRecorder.Eventf(baseIngress, api.EventTypeNormal, "FailedClusterUpdate", "Ingress update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v", ingress, err) ic.deliverIngress(ingress, ic.ingressReviewDelay, true) return } // Schedule another periodic reconciliation, only to account for possible bugs in watch processing. ic.deliverIngress(ingress, ic.ingressReviewDelay, false) }
func (frsc *ReplicaSetController) reconcileReplicaSet(key string) (reconciliationStatus, error) { if !frsc.isSynced() { return statusNotSynced, nil } glog.V(4).Infof("Start reconcile replicaset %q", key) startTime := time.Now() defer glog.V(4).Infof("Finished reconcile replicaset %q (%v)", key, time.Now().Sub(startTime)) objFromStore, exists, err := frsc.replicaSetStore.Indexer.GetByKey(key) if err != nil { return statusError, err } if !exists { // don't delete local replicasets for now. Do not reconcile it anymore. return statusAllOk, nil } obj, err := conversion.NewCloner().DeepCopy(objFromStore) frs, ok := obj.(*extensionsv1.ReplicaSet) if err != nil || !ok { glog.Errorf("Error in retrieving obj from store: %v, %v", ok, err) frsc.deliverReplicaSetByKey(key, 0, true) return statusError, err } if frs.DeletionTimestamp != nil { if err := frsc.delete(frs); err != nil { glog.Errorf("Failed to delete %s: %v", frs, err) frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "DeleteFailed", "ReplicaSet delete failed: %v", err) frsc.deliverReplicaSetByKey(key, 0, true) return statusError, err } return statusAllOk, nil } glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for replicaset: %s", frs.Name) // Add the required finalizers before creating a replicaset in underlying clusters. updatedRsObj, err := frsc.deletionHelper.EnsureFinalizers(frs) if err != nil { glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in replicaset %s: %v", frs.Name, err) frsc.deliverReplicaSetByKey(key, 0, false) return statusError, err } frs = updatedRsObj.(*extensionsv1.ReplicaSet) glog.V(3).Infof("Syncing replicaset %s in underlying clusters", frs.Name) clusters, err := frsc.fedReplicaSetInformer.GetReadyClusters() if err != nil { return statusError, err } // collect current status and do schedule allPods, err := frsc.fedPodInformer.GetTargetStore().List() if err != nil { return statusError, err } podStatus, err := podanalyzer.AnalysePods(frs.Spec.Selector, allPods, time.Now()) current := make(map[string]int64) estimatedCapacity := make(map[string]int64) for _, cluster := range clusters { lrsObj, exists, err := frsc.fedReplicaSetInformer.GetTargetStore().GetByKey(cluster.Name, key) if err != nil { return statusError, err } if exists { lrs := lrsObj.(*extensionsv1.ReplicaSet) current[cluster.Name] = int64(podStatus[cluster.Name].RunningAndReady) // include pending as well? unschedulable := int64(podStatus[cluster.Name].Unschedulable) if unschedulable > 0 { estimatedCapacity[cluster.Name] = int64(*lrs.Spec.Replicas) - unschedulable } } } scheduleResult := frsc.schedule(frs, clusters, current, estimatedCapacity) glog.V(4).Infof("Start syncing local replicaset %s: %v", key, scheduleResult) fedStatus := extensionsv1.ReplicaSetStatus{ObservedGeneration: frs.Generation} operations := make([]fedutil.FederatedOperation, 0) for clusterName, replicas := range scheduleResult { lrsObj, exists, err := frsc.fedReplicaSetInformer.GetTargetStore().GetByKey(clusterName, key) if err != nil { return statusError, err } lrs := &extensionsv1.ReplicaSet{ ObjectMeta: fedutil.CopyObjectMeta(frs.ObjectMeta), Spec: frs.Spec, } specReplicas := int32(replicas) lrs.Spec.Replicas = &specReplicas if !exists { if replicas > 0 { frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "CreateInCluster", "Creating replicaset in cluster %s", clusterName) operations = append(operations, fedutil.FederatedOperation{ Type: fedutil.OperationTypeAdd, Obj: lrs, ClusterName: clusterName, }) } } else { currentLrs := lrsObj.(*extensionsv1.ReplicaSet) // Update existing replica set, if needed. if !fedutil.ObjectMetaAndSpecEquivalent(lrs, currentLrs) { frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "UpdateInCluster", "Updating replicaset in cluster %s", clusterName) operations = append(operations, fedutil.FederatedOperation{ Type: fedutil.OperationTypeUpdate, Obj: lrs, ClusterName: clusterName, }) } fedStatus.Replicas += currentLrs.Status.Replicas fedStatus.FullyLabeledReplicas += currentLrs.Status.FullyLabeledReplicas // leave the replicaset even the replicas dropped to 0 } } if fedStatus.Replicas != frs.Status.Replicas || fedStatus.FullyLabeledReplicas != frs.Status.FullyLabeledReplicas { frs.Status = fedStatus _, err = frsc.fedClient.Extensions().ReplicaSets(frs.Namespace).UpdateStatus(frs) if err != nil { return statusError, err } } if len(operations) == 0 { // Everything is in order return statusAllOk, nil } err = frsc.fedUpdater.UpdateWithOnError(operations, updateTimeout, func(op fedutil.FederatedOperation, operror error) { frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "FailedUpdateInCluster", "Replicaset update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v", key, err) return statusError, err } // Some operations were made, reconcile after a while. return statusNeedRecheck, nil }
func (fdc *DeploymentController) reconcileDeployment(key string) (reconciliationStatus, error) { if !fdc.isSynced() { return statusNotSynced, nil } glog.V(4).Infof("Start reconcile deployment %q", key) startTime := time.Now() defer glog.V(4).Infof("Finished reconcile deployment %q (%v)", key, time.Now().Sub(startTime)) objFromStore, exists, err := fdc.deploymentStore.GetByKey(key) if err != nil { return statusError, err } if !exists { // don't delete local deployments for now. Do not reconcile it anymore. return statusAllOk, nil } obj, err := conversion.NewCloner().DeepCopy(objFromStore) fd, ok := obj.(*extensionsv1.Deployment) if err != nil || !ok { glog.Errorf("Error in retrieving obj from store: %v, %v", ok, err) return statusError, err } if fd.DeletionTimestamp != nil { if err := fdc.delete(fd); err != nil { glog.Errorf("Failed to delete %s: %v", fd.Name, err) fdc.eventRecorder.Eventf(fd, api.EventTypeNormal, "DeleteFailed", "Deployment delete failed: %v", err) return statusError, err } return statusAllOk, nil } glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for deployment: %s", fd.Name) // Add the required finalizers before creating a deployment in underlying clusters. updatedDeploymentObj, err := fdc.deletionHelper.EnsureFinalizers(fd) if err != nil { glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in deployment %s: %v", fd.Name, err) return statusError, err } fd = updatedDeploymentObj.(*extensionsv1.Deployment) glog.V(3).Infof("Syncing deployment %s in underlying clusters", fd.Name) clusters, err := fdc.fedDeploymentInformer.GetReadyClusters() if err != nil { return statusError, err } // collect current status and do schedule allPods, err := fdc.fedPodInformer.GetTargetStore().List() if err != nil { return statusError, err } podStatus, err := podanalyzer.AnalysePods(fd.Spec.Selector, allPods, time.Now()) current := make(map[string]int64) estimatedCapacity := make(map[string]int64) for _, cluster := range clusters { ldObj, exists, err := fdc.fedDeploymentInformer.GetTargetStore().GetByKey(cluster.Name, key) if err != nil { return statusError, err } if exists { ld := ldObj.(*extensionsv1.Deployment) current[cluster.Name] = int64(podStatus[cluster.Name].RunningAndReady) // include pending as well? unschedulable := int64(podStatus[cluster.Name].Unschedulable) if unschedulable > 0 { estimatedCapacity[cluster.Name] = int64(*ld.Spec.Replicas) - unschedulable } } } scheduleResult := fdc.schedule(fd, clusters, current, estimatedCapacity) glog.V(4).Infof("Start syncing local deployment %s: %v", key, scheduleResult) fedStatus := extensionsv1.DeploymentStatus{ObservedGeneration: fd.Generation} operations := make([]fedutil.FederatedOperation, 0) for clusterName, replicas := range scheduleResult { ldObj, exists, err := fdc.fedDeploymentInformer.GetTargetStore().GetByKey(clusterName, key) if err != nil { return statusError, err } // The object can be modified. ld := &extensionsv1.Deployment{ ObjectMeta: fedutil.DeepCopyRelevantObjectMeta(fd.ObjectMeta), Spec: fedutil.DeepCopyApiTypeOrPanic(fd.Spec).(extensionsv1.DeploymentSpec), } specReplicas := int32(replicas) ld.Spec.Replicas = &specReplicas if !exists { if replicas > 0 { fdc.eventRecorder.Eventf(fd, api.EventTypeNormal, "CreateInCluster", "Creating deployment in cluster %s", clusterName) operations = append(operations, fedutil.FederatedOperation{ Type: fedutil.OperationTypeAdd, Obj: ld, ClusterName: clusterName, }) } } else { // TODO: Update only one deployment at a time if update strategy is rolling udpate. currentLd := ldObj.(*extensionsv1.Deployment) // Update existing replica set, if needed. if !fedutil.ObjectMetaAndSpecEquivalent(ld, currentLd) { fdc.eventRecorder.Eventf(fd, api.EventTypeNormal, "UpdateInCluster", "Updating deployment in cluster %s", clusterName) operations = append(operations, fedutil.FederatedOperation{ Type: fedutil.OperationTypeUpdate, Obj: ld, ClusterName: clusterName, }) glog.Infof("Updating %s in %s", currentLd.Name, clusterName) } fedStatus.Replicas += currentLd.Status.Replicas fedStatus.AvailableReplicas += currentLd.Status.AvailableReplicas fedStatus.UnavailableReplicas += currentLd.Status.UnavailableReplicas } } if fedStatus.Replicas != fd.Status.Replicas || fedStatus.AvailableReplicas != fd.Status.AvailableReplicas || fedStatus.UnavailableReplicas != fd.Status.UnavailableReplicas { fd.Status = fedStatus _, err = fdc.fedClient.Extensions().Deployments(fd.Namespace).UpdateStatus(fd) if err != nil { return statusError, err } } if len(operations) == 0 { // Everything is in order return statusAllOk, nil } err = fdc.fedUpdater.UpdateWithOnError(operations, updateTimeout, func(op fedutil.FederatedOperation, operror error) { fdc.eventRecorder.Eventf(fd, api.EventTypeNormal, "FailedUpdateInCluster", "Deployment update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v", key, err) return statusError, err } // Some operations were made, reconcile after a while. return statusNeedRecheck, nil }
func (nc *NamespaceController) reconcileNamespace(namespace string) { if !nc.isSynced() { nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false) return } baseNamespaceObj, exist, err := nc.namespaceInformerStore.GetByKey(namespace) if err != nil { glog.Errorf("Failed to query main namespace store for %v: %v", namespace, err) nc.deliverNamespace(namespace, 0, true) return } if !exist { // Not federated namespace, ignoring. return } baseNamespace := baseNamespaceObj.(*api_v1.Namespace) if baseNamespace.DeletionTimestamp != nil { if err := nc.delete(baseNamespace); err != nil { glog.Errorf("Failed to delete %s: %v", namespace, err) nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "DeleteFailed", "Namespace delete failed: %v", err) nc.deliverNamespace(namespace, 0, true) } return } clusters, err := nc.namespaceFederatedInformer.GetReadyClusters() if err != nil { glog.Errorf("Failed to get cluster list: %v", err) nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false) return } operations := make([]util.FederatedOperation, 0) for _, cluster := range clusters { clusterNamespaceObj, found, err := nc.namespaceFederatedInformer.GetTargetStore().GetByKey(cluster.Name, namespace) if err != nil { glog.Errorf("Failed to get %s from %s: %v", namespace, cluster.Name, err) nc.deliverNamespace(namespace, 0, true) return } desiredNamespace := &api_v1.Namespace{ ObjectMeta: util.CopyObjectMeta(baseNamespace.ObjectMeta), Spec: baseNamespace.Spec, } if !found { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "CreateInCluster", "Creating namespace in cluster %s", cluster.Name) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeAdd, Obj: desiredNamespace, ClusterName: cluster.Name, }) } else { clusterNamespace := clusterNamespaceObj.(*api_v1.Namespace) // Update existing namespace, if needed. if !util.ObjectMetaAndSpecEquivalent(desiredNamespace, clusterNamespace) { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInCluster", "Updating namespace in cluster %s", cluster.Name) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeUpdate, Obj: desiredNamespace, ClusterName: cluster.Name, }) } } } if len(operations) == 0 { // Everything is in order return } err = nc.federatedUpdater.UpdateWithOnError(operations, nc.updateTimeout, func(op util.FederatedOperation, operror error) { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInClusterFailed", "Namespace update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v", namespace, err) nc.deliverNamespace(namespace, 0, true) return } // Evertyhing is in order but lets be double sure nc.deliverNamespace(namespace, nc.namespaceReviewDelay, false) }