// updateFederationService Returns whatever error occurred along with a boolean indicator of whether it // should be retried. func (s *ServiceController) updateFederationService(key string, cachedService *cachedService) (error, bool) { // Clone federation service, and create them in underlying k8s cluster desiredService := &v1.Service{ ObjectMeta: util.DeepCopyRelevantObjectMeta(cachedService.lastState.ObjectMeta), Spec: *(util.DeepCopyApiTypeOrPanic(&cachedService.lastState.Spec).(*v1.ServiceSpec)), } // handle available clusters one by one var hasErr bool for clusterName, cache := range s.clusterCache.clientMap { go func(cache *clusterCache, clusterName string) { err := s.processServiceForCluster(cachedService, clusterName, desiredService, cache.clientset) if err != nil { hasErr = true } }(cache, clusterName) } if hasErr { // detail error has been dumped inside the loop return fmt.Errorf("Service %s/%s was not successfully updated to all clusters", desiredService.Namespace, desiredService.Name), retryable } return nil, !retryable }
func (nc *NamespaceController) reconcileNamespace(namespace string) { if !nc.isSynced() { nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false) return } namespaceObjFromStore, exist, err := nc.namespaceInformerStore.GetByKey(namespace) if err != nil { glog.Errorf("Failed to query main namespace store for %v: %v", namespace, err) nc.deliverNamespace(namespace, 0, true) return } if !exist { // Not federated namespace, ignoring. return } // Create a copy before modifying the namespace to prevent race condition with // other readers of namespace from store. namespaceObj, err := api.Scheme.DeepCopy(namespaceObjFromStore) baseNamespace, ok := namespaceObj.(*apiv1.Namespace) if err != nil || !ok { glog.Errorf("Error in retrieving obj from store: %v, %v", ok, err) nc.deliverNamespace(namespace, 0, true) return } if baseNamespace.DeletionTimestamp != nil { if err := nc.delete(baseNamespace); err != nil { glog.Errorf("Failed to delete %s: %v", namespace, err) nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "DeleteFailed", "Namespace delete failed: %v", err) nc.deliverNamespace(namespace, 0, true) } return } glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for namespace: %s", baseNamespace.Name) // Add the required finalizers before creating a namespace in // underlying clusters. // This ensures that the dependent namespaces are deleted in underlying // clusters when the federated namespace is deleted. updatedNamespaceObj, err := nc.deletionHelper.EnsureFinalizers(baseNamespace) if err != nil { glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in namespace %s: %v", baseNamespace.Name, err) nc.deliverNamespace(namespace, 0, false) return } baseNamespace = updatedNamespaceObj.(*apiv1.Namespace) glog.V(3).Infof("Syncing namespace %s in underlying clusters", baseNamespace.Name) // Sync the namespace in all underlying clusters. clusters, err := nc.namespaceFederatedInformer.GetReadyClusters() if err != nil { glog.Errorf("Failed to get cluster list: %v", err) nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false) return } operations := make([]util.FederatedOperation, 0) for _, cluster := range clusters { clusterNamespaceObj, found, err := nc.namespaceFederatedInformer.GetTargetStore().GetByKey(cluster.Name, namespace) if err != nil { glog.Errorf("Failed to get %s from %s: %v", namespace, cluster.Name, err) nc.deliverNamespace(namespace, 0, true) return } // The object should not be modified. desiredNamespace := &apiv1.Namespace{ ObjectMeta: util.DeepCopyRelevantObjectMeta(baseNamespace.ObjectMeta), Spec: *(util.DeepCopyApiTypeOrPanic(&baseNamespace.Spec).(*apiv1.NamespaceSpec)), } glog.V(5).Infof("Desired namespace in underlying clusters: %+v", desiredNamespace) if !found { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "CreateInCluster", "Creating namespace in cluster %s", cluster.Name) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeAdd, Obj: desiredNamespace, ClusterName: cluster.Name, }) } else { clusterNamespace := clusterNamespaceObj.(*apiv1.Namespace) // Update existing namespace, if needed. if !util.ObjectMetaAndSpecEquivalent(desiredNamespace, clusterNamespace) { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInCluster", "Updating namespace in cluster %s. Desired: %+v\n Actual: %+v\n", cluster.Name, desiredNamespace, clusterNamespace) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeUpdate, Obj: desiredNamespace, ClusterName: cluster.Name, }) } } } if len(operations) == 0 { // Everything is in order return } glog.V(2).Infof("Updating namespace %s in underlying clusters. Operations: %d", baseNamespace.Name, len(operations)) err = nc.federatedUpdater.UpdateWithOnError(operations, nc.updateTimeout, func(op util.FederatedOperation, operror error) { nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInClusterFailed", "Namespace update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v", namespace, err) nc.deliverNamespace(namespace, 0, true) return } // Everything is in order but lets be double sure nc.deliverNamespace(namespace, nc.namespaceReviewDelay, false) }
func (daemonsetcontroller *DaemonSetController) reconcileDaemonSet(namespace string, daemonsetName string) { glog.V(4).Infof("Reconciling daemonset %s/%s", namespace, daemonsetName) if !daemonsetcontroller.isSynced() { glog.V(4).Infof("Daemonset controller is not synced") daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, daemonsetcontroller.clusterAvailableDelay, false) return } key := getDaemonSetKey(namespace, daemonsetName) baseDaemonSetObjFromStore, exist, err := daemonsetcontroller.daemonsetInformerStore.GetByKey(key) if err != nil { glog.Errorf("Failed to query main daemonset store for %v: %v", key, err) daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true) return } if !exist { glog.V(4).Infof("Skipping daemonset %s/%s - not federated", namespace, daemonsetName) // Not federated daemonset, ignoring. return } baseDaemonSetObj, err := conversion.NewCloner().DeepCopy(baseDaemonSetObjFromStore) baseDaemonSet, ok := baseDaemonSetObj.(*extensionsv1.DaemonSet) if err != nil || !ok { glog.Errorf("Error in retrieving obj %s from store: %v, %v", daemonsetName, ok, err) daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true) return } if baseDaemonSet.DeletionTimestamp != nil { if err := daemonsetcontroller.delete(baseDaemonSet); err != nil { glog.Errorf("Failed to delete %s: %v", daemonsetName, err) daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "DeleteFailed", "DaemonSet delete failed: %v", err) daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true) } return } glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for daemonset: %s", baseDaemonSet.Name) // Add the required finalizers before creating a daemonset in underlying clusters. updatedDaemonSetObj, err := daemonsetcontroller.deletionHelper.EnsureFinalizers(baseDaemonSet) if err != nil { glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in daemonset %s: %v", baseDaemonSet.Name, err) daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, false) return } baseDaemonSet = updatedDaemonSetObj.(*extensionsv1.DaemonSet) glog.V(3).Infof("Syncing daemonset %s in underlying clusters", baseDaemonSet.Name) clusters, err := daemonsetcontroller.daemonsetFederatedInformer.GetReadyClusters() if err != nil { glog.Errorf("Failed to get cluster list: %v", err) daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, daemonsetcontroller.clusterAvailableDelay, false) return } operations := make([]util.FederatedOperation, 0) for _, cluster := range clusters { clusterDaemonSetObj, found, err := daemonsetcontroller.daemonsetFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key) if err != nil { glog.Errorf("Failed to get %s from %s: %v", key, cluster.Name, err) daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true) return } // Do not modify. Otherwise make a deep copy. desiredDaemonSet := &extensionsv1.DaemonSet{ ObjectMeta: util.DeepCopyRelevantObjectMeta(baseDaemonSet.ObjectMeta), Spec: util.DeepCopyApiTypeOrPanic(baseDaemonSet.Spec).(extensionsv1.DaemonSetSpec), } if !found { glog.V(4).Infof("Creating daemonset %s/%s in cluster %s", namespace, daemonsetName, cluster.Name) daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "CreateInCluster", "Creating daemonset in cluster %s", cluster.Name) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeAdd, Obj: desiredDaemonSet, ClusterName: cluster.Name, }) } else { clusterDaemonSet := clusterDaemonSetObj.(*extensionsv1.DaemonSet) // Update existing daemonset, if needed. if !util.ObjectMetaEquivalent(desiredDaemonSet.ObjectMeta, clusterDaemonSet.ObjectMeta) || !reflect.DeepEqual(desiredDaemonSet.Spec, clusterDaemonSet.Spec) { glog.V(4).Infof("Upadting daemonset %s/%s in cluster %s", namespace, daemonsetName, cluster.Name) daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "UpdateInCluster", "Updating daemonset in cluster %s", cluster.Name) operations = append(operations, util.FederatedOperation{ Type: util.OperationTypeUpdate, Obj: desiredDaemonSet, ClusterName: cluster.Name, }) } } } if len(operations) == 0 { glog.V(4).Infof("No operation needed for %s/%s", namespace, daemonsetName) // Everything is in order return } err = daemonsetcontroller.federatedUpdater.UpdateWithOnError(operations, daemonsetcontroller.updateTimeout, func(op util.FederatedOperation, operror error) { daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "UpdateInClusterFailed", "DaemonSet update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v, retrying shortly", key, err) daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true) return } }
func (frsc *ReplicaSetController) reconcileReplicaSet(key string) (reconciliationStatus, error) { if !frsc.isSynced() { return statusNotSynced, nil } glog.V(4).Infof("Start reconcile replicaset %q", key) startTime := time.Now() defer glog.V(4).Infof("Finished reconcile replicaset %q (%v)", key, time.Now().Sub(startTime)) objFromStore, exists, err := frsc.replicaSetStore.Indexer.GetByKey(key) if err != nil { return statusError, err } if !exists { // don't delete local replicasets for now. Do not reconcile it anymore. return statusAllOk, nil } obj, err := conversion.NewCloner().DeepCopy(objFromStore) frs, ok := obj.(*extensionsv1.ReplicaSet) if err != nil || !ok { glog.Errorf("Error in retrieving obj from store: %v, %v", ok, err) frsc.deliverReplicaSetByKey(key, 0, true) return statusError, err } if frs.DeletionTimestamp != nil { if err := frsc.delete(frs); err != nil { glog.Errorf("Failed to delete %s: %v", frs, err) frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "DeleteFailed", "ReplicaSet delete failed: %v", err) frsc.deliverReplicaSetByKey(key, 0, true) return statusError, err } return statusAllOk, nil } glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for replicaset: %s", frs.Name) // Add the required finalizers before creating a replicaset in underlying clusters. updatedRsObj, err := frsc.deletionHelper.EnsureFinalizers(frs) if err != nil { glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in replicaset %s: %v", frs.Name, err) frsc.deliverReplicaSetByKey(key, 0, false) return statusError, err } frs = updatedRsObj.(*extensionsv1.ReplicaSet) glog.V(3).Infof("Syncing replicaset %s in underlying clusters", frs.Name) clusters, err := frsc.fedReplicaSetInformer.GetReadyClusters() if err != nil { return statusError, err } // collect current status and do schedule allPods, err := frsc.fedPodInformer.GetTargetStore().List() if err != nil { return statusError, err } podStatus, err := podanalyzer.AnalysePods(frs.Spec.Selector, allPods, time.Now()) current := make(map[string]int64) estimatedCapacity := make(map[string]int64) for _, cluster := range clusters { lrsObj, exists, err := frsc.fedReplicaSetInformer.GetTargetStore().GetByKey(cluster.Name, key) if err != nil { return statusError, err } if exists { lrs := lrsObj.(*extensionsv1.ReplicaSet) current[cluster.Name] = int64(podStatus[cluster.Name].RunningAndReady) // include pending as well? unschedulable := int64(podStatus[cluster.Name].Unschedulable) if unschedulable > 0 { estimatedCapacity[cluster.Name] = int64(*lrs.Spec.Replicas) - unschedulable } } } scheduleResult := frsc.schedule(frs, clusters, current, estimatedCapacity) glog.V(4).Infof("Start syncing local replicaset %s: %v", key, scheduleResult) fedStatus := extensionsv1.ReplicaSetStatus{ObservedGeneration: frs.Generation} operations := make([]fedutil.FederatedOperation, 0) for clusterName, replicas := range scheduleResult { lrsObj, exists, err := frsc.fedReplicaSetInformer.GetTargetStore().GetByKey(clusterName, key) if err != nil { return statusError, err } // The object can be modified. lrs := &extensionsv1.ReplicaSet{ ObjectMeta: fedutil.DeepCopyRelevantObjectMeta(frs.ObjectMeta), Spec: fedutil.DeepCopyApiTypeOrPanic(frs.Spec).(extensionsv1.ReplicaSetSpec), } specReplicas := int32(replicas) lrs.Spec.Replicas = &specReplicas if !exists { if replicas > 0 { frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "CreateInCluster", "Creating replicaset in cluster %s", clusterName) operations = append(operations, fedutil.FederatedOperation{ Type: fedutil.OperationTypeAdd, Obj: lrs, ClusterName: clusterName, }) } } else { currentLrs := lrsObj.(*extensionsv1.ReplicaSet) // Update existing replica set, if needed. if !fedutil.ObjectMetaAndSpecEquivalent(lrs, currentLrs) { frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "UpdateInCluster", "Updating replicaset in cluster %s", clusterName) operations = append(operations, fedutil.FederatedOperation{ Type: fedutil.OperationTypeUpdate, Obj: lrs, ClusterName: clusterName, }) } fedStatus.Replicas += currentLrs.Status.Replicas fedStatus.FullyLabeledReplicas += currentLrs.Status.FullyLabeledReplicas fedStatus.ReadyReplicas += currentLrs.Status.ReadyReplicas fedStatus.AvailableReplicas += currentLrs.Status.AvailableReplicas } } if fedStatus.Replicas != frs.Status.Replicas || fedStatus.FullyLabeledReplicas != frs.Status.FullyLabeledReplicas || fedStatus.ReadyReplicas != frs.Status.ReadyReplicas || fedStatus.AvailableReplicas != frs.Status.AvailableReplicas { frs.Status = fedStatus _, err = frsc.fedClient.Extensions().ReplicaSets(frs.Namespace).UpdateStatus(frs) if err != nil { return statusError, err } } if len(operations) == 0 { // Everything is in order return statusAllOk, nil } err = frsc.fedUpdater.UpdateWithOnError(operations, updateTimeout, func(op fedutil.FederatedOperation, operror error) { frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "FailedUpdateInCluster", "Replicaset update in cluster %s failed: %v", op.ClusterName, operror) }) if err != nil { glog.Errorf("Failed to execute updates for %s: %v", key, err) return statusError, err } // Some operations were made, reconcile after a while. return statusNeedRecheck, nil }