func (nc *NamespaceController) reconcileNamespace(namespace string) {
if !nc.isSynced() {
nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false)
return
}
baseNamespaceObj, exist, err := nc.namespaceInformerStore.GetByKey(namespace)
if err != nil {
glog.Errorf("Failed to query main namespace store for %v: %v", namespace, err)
nc.deliverNamespace(namespace, 0, true)
return
}
if !exist {
// Not federated namespace, ignoring.
return
}
baseNamespace := baseNamespaceObj.(*api_v1.Namespace)
if baseNamespace.DeletionTimestamp != nil {
if err := nc.delete(baseNamespace); err != nil {
glog.Errorf("Failed to delete %s: %v", namespace, err)
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "DeleteFailed",
"Namespace delete failed: %v", err)
nc.deliverNamespace(namespace, 0, true)
}
return
}
glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for namespace: %s",
baseNamespace.Name)
// Add the DeleteFromUnderlyingClusters finalizer before creating a namespace in
// underlying clusters.
// This ensures that the dependent namespaces are deleted in underlying
// clusters when the federated namespace is deleted.
updatedNamespaceObj, err := nc.deletionHelper.EnsureDeleteFromUnderlyingClustersFinalizer(baseNamespace)
if err != nil {
glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in namespace %s: %v",
baseNamespace.Name, err)
nc.deliverNamespace(namespace, 0, false)
return
}
baseNamespace = updatedNamespaceObj.(*api_v1.Namespace)
glog.V(3).Infof("Syncing namespace %s in underlying clusters", baseNamespace.Name)
// Sync the namespace in all underlying clusters.
clusters, err := nc.namespaceFederatedInformer.GetReadyClusters()
if err != nil {
glog.Errorf("Failed to get cluster list: %v", err)
nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false)
return
}
operations := make([]util.FederatedOperation, 0)
for _, cluster := range clusters {
clusterNamespaceObj, found, err := nc.namespaceFederatedInformer.GetTargetStore().GetByKey(cluster.Name, namespace)
if err != nil {
glog.Errorf("Failed to get %s from %s: %v", namespace, cluster.Name, err)
nc.deliverNamespace(namespace, 0, true)
return
}
desiredNamespace := &api_v1.Namespace{
ObjectMeta: util.CopyObjectMeta(baseNamespace.ObjectMeta),
Spec: baseNamespace.Spec,
}
glog.V(5).Infof("Desired namespace in underlying clusters: %+v", desiredNamespace)
if !found {
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "CreateInCluster",
"Creating namespace in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeAdd,
Obj: desiredNamespace,
ClusterName: cluster.Name,
})
} else {
clusterNamespace := clusterNamespaceObj.(*api_v1.Namespace)
// Update existing namespace, if needed.
if !util.ObjectMetaAndSpecEquivalent(desiredNamespace, clusterNamespace) {
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInCluster",
"Updating namespace in cluster %s. Desired: %+v\n Actual: %+v\n", cluster.Name, desiredNamespace, clusterNamespace)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeUpdate,
Obj: desiredNamespace,
ClusterName: cluster.Name,
})
}
}
}
if len(operations) == 0 {
// Everything is in order
return
}
glog.V(2).Infof("Updating namespace %s in underlying clusters. Operations: %d", baseNamespace.Name, len(operations))
err = nc.federatedUpdater.UpdateWithOnError(operations, nc.updateTimeout, func(op util.FederatedOperation, operror error) {
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInClusterFailed",
"Namespace update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v", namespace, err)
nc.deliverNamespace(namespace, 0, true)
return
}
// Evertyhing is in order but lets be double sure
nc.deliverNamespace(namespace, nc.namespaceReviewDelay, false)
}
func (nc *NamespaceController) reconcileNamespace(namespace string) {
if !nc.isSynced() {
nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false)
return
}
baseNamespaceObj, exist, err := nc.namespaceInformerStore.GetByKey(namespace)
if err != nil {
glog.Errorf("Failed to query main namespace store for %v: %v", namespace, err)
nc.deliverNamespace(namespace, 0, true)
return
}
if !exist {
// Not federated namespace, ignoring.
return
}
baseNamespace := baseNamespaceObj.(*api_v1.Namespace)
if baseNamespace.DeletionTimestamp != nil {
if err := nc.delete(baseNamespace); err != nil {
glog.Errorf("Failed to delete %s: %v", namespace, err)
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "DeleteFailed",
"Namespace delete failed: %v", err)
nc.deliverNamespace(namespace, 0, true)
}
return
}
clusters, err := nc.namespaceFederatedInformer.GetReadyClusters()
if err != nil {
glog.Errorf("Failed to get cluster list: %v", err)
nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false)
return
}
operations := make([]util.FederatedOperation, 0)
for _, cluster := range clusters {
clusterNamespaceObj, found, err := nc.namespaceFederatedInformer.GetTargetStore().GetByKey(cluster.Name, namespace)
if err != nil {
glog.Errorf("Failed to get %s from %s: %v", namespace, cluster.Name, err)
nc.deliverNamespace(namespace, 0, true)
return
}
desiredNamespace := &api_v1.Namespace{
ObjectMeta: util.CopyObjectMeta(baseNamespace.ObjectMeta),
Spec: baseNamespace.Spec,
}
if !found {
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "CreateInCluster",
"Creating namespace in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeAdd,
Obj: desiredNamespace,
ClusterName: cluster.Name,
})
} else {
clusterNamespace := clusterNamespaceObj.(*api_v1.Namespace)
// Update existing namespace, if needed.
if !util.ObjectMetaEquivalent(desiredNamespace.ObjectMeta, clusterNamespace.ObjectMeta) ||
!reflect.DeepEqual(desiredNamespace.Spec, clusterNamespace.Spec) {
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInCluster",
"Updating namespace in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeUpdate,
Obj: desiredNamespace,
ClusterName: cluster.Name,
})
}
}
}
if len(operations) == 0 {
// Everything is in order
return
}
err = nc.federatedUpdater.UpdateWithOnError(operations, nc.updateTimeout, func(op util.FederatedOperation, operror error) {
nc.eventRecorder.Eventf(baseNamespace, api.EventTypeNormal, "UpdateInClusterFailed",
"Namespace update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v", namespace, err)
nc.deliverNamespace(namespace, 0, true)
return
}
// Evertyhing is in order but lets be double sure
nc.deliverNamespace(namespace, nc.namespaceReviewDelay, false)
}
func (secretcontroller *SecretController) reconcileSecret(namespace string, secretName string) {
if !secretcontroller.isSynced() {
secretcontroller.deliverSecret(namespace, secretName, secretcontroller.clusterAvailableDelay, false)
return
}
key := getSecretKey(namespace, secretName)
baseSecretObj, exist, err := secretcontroller.secretInformerStore.GetByKey(key)
if err != nil {
glog.Errorf("Failed to query main secret store for %v: %v", key, err)
secretcontroller.deliverSecret(namespace, secretName, 0, true)
return
}
if !exist {
// Not federated secret, ignoring.
return
}
baseSecret := baseSecretObj.(*api_v1.Secret)
clusters, err := secretcontroller.secretFederatedInformer.GetReadyClusters()
if err != nil {
glog.Errorf("Failed to get cluster list: %v", err)
secretcontroller.deliverSecret(namespace, secretName, secretcontroller.clusterAvailableDelay, false)
return
}
operations := make([]util.FederatedOperation, 0)
for _, cluster := range clusters {
clusterSecretObj, found, err := secretcontroller.secretFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
if err != nil {
glog.Errorf("Failed to get %s from %s: %v", key, cluster.Name, err)
secretcontroller.deliverSecret(namespace, secretName, 0, true)
return
}
desiredSecret := &api_v1.Secret{
ObjectMeta: util.CopyObjectMeta(baseSecret.ObjectMeta),
Data: baseSecret.Data,
Type: baseSecret.Type,
}
if !found {
secretcontroller.eventRecorder.Eventf(baseSecret, api.EventTypeNormal, "CreateInCluster",
"Creating secret in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeAdd,
Obj: desiredSecret,
ClusterName: cluster.Name,
})
} else {
clusterSecret := clusterSecretObj.(*api_v1.Secret)
// Update existing secret, if needed.
if !util.SecretEquivalent(*desiredSecret, *clusterSecret) {
secretcontroller.eventRecorder.Eventf(baseSecret, api.EventTypeNormal, "UpdateInCluster",
"Updating secret in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeUpdate,
Obj: desiredSecret,
ClusterName: cluster.Name,
})
}
}
}
if len(operations) == 0 {
// Everything is in order
return
}
err = secretcontroller.federatedUpdater.UpdateWithOnError(operations, secretcontroller.updateTimeout,
func(op util.FederatedOperation, operror error) {
secretcontroller.eventRecorder.Eventf(baseSecret, api.EventTypeNormal, "UpdateInClusterFailed",
"Secret update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v", key, err)
secretcontroller.deliverSecret(namespace, secretName, 0, true)
return
}
// Evertyhing is in order but lets be double sure
secretcontroller.deliverSecret(namespace, secretName, secretcontroller.secretReviewDelay, false)
}
func (frsc *ReplicaSetController) reconcileReplicaSet(key string) (reconciliationStatus, error) {
if !frsc.isSynced() {
return statusNotSynced, nil
}
glog.V(4).Infof("Start reconcile replicaset %q", key)
startTime := time.Now()
defer glog.V(4).Infof("Finished reconcile replicaset %q (%v)", key, time.Now().Sub(startTime))
objFromStore, exists, err := frsc.replicaSetStore.Indexer.GetByKey(key)
if err != nil {
return statusError, err
}
if !exists {
// don't delete local replicasets for now. Do not reconcile it anymore.
return statusAllOk, nil
}
obj, err := conversion.NewCloner().DeepCopy(objFromStore)
frs, ok := obj.(*extensionsv1.ReplicaSet)
if err != nil || !ok {
glog.Errorf("Error in retrieving obj from store: %v, %v", ok, err)
frsc.deliverReplicaSetByKey(key, 0, true)
return statusError, err
}
if frs.DeletionTimestamp != nil {
if err := frsc.delete(frs); err != nil {
glog.Errorf("Failed to delete %s: %v", frs, err)
frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "DeleteFailed",
"ReplicaSet delete failed: %v", err)
frsc.deliverReplicaSetByKey(key, 0, true)
return statusError, err
}
return statusAllOk, nil
}
glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for replicaset: %s",
frs.Name)
// Add the required finalizers before creating a replicaset in underlying clusters.
updatedRsObj, err := frsc.deletionHelper.EnsureFinalizers(frs)
if err != nil {
glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in replicaset %s: %v",
frs.Name, err)
frsc.deliverReplicaSetByKey(key, 0, false)
return statusError, err
}
frs = updatedRsObj.(*extensionsv1.ReplicaSet)
glog.V(3).Infof("Syncing replicaset %s in underlying clusters", frs.Name)
clusters, err := frsc.fedReplicaSetInformer.GetReadyClusters()
if err != nil {
return statusError, err
}
// collect current status and do schedule
allPods, err := frsc.fedPodInformer.GetTargetStore().List()
if err != nil {
return statusError, err
}
podStatus, err := podanalyzer.AnalysePods(frs.Spec.Selector, allPods, time.Now())
current := make(map[string]int64)
estimatedCapacity := make(map[string]int64)
for _, cluster := range clusters {
lrsObj, exists, err := frsc.fedReplicaSetInformer.GetTargetStore().GetByKey(cluster.Name, key)
if err != nil {
return statusError, err
}
if exists {
lrs := lrsObj.(*extensionsv1.ReplicaSet)
current[cluster.Name] = int64(podStatus[cluster.Name].RunningAndReady) // include pending as well?
unschedulable := int64(podStatus[cluster.Name].Unschedulable)
if unschedulable > 0 {
estimatedCapacity[cluster.Name] = int64(*lrs.Spec.Replicas) - unschedulable
}
}
}
scheduleResult := frsc.schedule(frs, clusters, current, estimatedCapacity)
glog.V(4).Infof("Start syncing local replicaset %s: %v", key, scheduleResult)
fedStatus := extensionsv1.ReplicaSetStatus{ObservedGeneration: frs.Generation}
operations := make([]fedutil.FederatedOperation, 0)
for clusterName, replicas := range scheduleResult {
lrsObj, exists, err := frsc.fedReplicaSetInformer.GetTargetStore().GetByKey(clusterName, key)
if err != nil {
return statusError, err
}
lrs := &extensionsv1.ReplicaSet{
ObjectMeta: fedutil.CopyObjectMeta(frs.ObjectMeta),
Spec: frs.Spec,
}
specReplicas := int32(replicas)
lrs.Spec.Replicas = &specReplicas
if !exists {
if replicas > 0 {
frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "CreateInCluster",
"Creating replicaset in cluster %s", clusterName)
operations = append(operations, fedutil.FederatedOperation{
Type: fedutil.OperationTypeAdd,
Obj: lrs,
ClusterName: clusterName,
})
}
} else {
currentLrs := lrsObj.(*extensionsv1.ReplicaSet)
// Update existing replica set, if needed.
if !fedutil.ObjectMetaAndSpecEquivalent(lrs, currentLrs) {
frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "UpdateInCluster",
"Updating replicaset in cluster %s", clusterName)
operations = append(operations, fedutil.FederatedOperation{
Type: fedutil.OperationTypeUpdate,
Obj: lrs,
ClusterName: clusterName,
})
}
fedStatus.Replicas += currentLrs.Status.Replicas
fedStatus.FullyLabeledReplicas += currentLrs.Status.FullyLabeledReplicas
// leave the replicaset even the replicas dropped to 0
}
}
if fedStatus.Replicas != frs.Status.Replicas || fedStatus.FullyLabeledReplicas != frs.Status.FullyLabeledReplicas {
frs.Status = fedStatus
_, err = frsc.fedClient.Extensions().ReplicaSets(frs.Namespace).UpdateStatus(frs)
if err != nil {
return statusError, err
}
}
if len(operations) == 0 {
// Everything is in order
return statusAllOk, nil
}
err = frsc.fedUpdater.UpdateWithOnError(operations, updateTimeout, func(op fedutil.FederatedOperation, operror error) {
frsc.eventRecorder.Eventf(frs, api.EventTypeNormal, "FailedUpdateInCluster",
"Replicaset update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v", key, err)
return statusError, err
}
// Some operations were made, reconcile after a while.
return statusNeedRecheck, nil
}
func (fdc *DeploymentController) reconcileDeployment(key string) (reconciliationStatus, error) {
if !fdc.isSynced() {
return statusNotSynced, nil
}
glog.V(4).Infof("Start reconcile deployment %q", key)
startTime := time.Now()
defer glog.V(4).Infof("Finished reconcile deployment %q (%v)", key, time.Now().Sub(startTime))
obj, exists, err := fdc.deploymentStore.GetByKey(key)
if err != nil {
return statusError, err
}
if !exists {
// don't delete local deployments for now. Do not reconcile it anymore.
return statusAllOk, nil
}
fd := obj.(*extensionsv1.Deployment)
clusters, err := fdc.fedDeploymentInformer.GetReadyClusters()
if err != nil {
return statusError, err
}
// collect current status and do schedule
allPods, err := fdc.fedPodInformer.GetTargetStore().List()
if err != nil {
return statusError, err
}
podStatus, err := podanalyzer.AnalysePods(fd.Spec.Selector, allPods, time.Now())
current := make(map[string]int64)
estimatedCapacity := make(map[string]int64)
for _, cluster := range clusters {
ldObj, exists, err := fdc.fedDeploymentInformer.GetTargetStore().GetByKey(cluster.Name, key)
if err != nil {
return statusError, err
}
if exists {
ld := ldObj.(*extensionsv1.Deployment)
current[cluster.Name] = int64(podStatus[cluster.Name].RunningAndReady) // include pending as well?
unschedulable := int64(podStatus[cluster.Name].Unschedulable)
if unschedulable > 0 {
estimatedCapacity[cluster.Name] = int64(*ld.Spec.Replicas) - unschedulable
}
}
}
scheduleResult := fdc.schedule(fd, clusters, current, estimatedCapacity)
glog.V(4).Infof("Start syncing local deployment %s: %v", key, scheduleResult)
fedStatus := extensionsv1.DeploymentStatus{ObservedGeneration: fd.Generation}
operations := make([]fedutil.FederatedOperation, 0)
for clusterName, replicas := range scheduleResult {
ldObj, exists, err := fdc.fedDeploymentInformer.GetTargetStore().GetByKey(clusterName, key)
if err != nil {
return statusError, err
}
ld := &extensionsv1.Deployment{
ObjectMeta: fedutil.CopyObjectMeta(fd.ObjectMeta),
Spec: fd.Spec,
}
specReplicas := int32(replicas)
ld.Spec.Replicas = &specReplicas
if !exists {
if replicas > 0 {
fdc.eventRecorder.Eventf(fd, api.EventTypeNormal, "CreateInCluster",
"Creating deployment in cluster %s", clusterName)
operations = append(operations, fedutil.FederatedOperation{
Type: fedutil.OperationTypeAdd,
Obj: ld,
ClusterName: clusterName,
})
}
} else {
// TODO: Update only one deployment at a time if update strategy is rolling udpate.
currentLd := ldObj.(*extensionsv1.Deployment)
// Update existing replica set, if needed.
if !fedutil.ObjectMetaAndSpecEquivalent(ld, currentLd) {
fdc.eventRecorder.Eventf(fd, api.EventTypeNormal, "UpdateInCluster",
"Updating deployment in cluster %s", clusterName)
operations = append(operations, fedutil.FederatedOperation{
Type: fedutil.OperationTypeUpdate,
Obj: ld,
ClusterName: clusterName,
})
glog.Infof("Updating %s in %s", currentLd.Name, clusterName)
}
fedStatus.Replicas += currentLd.Status.Replicas
fedStatus.AvailableReplicas += currentLd.Status.AvailableReplicas
fedStatus.UnavailableReplicas += currentLd.Status.UnavailableReplicas
}
}
if fedStatus.Replicas != fd.Status.Replicas ||
fedStatus.AvailableReplicas != fd.Status.AvailableReplicas ||
fedStatus.UnavailableReplicas != fd.Status.UnavailableReplicas {
fd.Status = fedStatus
_, err = fdc.fedClient.Extensions().Deployments(fd.Namespace).UpdateStatus(fd)
if err != nil {
return statusError, err
}
}
if len(operations) == 0 {
// Everything is in order
return statusAllOk, nil
}
err = fdc.fedUpdater.UpdateWithOnError(operations, updateTimeout, func(op fedutil.FederatedOperation, operror error) {
fdc.eventRecorder.Eventf(fd, api.EventTypeNormal, "FailedUpdateInCluster",
"Deployment update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v", key, err)
return statusError, err
}
// Some operations were made, reconcile after a while.
return statusNeedRecheck, nil
}
func (daemonsetcontroller *DaemonSetController) reconcileDaemonSet(namespace string, daemonsetName string) {
glog.V(4).Infof("Reconciling daemonset %s/%s", namespace, daemonsetName)
if !daemonsetcontroller.isSynced() {
glog.V(4).Infof("Daemonset controller is not synced")
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, daemonsetcontroller.clusterAvailableDelay, false)
return
}
key := getDaemonSetKey(namespace, daemonsetName)
baseDaemonSetObjFromStore, exist, err := daemonsetcontroller.daemonsetInformerStore.GetByKey(key)
if err != nil {
glog.Errorf("Failed to query main daemonset store for %v: %v", key, err)
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true)
return
}
if !exist {
glog.V(4).Infof("Skipping daemonset %s/%s - not federated", namespace, daemonsetName)
// Not federated daemonset, ignoring.
return
}
baseDaemonSetObj, err := conversion.NewCloner().DeepCopy(baseDaemonSetObjFromStore)
baseDaemonSet, ok := baseDaemonSetObj.(*extensionsv1.DaemonSet)
if err != nil || !ok {
glog.Errorf("Error in retrieving obj %s from store: %v, %v", daemonsetName, ok, err)
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true)
return
}
if baseDaemonSet.DeletionTimestamp != nil {
if err := daemonsetcontroller.delete(baseDaemonSet); err != nil {
glog.Errorf("Failed to delete %s: %v", daemonsetName, err)
daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "DeleteFailed",
"DaemonSet delete failed: %v", err)
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true)
}
return
}
glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for daemonset: %s",
baseDaemonSet.Name)
// Add the required finalizers before creating a daemonset in underlying clusters.
updatedDaemonSetObj, err := daemonsetcontroller.deletionHelper.EnsureFinalizers(baseDaemonSet)
if err != nil {
glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in daemonset %s: %v",
baseDaemonSet.Name, err)
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, false)
return
}
baseDaemonSet = updatedDaemonSetObj.(*extensionsv1.DaemonSet)
glog.V(3).Infof("Syncing daemonset %s in underlying clusters", baseDaemonSet.Name)
clusters, err := daemonsetcontroller.daemonsetFederatedInformer.GetReadyClusters()
if err != nil {
glog.Errorf("Failed to get cluster list: %v", err)
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, daemonsetcontroller.clusterAvailableDelay, false)
return
}
operations := make([]util.FederatedOperation, 0)
for _, cluster := range clusters {
clusterDaemonSetObj, found, err := daemonsetcontroller.daemonsetFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
if err != nil {
glog.Errorf("Failed to get %s from %s: %v", key, cluster.Name, err)
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true)
return
}
desiredDaemonSet := &extensionsv1.DaemonSet{
ObjectMeta: util.CopyObjectMeta(baseDaemonSet.ObjectMeta),
Spec: baseDaemonSet.Spec,
}
if !found {
glog.V(4).Infof("Creating daemonset %s/%s in cluster %s", namespace, daemonsetName, cluster.Name)
daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "CreateInCluster",
"Creating daemonset in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeAdd,
Obj: desiredDaemonSet,
ClusterName: cluster.Name,
})
} else {
clusterDaemonSet := clusterDaemonSetObj.(*extensionsv1.DaemonSet)
// Update existing daemonset, if needed.
if !util.ObjectMetaEquivalent(desiredDaemonSet.ObjectMeta, clusterDaemonSet.ObjectMeta) ||
!reflect.DeepEqual(desiredDaemonSet.Spec, clusterDaemonSet.Spec) {
glog.V(4).Infof("Upadting daemonset %s/%s in cluster %s", namespace, daemonsetName, cluster.Name)
daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "UpdateInCluster",
"Updating daemonset in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeUpdate,
Obj: desiredDaemonSet,
ClusterName: cluster.Name,
})
}
}
}
if len(operations) == 0 {
glog.V(4).Infof("No operation needed for %s/%s", namespace, daemonsetName)
// Everything is in order
return
}
err = daemonsetcontroller.federatedUpdater.UpdateWithOnError(operations, daemonsetcontroller.updateTimeout,
func(op util.FederatedOperation, operror error) {
daemonsetcontroller.eventRecorder.Eventf(baseDaemonSet, api.EventTypeNormal, "UpdateInClusterFailed",
"DaemonSet update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v, retrying shortly", key, err)
daemonsetcontroller.deliverDaemonSet(namespace, daemonsetName, 0, true)
return
}
}
func (configmapcontroller *ConfigMapController) reconcileConfigMap(configmap types.NamespacedName) {
if !configmapcontroller.isSynced() {
glog.V(4).Infof("Configmap controller not synced")
configmapcontroller.deliverConfigMap(configmap, configmapcontroller.clusterAvailableDelay, false)
return
}
key := configmap.String()
baseConfigMapObj, exist, err := configmapcontroller.configmapInformerStore.GetByKey(key)
if err != nil {
glog.Errorf("Failed to query main configmap store for %v: %v", key, err)
configmapcontroller.deliverConfigMap(configmap, 0, true)
return
}
if !exist {
// Not federated configmap, ignoring.
glog.V(8).Infof("Skipping not federated config map: %s", key)
return
}
baseConfigMap := baseConfigMapObj.(*api_v1.ConfigMap)
clusters, err := configmapcontroller.configmapFederatedInformer.GetReadyClusters()
if err != nil {
glog.Errorf("Failed to get cluster list: %v, retrying shortly", err)
configmapcontroller.deliverConfigMap(configmap, configmapcontroller.clusterAvailableDelay, false)
return
}
operations := make([]util.FederatedOperation, 0)
for _, cluster := range clusters {
clusterConfigMapObj, found, err := configmapcontroller.configmapFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
if err != nil {
glog.Errorf("Failed to get %s from %s: %v, retrying shortly", key, cluster.Name, err)
configmapcontroller.deliverConfigMap(configmap, 0, true)
return
}
desiredConfigMap := &api_v1.ConfigMap{
ObjectMeta: util.CopyObjectMeta(baseConfigMap.ObjectMeta),
Data: baseConfigMap.Data,
}
if !found {
configmapcontroller.eventRecorder.Eventf(baseConfigMap, api.EventTypeNormal, "CreateInCluster",
"Creating configmap in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeAdd,
Obj: desiredConfigMap,
ClusterName: cluster.Name,
})
} else {
clusterConfigMap := clusterConfigMapObj.(*api_v1.ConfigMap)
// Update existing configmap, if needed.
if !util.ConfigMapEquivalent(desiredConfigMap, clusterConfigMap) {
configmapcontroller.eventRecorder.Eventf(baseConfigMap, api.EventTypeNormal, "UpdateInCluster",
"Updating configmap in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeUpdate,
Obj: desiredConfigMap,
ClusterName: cluster.Name,
})
}
}
}
if len(operations) == 0 {
// Everything is in order
glog.V(8).Infof("No operations needed for %s", key)
return
}
err = configmapcontroller.federatedUpdater.UpdateWithOnError(operations, configmapcontroller.updateTimeout,
func(op util.FederatedOperation, operror error) {
configmapcontroller.eventRecorder.Eventf(baseConfigMap, api.EventTypeNormal, "UpdateInClusterFailed",
"ConfigMap update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v, retrying shortly", key, err)
configmapcontroller.deliverConfigMap(configmap, 0, true)
return
}
}
func (secretcontroller *SecretController) reconcileSecret(secret types.NamespacedName) {
if !secretcontroller.isSynced() {
secretcontroller.deliverSecret(secret, secretcontroller.clusterAvailableDelay, false)
return
}
key := secret.String()
baseSecretObjFromStore, exist, err := secretcontroller.secretInformerStore.GetByKey(key)
if err != nil {
glog.Errorf("Failed to query main secret store for %v: %v", key, err)
secretcontroller.deliverSecret(secret, 0, true)
return
}
if !exist {
// Not federated secret, ignoring.
return
}
// Create a copy before modifying the obj to prevent race condition with
// other readers of obj from store.
baseSecretObj, err := conversion.NewCloner().DeepCopy(baseSecretObjFromStore)
baseSecret, ok := baseSecretObj.(*api_v1.Secret)
if err != nil || !ok {
glog.Errorf("Error in retrieving obj from store: %v, %v", ok, err)
secretcontroller.deliverSecret(secret, 0, true)
return
}
if baseSecret.DeletionTimestamp != nil {
if err := secretcontroller.delete(baseSecret); err != nil {
glog.Errorf("Failed to delete %s: %v", secret, err)
secretcontroller.eventRecorder.Eventf(baseSecret, api.EventTypeNormal, "DeleteFailed",
"Secret delete failed: %v", err)
secretcontroller.deliverSecret(secret, 0, true)
}
return
}
glog.V(3).Infof("Ensuring delete object from underlying clusters finalizer for secret: %s",
baseSecret.Name)
// Add the required finalizers before creating a secret in underlying clusters.
updatedSecretObj, err := secretcontroller.deletionHelper.EnsureFinalizers(baseSecret)
if err != nil {
glog.Errorf("Failed to ensure delete object from underlying clusters finalizer in secret %s: %v",
baseSecret.Name, err)
secretcontroller.deliverSecret(secret, 0, false)
return
}
baseSecret = updatedSecretObj.(*api_v1.Secret)
glog.V(3).Infof("Syncing secret %s in underlying clusters", baseSecret.Name)
clusters, err := secretcontroller.secretFederatedInformer.GetReadyClusters()
if err != nil {
glog.Errorf("Failed to get cluster list: %v", err)
secretcontroller.deliverSecret(secret, secretcontroller.clusterAvailableDelay, false)
return
}
operations := make([]util.FederatedOperation, 0)
for _, cluster := range clusters {
clusterSecretObj, found, err := secretcontroller.secretFederatedInformer.GetTargetStore().GetByKey(cluster.Name, key)
if err != nil {
glog.Errorf("Failed to get %s from %s: %v", key, cluster.Name, err)
secretcontroller.deliverSecret(secret, 0, true)
return
}
desiredSecret := &api_v1.Secret{
ObjectMeta: util.CopyObjectMeta(baseSecret.ObjectMeta),
Data: baseSecret.Data,
Type: baseSecret.Type,
}
if !found {
secretcontroller.eventRecorder.Eventf(baseSecret, api.EventTypeNormal, "CreateInCluster",
"Creating secret in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeAdd,
Obj: desiredSecret,
ClusterName: cluster.Name,
})
} else {
clusterSecret := clusterSecretObj.(*api_v1.Secret)
// Update existing secret, if needed.
if !util.SecretEquivalent(*desiredSecret, *clusterSecret) {
secretcontroller.eventRecorder.Eventf(baseSecret, api.EventTypeNormal, "UpdateInCluster",
"Updating secret in cluster %s", cluster.Name)
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeUpdate,
Obj: desiredSecret,
ClusterName: cluster.Name,
})
}
}
}
if len(operations) == 0 {
// Everything is in order
return
}
err = secretcontroller.federatedUpdater.UpdateWithOnError(operations, secretcontroller.updateTimeout,
func(op util.FederatedOperation, operror error) {
secretcontroller.eventRecorder.Eventf(baseSecret, api.EventTypeNormal, "UpdateInClusterFailed",
"Secret update in cluster %s failed: %v", op.ClusterName, operror)
})
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v", key, err)
secretcontroller.deliverSecret(secret, 0, true)
return
}
// Evertyhing is in order but lets be double sure
secretcontroller.deliverSecret(secret, secretcontroller.secretReviewDelay, false)
}
func (nc *NamespaceController) reconcileNamespace(namespace string) {
if !nc.isSynced() {
nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false)
return
}
baseNamespaceObj, exist, err := nc.namespaceInformerStore.GetByKey(namespace)
if err != nil {
glog.Errorf("Failed to query main namespace store for %v: %v", namespace, err)
nc.deliverNamespace(namespace, 0, true)
return
}
if !exist {
// Not federated namespace, ignoring.
return
}
baseNamespace := baseNamespaceObj.(*api_v1.Namespace)
if baseNamespace.Status.Phase == api_v1.NamespaceTerminating {
// TODO: What about namespaces in subclusters ???
err = nc.federatedApiClient.Core().Namespaces().Delete(baseNamespace.Name, &api.DeleteOptions{})
if err != nil {
glog.Errorf("Failed to delete namespace %s: %v", baseNamespace.Name, err)
nc.deliverNamespace(namespace, 0, true)
}
return
}
clusters, err := nc.namespaceFederatedInformer.GetReadyClusters()
if err != nil {
glog.Errorf("Failed to get cluster list: %v", err)
nc.deliverNamespace(namespace, nc.clusterAvailableDelay, false)
return
}
operations := make([]util.FederatedOperation, 0)
for _, cluster := range clusters {
clusterNamespaceObj, found, err := nc.namespaceFederatedInformer.GetTargetStore().GetByKey(cluster.Name, namespace)
if err != nil {
glog.Errorf("Failed to get %s from %s: %v", namespace, cluster.Name, err)
nc.deliverNamespace(namespace, 0, true)
return
}
desiredNamespace := &api_v1.Namespace{
ObjectMeta: util.CopyObjectMeta(baseNamespace.ObjectMeta),
Spec: baseNamespace.Spec,
}
if !found {
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeAdd,
Obj: desiredNamespace,
ClusterName: cluster.Name,
})
} else {
clusterNamespace := clusterNamespaceObj.(*api_v1.Namespace)
// Update existing namespace, if needed.
if !util.ObjectMetaEquivalent(desiredNamespace.ObjectMeta, clusterNamespace.ObjectMeta) ||
!reflect.DeepEqual(desiredNamespace.Spec, clusterNamespace.Spec) {
operations = append(operations, util.FederatedOperation{
Type: util.OperationTypeUpdate,
Obj: desiredNamespace,
ClusterName: cluster.Name,
})
}
}
}
if len(operations) == 0 {
// Everything is in order
return
}
err = nc.federatedUpdater.Update(operations, nc.updateTimeout)
if err != nil {
glog.Errorf("Failed to execute updates for %s: %v", namespace, err)
nc.deliverNamespace(namespace, 0, true)
return
}
// Evertyhing is in order but lets be double sure
nc.deliverNamespace(namespace, nc.namespaceReviewDelay, false)
}