// NewNamespaceManager creates a new NamespaceManager
func NewNamespaceManager(qingClient client.Interface, resyncPeriod time.Duration) *NamespaceManager {
_, controller := framework.NewInformer(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return qingClient.Namespaces().List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return qingClient.Namespaces().Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
},
&api.Namespace{},
resyncPeriod,
framework.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
namespace := obj.(*api.Namespace)
err := syncNamespace(qingClient, *namespace)
if err != nil {
glog.Error(err)
}
},
UpdateFunc: func(oldObj, newObj interface{}) {
namespace := newObj.(*api.Namespace)
err := syncNamespace(qingClient, *namespace)
if err != nil {
glog.Error(err)
}
},
},
)
return &NamespaceManager{
controller: controller,
}
}
func GetApiVersions(w io.Writer, qingClient client.Interface) {
apiVersions, err := qingClient.ServerAPIVersions()
if err != nil {
fmt.Printf("Couldn't get available api versions from server: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(w, "Available Server Api Versions: %#v\n", *apiVersions)
}
func UpdateExistingReplicationController(c client.Interface, oldRc *api.ReplicationController, namespace, newName, deploymentKey, deploymentValue string, out io.Writer) (*api.ReplicationController, error) {
SetNextControllerAnnotation(oldRc, newName)
if _, found := oldRc.Spec.Selector[deploymentKey]; !found {
return AddDeploymentKeyToReplicationController(oldRc, c, deploymentKey, deploymentValue, namespace, out)
} else {
// If we didn't need to update the controller for the deployment key, we still need to write
// the "next" controller.
return c.ReplicationControllers(namespace).Update(oldRc)
}
}
func GetVersion(w io.Writer, qingClient client.Interface) {
GetClientVersion(w)
serverVersion, err := qingClient.ServerVersion()
if err != nil {
fmt.Printf("Couldn't read version from server: %v\n", err)
os.Exit(1)
}
fmt.Fprintf(w, "Server Version: %#v\n", *serverVersion)
}
func deletePersistentVolumeClaims(qingClient client.Interface, ns string) error {
items, err := qingClient.PersistentVolumeClaims(ns).List(labels.Everything(), fields.Everything())
if err != nil {
return err
}
for i := range items.Items {
err := qingClient.PersistentVolumeClaims(ns).Delete(items.Items[i].Name)
if err != nil && !errors.IsNotFound(err) {
return err
}
}
return nil
}
func deleteReplicationControllers(qingClient client.Interface, ns string) error {
items, err := qingClient.ReplicationControllers(ns).List(labels.Everything())
if err != nil {
return err
}
for i := range items.Items {
err := qingClient.ReplicationControllers(ns).Delete(items.Items[i].Name)
if err != nil && !errors.IsNotFound(err) {
return err
}
}
return nil
}
func deleteResourceQuotas(qingClient client.Interface, ns string) error {
resourceQuotas, err := qingClient.ResourceQuotas(ns).List(labels.Everything())
if err != nil {
return err
}
for i := range resourceQuotas.Items {
err := qingClient.ResourceQuotas(ns).Delete(resourceQuotas.Items[i].Name)
if err != nil && !errors.IsNotFound(err) {
return err
}
}
return nil
}
// NewResourceQuota creates a new resource quota admission control handler
func NewResourceQuota(client client.Interface) admission.Interface {
lw := &cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return client.ResourceQuotas(api.NamespaceAll).List(labels.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return client.ResourceQuotas(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
indexer, reflector := cache.NewNamespaceKeyedIndexerAndReflector(lw, &api.ResourceQuota{}, 0)
reflector.Run()
return createResourceQuota(client, indexer)
}
// New returns a new service controller to keep cloud provider service resources
// (like external load balancers) in sync with the registry.
func New(cloud cloudprovider.Interface, qingClient client.Interface, clusterName string) *ServiceController {
broadcaster := record.NewBroadcaster()
broadcaster.StartRecordingToSink(qingClient.Events(""))
recorder := broadcaster.NewRecorder(api.EventSource{Component: "service-controller"})
return &ServiceController{
cloud: cloud,
qingClient: qingClient,
clusterName: clusterName,
cache: &serviceCache{serviceMap: make(map[string]*cachedService)},
eventBroadcaster: broadcaster,
eventRecorder: recorder,
}
}
// finalize will finalize the namespace for qingyuan
func finalize(qingClient client.Interface, namespace api.Namespace) (*api.Namespace, error) {
namespaceFinalize := api.Namespace{}
namespaceFinalize.ObjectMeta = namespace.ObjectMeta
namespaceFinalize.Spec = namespace.Spec
finalizerSet := util.NewStringSet()
for i := range namespace.Spec.Finalizers {
if namespace.Spec.Finalizers[i] != api.FinalizerQingYuan {
finalizerSet.Insert(string(namespace.Spec.Finalizers[i]))
}
}
namespaceFinalize.Spec.Finalizers = make([]api.FinalizerName, 0, len(finalizerSet))
for _, value := range finalizerSet.List() {
namespaceFinalize.Spec.Finalizers = append(namespaceFinalize.Spec.Finalizers, api.FinalizerName(value))
}
return qingClient.Namespaces().Finalize(&namespaceFinalize)
}
// NewLimitRanger returns an object that enforces limits based on the supplied limit function
func NewLimitRanger(client client.Interface, limitFunc LimitFunc) admission.Interface {
lw := &cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return client.LimitRanges(api.NamespaceAll).List(labels.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return client.LimitRanges(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
}
indexer, reflector := cache.NewNamespaceKeyedIndexerAndReflector(lw, &api.LimitRange{}, 0)
reflector.Run()
return &limitRanger{
Handler: admission.NewHandler(admission.Create, admission.Update),
client: client,
limitFunc: limitFunc,
indexer: indexer,
}
}
// NewProvision creates a new namespace provision admission control handler
func NewProvision(c client.Interface) admission.Interface {
store := cache.NewStore(cache.MetaNamespaceKeyFunc)
reflector := cache.NewReflector(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return c.Namespaces().List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return c.Namespaces().Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
},
&api.Namespace{},
store,
0,
)
reflector.Run()
return createProvision(c, store)
}
// NewPersistentVolumeClaimBinder creates a new PersistentVolumeClaimBinder
func NewPersistentVolumeClaimBinder(qingClient client.Interface, syncPeriod time.Duration) *PersistentVolumeClaimBinder {
volumeIndex := NewPersistentVolumeOrderedIndex()
binderClient := NewBinderClient(qingClient)
binder := &PersistentVolumeClaimBinder{
volumeIndex: volumeIndex,
client: binderClient,
}
_, volumeController := framework.NewInformer(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return qingClient.PersistentVolumes().List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return qingClient.PersistentVolumes().Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
},
&api.PersistentVolume{},
syncPeriod,
framework.ResourceEventHandlerFuncs{
AddFunc: binder.addVolume,
UpdateFunc: binder.updateVolume,
DeleteFunc: binder.deleteVolume,
},
)
_, claimController := framework.NewInformer(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return qingClient.PersistentVolumeClaims(api.NamespaceAll).List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return qingClient.PersistentVolumeClaims(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
},
&api.PersistentVolumeClaim{},
syncPeriod,
framework.ResourceEventHandlerFuncs{
AddFunc: binder.addClaim,
UpdateFunc: binder.updateClaim,
// no DeleteFunc needed. a claim requires no clean-up.
// syncVolume handles the missing claim
},
)
binder.claimController = claimController
binder.volumeController = volumeController
return binder
}
// NewLifecycle creates a new namespace lifecycle admission control handler
func NewLifecycle(c client.Interface) admission.Interface {
store := cache.NewStore(cache.MetaNamespaceKeyFunc)
reflector := cache.NewReflector(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return c.Namespaces().List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return c.Namespaces().Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
},
&api.Namespace{},
store,
0,
)
reflector.Run()
return &lifecycle{
Handler: admission.NewHandler(admission.Create, admission.Delete),
client: c,
store: store,
immortalNamespaces: util.NewStringSet(api.NamespaceDefault),
}
}
// PersistentVolumeRecycler creates a new PersistentVolumeRecycler
func NewPersistentVolumeRecycler(qingClient client.Interface, syncPeriod time.Duration, plugins []volume.VolumePlugin) (*PersistentVolumeRecycler, error) {
recyclerClient := NewRecyclerClient(qingClient)
recycler := &PersistentVolumeRecycler{
client: recyclerClient,
qingClient: qingClient,
}
if err := recycler.pluginMgr.InitPlugins(plugins, recycler); err != nil {
return nil, fmt.Errorf("Could not initialize volume plugins for PVClaimBinder: %+v", err)
}
_, volumeController := framework.NewInformer(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return qingClient.PersistentVolumes().List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return qingClient.PersistentVolumes().Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
},
&api.PersistentVolume{},
syncPeriod,
framework.ResourceEventHandlerFuncs{
AddFunc: func(obj interface{}) {
pv := obj.(*api.PersistentVolume)
recycler.reclaimVolume(pv)
},
UpdateFunc: func(oldObj, newObj interface{}) {
pv := newObj.(*api.PersistentVolume)
recycler.reclaimVolume(pv)
},
},
)
recycler.volumeController = volumeController
return recycler, nil
}
// NewServiceAccount returns an admission.Interface implementation which limits admission of Pod CREATE requests based on the pod's ServiceAccount:
// 1. If the pod does not specify a ServiceAccount, it sets the pod's ServiceAccount to "default"
// 2. It ensures the ServiceAccount referenced by the pod exists
// 3. If LimitSecretReferences is true, it rejects the pod if the pod references Secret objects which the pod's ServiceAccount does not reference
// 4. If the pod does not contain any ImagePullSecrets, the ImagePullSecrets of the service account are added.
// 5. If MountServiceAccountToken is true, it adds a VolumeMount with the pod's ServiceAccount's api token secret to containers
func NewServiceAccount(cl client.Interface) *serviceAccount {
serviceAccountsIndexer, serviceAccountsReflector := cache.NewNamespaceKeyedIndexerAndReflector(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return cl.ServiceAccounts(api.NamespaceAll).List(labels.Everything(), fields.Everything())
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return cl.ServiceAccounts(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), resourceVersion)
},
},
&api.ServiceAccount{},
0,
)
tokenSelector := fields.SelectorFromSet(map[string]string{client.SecretType: string(api.SecretTypeServiceAccountToken)})
secretsIndexer, secretsReflector := cache.NewNamespaceKeyedIndexerAndReflector(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return cl.Secrets(api.NamespaceAll).List(labels.Everything(), tokenSelector)
},
WatchFunc: func(resourceVersion string) (watch.Interface, error) {
return cl.Secrets(api.NamespaceAll).Watch(labels.Everything(), tokenSelector, resourceVersion)
},
},
&api.Secret{},
0,
)
return &serviceAccount{
Handler: admission.NewHandler(admission.Create),
// TODO: enable this once we've swept secret usage to account for adding secret references to service accounts
LimitSecretReferences: false,
// Auto mount service account API token secrets
MountServiceAccountToken: true,
client: cl,
serviceAccounts: serviceAccountsIndexer,
serviceAccountsReflector: serviceAccountsReflector,
secrets: secretsIndexer,
secretsReflector: secretsReflector,
}
}
// syncNamespace makes namespace life-cycle decisions
func syncNamespace(qingClient client.Interface, namespace api.Namespace) (err error) {
if namespace.DeletionTimestamp == nil {
return nil
}
// if there is a deletion timestamp, and the status is not terminating, then update status
if !namespace.DeletionTimestamp.IsZero() && namespace.Status.Phase != api.NamespaceTerminating {
newNamespace := api.Namespace{}
newNamespace.ObjectMeta = namespace.ObjectMeta
newNamespace.Status = namespace.Status
newNamespace.Status.Phase = api.NamespaceTerminating
result, err := qingClient.Namespaces().Status(&newNamespace)
if err != nil {
return err
}
// work with the latest copy so we can proceed to clean up right away without another interval
namespace = *result
}
// if the namespace is already finalized, delete it
if finalized(namespace) {
err = qingClient.Namespaces().Delete(namespace.Name)
if err != nil && !errors.IsNotFound(err) {
return err
}
return nil
}
// there may still be content for us to remove
err = deleteAllContent(qingClient, namespace.Name)
if err != nil {
return err
}
// we have removed content, so mark it finalized by us
result, err := finalize(qingClient, namespace)
if err != nil {
return err
}
// now check if all finalizers have reported that we delete now
if finalized(*result) {
err = qingClient.Namespaces().Delete(namespace.Name)
if err != nil && !errors.IsNotFound(err) {
return err
}
}
return nil
}
func AddDeploymentKeyToReplicationController(oldRc *api.ReplicationController, client client.Interface, deploymentKey, deploymentValue, namespace string, out io.Writer) (*api.ReplicationController, error) {
var err error
// First, update the template label. This ensures that any newly created pods will have the new label
if oldRc, err = updateWithRetries(client.ReplicationControllers(namespace), oldRc, func(rc *api.ReplicationController) {
if rc.Spec.Template.Labels == nil {
rc.Spec.Template.Labels = map[string]string{}
}
rc.Spec.Template.Labels[deploymentKey] = deploymentValue
}); err != nil {
return nil, err
}
// Update all pods managed by the rc to have the new hash label, so they are correctly adopted
// TODO: extract the code from the label command and re-use it here.
podList, err := client.Pods(namespace).List(labels.SelectorFromSet(oldRc.Spec.Selector), fields.Everything())
if err != nil {
return nil, err
}
for ix := range podList.Items {
pod := &podList.Items[ix]
if pod.Labels == nil {
pod.Labels = map[string]string{
deploymentKey: deploymentValue,
}
} else {
pod.Labels[deploymentKey] = deploymentValue
}
err = nil
delay := 3
for i := 0; i < MaxRetries; i++ {
_, err = client.Pods(namespace).Update(pod)
if err != nil {
fmt.Fprintf(out, "Error updating pod (%v), retrying after %d seconds", err, delay)
time.Sleep(time.Second * time.Duration(delay))
delay *= delay
} else {
break
}
}
if err != nil {
return nil, err
}
}
if oldRc.Spec.Selector == nil {
oldRc.Spec.Selector = map[string]string{}
}
// Copy the old selector, so that we can scrub out any orphaned pods
selectorCopy := map[string]string{}
for k, v := range oldRc.Spec.Selector {
selectorCopy[k] = v
}
oldRc.Spec.Selector[deploymentKey] = deploymentValue
// Update the selector of the rc so it manages all the pods we updated above
if oldRc, err = updateWithRetries(client.ReplicationControllers(namespace), oldRc, func(rc *api.ReplicationController) {
rc.Spec.Selector[deploymentKey] = deploymentValue
}); err != nil {
return nil, err
}
// Clean up any orphaned pods that don't have the new label, this can happen if the rc manager
// doesn't see the update to its pod template and creates a new pod with the old labels after
// we've finished re-adopting existing pods to the rc.
podList, err = client.Pods(namespace).List(labels.SelectorFromSet(selectorCopy), fields.Everything())
for ix := range podList.Items {
pod := &podList.Items[ix]
if value, found := pod.Labels[deploymentKey]; !found || value != deploymentValue {
if err := client.Pods(namespace).Delete(pod.Name, nil); err != nil {
return nil, err
}
}
}
return oldRc, nil
}
// NewReplicationManager creates a new ReplicationManager.
func NewReplicationManager(qingClient client.Interface, burstReplicas int) *ReplicationManager {
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(qingClient.Events(""))
rm := &ReplicationManager{
qingClient: qingClient,
podControl: RealPodControl{
qingClient: qingClient,
recorder: eventBroadcaster.NewRecorder(api.EventSource{Component: "replication-controller"}),
},
burstReplicas: burstReplicas,
expectations: NewRCExpectations(),
queue: workqueue.New(),
}
rm.controllerStore.Store, rm.rcController = framework.NewInformer(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return rm.qingClient.ReplicationControllers(api.NamespaceAll).List(labels.Everything())
},
WatchFunc: func(rv string) (watch.Interface, error) {
return rm.qingClient.ReplicationControllers(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), rv)
},
},
&api.ReplicationController{},
FullControllerResyncPeriod,
framework.ResourceEventHandlerFuncs{
AddFunc: rm.enqueueController,
UpdateFunc: func(old, cur interface{}) {
// We only really need to do this when spec changes, but for correctness it is safer to
// periodically double check. It is overkill for 2 reasons:
// 1. Status.Replica updates will cause a sync
// 2. Every 30s we will get a full resync (this will happen anyway every 5 minutes when pods relist)
// However, it shouldn't be that bad as rcs that haven't met expectations won't sync, and all
// the listing is done using local stores.
oldRC := old.(*api.ReplicationController)
curRC := cur.(*api.ReplicationController)
if oldRC.Status.Replicas != curRC.Status.Replicas {
glog.V(4).Infof("Observed updated replica count for rc: %v, %d->%d", curRC.Name, oldRC.Status.Replicas, curRC.Status.Replicas)
}
rm.enqueueController(cur)
},
// This will enter the sync loop and no-op, becuase the controller has been deleted from the store.
// Note that deleting a controller immediately after scaling it to 0 will not work. The recommended
// way of achieving this is by performing a `stop` operation on the controller.
DeleteFunc: rm.enqueueController,
},
)
rm.podStore.Store, rm.podController = framework.NewInformer(
&cache.ListWatch{
ListFunc: func() (runtime.Object, error) {
return rm.qingClient.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
},
WatchFunc: func(rv string) (watch.Interface, error) {
return rm.qingClient.Pods(api.NamespaceAll).Watch(labels.Everything(), fields.Everything(), rv)
},
},
&api.Pod{},
PodRelistPeriod,
framework.ResourceEventHandlerFuncs{
AddFunc: rm.addPod,
// This invokes the rc for every pod change, eg: host assignment. Though this might seem like overkill
// the most frequent pod update is status, and the associated rc will only list from local storage, so
// it should be ok.
UpdateFunc: rm.updatePod,
DeleteFunc: rm.deletePod,
},
)
rm.syncHandler = rm.syncReplicationController
rm.podStoreSynced = rm.podController.HasSynced
return rm
}
// IncrementUsage updates the supplied ResourceQuotaStatus object based on the incoming operation
// Return true if the usage must be recorded prior to admitting the new resource
// Return an error if the operation should not pass admission control
func IncrementUsage(a admission.Attributes, status *api.ResourceQuotaStatus, client client.Interface) (bool, error) {
dirty := false
set := map[api.ResourceName]bool{}
for k := range status.Hard {
set[k] = true
}
obj := a.GetObject()
// handle max counts for each kind of resource (pods, services, replicationControllers, etc.)
if a.GetOperation() == admission.Create {
// TODO v1beta1 had camel case, v1beta3 went to all lower, we can remove this line when we deprecate v1beta1
resourceNormalized := strings.ToLower(a.GetResource())
resourceName := resourceToResourceName[resourceNormalized]
hard, hardFound := status.Hard[resourceName]
if hardFound {
used, usedFound := status.Used[resourceName]
if !usedFound {
return false, fmt.Errorf("Quota usage stats are not yet known, unable to admit resource until an accurate count is completed.")
}
if used.Value() >= hard.Value() {
return false, fmt.Errorf("Limited to %s %s", hard.String(), resourceName)
} else {
status.Used[resourceName] = *resource.NewQuantity(used.Value()+int64(1), resource.DecimalSI)
dirty = true
}
}
}
// handle memory/cpu constraints, and any diff of usage based on memory/cpu on updates
if a.GetResource() == "pods" && (set[api.ResourceMemory] || set[api.ResourceCPU]) {
pod := obj.(*api.Pod)
deltaCPU := resourcequota.PodCPU(pod)
deltaMemory := resourcequota.PodMemory(pod)
// if this is an update, we need to find the delta cpu/memory usage from previous state
if a.GetOperation() == admission.Update {
oldPod, err := client.Pods(a.GetNamespace()).Get(pod.Name)
if err != nil {
return false, err
}
oldCPU := resourcequota.PodCPU(oldPod)
oldMemory := resourcequota.PodMemory(oldPod)
deltaCPU = resource.NewMilliQuantity(deltaCPU.MilliValue()-oldCPU.MilliValue(), resource.DecimalSI)
deltaMemory = resource.NewQuantity(deltaMemory.Value()-oldMemory.Value(), resource.DecimalSI)
}
hardMem, hardMemFound := status.Hard[api.ResourceMemory]
if hardMemFound {
if set[api.ResourceMemory] && resourcequota.IsPodMemoryUnbounded(pod) {
return false, fmt.Errorf("Limited to %s memory, but pod has no specified memory limit", hardMem.String())
}
used, usedFound := status.Used[api.ResourceMemory]
if !usedFound {
return false, fmt.Errorf("Quota usage stats are not yet known, unable to admit resource until an accurate count is completed.")
}
if used.Value()+deltaMemory.Value() > hardMem.Value() {
return false, fmt.Errorf("Limited to %s memory", hardMem.String())
} else {
status.Used[api.ResourceMemory] = *resource.NewQuantity(used.Value()+deltaMemory.Value(), resource.DecimalSI)
dirty = true
}
}
hardCPU, hardCPUFound := status.Hard[api.ResourceCPU]
if hardCPUFound {
if set[api.ResourceCPU] && resourcequota.IsPodCPUUnbounded(pod) {
return false, fmt.Errorf("Limited to %s CPU, but pod has no specified cpu limit", hardCPU.String())
}
used, usedFound := status.Used[api.ResourceCPU]
if !usedFound {
return false, fmt.Errorf("Quota usage stats are not yet known, unable to admit resource until an accurate count is completed.")
}
if used.MilliValue()+deltaCPU.MilliValue() > hardCPU.MilliValue() {
return false, fmt.Errorf("Limited to %s CPU", hardCPU.String())
} else {
status.Used[api.ResourceCPU] = *resource.NewMilliQuantity(used.MilliValue()+deltaCPU.MilliValue(), resource.DecimalSI)
dirty = true
}
}
}
return dirty, nil
}