func (c *MasterConfig) RunPersistentVolumeController(client *client.Client, namespace, recyclerImageName, recyclerServiceAccountName string) {
s := c.ControllerManager
provisioner, err := kctrlmgr.NewVolumeProvisioner(c.CloudProvider, s.VolumeConfiguration)
if err != nil {
glog.Fatal("A Provisioner could not be created, but one was expected. Provisioning will not work. This functionality is considered an early Alpha version.")
}
volumeController := persistentvolumecontroller.NewPersistentVolumeController(
clientadapter.FromUnversionedClient(client),
s.PVClaimBinderSyncPeriod.Duration,
provisioner,
probeRecyclableVolumePlugins(s.VolumeConfiguration, namespace, recyclerImageName, recyclerServiceAccountName),
c.CloudProvider,
s.ClusterName,
nil, nil, nil,
s.VolumeConfiguration.EnableDynamicProvisioning,
)
volumeController.Run()
attachDetachController, err :=
volumecontroller.NewAttachDetachController(
clientadapter.FromUnversionedClient(client),
c.Informers.Pods().Informer(),
c.Informers.Nodes().Informer(),
c.Informers.PersistentVolumeClaims().Informer(),
c.Informers.PersistentVolumes().Informer(),
c.CloudProvider,
kctrlmgr.ProbeAttachableVolumePlugins(s.VolumeConfiguration))
if err != nil {
glog.Fatalf("Failed to start attach/detach controller: %v", err)
} else {
go attachDetachController.Run(utilwait.NeverStop)
}
}
// RunServiceAccountTokensController starts the service account token controller
func (c *MasterConfig) RunServiceAccountTokensController() {
if len(c.Options.ServiceAccountConfig.PrivateKeyFile) == 0 {
glog.Infof("Skipped starting Service Account Token Manager, no private key specified")
return
}
privateKey, err := serviceaccount.ReadPrivateKey(c.Options.ServiceAccountConfig.PrivateKeyFile)
if err != nil {
glog.Fatalf("Error reading signing key for Service Account Token Manager: %v", err)
}
rootCA := []byte{}
if len(c.Options.ServiceAccountConfig.MasterCA) > 0 {
rootCA, err = ioutil.ReadFile(c.Options.ServiceAccountConfig.MasterCA)
if err != nil {
glog.Fatalf("Error reading master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
}
if _, err := kcrypto.CertsFromPEM(rootCA); err != nil {
glog.Fatalf("Error parsing master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
}
}
options := sacontroller.TokensControllerOptions{
TokenGenerator: serviceaccount.JWTTokenGenerator(privateKey),
RootCA: rootCA,
}
sacontroller.NewTokensController(clientadapter.FromUnversionedClient(c.KubeClient()), options).Run()
}
func (o *SCCModificationOptions) CompleteUsers(f *clientcmd.Factory, args []string, saNames []string) error {
if len(args) < 1 {
return errors.New("you must specify a scc")
}
o.SCCName = args[0]
o.Subjects = authorizationapi.BuildSubjects(args[1:], []string{}, uservalidation.ValidateUserName, uservalidation.ValidateGroupName)
if (len(o.Subjects) == 0) && (len(saNames) == 0) {
return errors.New("you must specify at least one user or service account")
}
_, kc, _, err := f.Clients()
if err != nil {
return err
}
o.SCCInterface = adapter.FromUnversionedClient(kc).Core()
o.DefaultSubjectNamespace, _, err = f.DefaultNamespace()
if err != nil {
return err
}
for _, sa := range saNames {
o.Subjects = append(o.Subjects, kapi.ObjectReference{Namespace: o.DefaultSubjectNamespace, Name: sa, Kind: "ServiceAccount"})
}
return nil
}
// NewRollingDeploymentStrategy makes a new RollingDeploymentStrategy.
func NewRollingDeploymentStrategy(namespace string, oldClient kclient.Interface, tags client.ImageStreamTagsNamespacer, events record.EventSink, decoder runtime.Decoder, initialStrategy acceptingDeploymentStrategy, out, errOut io.Writer, until string) *RollingDeploymentStrategy {
if out == nil {
out = ioutil.Discard
}
if errOut == nil {
errOut = ioutil.Discard
}
// TODO internalclientset: get rid of oldClient after next rebase
client := adapter.FromUnversionedClient(oldClient.(*kclient.Client))
return &RollingDeploymentStrategy{
out: out,
errOut: errOut,
until: until,
decoder: decoder,
initialStrategy: initialStrategy,
rcClient: client.Core(),
eventClient: client.Core(),
tags: tags,
apiRetryPeriod: defaultApiRetryPeriod,
apiRetryTimeout: defaultApiRetryTimeout,
rollingUpdate: func(config *kubectl.RollingUpdaterConfig) error {
updater := kubectl.NewRollingUpdater(namespace, oldClient)
return updater.Update(config)
},
hookExecutor: stratsupport.NewHookExecutor(client.Core(), tags, client.Core(), os.Stdout, decoder),
getUpdateAcceptor: func(timeout time.Duration, minReadySeconds int32) strat.UpdateAcceptor {
return stratsupport.NewAcceptAvailablePods(out, client.Core(), timeout, acceptorInterval, minReadySeconds)
},
}
}
func (c *MasterConfig) RunPersistentVolumeProvisioner(client *client.Client) {
provisioner, err := kctrlmgr.NewVolumeProvisioner(c.CloudProvider, c.ControllerManager.VolumeConfiguration)
if err != nil {
// a provisioner was expected but encountered an error
glog.Fatal(err)
}
// not all cloud providers have a provisioner.
if provisioner != nil {
allPlugins := []volume.VolumePlugin{}
allPlugins = append(allPlugins, aws_ebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gce_pd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
controllerClient := volumeclaimbinder.NewControllerClient(clientadapter.FromUnversionedClient(client))
provisionerController, err := volumeclaimbinder.NewPersistentVolumeProvisionerController(
controllerClient,
c.ControllerManager.PVClaimBinderSyncPeriod.Duration,
c.ControllerManager.ClusterName,
allPlugins,
provisioner,
c.CloudProvider,
)
if err != nil {
glog.Fatalf("Unable to start persistent volume provisioner: %+v", err)
}
provisionerController.Run()
}
}
func (c *MasterConfig) RunDeploymentController(client *client.Client) {
controller := deployment.NewDeploymentController(
clientadapter.FromUnversionedClient(client),
kctrlmgr.ResyncPeriod(c.ControllerManager),
)
go controller.Run(int(c.ControllerManager.ConcurrentDeploymentSyncs), utilwait.NeverStop)
}
// NewRecreateDeploymentStrategy makes a RecreateDeploymentStrategy backed by
// a real HookExecutor and client.
func NewRecreateDeploymentStrategy(oldClient kclient.Interface, tagClient client.ImageStreamTagsNamespacer, events record.EventSink, decoder runtime.Decoder, out, errOut io.Writer, until string) *RecreateDeploymentStrategy {
if out == nil {
out = ioutil.Discard
}
if errOut == nil {
errOut = ioutil.Discard
}
scaler, _ := kubectl.ScalerFor(kapi.Kind("ReplicationController"), oldClient)
// TODO internalclientset: get rid of oldClient after next rebase
client := adapter.FromUnversionedClient(oldClient.(*kclient.Client))
return &RecreateDeploymentStrategy{
out: out,
errOut: errOut,
events: events,
until: until,
rcClient: client.Core(),
eventClient: client.Core(),
getUpdateAcceptor: func(timeout time.Duration, minReadySeconds int32) strat.UpdateAcceptor {
return stratsupport.NewAcceptAvailablePods(out, client.Core(), timeout, acceptorInterval, minReadySeconds)
},
scaler: scaler,
decoder: decoder,
hookExecutor: stratsupport.NewHookExecutor(client.Core(), tagClient, client.Core(), os.Stdout, decoder),
retryTimeout: 120 * time.Second,
retryPeriod: 1 * time.Second,
}
}
func newServiceAccountTokenGetter(options configapi.MasterConfig) (serviceaccount.ServiceAccountTokenGetter, error) {
if options.KubernetesMasterConfig == nil {
// When we're running against an external Kubernetes, use the external kubernetes client to validate service account tokens
// This prevents infinite auth loops if the privilegedLoopbackKubeClient authenticates using a service account token
kubeClient, _, err := configapi.GetKubeClient(options.MasterClients.ExternalKubernetesKubeConfig, options.MasterClients.ExternalKubernetesClientConnectionOverrides)
if err != nil {
return nil, err
}
return sacontroller.NewGetterFromClient(clientadapter.FromUnversionedClient(kubeClient)), nil
}
// TODO: could be hoisted if other Origin code needs direct access to etcd, otherwise discourage this access pattern
// as we move to be more on top of Kube.
_, kubeStorageFactory, err := kubernetes.BuildDefaultAPIServer(options)
if err != nil {
return nil, err
}
storageConfig, err := kubeStorageFactory.NewConfig(kapi.Resource("serviceaccounts"))
if err != nil {
return nil, err
}
// TODO: by doing this we will not be able to authenticate while a master quorum is not present - reimplement
// as two storages called in succession (non quorum and then quorum).
storageConfig.Quorum = true
return sacontroller.NewGetterFromStorageInterface(storageConfig, kubeStorageFactory.ResourcePrefix(kapi.Resource("serviceaccounts")), kubeStorageFactory.ResourcePrefix(kapi.Resource("secrets"))), nil
}
func testNewDeployment(f *framework.Framework) {
ns := f.Namespace.Name
// TODO: remove unversionedClient when the refactoring is done. Currently some
// functions like verifyPod still expects a unversioned#Client.
c := adapter.FromUnversionedClient(f.Client)
deploymentName := "test-new-deployment"
podLabels := map[string]string{"name": nginxImageName}
replicas := 1
framework.Logf("Creating simple deployment %s", deploymentName)
d := newDeployment(deploymentName, replicas, podLabels, nginxImageName, nginxImage, extensions.RollingUpdateDeploymentStrategyType, nil)
d.Annotations = map[string]string{"test": "should-copy-to-replica-set", kubectl.LastAppliedConfigAnnotation: "should-not-copy-to-replica-set"}
_, err := c.Extensions().Deployments(ns).Create(d)
Expect(err).NotTo(HaveOccurred())
defer stopDeployment(c, f.Client, ns, deploymentName)
// Wait for it to be updated to revision 1
err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", nginxImage)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForDeploymentStatus(c, ns, deploymentName, replicas, replicas-1, replicas+1, 0)
Expect(err).NotTo(HaveOccurred())
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
newRS, err := deploymentutil.GetNewReplicaSet(deployment, c)
Expect(err).NotTo(HaveOccurred())
// Check new RS annotations
Expect(newRS.Annotations["test"]).Should(Equal("should-copy-to-replica-set"))
Expect(newRS.Annotations[kubectl.LastAppliedConfigAnnotation]).Should(Equal(""))
Expect(deployment.Annotations["test"]).Should(Equal("should-copy-to-replica-set"))
Expect(deployment.Annotations[kubectl.LastAppliedConfigAnnotation]).Should(Equal("should-not-copy-to-replica-set"))
}
// RunNodeController starts the node controller
// TODO: handle node CIDR and route allocation
func (c *MasterConfig) RunNodeController() {
s := c.ControllerManager
// this cidr has been validated already
_, clusterCIDR, _ := net.ParseCIDR(s.ClusterCIDR)
_, serviceCIDR, _ := net.ParseCIDR(s.ServiceCIDR)
controller, err := nodecontroller.NewNodeController(
c.Informers.Pods().Informer(),
c.CloudProvider,
clientadapter.FromUnversionedClient(c.KubeClient),
s.PodEvictionTimeout.Duration,
s.NodeEvictionRate,
s.SecondaryNodeEvictionRate,
s.LargeClusterSizeThreshold,
s.UnhealthyZoneThreshold,
s.NodeMonitorGracePeriod.Duration,
s.NodeStartupGracePeriod.Duration,
s.NodeMonitorPeriod.Duration,
clusterCIDR,
serviceCIDR,
int(s.NodeCIDRMaskSize),
s.AllocateNodeCIDRs,
)
if err != nil {
glog.Fatalf("Unable to start node controller: %v", err)
}
controller.Run()
}
// RunNodeController starts the node controller
// TODO: handle node CIDR and route allocation
func (c *MasterConfig) RunNodeController() {
s := c.ControllerManager
// this cidr has been validated already
_, clusterCIDR, _ := net.ParseCIDR(s.ClusterCIDR)
_, serviceCIDR, _ := net.ParseCIDR(s.ServiceCIDR)
controller, err := nodecontroller.NewNodeController(
c.CloudProvider,
clientadapter.FromUnversionedClient(c.KubeClient),
s.PodEvictionTimeout.Duration,
flowcontrol.NewTokenBucketRateLimiter(s.DeletingPodsQps, int(s.DeletingPodsBurst)),
flowcontrol.NewTokenBucketRateLimiter(s.DeletingPodsQps, int(s.DeletingPodsBurst)), // upstream uses the same ones too
s.NodeMonitorGracePeriod.Duration,
s.NodeStartupGracePeriod.Duration,
s.NodeMonitorPeriod.Duration,
clusterCIDR,
serviceCIDR,
int(s.NodeCIDRMaskSize),
s.AllocateNodeCIDRs,
)
if err != nil {
glog.Fatalf("Unable to start node controller: %v", err)
}
controller.Run(s.NodeSyncPeriod.Duration)
}
func testDeploymentLabelAdopted(f *framework.Framework) {
ns := f.Namespace.Name
// TODO: remove unversionedClient when the refactoring is done. Currently some
// functions like verifyPod still expects a unversioned#Client.
unversionedClient := f.Client
c := adapter.FromUnversionedClient(unversionedClient)
// Create nginx pods.
podName := "nginx"
podLabels := map[string]string{"name": podName}
rsName := "test-adopted-controller"
replicas := 3
image := nginxImage
_, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, podLabels, podName, image))
Expect(err).NotTo(HaveOccurred())
// Verify that the required pods have come up.
err = framework.VerifyPods(unversionedClient, ns, podName, false, 3)
if err != nil {
framework.Logf("error in waiting for pods to come up: %s", err)
Expect(err).NotTo(HaveOccurred())
}
// Create a nginx deployment to adopt the old rs.
deploymentName := "test-adopted-deployment"
framework.Logf("Creating deployment %s", deploymentName)
_, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, podLabels, podName, image, extensions.RollingUpdateDeploymentStrategyType, nil))
Expect(err).NotTo(HaveOccurred())
defer stopDeployment(c, f.Client, ns, deploymentName)
// Wait for it to be updated to revision 1
err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", image)
Expect(err).NotTo(HaveOccurred())
// The RS and pods should be relabeled before the status is updated by syncRollingUpdateDeployment
err = framework.WaitForDeploymentStatus(c, ns, deploymentName, replicas, replicas-1, replicas+1, 0)
Expect(err).NotTo(HaveOccurred())
// There should be no old RSs (overlapping RS)
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
oldRSs, allOldRSs, err := deploymentutil.GetOldReplicaSets(deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(len(oldRSs)).Should(Equal(0))
Expect(len(allOldRSs)).Should(Equal(0))
// New RS should contain pod-template-hash in its selector, label, and template label
newRS, err := deploymentutil.GetNewReplicaSet(deployment, c)
Expect(err).NotTo(HaveOccurred())
err = framework.CheckRSHashLabel(newRS)
Expect(err).NotTo(HaveOccurred())
// All pods targeted by the deployment should contain pod-template-hash in their labels, and there should be only 3 pods
selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
Expect(err).NotTo(HaveOccurred())
options := api.ListOptions{LabelSelector: selector}
pods, err := c.Core().Pods(ns).List(options)
Expect(err).NotTo(HaveOccurred())
err = framework.CheckPodHashLabel(pods)
Expect(err).NotTo(HaveOccurred())
Expect(len(pods.Items)).Should(Equal(replicas))
}
func (c *MasterConfig) RunDaemonSetsController(client *client.Client) {
controller := daemon.NewDaemonSetsController(
clientadapter.FromUnversionedClient(client),
kctrlmgr.ResyncPeriod(c.ControllerManager),
c.ControllerManager.LookupCacheSizeForDaemonSet,
)
go controller.Run(c.ControllerManager.ConcurrentDaemonSetSyncs, utilwait.NeverStop)
}
func (c *MasterConfig) RunReplicaSetController(client *client.Client) {
controller := replicasetcontroller.NewReplicaSetController(
clientadapter.FromUnversionedClient(client),
kctrlmgr.ResyncPeriod(c.ControllerManager),
replicasetcontroller.BurstReplicas,
int(c.ControllerManager.LookupCacheSizeForRC),
)
go controller.Run(int(c.ControllerManager.ConcurrentRSSyncs), utilwait.NeverStop)
}
// RunServiceLoadBalancerController starts the service loadbalancer controller if the cloud provider is configured.
func (c *MasterConfig) RunServiceLoadBalancerController(client *client.Client) {
if c.CloudProvider == nil {
glog.V(2).Infof("Service controller will not start - no cloud provider configured")
return
}
serviceController := servicecontroller.New(c.CloudProvider, clientadapter.FromUnversionedClient(client), c.ControllerManager.ClusterName)
if err := serviceController.Run(c.ControllerManager.ServiceSyncPeriod.Duration, c.ControllerManager.NodeSyncPeriod.Duration); err != nil {
glog.Fatalf("Unable to start service controller: %v", err)
}
}
// RunUnidlingController starts the unidling controller
func (c *MasterConfig) RunUnidlingController() {
oc, kc := c.UnidlingControllerClients()
resyncPeriod := 2 * time.Hour
scaleNamespacer := osclient.NewDelegatingScaleNamespacer(oc, kc)
coreClient := clientadapter.FromUnversionedClient(kc).Core()
dcCoreClient := deployclient.New(oc.RESTClient)
cont := unidlingcontroller.NewUnidlingController(scaleNamespacer, coreClient, coreClient, dcCoreClient, coreClient, resyncPeriod)
cont.Run(utilwait.NeverStop)
}
func (c *MasterConfig) RunPersistentVolumeClaimRecycler(recyclerImageName string, client *client.Client, namespace string) {
uid := int64(0)
defaultScrubPod := volume.NewPersistentVolumeRecyclerPodTemplate()
defaultScrubPod.Namespace = namespace
defaultScrubPod.Spec.Containers[0].Image = recyclerImageName
defaultScrubPod.Spec.Containers[0].Command = []string{"/usr/bin/recycle"}
defaultScrubPod.Spec.Containers[0].Args = []string{"/scrub"}
defaultScrubPod.Spec.Containers[0].SecurityContext = &kapi.SecurityContext{RunAsUser: &uid}
defaultScrubPod.Spec.Containers[0].ImagePullPolicy = kapi.PullIfNotPresent
volumeConfig := c.ControllerManager.VolumeConfiguration
hostPathConfig := volume.VolumeConfig{
RecyclerMinimumTimeout: int(volumeConfig.PersistentVolumeRecyclerConfiguration.MinimumTimeoutHostPath),
RecyclerTimeoutIncrement: int(volumeConfig.PersistentVolumeRecyclerConfiguration.IncrementTimeoutHostPath),
RecyclerPodTemplate: defaultScrubPod,
}
if len(volumeConfig.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath) != 0 {
if err := attemptToLoadRecycler(volumeConfig.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, &hostPathConfig); err != nil {
glog.Fatalf("Could not create hostpath recycler pod from file %s: %+v", volumeConfig.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathHostPath, err)
}
}
nfsConfig := volume.VolumeConfig{
RecyclerMinimumTimeout: int(volumeConfig.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS),
RecyclerTimeoutIncrement: int(volumeConfig.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS),
RecyclerPodTemplate: defaultScrubPod,
}
if len(volumeConfig.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS) != 0 {
if err := attemptToLoadRecycler(volumeConfig.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, &nfsConfig); err != nil {
glog.Fatalf("Could not create NFS recycler pod from file %s: %+v", volumeConfig.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, err)
}
}
allPlugins := []volume.VolumePlugin{}
allPlugins = append(allPlugins, host_path.ProbeVolumePlugins(hostPathConfig)...)
allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(nfsConfig)...)
// dynamic provisioning allows deletion of volumes as a recycling operation after a claim is deleted
allPlugins = append(allPlugins, aws_ebs.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, gce_pd.ProbeVolumePlugins()...)
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
recycler, err := volumeclaimbinder.NewPersistentVolumeRecycler(
clientadapter.FromUnversionedClient(client),
c.ControllerManager.PVClaimBinderSyncPeriod.Duration,
int(volumeConfig.PersistentVolumeRecyclerConfiguration.MaximumRetry),
allPlugins,
c.CloudProvider,
)
if err != nil {
glog.Fatalf("Could not start Persistent Volume Recycler: %+v", err)
}
recycler.Run()
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
config := HollowNodeConfig{}
config.addFlags(pflag.CommandLine)
flag.InitFlags()
if !knownMorphs.Has(config.Morph) {
glog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List())
}
// create a client to communicate with API server.
cl, err := config.createClientFromFile()
clientset := clientset.FromUnversionedClient(cl)
if err != nil {
glog.Fatal("Failed to create a Client. Exiting.")
}
if config.Morph == "kubelet" {
cadvisorInterface := new(cadvisortest.Fake)
containerManager := cm.NewStubContainerManager()
fakeDockerClient := dockertools.NewFakeDockerClient()
fakeDockerClient.EnableSleep = true
hollowKubelet := kubemark.NewHollowKubelet(
config.NodeName,
clientset,
cadvisorInterface,
fakeDockerClient,
config.KubeletPort,
config.KubeletReadOnlyPort,
containerManager,
maxPods,
podsPerCore,
)
hollowKubelet.Run()
}
if config.Morph == "proxy" {
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(api.EventSource{Component: "kube-proxy", Host: config.NodeName})
iptInterface := fakeiptables.NewFake()
serviceConfig := proxyconfig.NewServiceConfig()
serviceConfig.RegisterHandler(&kubemark.FakeProxyHandler{})
endpointsConfig := proxyconfig.NewEndpointsConfig()
endpointsConfig.RegisterHandler(&kubemark.FakeProxyHandler{})
hollowProxy := kubemark.NewHollowProxyOrDie(config.NodeName, cl, endpointsConfig, serviceConfig, iptInterface, eventBroadcaster, recorder)
hollowProxy.Run()
}
}
func testRecreateDeployment(f *framework.Framework) {
ns := f.Namespace.Name
// TODO: remove unversionedClient when the refactoring is done. Currently some
// functions like verifyPod still expects a unversioned#Client.
unversionedClient := f.Client
c := adapter.FromUnversionedClient(unversionedClient)
// Create nginx pods.
deploymentPodLabels := map[string]string{"name": "sample-pod-3"}
rsPodLabels := map[string]string{
"name": "sample-pod-3",
"pod": nginxImageName,
}
rsName := "test-recreate-controller"
replicas := 3
_, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, rsPodLabels, nginxImageName, nginxImage))
Expect(err).NotTo(HaveOccurred())
// Verify that the required pods have come up.
err = framework.VerifyPods(unversionedClient, ns, "sample-pod-3", false, 3)
if err != nil {
framework.Logf("error in waiting for pods to come up: %s", err)
Expect(err).NotTo(HaveOccurred())
}
// Create a deployment to delete nginx pods and instead bring up redis pods.
deploymentName := "test-recreate-deployment"
framework.Logf("Creating deployment %s", deploymentName)
_, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, redisImageName, redisImage, extensions.RecreateDeploymentStrategyType, nil))
Expect(err).NotTo(HaveOccurred())
defer stopDeployment(c, f.Client, ns, deploymentName)
// Wait for it to be updated to revision 1
err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", redisImage)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForDeploymentStatus(c, ns, deploymentName, replicas, 0, replicas, 0)
Expect(err).NotTo(HaveOccurred())
// Verify that the pods were scaled up and down as expected. We use events to verify that.
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
framework.WaitForEvents(unversionedClient, ns, deployment, 2)
events, err := c.Core().Events(ns).Search(deployment)
if err != nil {
framework.Logf("error in listing events: %s", err)
Expect(err).NotTo(HaveOccurred())
}
// There should be 2 events, one to scale up the new ReplicaSet and then to scale down the old ReplicaSet.
Expect(len(events.Items)).Should(Equal(2))
newRS, err := deploymentutil.GetNewReplicaSet(deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(newRS).NotTo(Equal(nil))
Expect(events.Items[0].Message).Should(Equal(fmt.Sprintf("Scaled down replica set %s to 0", rsName)))
Expect(events.Items[1].Message).Should(Equal(fmt.Sprintf("Scaled up replica set %s to 3", newRS.Name)))
}
// RunBuildController starts the build sync loop for builds and buildConfig processing.
func (c *MasterConfig) RunBuildController(informers shared.InformerFactory) error {
// initialize build controller
dockerImage := c.ImageFor("docker-builder")
stiImage := c.ImageFor("sti-builder")
storageVersion := c.Options.EtcdStorageConfig.OpenShiftStorageVersion
groupVersion := unversioned.GroupVersion{Group: "", Version: storageVersion}
codec := kapi.Codecs.LegacyCodec(groupVersion)
admissionControl := admission.InitPlugin("SecurityContextConstraint", clientadapter.FromUnversionedClient(c.PrivilegedLoopbackKubernetesClient), "")
if wantsInformers, ok := admissionControl.(cmdadmission.WantsInformers); ok {
wantsInformers.SetInformers(informers)
}
buildDefaults, err := builddefaults.NewBuildDefaults(c.Options.AdmissionConfig.PluginConfig)
if err != nil {
return err
}
buildOverrides, err := buildoverrides.NewBuildOverrides(c.Options.AdmissionConfig.PluginConfig)
if err != nil {
return err
}
osclient, kclient := c.BuildControllerClients()
factory := buildcontrollerfactory.BuildControllerFactory{
KubeClient: kclient,
OSClient: osclient,
BuildUpdater: buildclient.NewOSClientBuildClient(osclient),
BuildLister: buildclient.NewOSClientBuildClient(osclient),
DockerBuildStrategy: &buildstrategy.DockerBuildStrategy{
Image: dockerImage,
// TODO: this will be set to --storage-version (the internal schema we use)
Codec: codec,
},
SourceBuildStrategy: &buildstrategy.SourceBuildStrategy{
Image: stiImage,
// TODO: this will be set to --storage-version (the internal schema we use)
Codec: codec,
AdmissionControl: admissionControl,
},
CustomBuildStrategy: &buildstrategy.CustomBuildStrategy{
// TODO: this will be set to --storage-version (the internal schema we use)
Codec: codec,
},
BuildDefaults: buildDefaults,
BuildOverrides: buildOverrides,
}
controller := factory.Create()
controller.Run()
deleteController := factory.CreateDeleteController()
deleteController.Run()
return nil
}
// StopRC stops the rc via kubectl's stop library
func StopRC(rc *api.ReplicationController, restClient *client.Client) error {
reaper, err := kubectl.ReaperFor(api.Kind("ReplicationController"), clientsetadapter.FromUnversionedClient(restClient))
if err != nil || reaper == nil {
return err
}
err = reaper.Stop(rc.Namespace, rc.Name, 0, nil)
if err != nil {
return err
}
return nil
}
// RunReplicationController starts the Kubernetes replication controller sync loop
func (c *MasterConfig) RunReplicationController(client *client.Client) {
controllerManager := replicationcontroller.NewReplicationManager(
c.Informers.Pods().Informer(),
clientadapter.FromUnversionedClient(client),
kctrlmgr.ResyncPeriod(c.ControllerManager),
replicationcontroller.BurstReplicas,
int(c.ControllerManager.LookupCacheSizeForRC),
c.ControllerManager.EnableGarbageCollector,
)
go controllerManager.Run(int(c.ControllerManager.ConcurrentRCSyncs), utilwait.NeverStop)
}
// Clients returns an OpenShift and Kubernetes client.
func (f *Factory) Clients() (*client.Client, *kclient.Client, *kclientset.Clientset, error) {
kClient, err := f.Client()
if err != nil {
return nil, nil, nil, err
}
kClientset := adapter.FromUnversionedClient(kClient)
osClient, err := f.clients.ClientForVersion(nil)
if err != nil {
return nil, nil, nil, err
}
return osClient, kClient, kClientset, nil
}
// RunServiceLoadBalancerController starts the service loadbalancer controller if the cloud provider is configured.
func (c *MasterConfig) RunServiceLoadBalancerController(client *client.Client) {
if c.CloudProvider == nil {
glog.V(2).Infof("Service controller will not start - no cloud provider configured")
return
}
serviceController, err := servicecontroller.New(c.CloudProvider, clientadapter.FromUnversionedClient(client), c.ControllerManager.ClusterName)
if err != nil {
glog.Errorf("Unable to start service controller: %v", err)
} else {
serviceController.Run(int(c.ControllerManager.ConcurrentServiceSyncs))
}
}
// RunHPAController starts the Kubernetes hpa controller sync loop
func (c *MasterConfig) RunHPAController(oc *osclient.Client, kc *client.Client, heapsterNamespace string) {
clientsetClient := clientadapter.FromUnversionedClient(kc)
delegatingScaleNamespacer := osclient.NewDelegatingScaleNamespacer(oc, kc)
podautoscaler := podautoscalercontroller.NewHorizontalController(
clientsetClient,
delegatingScaleNamespacer,
clientsetClient,
metrics.NewHeapsterMetricsClient(clientsetClient, heapsterNamespace, "https", "heapster", ""),
c.ControllerManager.HorizontalPodAutoscalerSyncPeriod.Duration,
)
go podautoscaler.Run(utilwait.NeverStop)
}
func testScalePausedDeployment(f *framework.Framework) {
ns := f.Namespace.Name
c := adapter.FromUnversionedClient(f.Client)
podLabels := map[string]string{"name": nginxImageName}
replicas := int32(3)
// Create a nginx deployment.
deploymentName := "nginx-deployment"
d := newDeployment(deploymentName, replicas, podLabels, nginxImageName, nginxImage, extensions.RollingUpdateDeploymentStrategyType, nil)
framework.Logf("Creating deployment %q", deploymentName)
_, err := c.Extensions().Deployments(ns).Create(d)
Expect(err).NotTo(HaveOccurred())
defer stopDeployment(c, f.Client, ns, deploymentName)
// Check that deployment is created fine.
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForObservedDeployment(c, ns, deploymentName, deployment.Generation)
Expect(err).NotTo(HaveOccurred())
rs, err := deploymentutil.GetNewReplicaSet(deployment, c)
Expect(err).NotTo(HaveOccurred())
// Pause the deployment and try to scale it.
deployment, err = framework.UpdateDeploymentWithRetries(c, ns, d.Name, func(update *extensions.Deployment) {
update.Spec.Paused = true
})
Expect(err).NotTo(HaveOccurred())
// Scale the paused deployment.
framework.Logf("Scaling up the paused deployment %q", deploymentName)
newReplicas := int32(5)
deployment, err = framework.UpdateDeploymentWithRetries(c, ns, deployment.Name, func(update *extensions.Deployment) {
update.Spec.Replicas = newReplicas
})
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForObservedDeployment(c, ns, deploymentName, deployment.Generation)
Expect(err).NotTo(HaveOccurred())
rs, err = deploymentutil.GetNewReplicaSet(deployment, c)
Expect(err).NotTo(HaveOccurred())
if rs.Spec.Replicas != newReplicas {
err = fmt.Errorf("Expected %d replicas for the new replica set, got %d", newReplicas, rs.Spec.Replicas)
Expect(err).NotTo(HaveOccurred())
}
}
func testRollingUpdateDeployment(f *framework.Framework) {
ns := f.Namespace.Name
// TODO: remove unversionedClient when the refactoring is done. Currently some
// functions like verifyPod still expects a unversioned#Client.
unversionedClient := f.Client
c := adapter.FromUnversionedClient(unversionedClient)
// Create nginx pods.
deploymentPodLabels := map[string]string{"name": "sample-pod"}
rsPodLabels := map[string]string{
"name": "sample-pod",
"pod": nginxImageName,
}
rsName := "test-rolling-update-controller"
replicas := 3
_, err := c.Extensions().ReplicaSets(ns).Create(newRS(rsName, replicas, rsPodLabels, nginxImageName, nginxImage))
Expect(err).NotTo(HaveOccurred())
// Verify that the required pods have come up.
err = framework.VerifyPods(unversionedClient, ns, "sample-pod", false, 3)
if err != nil {
framework.Logf("error in waiting for pods to come up: %s", err)
Expect(err).NotTo(HaveOccurred())
}
// Create a deployment to delete nginx pods and instead bring up redis pods.
deploymentName := "test-rolling-update-deployment"
framework.Logf("Creating deployment %s", deploymentName)
_, err = c.Extensions().Deployments(ns).Create(newDeployment(deploymentName, replicas, deploymentPodLabels, redisImageName, redisImage, extensions.RollingUpdateDeploymentStrategyType, nil))
Expect(err).NotTo(HaveOccurred())
defer stopDeployment(c, f.Client, ns, deploymentName)
// Wait for it to be updated to revision 1
err = framework.WaitForDeploymentRevisionAndImage(c, ns, deploymentName, "1", redisImage)
Expect(err).NotTo(HaveOccurred())
err = framework.WaitForDeploymentStatus(c, ns, deploymentName, replicas, replicas-1, replicas+1, 0)
Expect(err).NotTo(HaveOccurred())
// There should be 1 old RS (nginx-controller, which is adopted)
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
_, allOldRSs, err := deploymentutil.GetOldReplicaSets(deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(len(allOldRSs)).Should(Equal(1))
// The old RS should contain pod-template-hash in its selector, label, and template label
Expect(len(allOldRSs[0].Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
Expect(len(allOldRSs[0].Spec.Selector.MatchLabels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
Expect(len(allOldRSs[0].Spec.Template.Labels[extensions.DefaultDeploymentUniqueLabelKey])).Should(BeNumerically(">", 0))
}
func GetClientForServiceAccount(adminClient *kclientset.Clientset, clientConfig restclient.Config, namespace, name string) (*client.Client, *kclientset.Clientset, *restclient.Config, error) {
_, err := adminClient.Core().Namespaces().Create(&kapi.Namespace{ObjectMeta: kapi.ObjectMeta{Name: namespace}})
if err != nil && !kerrs.IsAlreadyExists(err) {
return nil, nil, nil, err
}
sa, err := adminClient.Core().ServiceAccounts(namespace).Create(&kapi.ServiceAccount{ObjectMeta: kapi.ObjectMeta{Name: name}})
if kerrs.IsAlreadyExists(err) {
sa, err = adminClient.Core().ServiceAccounts(namespace).Get(name)
}
if err != nil {
return nil, nil, nil, err
}
token := ""
err = wait.Poll(time.Second, 30*time.Second, func() (bool, error) {
selector := fields.OneTermEqualSelector(kapi.SecretTypeField, string(kapi.SecretTypeServiceAccountToken))
secrets, err := adminClient.Core().Secrets(namespace).List(kapi.ListOptions{FieldSelector: selector})
if err != nil {
return false, err
}
for _, secret := range secrets.Items {
if serviceaccounts.IsValidServiceAccountToken(sa, &secret) {
token = string(secret.Data[kapi.ServiceAccountTokenKey])
return true, nil
}
}
return false, nil
})
if err != nil {
return nil, nil, nil, err
}
saClientConfig := clientcmd.AnonymousClientConfig(&clientConfig)
saClientConfig.BearerToken = token
kubeClient, err := kclient.New(&saClientConfig)
if err != nil {
return nil, nil, nil, err
}
kubeClientset := adapter.FromUnversionedClient(kubeClient)
osClient, err := client.New(&saClientConfig)
if err != nil {
return nil, nil, nil, err
}
return osClient, kubeClientset, &saClientConfig, nil
}
// RunServiceAccountTokensController starts the service account token controller
func (c *MasterConfig) RunServiceAccountTokensController(cm *cmapp.CMServer) {
if len(c.Options.ServiceAccountConfig.PrivateKeyFile) == 0 {
glog.Infof("Skipped starting Service Account Token Manager, no private key specified")
return
}
privateKey, err := serviceaccount.ReadPrivateKey(c.Options.ServiceAccountConfig.PrivateKeyFile)
if err != nil {
glog.Fatalf("Error reading signing key for Service Account Token Manager: %v", err)
}
rootCA := []byte{}
if len(c.Options.ServiceAccountConfig.MasterCA) > 0 {
rootCA, err = ioutil.ReadFile(c.Options.ServiceAccountConfig.MasterCA)
if err != nil {
glog.Fatalf("Error reading master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
}
if _, err := kcrypto.CertsFromPEM(rootCA); err != nil {
glog.Fatalf("Error parsing master ca file for Service Account Token Manager: %s: %v", c.Options.ServiceAccountConfig.MasterCA, err)
}
}
servingServingCABundle := []byte{}
if c.Options.ControllerConfig.ServiceServingCert.Signer != nil && len(c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile) > 0 {
servingServingCA, err := ioutil.ReadFile(c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile)
if err != nil {
glog.Fatalf("Error reading ca file for Service Serving Certificate Signer: %s: %v", c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile, err)
}
if _, err := kcrypto.CertsFromPEM(servingServingCA); err != nil {
glog.Fatalf("Error parsing ca file for Service Serving Certificate Signer: %s: %v", c.Options.ControllerConfig.ServiceServingCert.Signer.CertFile, err)
}
// if we have a rootCA bundle add that too. The rootCA will be used when hitting the default master service, since those are signed
// using a different CA by default. The rootCA's key is more closely guarded than ours and if it is compromised, that power could
// be used to change the trusted signers for every pod anyway, so we're already effectively trusting it.
if len(rootCA) > 0 {
servingServingCABundle = append(servingServingCABundle, rootCA...)
servingServingCABundle = append(servingServingCABundle, []byte("\n")...)
}
servingServingCABundle = append(servingServingCABundle, servingServingCA...)
}
options := sacontroller.TokensControllerOptions{
TokenGenerator: serviceaccount.JWTTokenGenerator(privateKey),
RootCA: rootCA,
ServiceServingCA: servingServingCABundle,
}
go sacontroller.NewTokensController(clientadapter.FromUnversionedClient(c.KubeClient()), options).Run(int(cm.ConcurrentSATokenSyncs), utilwait.NeverStop)
}
// RunServiceAccountsController starts the service account controller
func (c *MasterConfig) RunServiceAccountsController() {
if len(c.Options.ServiceAccountConfig.ManagedNames) == 0 {
glog.Infof("Skipped starting Service Account Manager, no managed names specified")
return
}
options := sacontroller.DefaultServiceAccountsControllerOptions()
options.ServiceAccounts = []kapi.ServiceAccount{}
for _, saName := range c.Options.ServiceAccountConfig.ManagedNames {
sa := kapi.ServiceAccount{}
sa.Name = saName
options.ServiceAccounts = append(options.ServiceAccounts, sa)
}
sacontroller.NewServiceAccountsController(clientadapter.FromUnversionedClient(c.KubeClient()), options).Run()
}