func CreateEssentialAddons(s *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
arches := [3]string{"amd64", "arm", "arm64"}
for _, arch := range arches {
kubeProxyDaemonSet := NewDaemonSet(kubeProxy+"-"+arch, createKubeProxyPodSpec(s, arch))
SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta)
if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-proxy addon [%v]", err)
}
}
fmt.Println("<master/addons> created essential addon: kube-proxy")
kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(s))
SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta)
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon [%v]", err)
}
kubeDNSServiceSpec, err := createKubeDNSServiceSpec(s)
if err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon - %v", err)
}
kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec)
if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon [%v]", err)
}
fmt.Println("<master/addons> created essential addon: kube-dns")
return nil
}
func CreateDiscoveryDeploymentAndSecret(s *kubeadmapi.MasterConfiguration, client *clientset.Clientset, caCert *x509.Certificate) error {
kd := newKubeDiscovery(s, caCert)
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil {
return fmt.Errorf("<master/discovery> failed to create %q deployment [%v]", kubeDiscoveryName, err)
}
if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil {
return fmt.Errorf("<master/discovery> failed to create %q secret [%v]", kubeDiscoverySecretName, err)
}
fmt.Println("<master/discovery> created essential addon: kube-discovery, waiting for it to become ready")
start := time.Now()
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
d, err := client.Extensions().Deployments(api.NamespaceSystem).Get(kubeDiscoveryName)
if err != nil {
return false, nil
}
if d.Status.AvailableReplicas < 1 {
return false, nil
}
return true, nil
})
fmt.Printf("<master/discovery> kube-discovery is ready after %f seconds\n", time.Since(start).Seconds())
return nil
}
func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(cfg))
SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta)
SetNodeAffinity(&kubeProxyDaemonSet.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity())
if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-proxy addon [%v]", err)
}
fmt.Println("<master/addons> created essential addon: kube-proxy")
kubeDNSDeployment := NewDeployment("kube-dns", 1, createKubeDNSPodSpec(cfg))
SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta)
SetNodeAffinity(&kubeDNSDeployment.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity())
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon [%v]", err)
}
kubeDNSServiceSpec, err := createKubeDNSServiceSpec(cfg)
if err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon - %v", err)
}
kubeDNSService := NewService("kube-dns", *kubeDNSServiceSpec)
if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil {
return fmt.Errorf("<master/addons> failed creating essential kube-dns addon [%v]", err)
}
fmt.Println("<master/addons> created essential addon: kube-dns")
return nil
}
// checkDeploymentRevision checks if the input deployment's and its new replica set's revision and images are as expected.
func checkDeploymentRevision(c *clientset.Clientset, ns, deploymentName, revision, imageName, image string) (*extensions.Deployment, *extensions.ReplicaSet) {
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
// Check revision of the new replica set of this deployment
newRS, err := deploymentutil.GetNewReplicaSet(deployment, c)
Expect(err).NotTo(HaveOccurred())
Expect(newRS.Annotations).NotTo(Equal(nil))
Expect(newRS.Annotations[deploymentutil.RevisionAnnotation]).Should(Equal(revision))
// Check revision of This deployment
Expect(deployment.Annotations).NotTo(Equal(nil))
Expect(deployment.Annotations[deploymentutil.RevisionAnnotation]).Should(Equal(revision))
if len(imageName) > 0 {
// Check the image the new replica set creates
Expect(newRS.Spec.Template.Spec.Containers[0].Name).Should(Equal(imageName))
Expect(newRS.Spec.Template.Spec.Containers[0].Image).Should(Equal(image))
// Check the image the deployment creates
Expect(deployment.Spec.Template.Spec.Containers[0].Name).Should(Equal(imageName))
Expect(deployment.Spec.Template.Spec.Containers[0].Image).Should(Equal(image))
}
return deployment, newRS
}
func stopDeployment(c *clientset.Clientset, oldC client.Interface, ns, deploymentName string) {
deployment, err := c.Extensions().Deployments(ns).Get(deploymentName)
Expect(err).NotTo(HaveOccurred())
Logf("deleting deployment %s", deploymentName)
reaper, err := kubectl.ReaperFor(extensions.Kind("Deployment"), oldC)
Expect(err).NotTo(HaveOccurred())
timeout := 1 * time.Minute
err = reaper.Stop(ns, deployment.Name, timeout, api.NewDeleteOptions(0))
Expect(err).NotTo(HaveOccurred())
Logf("ensuring deployment %s was deleted", deploymentName)
_, err = c.Extensions().Deployments(ns).Get(deployment.Name)
Expect(err).To(HaveOccurred())
Expect(errors.IsNotFound(err)).To(BeTrue())
Logf("ensuring deployment %s rcs were deleted", deploymentName)
selector, err := unversioned.LabelSelectorAsSelector(deployment.Spec.Selector)
Expect(err).NotTo(HaveOccurred())
options := api.ListOptions{LabelSelector: selector}
rss, err := c.Extensions().ReplicaSets(ns).List(options)
Expect(err).NotTo(HaveOccurred())
Expect(rss.Items).Should(HaveLen(0))
Logf("ensuring deployment %s pods were deleted", deploymentName)
var pods *api.PodList
if err := wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
pods, err = c.Core().Pods(ns).List(api.ListOptions{})
if err != nil {
return false, err
}
if len(pods.Items) == 0 {
return true, nil
}
return false, nil
}); err != nil {
Failf("Err : %s\n. Failed to remove deployment %s pods : %+v", err, deploymentName, pods)
}
}
func createDummyDeployment(client *clientset.Clientset) {
fmt.Println("<master/apiclient> attempting a test deployment")
dummyDeployment := NewDeployment("dummy", 1, api.PodSpec{
SecurityContext: &api.PodSecurityContext{HostNetwork: true},
Containers: []api.Container{{
Name: "dummy",
Image: images.GetAddonImage("pause"),
}},
})
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
// TODO: we should check the error, as some cases may be fatal
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(dummyDeployment); err != nil {
fmt.Printf("<master/apiclient> failed to create test deployment [%v] (will retry)", err)
return false, nil
}
return true, nil
})
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
d, err := client.Extensions().Deployments(api.NamespaceSystem).Get("dummy")
if err != nil {
fmt.Printf("<master/apiclient> failed to get test deployment [%v] (will retry)", err)
return false, nil
}
if d.Status.AvailableReplicas < 1 {
return false, nil
}
return true, nil
})
fmt.Println("<master/apiclient> test deployment succeeded")
if err := client.Extensions().Deployments(api.NamespaceSystem).Delete("dummy", &api.DeleteOptions{}); err != nil {
fmt.Printf("<master/apiclient> failed to delete test deployment [%v] (will ignore)", err)
}
}
func createControllerManager(clientset *client.Clientset, namespace, name, cmName, image, kubeconfigName, dnsZoneName, dnsProvider string) (*extensions.Deployment, error) {
dep := &extensions.Deployment{
ObjectMeta: api.ObjectMeta{
Name: cmName,
Namespace: namespace,
Labels: componentLabel,
},
Spec: extensions.DeploymentSpec{
Replicas: 1,
Template: api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Name: cmName,
Labels: controllerManagerPodLabels,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "controller-manager",
Image: image,
Command: []string{
"/hyperkube",
"federation-controller-manager",
"--master=https://federation-apiserver",
"--kubeconfig=/etc/federation/controller-manager/kubeconfig",
fmt.Sprintf("--dns-provider=%s", dnsProvider),
"--dns-provider-config=",
fmt.Sprintf("--federation-name=%s", name),
fmt.Sprintf("--zone-name=%s", dnsZoneName),
},
VolumeMounts: []api.VolumeMount{
{
Name: kubeconfigName,
MountPath: "/etc/federation/controller-manager",
ReadOnly: true,
},
},
Env: []api.EnvVar{
{
Name: "POD_NAMESPACE",
ValueFrom: &api.EnvVarSource{
FieldRef: &api.ObjectFieldSelector{
FieldPath: "metadata.namespace",
},
},
},
},
},
},
Volumes: []api.Volume{
{
Name: kubeconfigName,
VolumeSource: api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: kubeconfigName,
},
},
},
},
},
},
},
}
return clientset.Extensions().Deployments(namespace).Create(dep)
}
func createAPIServer(clientset *client.Clientset, namespace, name, image, credentialsName, pvcName, advertiseAddress string) (*extensions.Deployment, error) {
command := []string{
"/hyperkube",
"federation-apiserver",
"--bind-address=0.0.0.0",
"--etcd-servers=http://localhost:2379",
"--service-cluster-ip-range=10.0.0.0/16",
"--secure-port=443",
"--client-ca-file=/etc/federation/apiserver/ca.crt",
"--tls-cert-file=/etc/federation/apiserver/server.crt",
"--tls-private-key-file=/etc/federation/apiserver/server.key",
}
if advertiseAddress != "" {
command = append(command, fmt.Sprintf("--advertise-address=%s", advertiseAddress))
}
dataVolumeName := "etcddata"
dep := &extensions.Deployment{
ObjectMeta: api.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: componentLabel,
},
Spec: extensions.DeploymentSpec{
Replicas: 1,
Template: api.PodTemplateSpec{
ObjectMeta: api.ObjectMeta{
Name: name,
Labels: apiserverPodLabels,
},
Spec: api.PodSpec{
Containers: []api.Container{
{
Name: "apiserver",
Image: image,
Command: command,
Ports: []api.ContainerPort{
{
Name: "https",
ContainerPort: 443,
},
{
Name: "local",
ContainerPort: 8080,
},
},
VolumeMounts: []api.VolumeMount{
{
Name: credentialsName,
MountPath: "/etc/federation/apiserver",
ReadOnly: true,
},
},
},
{
Name: "etcd",
Image: "quay.io/coreos/etcd:v2.3.3",
Command: []string{
"/etcd",
"--data-dir",
"/var/etcd/data",
},
VolumeMounts: []api.VolumeMount{
{
Name: dataVolumeName,
MountPath: "/var/etcd",
},
},
},
},
Volumes: []api.Volume{
{
Name: credentialsName,
VolumeSource: api.VolumeSource{
Secret: &api.SecretVolumeSource{
SecretName: credentialsName,
},
},
},
{
Name: dataVolumeName,
VolumeSource: api.VolumeSource{
PersistentVolumeClaim: &api.PersistentVolumeClaimVolumeSource{
ClaimName: pvcName,
},
},
},
},
},
},
},
}
return clientset.Extensions().Deployments(namespace).Create(dep)
}