func attemptToUpdateMasterRoleLabelsAndTaints(client *clientset.Clientset, schedulable bool) error {
n, err := findMyself(client)
if err != nil {
return err
}
n.ObjectMeta.Labels[metav1.NodeLabelKubeadmAlphaRole] = metav1.NodeLabelRoleMaster
if !schedulable {
taintsAnnotation, _ := json.Marshal([]v1.Taint{{Key: "dedicated", Value: "master", Effect: "NoSchedule"}})
n.ObjectMeta.Annotations[v1.TaintsAnnotationKey] = string(taintsAnnotation)
}
if _, err := client.Nodes().Update(n); err != nil {
if apierrs.IsConflict(err) {
fmt.Println("[apiclient] Temporarily unable to update master node metadata due to conflict (will retry)")
time.Sleep(apiCallRetryInterval)
attemptToUpdateMasterRoleLabelsAndTaints(client, schedulable)
} else {
return err
}
}
return nil
}
func createConfig(s *options.SchedulerServer, kubecli *clientset.Clientset) (*scheduler.Config, error) {
configFactory := factory.NewConfigFactory(kubecli, s.SchedulerName, s.HardPodAffinitySymmetricWeight, s.FailureDomains)
if _, err := os.Stat(s.PolicyConfigFile); err == nil {
var (
policy schedulerapi.Policy
configData []byte
)
configData, err := ioutil.ReadFile(s.PolicyConfigFile)
if err != nil {
return nil, fmt.Errorf("unable to read policy config: %v", err)
}
if err := runtime.DecodeInto(latestschedulerapi.Codec, configData, &policy); err != nil {
return nil, fmt.Errorf("invalid configuration: %v", err)
}
return configFactory.CreateFromConfig(policy)
}
// if the config file isn't provided, use the specified (or default) provider
config, err := configFactory.CreateFromProvider(s.AlgorithmProvider)
if err != nil {
return nil, err
}
eventBroadcaster := record.NewBroadcaster()
config.Recorder = eventBroadcaster.NewRecorder(v1.EventSource{Component: s.SchedulerName})
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubecli.Core().Events("")})
return config, nil
}
// TODO: quinton: This is largely a cut 'n paste of the above. Yuck! Refactor as soon as we have a common interface implmented by both fedclientset.Clientset and kubeclientset.Clientset
func deleteClusterIngressOrFail(clusterName string, clientset *kubeclientset.Clientset, namespace string, ingressName string) {
if clientset == nil || len(namespace) == 0 || len(ingressName) == 0 {
Fail(fmt.Sprintf("Internal error: invalid parameters passed to deleteClusterIngressOrFail: cluster: %q, clientset: %v, namespace: %v, ingress: %v", clusterName, clientset, namespace, ingressName))
}
err := clientset.Ingresses(namespace).Delete(ingressName, v1.NewDeleteOptions(0))
framework.ExpectNoError(err, "Error deleting cluster ingress %q/%q from cluster %q", namespace, ingressName, clusterName)
}
func waitForQuota(t *testing.T, quota *v1.ResourceQuota, clientset *clientset.Clientset) {
w, err := clientset.Core().ResourceQuotas(quota.Namespace).Watch(v1.SingleObject(v1.ObjectMeta{Name: quota.Name}))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := clientset.Core().ResourceQuotas(quota.Namespace).Create(quota); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = watch.Until(1*time.Minute, w, func(event watch.Event) (bool, error) {
switch event.Type {
case watch.Modified:
default:
return false, nil
}
switch cast := event.Object.(type) {
case *v1.ResourceQuota:
if len(cast.Status.Hard) > 0 {
return true, nil
}
}
return false, nil
})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
// CreateBootstrapRBACClusterRole creates the necessary ClusterRole for bootstrapping
func CreateBootstrapRBACClusterRole(clientset *clientset.Clientset) error {
clusterRole := rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: "kubeadm:kubelet-bootstrap"},
Rules: []rbac.PolicyRule{
rbac.NewRule("get").Groups("").Resources("nodes").RuleOrDie(),
rbac.NewRule("create", "watch").Groups("certificates.k8s.io").Resources("certificatesigningrequests").RuleOrDie(),
},
}
if _, err := clientset.Rbac().ClusterRoles().Create(&clusterRole); err != nil {
return err
}
subject := rbac.Subject{
Kind: "Group",
Name: "kubeadm:kubelet-bootstrap",
}
clusterRoleBinding := rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "kubeadm:kubelet-bootstrap",
},
RoleRef: rbac.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "kubeadm:kubelet-bootstrap",
},
Subjects: []rbac.Subject{subject},
}
if _, err := clientset.Rbac().ClusterRoleBindings().Create(&clusterRoleBinding); err != nil {
return err
}
fmt.Println("[apiconfig] Created kubelet-bootstrap RBAC rules")
return nil
}
func scale(t *testing.T, namespace string, clientset *clientset.Clientset) {
target := int32(100)
rc := &v1.ReplicationController{
ObjectMeta: v1.ObjectMeta{
Name: "foo",
Namespace: namespace,
},
Spec: v1.ReplicationControllerSpec{
Replicas: &target,
Selector: map[string]string{"foo": "bar"},
Template: &v1.PodTemplateSpec{
ObjectMeta: v1.ObjectMeta{
Labels: map[string]string{
"foo": "bar",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "container",
Image: "busybox",
},
},
},
},
},
}
w, err := clientset.Core().ReplicationControllers(namespace).Watch(v1.SingleObject(v1.ObjectMeta{Name: rc.Name}))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if _, err := clientset.Core().ReplicationControllers(namespace).Create(rc); err != nil {
t.Fatalf("unexpected error: %v", err)
}
_, err = watch.Until(3*time.Minute, w, func(event watch.Event) (bool, error) {
switch event.Type {
case watch.Modified:
default:
return false, nil
}
switch cast := event.Object.(type) {
case *v1.ReplicationController:
fmt.Printf("Found %v of %v replicas\n", int(cast.Status.Replicas), target)
if cast.Status.Replicas == target {
return true, nil
}
}
return false, nil
})
if err != nil {
pods, _ := clientset.Core().Pods(namespace).List(v1.ListOptions{LabelSelector: labels.Everything().String(), FieldSelector: fields.Everything().String()})
t.Fatalf("unexpected error: %v, ended with %v pods", err, len(pods.Items))
}
}
// checkForNodeNameDuplicates checks whether there are other nodes in the cluster with identical node names.
func checkForNodeNameDuplicates(clientSet *clientset.Clientset) error {
hostName, err := os.Hostname()
if err != nil {
return fmt.Errorf("Failed to get node hostname [%v]", err)
}
_, err = clientSet.Nodes().Get(hostName, metav1.GetOptions{})
if err != nil && !apierrs.IsNotFound(err) {
return err
}
return nil
}
// It's safe to do this for alpha, as we don't have HA and there is no way we can get
// more then one node here (TODO(phase1+) use os.Hostname)
func findMyself(client *clientset.Clientset) (*v1.Node, error) {
nodeList, err := client.Nodes().List(v1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("unable to list nodes [%v]", err)
}
if len(nodeList.Items) < 1 {
return nil, fmt.Errorf("no nodes found")
}
node := &nodeList.Items[0]
return node, nil
}
// getNode gets node object from the apiserver.
func getNode(c *clientset.Clientset) (*v1.Node, error) {
nodes, err := c.Nodes().List(v1.ListOptions{})
Expect(err).NotTo(HaveOccurred(), "should be able to list nodes.")
if nodes == nil {
return nil, fmt.Errorf("the node list is nil.")
}
Expect(len(nodes.Items) > 1).NotTo(BeTrue(), "should not be more than 1 nodes.")
if len(nodes.Items) == 0 {
return nil, fmt.Errorf("empty node list: %+v", nodes)
}
return &nodes.Items[0], nil
}
func getReferencedServiceAccountToken(c *clientset.Clientset, ns string, name string, shouldWait bool) (string, string, error) {
tokenName := ""
token := ""
findToken := func() (bool, error) {
user, err := c.Core().ServiceAccounts(ns).Get(name, metav1.GetOptions{})
if errors.IsNotFound(err) {
return false, nil
}
if err != nil {
return false, err
}
for _, ref := range user.Secrets {
secret, err := c.Core().Secrets(ns).Get(ref.Name, metav1.GetOptions{})
if errors.IsNotFound(err) {
continue
}
if err != nil {
return false, err
}
if secret.Type != v1.SecretTypeServiceAccountToken {
continue
}
name := secret.Annotations[v1.ServiceAccountNameKey]
uid := secret.Annotations[v1.ServiceAccountUIDKey]
tokenData := secret.Data[v1.ServiceAccountTokenKey]
if name == user.Name && uid == string(user.UID) && len(tokenData) > 0 {
tokenName = secret.Name
token = string(tokenData)
return true, nil
}
}
return false, nil
}
if shouldWait {
err := wait.Poll(time.Second, 10*time.Second, findToken)
if err != nil {
return "", "", err
}
} else {
ok, err := findToken()
if err != nil {
return "", "", err
}
if !ok {
return "", "", fmt.Errorf("No token found for %s/%s", ns, name)
}
}
return tokenName, token, nil
}
func cleanupServiceShard(clientset *kubeclientset.Clientset, clusterName, namespace string, service *v1.Service, timeout time.Duration) error {
err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
err := clientset.Services(namespace).Delete(service.Name, &v1.DeleteOptions{})
if err != nil && !errors.IsNotFound(err) {
// Deletion failed with an error, try again.
framework.Logf("Failed to delete service %q in namespace %q, in cluster %q", service.Name, namespace, clusterName)
return false, nil
}
By(fmt.Sprintf("Service %q in namespace %q in cluster %q deleted", service.Name, namespace, clusterName))
return true, nil
})
return err
}
// CreateKubeProxyClusterRoleBinding creates the necessary ClusterRole for kube-dns
func CreateKubeProxyClusterRoleBinding(clientset *clientset.Clientset) error {
systemKubeProxySubject := rbac.Subject{
Kind: "User",
Name: "system:kube-proxy",
Namespace: api.NamespaceSystem,
}
systemNodesSubject := rbac.Subject{
Kind: "Group",
Name: "system:nodes",
Namespace: api.NamespaceSystem,
}
clusterRoleBinding := rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "system:node-proxier",
},
RoleRef: rbac.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "system:node-proxier",
},
Subjects: []rbac.Subject{systemKubeProxySubject, systemNodesSubject},
}
if _, err := clientset.Rbac().ClusterRoleBindings().Update(&clusterRoleBinding); err != nil {
return err
}
fmt.Println("[apiconfig] Created kube-proxy RBAC rules")
return nil
}
/*
waitForIngressUpdateOrFail waits until a ingress is updated in the specified cluster with same spec of federated ingress.
If the condition is not met within timeout, it fails the calling test.
*/
func waitForIngressUpdateOrFail(clientset *kubeclientset.Clientset, namespace string, ingress *v1beta1.Ingress, timeout time.Duration) {
By(fmt.Sprintf("Fetching a federated ingress shard of ingress %q in namespace %q from cluster", ingress.Name, namespace))
err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
clusterIngress, err := clientset.Ingresses(namespace).Get(ingress.Name, metav1.GetOptions{})
if err == nil { // We want it present, and the Get succeeded, so we're all good.
if equivalentIngress(*clusterIngress, *ingress) {
By(fmt.Sprintf("Success: shard of federated ingress %q in namespace %q in cluster is updated", ingress.Name, namespace))
return true, nil
}
By(fmt.Sprintf("Ingress %q in namespace %q in cluster, waiting for service being updated, trying again in %s (err=%v)", ingress.Name, namespace, framework.Poll, err))
return false, nil
}
By(fmt.Sprintf("Ingress %q in namespace %q in cluster, waiting for service being updated, trying again in %s (err=%v)", ingress.Name, namespace, framework.Poll, err))
return false, nil
})
framework.ExpectNoError(err, "Failed to verify ingress %q in namespace %q in cluster", ingress.Name, namespace)
}
func doServiceAccountAPIRequests(t *testing.T, c *clientset.Clientset, ns string, authenticated bool, canRead bool, canWrite bool) {
testSecret := &v1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "testSecret"},
Data: map[string][]byte{"test": []byte("data")},
}
readOps := []testOperation{
func() error {
_, err := c.Core().Secrets(ns).List(v1.ListOptions{})
return err
},
func() error {
_, err := c.Core().Pods(ns).List(v1.ListOptions{})
return err
},
}
writeOps := []testOperation{
func() error { _, err := c.Core().Secrets(ns).Create(testSecret); return err },
func() error { return c.Core().Secrets(ns).Delete(testSecret.Name, nil) },
}
for _, op := range readOps {
err := op()
unauthorizedError := errors.IsUnauthorized(err)
forbiddenError := errors.IsForbidden(err)
switch {
case !authenticated && !unauthorizedError:
t.Fatalf("expected unauthorized error, got %v", err)
case authenticated && unauthorizedError:
t.Fatalf("unexpected unauthorized error: %v", err)
case authenticated && canRead && forbiddenError:
t.Fatalf("unexpected forbidden error: %v", err)
case authenticated && !canRead && !forbiddenError:
t.Fatalf("expected forbidden error, got: %v", err)
}
}
for _, op := range writeOps {
err := op()
unauthorizedError := errors.IsUnauthorized(err)
forbiddenError := errors.IsForbidden(err)
switch {
case !authenticated && !unauthorizedError:
t.Fatalf("expected unauthorized error, got %v", err)
case authenticated && unauthorizedError:
t.Fatalf("unexpected unauthorized error: %v", err)
case authenticated && canWrite && forbiddenError:
t.Fatalf("unexpected forbidden error: %v", err)
case authenticated && !canWrite && !forbiddenError:
t.Fatalf("expected forbidden error, got: %v", err)
}
}
}
func getServiceAccount(c *clientset.Clientset, ns string, name string, shouldWait bool) (*v1.ServiceAccount, error) {
if !shouldWait {
return c.Core().ServiceAccounts(ns).Get(name, metav1.GetOptions{})
}
var user *v1.ServiceAccount
var err error
err = wait.Poll(time.Second, 10*time.Second, func() (bool, error) {
user, err = c.Core().ServiceAccounts(ns).Get(name, metav1.GetOptions{})
if errors.IsNotFound(err) {
return false, nil
}
if err != nil {
return false, err
}
return true, nil
})
return user, err
}
func waitForSecretUpdateOrFail(clientset *kubeclientset.Clientset, nsName string, secret *v1.Secret, timeout time.Duration) {
By(fmt.Sprintf("Fetching a federated secret shard of secret %q in namespace %q from cluster", secret.Name, nsName))
err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
clusterSecret, err := clientset.Core().Secrets(nsName).Get(secret.Name, metav1.GetOptions{})
if err == nil { // We want it present, and the Get succeeded, so we're all good.
if util.SecretEquivalent(*clusterSecret, *secret) {
By(fmt.Sprintf("Success: shard of federated secret %q in namespace %q in cluster is updated", secret.Name, nsName))
return true, nil
} else {
By(fmt.Sprintf("Expected equal secrets. expected: %+v\nactual: %+v", *secret, *clusterSecret))
}
By(fmt.Sprintf("Secret %q in namespace %q in cluster, waiting for secret being updated, trying again in %s (err=%v)", secret.Name, nsName, framework.Poll, err))
return false, nil
}
By(fmt.Sprintf("Secret %q in namespace %q in cluster, waiting for being updated, trying again in %s (err=%v)", secret.Name, nsName, framework.Poll, err))
return false, nil
})
framework.ExpectNoError(err, "Failed to verify secret %q in namespace %q in cluster", secret.Name, nsName)
}
func waitForPersistentVolumeClaimPhase(client *clientset.Clientset, claimName, namespace string, w watch.Interface, phase v1.PersistentVolumeClaimPhase) {
// Check if the claim is already in requested phase
claim, err := client.Core().PersistentVolumeClaims(namespace).Get(claimName, metav1.GetOptions{})
if err == nil && claim.Status.Phase == phase {
return
}
// Wait for the phase
for {
event := <-w.ResultChan()
claim, ok := event.Object.(*v1.PersistentVolumeClaim)
if !ok {
continue
}
if claim.Status.Phase == phase && claim.Name == claimName {
glog.V(2).Infof("claim %q is %s", claim.Name, phase)
break
}
}
}
func waitForPersistentVolumePhase(client *clientset.Clientset, pvName string, w watch.Interface, phase v1.PersistentVolumePhase) {
// Check if the volume is already in requested phase
volume, err := client.Core().PersistentVolumes().Get(pvName, metav1.GetOptions{})
if err == nil && volume.Status.Phase == phase {
return
}
// Wait for the phase
for {
event := <-w.ResultChan()
volume, ok := event.Object.(*v1.PersistentVolume)
if !ok {
continue
}
if volume.Status.Phase == phase && volume.Name == pvName {
glog.V(2).Infof("volume %q is %s", volume.Name, phase)
break
}
}
}
func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, caCert *x509.Certificate) error {
kd := newKubeDiscovery(cfg, caCert)
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil {
return fmt.Errorf("failed to create %q deployment [%v]", kubeDiscoveryName, err)
}
if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil {
return fmt.Errorf("failed to create %q secret [%v]", kubeDiscoverySecretName, err)
}
fmt.Println("[token-discovery] Created the kube-discovery deployment, waiting for it to become ready")
start := time.Now()
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
d, err := client.Extensions().Deployments(api.NamespaceSystem).Get(kubeDiscoveryName, metav1.GetOptions{})
if err != nil {
return false, nil
}
if d.Status.AvailableReplicas < 1 {
return false, nil
}
return true, nil
})
fmt.Printf("[token-discovery] kube-discovery is ready after %f seconds\n", time.Since(start).Seconds())
return nil
}
/*
waitForIngressOrFail waits until a ingress is either present or absent in the cluster specified by clientset.
If the condition is not met within timout, it fails the calling test.
*/
func waitForIngressOrFail(clientset *kubeclientset.Clientset, namespace string, ingress *v1beta1.Ingress, present bool, timeout time.Duration) {
By(fmt.Sprintf("Fetching a federated ingress shard of ingress %q in namespace %q from cluster", ingress.Name, namespace))
var clusterIngress *v1beta1.Ingress
err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
clusterIngress, err := clientset.Ingresses(namespace).Get(ingress.Name, metav1.GetOptions{})
if (!present) && errors.IsNotFound(err) { // We want it gone, and it's gone.
By(fmt.Sprintf("Success: shard of federated ingress %q in namespace %q in cluster is absent", ingress.Name, namespace))
return true, nil // Success
}
if present && err == nil { // We want it present, and the Get succeeded, so we're all good.
By(fmt.Sprintf("Success: shard of federated ingress %q in namespace %q in cluster is present", ingress.Name, namespace))
return true, nil // Success
}
By(fmt.Sprintf("Ingress %q in namespace %q in cluster. Found: %v, waiting for Found: %v, trying again in %s (err=%v)", ingress.Name, namespace, clusterIngress != nil && err == nil, present, framework.Poll, err))
return false, nil
})
framework.ExpectNoError(err, "Failed to verify ingress %q in namespace %q in cluster: Present=%v", ingress.Name, namespace, present)
if present && clusterIngress != nil {
Expect(equivalentIngress(*clusterIngress, *ingress))
}
}
func waitForSecretOrFail(clientset *kubeclientset.Clientset, nsName string, secret *v1.Secret, present bool, timeout time.Duration) {
By(fmt.Sprintf("Fetching a federated secret shard of secret %q in namespace %q from cluster", secret.Name, nsName))
var clusterSecret *v1.Secret
err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
clusterSecret, err := clientset.Core().Secrets(nsName).Get(secret.Name, metav1.GetOptions{})
if (!present) && errors.IsNotFound(err) { // We want it gone, and it's gone.
By(fmt.Sprintf("Success: shard of federated secret %q in namespace %q in cluster is absent", secret.Name, nsName))
return true, nil // Success
}
if present && err == nil { // We want it present, and the Get succeeded, so we're all good.
By(fmt.Sprintf("Success: shard of federated secret %q in namespace %q in cluster is present", secret.Name, nsName))
return true, nil // Success
}
By(fmt.Sprintf("Secret %q in namespace %q in cluster. Found: %v, waiting for Found: %v, trying again in %s (err=%v)", secret.Name, nsName, clusterSecret != nil && err == nil, present, framework.Poll, err))
return false, nil
})
framework.ExpectNoError(err, "Failed to verify secret %q in namespace %q in cluster: Present=%v", secret.Name, nsName, present)
if present && clusterSecret != nil {
Expect(util.SecretEquivalent(*clusterSecret, *secret))
}
}
// CreateKubeDNSRBACClusterRole creates the necessary ClusterRole for kube-dns
func CreateKubeDNSRBACClusterRole(clientset *clientset.Clientset) error {
clusterRole := rbac.ClusterRole{
ObjectMeta: metav1.ObjectMeta{Name: "kubeadm:" + master.KubeDNS},
Rules: []rbac.PolicyRule{
rbac.NewRule("list", "watch").Groups("").Resources("endpoints", "services").RuleOrDie(),
// TODO: remove watch rule when https://github.com/kubernetes/kubernetes/pull/38816 gets merged
rbac.NewRule("get", "list", "watch").Groups("").Resources("configmaps").RuleOrDie(),
},
}
if _, err := clientset.Rbac().ClusterRoles().Create(&clusterRole); err != nil {
return err
}
subject := rbac.Subject{
Kind: "ServiceAccount",
Name: master.KubeDNS,
Namespace: api.NamespaceSystem,
}
clusterRoleBinding := rbac.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "kubeadm:" + master.KubeDNS,
},
RoleRef: rbac.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "kubeadm:" + master.KubeDNS,
},
Subjects: []rbac.Subject{subject},
}
if _, err := clientset.Rbac().ClusterRoleBindings().Create(&clusterRoleBinding); err != nil {
return err
}
fmt.Println("[apiconfig] Created kube-dns RBAC rules")
return nil
}
func CreateEssentialAddons(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset) error {
kubeProxyDaemonSet := NewDaemonSet(kubeProxy, createKubeProxyPodSpec(cfg))
SetMasterTaintTolerations(&kubeProxyDaemonSet.Spec.Template.ObjectMeta)
SetNodeAffinity(&kubeProxyDaemonSet.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity())
if _, err := client.Extensions().DaemonSets(api.NamespaceSystem).Create(kubeProxyDaemonSet); err != nil {
return fmt.Errorf("failed creating essential kube-proxy addon [%v]", err)
}
fmt.Println("[addons] Created essential addon: kube-proxy")
kubeDNSDeployment := NewDeployment(KubeDNS, 1, createKubeDNSPodSpec(cfg))
SetMasterTaintTolerations(&kubeDNSDeployment.Spec.Template.ObjectMeta)
SetNodeAffinity(&kubeDNSDeployment.Spec.Template.ObjectMeta, NativeArchitectureNodeAffinity())
kubeDNSServiceAccount := &v1.ServiceAccount{}
kubeDNSServiceAccount.ObjectMeta.Name = KubeDNS
if _, err := client.ServiceAccounts(api.NamespaceSystem).Create(kubeDNSServiceAccount); err != nil {
return fmt.Errorf("failed creating kube-dns service account [%v]", err)
}
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kubeDNSDeployment); err != nil {
return fmt.Errorf("failed creating essential kube-dns addon [%v]", err)
}
kubeDNSServiceSpec, err := createKubeDNSServiceSpec(cfg)
if err != nil {
return fmt.Errorf("failed creating essential kube-dns addon [%v]", err)
}
kubeDNSService := NewService(KubeDNS, *kubeDNSServiceSpec)
kubeDNSService.ObjectMeta.Labels["kubernetes.io/name"] = "KubeDNS"
if _, err := client.Services(api.NamespaceSystem).Create(kubeDNSService); err != nil {
return fmt.Errorf("failed creating essential kube-dns addon [%v]", err)
}
fmt.Println("[addons] Created essential addon: kube-dns")
return nil
}
// UpdateOrCreateToken attempts to update a token with the given ID, or create if it does
// not already exist.
func UpdateOrCreateToken(client *clientset.Clientset, d *kubeadmapi.TokenDiscovery, tokenDuration time.Duration) error {
// Let's make sure
if valid, err := ValidateToken(d); !valid {
return err
}
secretName := fmt.Sprintf("%s%s", BootstrapTokenSecretPrefix, d.ID)
var lastErr error
for i := 0; i < tokenCreateRetries; i++ {
secret, err := client.Secrets(api.NamespaceSystem).Get(secretName, metav1.GetOptions{})
if err == nil {
// Secret with this ID already exists, update it:
secret.Data = encodeTokenSecretData(d, tokenDuration)
if _, err := client.Secrets(api.NamespaceSystem).Update(secret); err == nil {
return nil
} else {
lastErr = err
}
continue
}
// Secret does not already exist:
if apierrors.IsNotFound(err) {
secret = &v1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: secretName,
},
Type: api.SecretTypeBootstrapToken,
Data: encodeTokenSecretData(d, tokenDuration),
}
if _, err := client.Secrets(api.NamespaceSystem).Create(secret); err == nil {
return nil
} else {
lastErr = err
}
continue
}
}
return fmt.Errorf(
"unable to create bootstrap token after %d attempts [%v]",
tokenCreateRetries,
lastErr,
)
}
func createDummyDeployment(client *clientset.Clientset) {
fmt.Println("[apiclient] Creating a test deployment")
dummyDeployment := NewDeployment("dummy", 1, v1.PodSpec{
HostNetwork: true,
SecurityContext: &v1.PodSecurityContext{},
Containers: []v1.Container{{
Name: "dummy",
Image: images.GetAddonImage("pause"),
}},
})
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
// TODO: we should check the error, as some cases may be fatal
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(dummyDeployment); err != nil {
fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err)
return false, nil
}
return true, nil
})
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
d, err := client.Extensions().Deployments(api.NamespaceSystem).Get("dummy", metav1.GetOptions{})
if err != nil {
fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err)
return false, nil
}
if d.Status.AvailableReplicas < 1 {
return false, nil
}
return true, nil
})
fmt.Println("[apiclient] Test deployment succeeded")
// TODO: In the future, make sure the ReplicaSet and Pod are garbage collected
if err := client.Extensions().Deployments(api.NamespaceSystem).Delete("dummy", &v1.DeleteOptions{}); err != nil {
fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err)
}
}
// NewSourceApiserver creates a config source that watches and pulls from the apiserver.
func NewSourceApiserver(c *clientset.Clientset, nodeName types.NodeName, updates chan<- interface{}) {
lw := cache.NewListWatchFromClient(c.Core().RESTClient(), "pods", v1.NamespaceAll, fields.OneTermEqualSelector(api.PodHostField, string(nodeName)))
newSourceApiserverFromLW(lw, updates)
}
func (s *ServiceController) ensureClusterService(cachedService *cachedService, clusterName string, service *v1.Service, client *kubeclientset.Clientset) error {
var err error
var needUpdate bool
for i := 0; i < clientRetryCount; i++ {
svc, err := client.Core().Services(service.Namespace).Get(service.Name, metav1.GetOptions{})
if err == nil {
// service exists
glog.V(5).Infof("Found service %s/%s from cluster %s", service.Namespace, service.Name, clusterName)
//reserve immutable fields
service.Spec.ClusterIP = svc.Spec.ClusterIP
//reserve auto assigned field
for i, oldPort := range svc.Spec.Ports {
for _, port := range service.Spec.Ports {
if port.NodePort == 0 {
if !portEqualExcludeNodePort(&oldPort, &port) {
svc.Spec.Ports[i] = port
needUpdate = true
}
} else {
if !portEqualForLB(&oldPort, &port) {
svc.Spec.Ports[i] = port
needUpdate = true
}
}
}
}
if needUpdate {
// we only apply spec update
svc.Spec = service.Spec
_, err = client.Core().Services(svc.Namespace).Update(svc)
if err == nil {
glog.V(5).Infof("Service %s/%s successfully updated to cluster %s", svc.Namespace, svc.Name, clusterName)
return nil
} else {
glog.V(4).Infof("Failed to update %+v", err)
}
} else {
glog.V(5).Infof("Service %s/%s is not updated to cluster %s as the spec are identical", svc.Namespace, svc.Name, clusterName)
return nil
}
} else if errors.IsNotFound(err) {
// Create service if it is not found
glog.Infof("Service '%s/%s' is not found in cluster %s, trying to create new",
service.Namespace, service.Name, clusterName)
service.ResourceVersion = ""
_, err = client.Core().Services(service.Namespace).Create(service)
if err == nil {
glog.V(5).Infof("Service %s/%s successfully created to cluster %s", service.Namespace, service.Name, clusterName)
return nil
}
glog.V(4).Infof("Failed to create %+v", err)
if errors.IsAlreadyExists(err) {
glog.V(5).Infof("service %s/%s already exists in cluster %s", service.Namespace, service.Name, clusterName)
return nil
}
}
if errors.IsConflict(err) {
glog.V(4).Infof("Not persisting update to service '%s/%s' that has been changed since we received it: %v",
service.Namespace, service.Name, err)
}
// should we reuse same retry delay for all clusters?
time.Sleep(cachedService.nextRetryDelay())
}
return err
}