func (gc *GarbageCollector) Run(workers int, stopCh <-chan struct{}) {
glog.Infof("Garbage Collector: Initializing")
for _, monitor := range gc.monitors {
go monitor.controller.Run(stopCh)
}
wait.PollInfinite(10*time.Second, func() (bool, error) {
for _, monitor := range gc.monitors {
if !monitor.controller.HasSynced() {
glog.Infof("Garbage Collector: Waiting for resource monitors to be synced...")
return false, nil
}
}
return true, nil
})
glog.Infof("Garbage Collector: All monitored resources synced. Proceeding to collect garbage")
// worker
go wait.Until(gc.propagator.processEvent, 0, stopCh)
for i := 0; i < workers; i++ {
go wait.Until(gc.worker, 0, stopCh)
go wait.Until(gc.orphanFinalizer, 0, stopCh)
}
Register()
<-stopCh
glog.Infof("Garbage Collector: Shutting down")
gc.dirtyQueue.ShutDown()
gc.orphanQueue.ShutDown()
gc.propagator.eventQueue.ShutDown()
}
func CreateDiscoveryDeploymentAndSecret(cfg *kubeadmapi.MasterConfiguration, client *clientset.Clientset, caCert *x509.Certificate) error {
kd := newKubeDiscovery(cfg, caCert)
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(kd.Deployment); err != nil {
return fmt.Errorf("failed to create %q deployment [%v]", kubeDiscoveryName, err)
}
if _, err := client.Secrets(api.NamespaceSystem).Create(kd.Secret); err != nil {
return fmt.Errorf("failed to create %q secret [%v]", kubeDiscoverySecretName, err)
}
fmt.Println("[token-discovery] Created the kube-discovery deployment, waiting for it to become ready")
start := time.Now()
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
d, err := client.Extensions().Deployments(api.NamespaceSystem).Get(kubeDiscoveryName, metav1.GetOptions{})
if err != nil {
return false, nil
}
if d.Status.AvailableReplicas < 1 {
return false, nil
}
return true, nil
})
fmt.Printf("[token-discovery] kube-discovery is ready after %f seconds\n", time.Since(start).Seconds())
return nil
}
func createDummyDeployment(client *clientset.Clientset) {
fmt.Println("[apiclient] Creating a test deployment")
dummyDeployment := NewDeployment("dummy", 1, v1.PodSpec{
HostNetwork: true,
SecurityContext: &v1.PodSecurityContext{},
Containers: []v1.Container{{
Name: "dummy",
Image: images.GetAddonImage("pause"),
}},
})
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
// TODO: we should check the error, as some cases may be fatal
if _, err := client.Extensions().Deployments(api.NamespaceSystem).Create(dummyDeployment); err != nil {
fmt.Printf("[apiclient] Failed to create test deployment [%v] (will retry)\n", err)
return false, nil
}
return true, nil
})
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
d, err := client.Extensions().Deployments(api.NamespaceSystem).Get("dummy", metav1.GetOptions{})
if err != nil {
fmt.Printf("[apiclient] Failed to get test deployment [%v] (will retry)\n", err)
return false, nil
}
if d.Status.AvailableReplicas < 1 {
return false, nil
}
return true, nil
})
fmt.Println("[apiclient] Test deployment succeeded")
// TODO: In the future, make sure the ReplicaSet and Pod are garbage collected
if err := client.Extensions().Deployments(api.NamespaceSystem).Delete("dummy", &v1.DeleteOptions{}); err != nil {
fmt.Printf("[apiclient] Failed to delete test deployment [%v] (will ignore)\n", err)
}
}
func RetrieveTrustedClusterInfo(d *kubeadmapi.TokenDiscovery) (*kubeadmapi.ClusterInfo, error) {
requestURL := fmt.Sprintf("http://%s/cluster-info/v1/?token-id=%s", d.Addresses[0], d.ID)
req, err := http.NewRequest("GET", requestURL, nil)
if err != nil {
return nil, fmt.Errorf("failed to consturct an HTTP request [%v]", err)
}
fmt.Printf("[discovery] Created cluster info discovery client, requesting info from %q\n", requestURL)
var res *http.Response
wait.PollInfinite(discoveryRetryTimeout, func() (bool, error) {
res, err = http.DefaultClient.Do(req)
if err != nil {
fmt.Printf("[discovery] Failed to request cluster info, will try again: [%s]\n", err)
return false, nil
}
return true, nil
})
buf := new(bytes.Buffer)
io.Copy(buf, res.Body)
res.Body.Close()
object, err := jose.ParseSigned(buf.String())
if err != nil {
return nil, fmt.Errorf("failed to parse response as JWS object [%v]", err)
}
fmt.Println("[discovery] Cluster info object received, verifying signature using given token")
output, err := object.Verify([]byte(d.Secret))
if err != nil {
return nil, fmt.Errorf("failed to verify JWS signature of received cluster info object [%v]", err)
}
clusterInfo := kubeadmapi.ClusterInfo{}
if err := json.Unmarshal(output, &clusterInfo); err != nil {
return nil, fmt.Errorf("failed to decode received cluster info object [%v]", err)
}
if len(clusterInfo.CertificateAuthorities) == 0 || len(clusterInfo.Endpoints) == 0 {
return nil, fmt.Errorf("cluster info object is invalid - no endpoint(s) and/or root CA certificate(s) found")
}
// TODO(phase1+) print summary info about the CA certificate, along with the the checksum signature
// we also need an ability for the user to configure the client to validate received CA cert against a checksum
fmt.Printf("[discovery] Cluster info signature and contents are valid, will use API endpoints %v\n", clusterInfo.Endpoints)
return &clusterInfo, nil
}
func waitSrvHealthy(config util.AdminConfig, context, kubeconfig string) error {
fedClientSet, err := config.FederationClientset(context, kubeconfig)
if err != nil {
return err
}
fedDiscoveryClient := fedClientSet.Discovery()
err = wait.PollInfinite(podWaitInterval, func() (bool, error) {
body, err := fedDiscoveryClient.RESTClient().Get().AbsPath("/healthz").Do().Raw()
if err != nil {
return false, nil
}
if strings.EqualFold(string(body), "ok") {
return true, nil
}
return false, nil
})
return err
}
func startKubeletConfigSyncLoop(s *options.KubeletServer, currentKC string) {
glog.Infof("Starting Kubelet configuration sync loop")
go func() {
wait.PollInfinite(30*time.Second, func() (bool, error) {
glog.Infof("Checking API server for new Kubelet configuration.")
remoteKC, err := getRemoteKubeletConfig(s, nil)
if err == nil {
// Detect new config by comparing with the last JSON string we extracted.
if remoteKC != currentKC {
glog.Info("Found new Kubelet configuration via API server, restarting!")
os.Exit(0)
}
} else {
glog.Infof("Did not find a configuration for this Kubelet via API server: %v", err)
}
return false, nil // Always return (false, nil) so we poll forever.
})
}()
}
func waitForPods(clientset *client.Clientset, fedPods []string, namespace string) error {
err := wait.PollInfinite(podWaitInterval, func() (bool, error) {
podCheck := len(fedPods)
podList, err := clientset.Core().Pods(namespace).List(api.ListOptions{})
if err != nil {
return false, nil
}
for _, pod := range podList.Items {
for _, fedPod := range fedPods {
if strings.HasPrefix(pod.Name, fedPod) && pod.Status.Phase == "Running" {
podCheck -= 1
}
}
//ensure that all pods are in running state or keep waiting
if podCheck == 0 {
return true, nil
}
}
return false, nil
})
return err
}
func CreateClientAndWaitForAPI(file string) (*clientset.Clientset, error) {
client, err := CreateClientFromFile(file)
if err != nil {
return nil, err
}
fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready")
start := time.Now()
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
// TODO: use /healthz API instead of this
cs, err := client.ComponentStatuses().List(v1.ListOptions{})
if err != nil {
if apierrs.IsForbidden(err) {
fmt.Print("\r[apiclient] Waiting for the API server to create RBAC policies")
}
return false, nil
}
fmt.Println("\n[apiclient] RBAC policies created")
// TODO(phase2) must revisit this when we implement HA
if len(cs.Items) < 3 {
fmt.Println("[apiclient] Not all control plane components are ready yet")
return false, nil
}
for _, item := range cs.Items {
for _, condition := range item.Conditions {
if condition.Type != v1.ComponentHealthy {
fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions)
return false, nil
}
}
}
fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds())
return true, nil
})
fmt.Println("[apiclient] Waiting for at least one node to register and become ready")
start = time.Now()
wait.PollInfinite(apiCallRetryInterval, func() (bool, error) {
nodeList, err := client.Nodes().List(v1.ListOptions{})
if err != nil {
fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)")
return false, nil
}
if len(nodeList.Items) < 1 {
return false, nil
}
n := &nodeList.Items[0]
if !v1.IsNodeReady(n) {
fmt.Println("[apiclient] First node has registered, but is not ready yet")
return false, nil
}
fmt.Printf("[apiclient] First node is ready after %f seconds\n", time.Since(start).Seconds())
return true, nil
})
createDummyDeployment(client)
return client, nil
}