func makePodSourceConfig(kc *KubeletConfig) *config.PodConfig { // source of all configuration cfg := config.NewPodConfig(config.PodConfigNotificationSnapshotAndUpdates) // define file config source if kc.ConfigFile != "" { glog.Infof("Adding manifest file: %v", kc.ConfigFile) config.NewSourceFile(kc.ConfigFile, kc.FileCheckFrequency, cfg.Channel(kubelet.FileSource)) } // define url config source if kc.ManifestURL != "" { glog.Infof("Adding manifest url: %v", kc.ManifestURL) config.NewSourceURL(kc.ManifestURL, kc.HttpCheckFrequency, cfg.Channel(kubelet.HTTPSource)) } if kc.EtcdClient != nil { glog.Infof("Watching for etcd configs at %v", kc.EtcdClient.GetCluster()) config.NewSourceEtcd(config.EtcdKeyForHost(kc.Hostname), kc.EtcdClient, cfg.Channel(kubelet.EtcdSource)) } if kc.KubeClient != nil { glog.Infof("Watching apiserver") config.NewSourceApiserver(kc.KubeClient, kc.Hostname, cfg.Channel(kubelet.ApiserverSource)) } return cfg }
func makePodSourceConfig(kc *KubeletConfig) *config.PodConfig { // source of all configuration cfg := config.NewPodConfig(config.PodConfigNotificationSnapshotAndUpdates, kc.Recorder) // define file config source if kc.ConfigFile != "" { glog.Infof("Adding manifest file: %v", kc.ConfigFile) config.NewSourceFile(kc.ConfigFile, kc.NodeName, kc.FileCheckFrequency, cfg.Channel(kubelet.FileSource)) } // define url config source if kc.ManifestURL != "" { glog.Infof("Adding manifest url: %v", kc.ManifestURL) config.NewSourceURL(kc.ManifestURL, kc.NodeName, kc.HTTPCheckFrequency, cfg.Channel(kubelet.HTTPSource)) } if kc.KubeClient != nil { glog.Infof("Watching apiserver") config.NewSourceApiserver(kc.KubeClient, kc.NodeName, cfg.Channel(kubelet.ApiserverSource)) } return cfg }
// TestExecutorStaticPods test that the ExecutorInfo.data is parsed // as a zip archive with pod definitions. func TestExecutorStaticPods(t *testing.T) { // create some zip with static pod definition var buf bytes.Buffer zw := zip.NewWriter(&buf) createStaticPodFile := func(fileName, id, name string) { w, err := zw.Create(fileName) assert.NoError(t, err) spod := `{ "apiVersion": "v1beta3", "kind": "Pod", "metadata": { "name": "%v", "labels": { "name": "foo", "cluster": "bar" } }, "spec": { "containers": [{ "name": "%v", "image": "library/nginx", "ports": [{ "containerPort": 80, "name": "http" }], "livenessProbe": { "enabled": true, "type": "http", "initialDelaySeconds": 30, "httpGet": { "path": "/", "port": 80 } } }] } }` _, err = w.Write([]byte(fmt.Sprintf(spod, id, name))) assert.NoError(t, err) } createStaticPodFile("spod.json", "spod-id-01", "spod-01") createStaticPodFile("spod2.json", "spod-id-02", "spod-02") createStaticPodFile("dir/spod.json", "spod-id-03", "spod-03") // same file name as first one to check for overwriting expectedStaticPodsNum := 2 // subdirectories are ignored by FileSource, hence only 2 err := zw.Close() assert.NoError(t, err) // create fake apiserver testApiServer := NewTestServer(t, api.NamespaceDefault, nil) defer testApiServer.server.Close() // temporary directory which is normally located in the executor sandbox staticPodsConfigPath, err := ioutil.TempDir("/tmp", "executor-k8sm-archive") assert.NoError(t, err) defer os.RemoveAll(staticPodsConfigPath) mockDriver := &MockExecutorDriver{} updates := make(chan interface{}, 1024) config := Config{ Docker: dockertools.ConnectToDockerOrDie("fake://"), Updates: make(chan interface{}, 1), // allow kube-executor source to proceed past init APIClient: client.NewOrDie(&client.Config{ Host: testApiServer.server.URL, Version: testapi.Version(), }), Kubelet: &kubelet.Kubelet{}, PodStatusFunc: func(kl KubeletInterface, pod *api.Pod) (*api.PodStatus, error) { return &api.PodStatus{ ContainerStatuses: []api.ContainerStatus{ { Name: "foo", State: api.ContainerState{ Running: &api.ContainerStateRunning{}, }, }, }, Phase: api.PodRunning, }, nil }, StaticPodsConfigPath: staticPodsConfigPath, } executor := New(config) hostname := "h1" go executor.InitializeStaticPodsSource(func() { kconfig.NewSourceFile(staticPodsConfigPath, hostname, 1*time.Second, updates) }) // create ExecutorInfo with static pod zip in data field executorInfo := mesosutil.NewExecutorInfo( mesosutil.NewExecutorID("ex1"), mesosutil.NewCommandInfo("k8sm-executor"), ) executorInfo.Data = buf.Bytes() // start the executor with the static pod data executor.Init(mockDriver) executor.Registered(mockDriver, executorInfo, nil, nil) // wait for static pod to start seenPods := map[string]struct{}{} timeout := time.After(time.Second) defer mockDriver.AssertExpectations(t) for { // filter by PodUpdate type select { case <-timeout: t.Fatalf("Executor should send pod updates for %v pods, only saw %v", expectedStaticPodsNum, len(seenPods)) case update, ok := <-updates: if !ok { return } podUpdate, ok := update.(kubelet.PodUpdate) if !ok { continue } for _, pod := range podUpdate.Pods { seenPods[pod.Name] = struct{}{} } if len(seenPods) == expectedStaticPodsNum { return } } } }
func main() { flag.Parse() util.InitLogs() defer util.FlushLogs() rand.Seed(time.Now().UTC().UnixNano()) verflag.PrintAndExitIfRequested() etcd.SetLogger(util.NewLogger("etcd ")) dockerClient, err := docker.NewClient(getDockerEndpoint()) if err != nil { glog.Fatal("Couldn't connect to docker.") } cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194") if err != nil { glog.Errorf("Error on creating cadvisor client: %v", err) } hostname := getHostname() if *rootDirectory == "" { glog.Fatal("Invalid root directory path.") } *rootDirectory = path.Clean(*rootDirectory) os.MkdirAll(*rootDirectory, 0750) // source of all configuration cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates) // define file config source if *config != "" { kconfig.NewSourceFile(*config, *fileCheckFrequency, cfg.Channel("file")) } // define url config source if *manifestURL != "" { kconfig.NewSourceURL(*manifestURL, *httpCheckFrequency, cfg.Channel("http")) } // define etcd config source and initialize etcd client var etcdClient tools.EtcdClient if len(etcdServerList) > 0 { glog.Infof("Watching for etcd configs at %v", etcdServerList) etcdClient = etcd.NewClient(etcdServerList) kconfig.NewSourceEtcd(kconfig.EtcdKeyForHost(hostname), etcdClient, 30*time.Second, cfg.Channel("etcd")) } // TODO: block until all sources have delivered at least one update to the channel, or break the sync loop // up into "per source" synchronizations k := kubelet.NewMainKubelet( getHostname(), dockerClient, cadvisorClient, etcdClient, *rootDirectory) // start the kubelet go util.Forever(func() { k.Run(cfg.Updates()) }, 0) // resynchronize periodically // TODO: make this part of PodConfig so that it is only delivered after syncFrequency has elapsed without // an update go util.Forever(cfg.Sync, *syncFrequency) // start the kubelet server if *enableServer { go util.Forever(func() { kubelet.ListenAndServeKubeletServer(k, cfg.Channel("http"), http.DefaultServeMux, *address, *port) }, 0) } // runs forever select {} }
func (ks *KubeletExecutorServer) createAndInitKubelet( kc *app.KubeletConfig, hks hyperkube.Interface, clientConfig *client.Config, shutdownCloser io.Closer, ) (app.KubeletBootstrap, *kconfig.PodConfig, error) { // TODO(k8s): block until all sources have delivered at least one update to the channel, or break the sync loop // up into "per source" synchronizations // TODO(k8s): KubeletConfig.KubeClient should be a client interface, but client interface misses certain methods // used by kubelet. Since NewMainKubelet expects a client interface, we need to make sure we are not passing // a nil pointer to it when what we really want is a nil interface. var kubeClient client.Interface if kc.KubeClient == nil { kubeClient = nil } else { kubeClient = kc.KubeClient } gcPolicy := kubelet.ContainerGCPolicy{ MinAge: kc.MinimumGCAge, MaxPerPodContainer: kc.MaxPerPodContainerCount, MaxContainers: kc.MaxContainerCount, } pc := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates, kc.Recorder) updates := pc.Channel(MESOS_CFG_SOURCE) klet, err := kubelet.NewMainKubelet( kc.Hostname, kc.NodeName, kc.DockerClient, kubeClient, kc.RootDirectory, kc.PodInfraContainerImage, kc.SyncFrequency, float32(kc.RegistryPullQPS), kc.RegistryBurst, gcPolicy, pc.SeenAllSources, kc.RegisterNode, kc.StandaloneMode, kc.ClusterDomain, net.IP(kc.ClusterDNS), kc.MasterServiceNamespace, kc.VolumePlugins, kc.NetworkPlugins, kc.NetworkPluginName, kc.StreamingConnectionIdleTimeout, kc.Recorder, kc.CadvisorInterface, kc.ImageGCPolicy, kc.DiskSpacePolicy, kc.Cloud, kc.NodeStatusUpdateFrequency, kc.ResourceContainer, kc.OSInterface, kc.CgroupRoot, kc.ContainerRuntime, kc.Mounter, kc.DockerDaemonContainer, kc.SystemContainer, kc.ConfigureCBR0, kc.PodCIDR, kc.MaxPods, kc.DockerExecHandler, ) if err != nil { return nil, nil, err } //TODO(jdef) either configure Watch here with something useful, or else // get rid of it from executor.Config kubeletFinished := make(chan struct{}) staticPodsConfigPath := filepath.Join(kc.RootDirectory, "static-pods") exec := executor.New(executor.Config{ Kubelet: klet, Updates: updates, SourceName: MESOS_CFG_SOURCE, APIClient: kc.KubeClient, Docker: kc.DockerClient, SuicideTimeout: ks.SuicideTimeout, KubeletFinished: kubeletFinished, ShutdownAlert: func() { if shutdownCloser != nil { if e := shutdownCloser.Close(); e != nil { log.Warningf("failed to signal shutdown to external watcher: %v", e) } } }, ExitFunc: os.Exit, PodStatusFunc: func(_ executor.KubeletInterface, pod *api.Pod) (*api.PodStatus, error) { return klet.GetRuntime().GetPodStatus(pod) }, StaticPodsConfigPath: staticPodsConfigPath, }) go exec.InitializeStaticPodsSource(func() { // Create file source only when we are called back. Otherwise, it is never marked unseen. fileSourceUpdates := pc.Channel(kubelet.FileSource) kconfig.NewSourceFile(staticPodsConfigPath, kc.Hostname, kc.FileCheckFrequency, fileSourceUpdates) }) k := &kubeletExecutor{ Kubelet: klet, runProxy: ks.RunProxy, proxyLogV: ks.ProxyLogV, proxyExec: ks.ProxyExec, proxyLogfile: ks.ProxyLogfile, proxyBindall: ks.ProxyBindall, address: ks.Address, dockerClient: kc.DockerClient, hks: hks, kubeletFinished: kubeletFinished, executorDone: exec.Done(), clientConfig: clientConfig, } dconfig := bindings.DriverConfig{ Executor: exec, HostnameOverride: ks.HostnameOverride, BindingAddress: net.IP(ks.Address), } if driver, err := bindings.NewMesosExecutorDriver(dconfig); err != nil { log.Fatalf("failed to create executor driver: %v", err) } else { k.driver = driver } log.V(2).Infof("Initialize executor driver...") k.BirthCry() exec.Init(k.driver) k.StartGarbageCollection() return k, pc, nil }
func main() { flag.Parse() util.InitLogs() defer util.FlushLogs() rand.Seed(time.Now().UTC().UnixNano()) verflag.PrintAndExitIfRequested() if *runonce { exclusiveFlag := "invalid option: --runonce and %s are mutually exclusive" if len(etcdServerList) > 0 { glog.Fatalf(exclusiveFlag, "--etcd_servers") } if *enableServer { glog.Infof("--runonce is set, disabling server") *enableServer = false } } etcd.SetLogger(util.NewLogger("etcd ")) // Log the events locally too. record.StartLogging(glog.Infof) capabilities.Initialize(capabilities.Capabilities{ AllowPrivileged: *allowPrivileged, }) dockerClient, err := docker.NewClient(getDockerEndpoint()) if err != nil { glog.Fatal("Couldn't connect to docker.") } hostname := getHostname() if *rootDirectory == "" { glog.Fatal("Invalid root directory path.") } *rootDirectory = path.Clean(*rootDirectory) if err := os.MkdirAll(*rootDirectory, 0750); err != nil { glog.Fatalf("Error creating root directory: %v", err) } // source of all configuration cfg := kconfig.NewPodConfig(kconfig.PodConfigNotificationSnapshotAndUpdates) // define file config source if *config != "" { kconfig.NewSourceFile(*config, *fileCheckFrequency, cfg.Channel("file")) } // define url config source if *manifestURL != "" { kconfig.NewSourceURL(*manifestURL, *httpCheckFrequency, cfg.Channel("http")) } // define etcd config source and initialize etcd client var etcdClient *etcd.Client if len(etcdServerList) > 0 { etcdClient = etcd.NewClient(etcdServerList) } else if *etcdConfigFile != "" { var err error etcdClient, err = etcd.NewClientFromFile(*etcdConfigFile) if err != nil { glog.Fatalf("Error with etcd config file: %v", err) } } if etcdClient != nil { glog.Infof("Watching for etcd configs at %v", etcdClient.GetCluster()) kconfig.NewSourceEtcd(kconfig.EtcdKeyForHost(hostname), etcdClient, cfg.Channel("etcd")) } // TODO: block until all sources have delivered at least one update to the channel, or break the sync loop // up into "per source" synchronizations k := kubelet.NewMainKubelet( getHostname(), dockerClient, etcdClient, *rootDirectory, *networkContainerImage, *syncFrequency, float32(*registryPullQPS), *registryBurst, *minimumGCAge, *maxContainerCount) k.BirthCry() go func() { util.Forever(func() { err := k.GarbageCollectContainers() if err != nil { glog.Errorf("Garbage collect failed: %v", err) } }, time.Minute*1) }() go func() { defer util.HandleCrash() // TODO: Monitor this connection, reconnect if needed? glog.V(1).Infof("Trying to create cadvisor client.") cadvisorClient, err := cadvisor.NewClient("http://127.0.0.1:4194") if err != nil { glog.Errorf("Error on creating cadvisor client: %v", err) return } glog.V(1).Infof("Successfully created cadvisor client.") k.SetCadvisorClient(cadvisorClient) }() // TODO: These should probably become more plugin-ish: register a factory func // in each checker's init(), iterate those here. health.AddHealthChecker(health.NewExecHealthChecker(k)) health.AddHealthChecker(health.NewHTTPHealthChecker(&http.Client{})) health.AddHealthChecker(&health.TCPHealthChecker{}) // process pods and exit. if *runonce { if _, err := k.RunOnce(cfg.Updates()); err != nil { glog.Fatalf("--runonce failed: %v", err) } return } // start the kubelet go util.Forever(func() { k.Run(cfg.Updates()) }, 0) // start the kubelet server if *enableServer { go util.Forever(func() { kubelet.ListenAndServeKubeletServer(k, cfg.Channel("http"), net.IP(address), *port, *enableDebuggingHandlers) }, 0) } // runs forever select {} }