func NewHollowKubelet(
nodeName string,
client *clientset.Clientset,
cadvisorInterface cadvisor.Interface,
dockerClient dockertools.DockerInterface,
kubeletPort, kubeletReadOnlyPort int,
containerManager cm.ContainerManager,
maxPods int, podsPerCore int,
) *HollowKubelet {
testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
manifestFilePath := utils.MakeTempDirOrDie("manifest", testRootDir)
glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
return &HollowKubelet{
KubeletConfig: kubeletapp.SimpleKubelet(
client,
dockerClient,
nodeName,
testRootDir,
"", /* manifest-url */
"0.0.0.0", /* bind address */
uint(kubeletPort),
uint(kubeletReadOnlyPort),
api.NamespaceDefault,
empty_dir.ProbeVolumePlugins(),
nil, /* tls-options */
cadvisorInterface,
manifestFilePath,
nil, /* cloud-provider */
&containertest.FakeOS{}, /* os-interface */
20*time.Second, /* FileCheckFrequency */
20*time.Second, /* HTTPCheckFrequency */
1*time.Minute, /* MinimumGCAge */
10*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
5*time.Minute, /* OutOfDiskTransitionFrequency */
5*time.Minute, /* EvictionPressureTransitionPeriod */
maxPods,
podsPerCore,
containerManager,
nil,
),
}
}
func startComponents(firstManifestURL, secondManifestURL string) (string, string) {
// Setup
handler := delegateHandler{}
apiServer := httptest.NewServer(&handler)
cfg := etcd.Config{
Endpoints: []string{"http://127.0.0.1:4001"},
}
etcdClient, err := etcd.New(cfg)
if err != nil {
glog.Fatalf("Error creating etcd client: %v", err)
}
glog.Infof("Creating etcd client pointing to %v", cfg.Endpoints)
keysAPI := etcd.NewKeysAPI(etcdClient)
sleep := 4 * time.Second
ok := false
for i := 0; i < 3; i++ {
keys, err := keysAPI.Get(context.TODO(), "/", nil)
if err != nil {
glog.Warningf("Unable to list root etcd keys: %v", err)
if i < 2 {
time.Sleep(sleep)
sleep = sleep * sleep
}
continue
}
for _, node := range keys.Node.Nodes {
if _, err := keysAPI.Delete(context.TODO(), node.Key, &etcd.DeleteOptions{Recursive: true}); err != nil {
glog.Fatalf("Unable delete key: %v", err)
}
}
ok = true
break
}
if !ok {
glog.Fatalf("Failed to connect to etcd")
}
cl := client.NewOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
clientset := clientset.NewForConfigOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Default.GroupVersion()}})
// TODO: caesarxuchao: hacky way to specify version of Experimental client.
// We will fix this by supporting multiple group versions in Config
cl.ExtensionsClient = client.NewExtensionsOrDie(&restclient.Config{Host: apiServer.URL, ContentConfig: restclient.ContentConfig{GroupVersion: testapi.Extensions.GroupVersion()}})
// Master
host, port, err := net.SplitHostPort(strings.TrimLeft(apiServer.URL, "http://"))
if err != nil {
glog.Fatalf("Unable to parse URL '%v': %v", apiServer.URL, err)
}
portNumber, err := strconv.Atoi(port)
if err != nil {
glog.Fatalf("Nonnumeric port? %v", err)
}
publicAddress := net.ParseIP(host)
if publicAddress == nil {
glog.Fatalf("No public address for %s", host)
}
// The caller of master.New should guarantee pulicAddress is properly set
hostIP, err := utilnet.ChooseBindAddress(publicAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Fail to get a valid public address for master.", err)
}
masterConfig := framework.NewMasterConfig()
masterConfig.EnableCoreControllers = true
masterConfig.EnableProfiling = true
masterConfig.ReadWritePort = portNumber
masterConfig.PublicAddress = hostIP
masterConfig.CacheTimeout = 2 * time.Second
masterConfig.EnableWatchCache = watchCache
// Create a master and install handlers into mux.
m, err := master.New(masterConfig)
if err != nil {
glog.Fatalf("Error in bringing up the master: %v", err)
}
handler.delegate = m.Handler
// Scheduler
schedulerConfigFactory := factory.NewConfigFactory(cl, api.DefaultSchedulerName, api.DefaultHardPodAffinitySymmetricWeight, api.DefaultFailureDomains)
schedulerConfig, err := schedulerConfigFactory.Create()
if err != nil {
glog.Fatalf("Couldn't create scheduler config: %v", err)
}
eventBroadcaster := record.NewBroadcaster()
schedulerConfig.Recorder = eventBroadcaster.NewRecorder(api.EventSource{Component: api.DefaultSchedulerName})
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(cl.Events(""))
scheduler.New(schedulerConfig).Run()
podInformer := informers.CreateSharedPodIndexInformer(clientset, controller.NoResyncPeriodFunc())
// ensure the service endpoints are sync'd several times within the window that the integration tests wait
go endpointcontroller.NewEndpointController(podInformer, clientset).
Run(3, wait.NeverStop)
// TODO: Write an integration test for the replication controllers watch.
go replicationcontroller.NewReplicationManager(podInformer, clientset, controller.NoResyncPeriodFunc, replicationcontroller.BurstReplicas, 4096).
Run(3, wait.NeverStop)
go podInformer.Run(wait.NeverStop)
nodeController := nodecontroller.NewNodeController(nil, clientset, 5*time.Minute, flowcontrol.NewFakeAlwaysRateLimiter(), flowcontrol.NewFakeAlwaysRateLimiter(),
40*time.Second, 60*time.Second, 5*time.Second, nil, nil, 0, false)
nodeController.Run(5 * time.Second)
cadvisorInterface := new(cadvisortest.Fake)
// Kubelet (localhost)
testRootDir := testutils.MakeTempDirOrDie("kubelet_integ_1.", "")
configFilePath := testutils.MakeTempDirOrDie("config", testRootDir)
glog.Infof("Using %s as root dir for kubelet #1", testRootDir)
cm := cm.NewStubContainerManager()
kcfg := kubeletapp.SimpleKubelet(
clientset,
fakeDocker1,
"localhost",
testRootDir,
firstManifestURL,
"127.0.0.1",
10250, /* KubeletPort */
0, /* ReadOnlyPort */
api.NamespaceDefault,
empty_dir.ProbeVolumePlugins(),
nil,
cadvisorInterface,
configFilePath,
nil,
&containertest.FakeOS{},
1*time.Second, /* FileCheckFrequency */
1*time.Second, /* HTTPCheckFrequency */
10*time.Second, /* MinimumGCAge */
3*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
10*time.Second, /* OutOfDiskTransitionFrequency */
10*time.Second, /* EvictionPressureTransitionPeriod */
40, /* MaxPods */
0, /* PodsPerCore*/
cm, net.ParseIP("127.0.0.1"))
kubeletapp.RunKubelet(kcfg)
// Kubelet (machine)
// Create a second kubelet so that the guestbook example's two redis slaves both
// have a place they can schedule.
testRootDir = testutils.MakeTempDirOrDie("kubelet_integ_2.", "")
glog.Infof("Using %s as root dir for kubelet #2", testRootDir)
kcfg = kubeletapp.SimpleKubelet(
clientset,
fakeDocker2,
"127.0.0.1",
testRootDir,
secondManifestURL,
"127.0.0.1",
10251, /* KubeletPort */
0, /* ReadOnlyPort */
api.NamespaceDefault,
empty_dir.ProbeVolumePlugins(),
nil,
cadvisorInterface,
"",
nil,
&containertest.FakeOS{},
1*time.Second, /* FileCheckFrequency */
1*time.Second, /* HTTPCheckFrequency */
10*time.Second, /* MinimumGCAge */
3*time.Second, /* NodeStatusUpdateFrequency */
10*time.Second, /* SyncFrequency */
10*time.Second, /* OutOfDiskTransitionFrequency */
10*time.Second, /* EvictionPressureTransitionPeriod */
40, /* MaxPods */
0, /* PodsPerCore*/
cm,
net.ParseIP("127.0.0.1"))
kubeletapp.RunKubelet(kcfg)
return apiServer.URL, configFilePath
}
// Builds a KubeletConfiguration for the HollowKubelet, ensuring that the
// usual defaults are applied for fields we do not override.
func GetHollowKubeletConfig(
nodeName string,
kubeletPort int,
kubeletReadOnlyPort int,
maxPods int,
podsPerCore int) *componentconfig.KubeletConfiguration {
testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
manifestFilePath := utils.MakeTempDirOrDie("manifest", testRootDir)
glog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
// Do the external -> internal conversion to make sure that defaults
// are set for fields not overridden in NewHollowKubelet.
tmp := &v1alpha1.KubeletConfiguration{}
api.Scheme.Default(tmp)
c := &componentconfig.KubeletConfiguration{}
api.Scheme.Convert(tmp, c, nil)
c.HostnameOverride = nodeName
c.RootDirectory = testRootDir
c.ManifestURL = ""
c.Address = "0.0.0.0" /* bind address */
c.Port = int32(kubeletPort)
c.ReadOnlyPort = int32(kubeletReadOnlyPort)
c.MasterServiceNamespace = api.NamespaceDefault
c.PodManifestPath = manifestFilePath
c.FileCheckFrequency.Duration = 20 * time.Second
c.HTTPCheckFrequency.Duration = 20 * time.Second
c.MinimumGCAge.Duration = 1 * time.Minute
c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
c.SyncFrequency.Duration = 10 * time.Second
c.OutOfDiskTransitionFrequency.Duration = 5 * time.Minute
c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
c.MaxPods = int32(maxPods)
c.PodsPerCore = int32(podsPerCore)
c.ClusterDNS = ""
c.DockerExecHandlerName = "native"
c.ImageGCHighThresholdPercent = 90
c.ImageGCLowThresholdPercent = 80
c.LowDiskSpaceThresholdMB = 256
c.VolumeStatsAggPeriod.Duration = time.Minute
c.CgroupRoot = ""
c.ContainerRuntime = "docker"
c.CPUCFSQuota = true
c.RuntimeCgroups = ""
c.EnableControllerAttachDetach = false
c.EnableCustomMetrics = false
c.EnableDebuggingHandlers = true
c.EnableServer = true
c.CgroupsPerQOS = false
// hairpin-veth is used to allow hairpin packets. Note that this deviates from
// what the "real" kubelet currently does, because there's no way to
// set promiscuous mode on docker0.
c.HairpinMode = componentconfig.HairpinVeth
c.MaxContainerCount = 100
c.MaxOpenFiles = 1024
c.MaxPerPodContainerCount = 2
c.NvidiaGPUs = 0
c.RegisterNode = true
c.RegisterSchedulable = true
c.RegistryBurst = 10
c.RegistryPullQPS = 5.0
c.ResolverConfig = kubetypes.ResolvConfDefault
c.KubeletCgroups = "/kubelet"
c.SerializeImagePulls = true
c.SystemCgroups = ""
c.ProtectKernelDefaults = false
// TODO(mtaufen): Note that PodInfraContainerImage was being set to the empty value before,
// but this may not have been intentional. (previous code (SimpleKubelet)
// was peeling it off of a componentconfig.KubeletConfiguration{}, but may
// have actually wanted the default).
// The default will be present in the KubeletConfiguration contstructed above.
return c
}