func rmSetup(t *testing.T, enableGarbageCollector bool) (*httptest.Server, *replication.ReplicationManager, cache.SharedIndexInformer, clientset.Interface) {
masterConfig := framework.NewIntegrationTestMasterConfig()
_, s := framework.RunAMaster(masterConfig)
config := restclient.Config{Host: s.URL}
clientSet, err := clientset.NewForConfig(&config)
if err != nil {
t.Fatalf("Error in create clientset: %v", err)
}
resyncPeriod := 12 * time.Hour
resyncPeriodFunc := func() time.Duration {
return resyncPeriod
}
podInformer := informers.NewPodInformer(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pod-informer")), resyncPeriod)
rm := replication.NewReplicationManager(
podInformer,
clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "replication-controller")),
resyncPeriodFunc,
replication.BurstReplicas,
4096,
enableGarbageCollector,
)
if err != nil {
t.Fatalf("Failed to create replication manager")
}
return s, rm, podInformer, clientSet
}
func setup(t *testing.T) (*httptest.Server, *garbagecollector.GarbageCollector, clientset.Interface) {
masterConfig := framework.NewIntegrationTestMasterConfig()
masterConfig.EnableCoreControllers = false
masterConfig.GenericConfig.EnableGarbageCollection = true
_, s := framework.RunAMaster(masterConfig)
clientSet, err := clientset.NewForConfig(&restclient.Config{Host: s.URL})
if err != nil {
t.Fatalf("Error in create clientset: %v", err)
}
preferredResources, err := clientSet.Discovery().ServerPreferredResources()
if err != nil {
t.Fatalf("Failed to get supported resources from server: %v", err)
}
deletableResources := discovery.FilteredBy(discovery.SupportsAllVerbs{Verbs: []string{"delete"}}, preferredResources)
deletableGroupVersionResources, err := discovery.GroupVersionResources(deletableResources)
if err != nil {
t.Fatalf("Failed to parse supported resources from server: %v", err)
}
config := &restclient.Config{Host: s.URL}
config.ContentConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: metaonly.NewMetadataCodecFactory()}
metaOnlyClientPool := dynamic.NewClientPool(config, registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc)
config.ContentConfig.NegotiatedSerializer = nil
clientPool := dynamic.NewClientPool(config, registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc)
gc, err := garbagecollector.NewGarbageCollector(metaOnlyClientPool, clientPool, registered.RESTMapper(), deletableGroupVersionResources)
if err != nil {
t.Fatalf("Failed to create garbage collector")
}
return s, gc, clientSet
}
// TODO: evaluate using pkg/client/clientcmd
func newKubeClient(dnsConfig *options.KubeDNSConfig) (clientset.Interface, error) {
var (
config *restclient.Config
err error
)
if dnsConfig.KubeMasterURL != "" && dnsConfig.KubeConfigFile == "" {
// Only --kube-master-url was provided.
config = &restclient.Config{
Host: dnsConfig.KubeMasterURL,
ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Version: "v1"}},
}
} else {
// We either have:
// 1) --kube-master-url and --kubecfg-file
// 2) just --kubecfg-file
// 3) neither flag
// In any case, the logic is the same. If (3), this will automatically
// fall back on the service account token.
overrides := &kclientcmd.ConfigOverrides{}
overrides.ClusterInfo.Server = dnsConfig.KubeMasterURL // might be "", but that is OK
rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: dnsConfig.KubeConfigFile} // might be "", but that is OK
if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil {
return nil, err
}
}
glog.V(0).Infof("Using %v for kubernetes master, kubernetes API: %v",
config.Host, config.GroupVersion)
return clientset.NewForConfig(config)
}
func (b SimpleControllerClientBuilder) Client(name string) (clientset.Interface, error) {
clientConfig, err := b.Config(name)
if err != nil {
return nil, err
}
return clientset.NewForConfig(clientConfig)
}
func rmSetup(t *testing.T, enableGarbageCollector bool) (*httptest.Server, *replicaset.ReplicaSetController, cache.SharedIndexInformer, cache.SharedIndexInformer, clientset.Interface) {
masterConfig := framework.NewIntegrationTestMasterConfig()
_, s := framework.RunAMaster(masterConfig)
config := restclient.Config{Host: s.URL}
clientSet, err := clientset.NewForConfig(&config)
if err != nil {
t.Fatalf("Error in create clientset: %v", err)
}
resyncPeriod := 12 * time.Hour
informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "rs-informers")), nil, resyncPeriod)
rm := replicaset.NewReplicaSetController(
informers.ReplicaSets(),
informers.Pods(),
clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "replicaset-controller")),
replicaset.BurstReplicas,
4096,
enableGarbageCollector,
)
if err != nil {
t.Fatalf("Failed to create replicaset controller")
}
return s, rm, informers.ReplicaSets().Informer(), informers.Pods().Informer(), clientSet
}
// getAPIServerClient gets a apiserver client.
func getAPIServerClient() (*clientset.Clientset, error) {
config, err := framework.LoadConfig()
if err != nil {
return nil, fmt.Errorf("failed to load config: %v", err)
}
client, err := clientset.NewForConfig(config)
if err != nil {
return nil, fmt.Errorf("failed to create client: %v", err)
}
return client, nil
}
func getKubeClient(s *options.KubeletServer) (*clientset.Clientset, error) {
clientConfig, err := CreateAPIServerClientConfig(s)
if err == nil {
kubeClient, err := clientset.NewForConfig(clientConfig)
if err != nil {
return nil, err
}
return kubeClient, nil
}
return nil, err
}
// Start starts the namespace controller.
func (n *NamespaceController) Start() error {
// Use the default QPS
config := restclient.AddUserAgent(&restclient.Config{Host: framework.TestContext.Host}, ncName)
client, err := clientset.NewForConfig(config)
if err != nil {
return err
}
clientPool := dynamic.NewClientPool(config, api.Registry.RESTMapper(), dynamic.LegacyAPIPathResolverFunc)
discoverResourcesFn := client.Discovery().ServerPreferredNamespacedResources
nc := namespacecontroller.NewNamespaceController(client, clientPool, discoverResourcesFn, ncResyncPeriod, v1.FinalizerKubernetes)
go nc.Run(ncConcurrency, n.stopCh)
return nil
}
func createClients(numberOfClients int) ([]*clientset.Clientset, []*internalclientset.Clientset, error) {
clients := make([]*clientset.Clientset, numberOfClients)
internalClients := make([]*internalclientset.Clientset, numberOfClients)
for i := 0; i < numberOfClients; i++ {
config, err := framework.LoadConfig()
Expect(err).NotTo(HaveOccurred())
config.QPS = 100
config.Burst = 200
if framework.TestContext.KubeAPIContentType != "" {
config.ContentType = framework.TestContext.KubeAPIContentType
}
// For the purpose of this test, we want to force that clients
// do not share underlying transport (which is a default behavior
// in Kubernetes). Thus, we are explicitly creating transport for
// each client here.
transportConfig, err := config.TransportConfig()
if err != nil {
return nil, nil, err
}
tlsConfig, err := transport.TLSConfigFor(transportConfig)
if err != nil {
return nil, nil, err
}
config.Transport = utilnet.SetTransportDefaults(&http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: tlsConfig,
MaxIdleConnsPerHost: 100,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
})
// Overwrite TLS-related fields from config to avoid collision with
// Transport field.
config.TLSClientConfig = restclient.TLSClientConfig{}
c, err := clientset.NewForConfig(config)
if err != nil {
return nil, nil, err
}
clients[i] = c
internalClient, err := internalclientset.NewForConfig(config)
if err != nil {
return nil, nil, err
}
internalClients[i] = internalClient
}
return clients, internalClients, nil
}
func createAPIClient(adminKubeconfig *clientcmdapi.Config) (*clientset.Clientset, error) {
adminClientConfig, err := clientcmd.NewDefaultClientConfig(
*adminKubeconfig,
&clientcmd.ConfigOverrides{},
).ClientConfig()
if err != nil {
return nil, fmt.Errorf("failed to create API client configuration [%v]", err)
}
client, err := clientset.NewForConfig(adminClientConfig)
if err != nil {
return nil, fmt.Errorf("failed to create API client [%v]", err)
}
return client, nil
}
func createClient(s *options.SchedulerServer) (*clientset.Clientset, error) {
kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig)
if err != nil {
return nil, fmt.Errorf("unable to build config from flags: %v", err)
}
kubeconfig.ContentType = s.ContentType
// Override kubeconfig qps/burst settings from flags
kubeconfig.QPS = s.KubeAPIQPS
kubeconfig.Burst = int(s.KubeAPIBurst)
cli, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "leader-election"))
if err != nil {
return nil, fmt.Errorf("invalid API configuration: %v", err)
}
return cli, nil
}
// creates a set of clients for this endpoint
func createClients(caCert []byte, endpoint, token string, nodeName types.NodeName) (*clientset.Clientset, error) {
bareClientConfig := kubeadmutil.CreateBasicClientConfig("kubernetes", endpoint, caCert)
bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig(
*kubeadmutil.MakeClientConfigWithToken(
bareClientConfig, "kubernetes", fmt.Sprintf("kubelet-%s", nodeName), token,
),
&clientcmd.ConfigOverrides{},
).ClientConfig()
if err != nil {
return nil, fmt.Errorf("failed to create API client configuration [%v]", err)
}
clientSet, err := clientset.NewForConfig(bootstrapClientConfig)
if err != nil {
return nil, fmt.Errorf("failed to create clients for the API endpoint %q: [%v]", endpoint, err)
}
return clientSet, nil
}
// PerformTLSBootstrap executes a node certificate signing request.
func PerformTLSBootstrap(cfg *clientcmdapi.Config) error {
hostName, err := os.Hostname()
if err != nil {
return err
}
name := types.NodeName(hostName)
rc, err := clientcmd.NewDefaultClientConfig(*cfg, &clientcmd.ConfigOverrides{}).ClientConfig()
if err != nil {
return err
}
c, err := clientset.NewForConfig(rc)
if err != nil {
return err
}
fmt.Println("[csr] Created API client to obtain unique certificate for this node, generating keys and certificate signing request")
key, err := certutil.MakeEllipticPrivateKeyPEM()
if err != nil {
return fmt.Errorf("failed to generate private key [%v]", err)
}
// Make sure there are no other nodes in the cluster with identical node name.
if err := checkForNodeNameDuplicates(c); err != nil {
return err
}
cert, err := csr.RequestNodeCertificate(c.Certificates().CertificateSigningRequests(), key, name)
if err != nil {
return fmt.Errorf("failed to request signed certificate from the API server [%v]", err)
}
fmt.Printf("[csr] Received signed certificate from the API server")
fmt.Println("[csr] Generating kubelet configuration")
cfg.AuthInfos["kubelet"] = &clientcmdapi.AuthInfo{
ClientKeyData: key,
ClientCertificateData: cert,
}
cfg.Contexts["kubelet"] = &clientcmdapi.Context{
AuthInfo: "kubelet",
Cluster: cfg.Contexts[cfg.CurrentContext].Cluster,
}
cfg.CurrentContext = "kubelet"
return nil
}
func rmSetup(t *testing.T) (*httptest.Server, *disruption.DisruptionController, cache.SharedIndexInformer, clientset.Interface) {
masterConfig := framework.NewIntegrationTestMasterConfig()
_, s := framework.RunAMaster(masterConfig)
config := restclient.Config{Host: s.URL}
clientSet, err := clientset.NewForConfig(&config)
if err != nil {
t.Fatalf("Error in create clientset: %v", err)
}
resyncPeriod := 12 * time.Hour
informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pdb-informers")), nil, resyncPeriod)
rm := disruption.NewDisruptionController(
informers.Pods().Informer(),
clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")),
)
return s, rm, informers.Pods().Informer(), clientSet
}
func rmSetup(t *testing.T, stopCh chan struct{}, enableGarbageCollector bool) (*httptest.Server, *replication.ReplicationManager, cache.SharedIndexInformer, clientset.Interface) {
masterConfig := framework.NewIntegrationTestMasterConfig()
_, s := framework.RunAMaster(masterConfig)
config := restclient.Config{Host: s.URL}
clientSet, err := clientset.NewForConfig(&config)
if err != nil {
t.Fatalf("Error in create clientset: %v", err)
}
resyncPeriod := 12 * time.Hour
informers := informers.NewSharedInformerFactory(clientSet, nil, resyncPeriod)
podInformer := informers.Pods().Informer()
rcInformer := informers.ReplicationControllers().Informer()
rm := replication.NewReplicationManager(podInformer, rcInformer, clientSet, replication.BurstReplicas, 4096, enableGarbageCollector)
informers.Start(stopCh)
return s, rm, podInformer, clientSet
}
// PerformTLSBootstrap executes a certificate signing request with the
// provided connection details.
func PerformTLSBootstrap(cfg *clientcmdapi.Config) error {
hostName, err := os.Hostname()
if err != nil {
return err
}
name := types.NodeName(hostName)
rc, err := clientcmd.NewDefaultClientConfig(*cfg, nil).ClientConfig()
if err != nil {
return err
}
c, err := clientset.NewForConfig(rc)
if err != nil {
return err
}
fmt.Println("<node/csr> created API client to obtain unique certificate for this node, generating keys and certificate signing request")
key, err := certutil.MakeEllipticPrivateKeyPEM()
if err != nil {
return fmt.Errorf("<node/csr> failed to generating private key [%v]", err)
}
cert, err := csr.RequestNodeCertificate(c.Certificates().CertificateSigningRequests(), key, name)
if err != nil {
return fmt.Errorf("<node/csr> failed to request signed certificate from the API server [%v]", err)
}
fmtCert, err := certutil.FormatBytesCert(cert)
if err != nil {
return fmt.Errorf("<node/csr> failed to format certificate [%v]", err)
}
fmt.Printf("<node/csr> received signed certificate from the API server:\n%s\n", fmtCert)
fmt.Println("<node/csr> generating kubelet configuration")
cfg.AuthInfos["kubelet"] = &clientcmdapi.AuthInfo{
ClientKeyData: key,
ClientCertificateData: []byte(fmtCert),
}
cfg.Contexts["kubelet"] = &clientcmdapi.Context{
AuthInfo: "kubelet",
Cluster: cfg.Contexts[cfg.CurrentContext].Cluster,
}
cfg.CurrentContext = "kubelet"
return nil
}
// creates a set of clients for this endpoint
func createClients(caCert []byte, endpoint, token string, nodeName types.NodeName) (*apiClient, error) {
clientConfig := kubeconfigphase.MakeClientConfigWithToken(
endpoint,
"kubernetes",
fmt.Sprintf("kubelet-%s", nodeName),
caCert,
token,
)
bootstrapClientConfig, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
if err != nil {
return nil, fmt.Errorf("failed to create API client configuration [%v]", err)
}
clientSet, err := clientset.NewForConfig(bootstrapClientConfig)
if err != nil {
return nil, fmt.Errorf("failed to create clients for the API endpoint %q: [%v]", endpoint, err)
}
ac := &apiClient{
clientSet: clientSet,
clientConfig: clientConfig,
}
return ac, nil
}
func TestConcurrentEvictionRequests(t *testing.T) {
podNameFormat := "test-pod-%d"
s, rm, podInformer, clientSet := rmSetup(t)
defer s.Close()
ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t)
defer framework.DeleteTestingNamespace(ns, s, t)
stopCh := make(chan struct{})
go podInformer.Run(stopCh)
go rm.Run(stopCh)
config := restclient.Config{Host: s.URL}
clientSet, err := clientset.NewForConfig(&config)
var gracePeriodSeconds int64 = 30
deleteOption := &v1.DeleteOptions{
GracePeriodSeconds: &gracePeriodSeconds,
}
// Generate 10 pods to evict
for i := 0; i < 10; i++ {
podName := fmt.Sprintf(podNameFormat, i)
pod := newPod(podName)
if _, err := clientSet.Core().Pods(ns.Name).Create(pod); err != nil {
t.Errorf("Failed to create pod: %v", err)
}
addPodConditionReady(pod)
if _, err := clientSet.Core().Pods(ns.Name).UpdateStatus(pod); err != nil {
t.Fatal(err)
}
}
waitToObservePods(t, podInformer, 10)
pdb := newPDB()
if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
t.Errorf("Failed to create PodDisruptionBudget: %v", err)
}
waitPDBStable(t, clientSet, 10, ns.Name, pdb.Name)
doneCh := make(chan bool, 10)
errCh := make(chan error, 1)
// spawn 10 goroutine to concurrently evict the pods
for i := 0; i < 10; i++ {
go func(id int, doneCh chan bool, errCh chan error) {
evictionName := fmt.Sprintf(podNameFormat, id)
eviction := newEviction(ns.Name, evictionName, deleteOption)
var e error
for {
e = clientSet.Policy().Evictions(ns.Name).Evict(eviction)
if errors.IsTooManyRequests(e) {
time.Sleep(5 * time.Second)
} else {
break
}
}
if e != nil {
if errors.IsConflict(err) {
fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
} else {
errCh <- e
}
return
}
doneCh <- true
}(i, doneCh, errCh)
}
doneCount := 0
for {
select {
case err := <-errCh:
t.Errorf("%v", err)
return
case <-doneCh:
doneCount++
if doneCount == 10 {
return
}
case <-time.After(defaultTimeout):
t.Errorf("Eviction did not complete within %v", defaultTimeout)
}
}
for i := 0; i < 10; i++ {
podName := fmt.Sprintf(podNameFormat, i)
_, err := clientSet.Core().Pods(ns.Name).Get(podName, metav1.GetOptions{})
if !errors.IsNotFound(err) {
t.Errorf("Pod %q is expected to be evicted", podName)
}
}
if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
t.Errorf("Failed to delete PodDisruptionBudget: %v", err)
}
close(stopCh)
}
// TestConcurrentEvictionRequests is to make sure pod disruption budgets (PDB) controller is able to
// handle concurrent eviction requests. Original issue:#37605
func TestConcurrentEvictionRequests(t *testing.T) {
podNameFormat := "test-pod-%d"
s, rm, podInformer, clientSet := rmSetup(t)
defer s.Close()
ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t)
defer framework.DeleteTestingNamespace(ns, s, t)
stopCh := make(chan struct{})
go podInformer.Run(stopCh)
go rm.Run(stopCh)
defer close(stopCh)
config := restclient.Config{Host: s.URL}
clientSet, err := clientset.NewForConfig(&config)
if err != nil {
t.Fatalf("Failed to create clientset: %v", err)
}
var gracePeriodSeconds int64 = 30
deleteOption := &v1.DeleteOptions{
GracePeriodSeconds: &gracePeriodSeconds,
}
// Generate numOfEvictions pods to evict
for i := 0; i < numOfEvictions; i++ {
podName := fmt.Sprintf(podNameFormat, i)
pod := newPod(podName)
if _, err := clientSet.Core().Pods(ns.Name).Create(pod); err != nil {
t.Errorf("Failed to create pod: %v", err)
}
addPodConditionReady(pod)
if _, err := clientSet.Core().Pods(ns.Name).UpdateStatus(pod); err != nil {
t.Fatal(err)
}
}
waitToObservePods(t, podInformer, numOfEvictions)
pdb := newPDB()
if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil {
t.Errorf("Failed to create PodDisruptionBudget: %v", err)
}
waitPDBStable(t, clientSet, numOfEvictions, ns.Name, pdb.Name)
var numberPodsEvicted uint32 = 0
errCh := make(chan error, 3*numOfEvictions)
var wg sync.WaitGroup
// spawn numOfEvictions goroutines to concurrently evict the pods
for i := 0; i < numOfEvictions; i++ {
wg.Add(1)
go func(id int, errCh chan error) {
defer wg.Done()
podName := fmt.Sprintf(podNameFormat, id)
eviction := newEviction(ns.Name, podName, deleteOption)
err := wait.PollImmediate(5*time.Second, 60*time.Second, func() (bool, error) {
e := clientSet.Policy().Evictions(ns.Name).Evict(eviction)
switch {
case errors.IsTooManyRequests(e):
return false, nil
case errors.IsConflict(e):
return false, fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e)
case e == nil:
return true, nil
default:
return false, e
}
})
if err != nil {
errCh <- err
// should not return here otherwise we would leak the pod
}
_, err = clientSet.Core().Pods(ns.Name).Get(podName, metav1.GetOptions{})
switch {
case errors.IsNotFound(err):
atomic.AddUint32(&numberPodsEvicted, 1)
// pod was evicted and deleted so return from goroutine immediately
return
case err == nil:
// this shouldn't happen if the pod was evicted successfully
errCh <- fmt.Errorf("Pod %q is expected to be evicted", podName)
default:
errCh <- err
}
// delete pod which still exists due to error
e := clientSet.Core().Pods(ns.Name).Delete(podName, deleteOption)
if e != nil {
errCh <- e
}
}(i, errCh)
}
wg.Wait()
close(errCh)
var errList []error
if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil {
errList = append(errList, fmt.Errorf("Failed to delete PodDisruptionBudget: %v", err))
}
for err := range errCh {
errList = append(errList, err)
}
if len(errList) > 0 {
t.Fatal(utilerrors.NewAggregate(errList))
}
if atomic.LoadUint32(&numberPodsEvicted) != numOfEvictions {
t.Fatalf("fewer number of successful evictions than expected :", numberPodsEvicted)
}
}
// Run runs the CMServer. This should never exit.
func Run(s *options.CMServer) error {
if err := s.Validate(KnownControllers(), ControllersDisabledByDefault.List()); err != nil {
return err
}
if c, err := configz.New("componentconfig"); err == nil {
c.Set(s.KubeControllerManagerConfiguration)
} else {
glog.Errorf("unable to register configz: %s", err)
}
kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig)
if err != nil {
return err
}
kubeconfig.ContentConfig.ContentType = s.ContentType
// Override kubeconfig qps/burst settings from flags
kubeconfig.QPS = s.KubeAPIQPS
kubeconfig.Burst = int(s.KubeAPIBurst)
kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "controller-manager"))
if err != nil {
glog.Fatalf("Invalid API configuration: %v", err)
}
leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election"))
go func() {
mux := http.NewServeMux()
healthz.InstallHandler(mux)
if s.EnableProfiling {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
}
configz.InstallHandler(mux)
mux.Handle("/metrics", prometheus.Handler())
server := &http.Server{
Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))),
Handler: mux,
}
glog.Fatal(server.ListenAndServe())
}()
eventBroadcaster := record.NewBroadcaster()
eventBroadcaster.StartLogging(glog.Infof)
eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.Core().Events("")})
recorder := eventBroadcaster.NewRecorder(v1.EventSource{Component: "controller-manager"})
run := func(stop <-chan struct{}) {
rootClientBuilder := controller.SimpleControllerClientBuilder{
ClientConfig: kubeconfig,
}
var clientBuilder controller.ControllerClientBuilder
if len(s.ServiceAccountKeyFile) > 0 && s.UseServiceAccountCredentials {
clientBuilder = controller.SAControllerClientBuilder{
ClientConfig: restclient.AnonymousClientConfig(kubeconfig),
CoreClient: kubeClient.Core(),
Namespace: "kube-system",
}
} else {
clientBuilder = rootClientBuilder
}
err := StartControllers(newControllerInitializers(), s, rootClientBuilder, clientBuilder, stop)
glog.Fatalf("error running controllers: %v", err)
panic("unreachable")
}
if !s.LeaderElection.LeaderElect {
run(nil)
panic("unreachable")
}
id, err := os.Hostname()
if err != nil {
return err
}
// TODO: enable other lock types
rl := resourcelock.EndpointsLock{
EndpointsMeta: metav1.ObjectMeta{
Namespace: "kube-system",
Name: "kube-controller-manager",
},
Client: leaderElectionClient,
LockConfig: resourcelock.ResourceLockConfig{
Identity: id,
EventRecorder: recorder,
},
}
leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{
Lock: &rl,
LeaseDuration: s.LeaderElection.LeaseDuration.Duration,
RenewDeadline: s.LeaderElection.RenewDeadline.Duration,
RetryPeriod: s.LeaderElection.RetryPeriod.Duration,
Callbacks: leaderelection.LeaderCallbacks{
OnStartedLeading: run,
OnStoppedLeading: func() {
glog.Fatalf("leaderelection lost")
},
},
})
panic("unreachable")
}
func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) {
// TODO: this should be replaced by a --standalone flag
standaloneMode := (len(s.APIServerList) == 0 && !s.RequireKubeConfig)
if s.ExitOnLockContention && s.LockFilePath == "" {
return errors.New("cannot exit on lock file contention: no lock file specified")
}
done := make(chan struct{})
if s.LockFilePath != "" {
glog.Infof("acquiring file lock on %q", s.LockFilePath)
if err := flock.Acquire(s.LockFilePath); err != nil {
return fmt.Errorf("unable to acquire file lock on %q: %v", s.LockFilePath, err)
}
if s.ExitOnLockContention {
glog.Infof("watching for inotify events for: %v", s.LockFilePath)
if err := watchForLockfileContention(s.LockFilePath, done); err != nil {
return err
}
}
}
// Set feature gates based on the value in KubeletConfiguration
err = utilconfig.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
if err != nil {
return err
}
// Register current configuration with /configz endpoint
cfgz, cfgzErr := initConfigz(&s.KubeletConfiguration)
if utilconfig.DefaultFeatureGate.DynamicKubeletConfig() {
// Look for config on the API server. If it exists, replace s.KubeletConfiguration
// with it and continue. initKubeletConfigSync also starts the background thread that checks for new config.
// Don't do dynamic Kubelet configuration in runonce mode
if s.RunOnce == false {
remoteKC, err := initKubeletConfigSync(s)
if err == nil {
// Update s (KubeletServer) with new config from API server
s.KubeletConfiguration = *remoteKC
// Ensure that /configz is up to date with the new config
if cfgzErr != nil {
glog.Errorf("was unable to register configz before due to %s, will not be able to set now", cfgzErr)
} else {
setConfigz(cfgz, &s.KubeletConfiguration)
}
// Update feature gates from the new config
err = utilconfig.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates)
if err != nil {
return err
}
} else {
glog.Errorf("failed to init dynamic Kubelet configuration sync: %v", err)
}
}
}
if kubeDeps == nil {
var kubeClient, eventClient *clientset.Clientset
var externalKubeClient clientgoclientset.Interface
var cloud cloudprovider.Interface
if s.CloudProvider != componentconfigv1alpha1.AutoDetectCloudProvider {
cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
return err
}
if cloud == nil {
glog.V(2).Infof("No cloud provider specified: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
} else {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile)
}
}
if s.BootstrapKubeconfig != "" {
nodeName, err := getNodeName(cloud, nodeutil.GetHostname(s.HostnameOverride))
if err != nil {
return err
}
if err := bootstrapClientCert(s.KubeConfig.Value(), s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil {
return err
}
}
clientConfig, err := CreateAPIServerClientConfig(s)
if err == nil {
kubeClient, err = clientset.NewForConfig(clientConfig)
if err != nil {
glog.Warningf("New kubeClient from clientConfig error: %v", err)
}
// make a separate client for events
eventClientConfig := *clientConfig
eventClientConfig.QPS = float32(s.EventRecordQPS)
eventClientConfig.Burst = int(s.EventBurst)
eventClient, err = clientset.NewForConfig(&eventClientConfig)
if err != nil {
glog.Warningf("Failed to create API Server client: %v", err)
}
} else {
if s.RequireKubeConfig {
return fmt.Errorf("invalid kubeconfig: %v", err)
}
if standaloneMode {
glog.Warningf("No API client: %v", err)
}
}
// client-go and kuberenetes generated clients are incompatible because the runtime
// and runtime/serializer types have been duplicated in client-go. This means that
// you can't reasonably convert from one to the other and its impossible for a single
// type to fulfill both interfaces. Because of that, we have to build the clients
// up from scratch twice.
// TODO eventually the kubelet should only use the client-go library
clientGoConfig, err := createAPIServerClientGoConfig(s)
if err == nil {
externalKubeClient, err = clientgoclientset.NewForConfig(clientGoConfig)
if err != nil {
glog.Warningf("New kubeClient from clientConfig error: %v", err)
}
} else {
if s.RequireKubeConfig {
return fmt.Errorf("invalid kubeconfig: %v", err)
}
if standaloneMode {
glog.Warningf("No API client: %v", err)
}
}
kubeDeps, err = UnsecuredKubeletDeps(s)
if err != nil {
return err
}
kubeDeps.Cloud = cloud
kubeDeps.KubeClient = kubeClient
kubeDeps.ExternalKubeClient = externalKubeClient
kubeDeps.EventClient = eventClient
}
if kubeDeps.Auth == nil {
nodeName, err := getNodeName(kubeDeps.Cloud, nodeutil.GetHostname(s.HostnameOverride))
if err != nil {
return err
}
auth, err := buildAuth(nodeName, kubeDeps.ExternalKubeClient, s.KubeletConfiguration)
if err != nil {
return err
}
kubeDeps.Auth = auth
}
if kubeDeps.CAdvisorInterface == nil {
kubeDeps.CAdvisorInterface, err = cadvisor.New(uint(s.CAdvisorPort), s.ContainerRuntime, s.RootDirectory)
if err != nil {
return err
}
}
if kubeDeps.ContainerManager == nil {
if s.SystemCgroups != "" && s.CgroupRoot == "" {
return fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified")
}
kubeDeps.ContainerManager, err = cm.NewContainerManager(
kubeDeps.Mounter,
kubeDeps.CAdvisorInterface,
cm.NodeConfig{
RuntimeCgroupsName: s.RuntimeCgroups,
SystemCgroupsName: s.SystemCgroups,
KubeletCgroupsName: s.KubeletCgroups,
ContainerRuntime: s.ContainerRuntime,
CgroupsPerQOS: s.ExperimentalCgroupsPerQOS,
CgroupRoot: s.CgroupRoot,
CgroupDriver: s.CgroupDriver,
ProtectKernelDefaults: s.ProtectKernelDefaults,
EnableCRI: s.EnableCRI,
},
s.ExperimentalFailSwapOn)
if err != nil {
return err
}
}
if err := checkPermissions(); err != nil {
glog.Error(err)
}
utilruntime.ReallyCrash = s.ReallyCrashForTesting
rand.Seed(time.Now().UTC().UnixNano())
// TODO(vmarmol): Do this through container config.
oomAdjuster := kubeDeps.OOMAdjuster
if err := oomAdjuster.ApplyOOMScoreAdj(0, int(s.OOMScoreAdj)); err != nil {
glog.Warning(err)
}
if err := RunKubelet(&s.KubeletConfiguration, kubeDeps, s.RunOnce, standaloneMode); err != nil {
return err
}
if s.HealthzPort > 0 {
healthz.DefaultHealthz()
go wait.Until(func() {
err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), nil)
if err != nil {
glog.Errorf("Starting health server failed: %v", err)
}
}, 5*time.Second, wait.NeverStop)
}
if s.RunOnce {
return nil
}
<-done
return nil
}
func TestServerRunWithSNI(t *testing.T) {
tests := map[string]struct {
Cert TestCertSpec
SNICerts []NamedTestCertSpec
ExpectedCertIndex int
// passed in the client hello info, "localhost" if unset
ServerName string
// optional ip or hostname to pass to NewSelfClientConfig
SelfClientBindAddressOverride string
ExpectSelfClientError bool
}{
"only one cert": {
Cert: TestCertSpec{
host: "localhost",
ips: []string{"127.0.0.1"},
},
ExpectedCertIndex: -1,
},
"cert with multiple alternate names": {
Cert: TestCertSpec{
host: "localhost",
names: []string{"test.com"},
ips: []string{"127.0.0.1"},
},
ExpectedCertIndex: -1,
ServerName: "test.com",
},
"one SNI and the default cert with the same name": {
Cert: TestCertSpec{
host: "localhost",
ips: []string{"127.0.0.1"},
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "localhost",
},
},
},
ExpectedCertIndex: 0,
},
"matching SNI cert": {
Cert: TestCertSpec{
host: "localhost",
ips: []string{"127.0.0.1"},
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
},
},
ExpectedCertIndex: 0,
ServerName: "test.com",
},
"matching IP in SNI cert and the server cert": {
// IPs must not be passed via SNI. Hence, the ServerName in the
// HELLO packet is empty and the server should select the non-SNI cert.
Cert: TestCertSpec{
host: "localhost",
ips: []string{"10.0.0.1", "127.0.0.1"},
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
ips: []string{"10.0.0.1"},
},
},
},
ExpectedCertIndex: -1,
ServerName: "10.0.0.1",
},
"wildcards": {
Cert: TestCertSpec{
host: "localhost",
ips: []string{"127.0.0.1"},
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
names: []string{"*.test.com"},
},
},
},
ExpectedCertIndex: 0,
ServerName: "www.test.com",
},
"loopback: IP for loopback client on SNI cert": {
Cert: TestCertSpec{
host: "localhost",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
ips: []string{"127.0.0.1"},
},
},
},
ExpectedCertIndex: -1,
ExpectSelfClientError: true,
},
"loopback: IP for loopback client on server and SNI cert": {
Cert: TestCertSpec{
ips: []string{"127.0.0.1"},
host: "localhost",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
ips: []string{"127.0.0.1"},
},
},
},
ExpectedCertIndex: -1,
},
"loopback: bind to 0.0.0.0 => loopback uses localhost; localhost on server cert": {
Cert: TestCertSpec{
host: "localhost",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "test.com",
},
},
},
ExpectedCertIndex: -1,
SelfClientBindAddressOverride: "0.0.0.0",
},
"loopback: bind to 0.0.0.0 => loopback uses localhost; localhost on SNI cert": {
Cert: TestCertSpec{
host: "test.com",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "localhost",
},
},
},
ExpectedCertIndex: 0,
SelfClientBindAddressOverride: "0.0.0.0",
},
"loopback: bind to 0.0.0.0 => loopback uses localhost; localhost on server and SNI cert": {
Cert: TestCertSpec{
host: "localhost",
},
SNICerts: []NamedTestCertSpec{
{
TestCertSpec: TestCertSpec{
host: "localhost",
},
},
},
ExpectedCertIndex: 0,
SelfClientBindAddressOverride: "0.0.0.0",
},
}
tempDir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
NextTest:
for title, test := range tests {
// create server cert
serverCertBundleFile, serverKeyFile, err := createTestCertFiles(tempDir, test.Cert)
if err != nil {
t.Errorf("%q - failed to create server cert: %v", title, err)
continue NextTest
}
ca, err := caCertFromBundle(serverCertBundleFile)
if err != nil {
t.Errorf("%q - failed to extract ca cert from server cert bundle: %v", title, err)
continue NextTest
}
caCerts := []*x509.Certificate{ca}
// create SNI certs
var namedCertKeys []config.NamedCertKey
serverSig, err := certFileSignature(serverCertBundleFile, serverKeyFile)
if err != nil {
t.Errorf("%q - failed to get server cert signature: %v", title, err)
continue NextTest
}
signatures := map[string]int{
serverSig: -1,
}
for j, c := range test.SNICerts {
certBundleFile, keyFile, err := createTestCertFiles(tempDir, c.TestCertSpec)
if err != nil {
t.Errorf("%q - failed to create SNI cert %d: %v", title, j, err)
continue NextTest
}
namedCertKeys = append(namedCertKeys, config.NamedCertKey{
KeyFile: keyFile,
CertFile: certBundleFile,
Names: c.explicitNames,
})
ca, err := caCertFromBundle(certBundleFile)
if err != nil {
t.Errorf("%q - failed to extract ca cert from SNI cert %d: %v", title, j, err)
continue NextTest
}
caCerts = append(caCerts, ca)
// store index in namedCertKeys with the signature as the key
sig, err := certFileSignature(certBundleFile, keyFile)
if err != nil {
t.Errorf("%q - failed get SNI cert %d signature: %v", title, j, err)
continue NextTest
}
signatures[sig] = j
}
stopCh := make(chan struct{})
// launch server
etcdserver, config, _ := setUp(t)
defer etcdserver.Terminate(t)
v := fakeVersion()
config.Version = &v
config.EnableIndex = true
_, err = config.ApplySecureServingOptions(&options.SecureServingOptions{
ServingOptions: options.ServingOptions{
BindAddress: net.ParseIP("127.0.0.1"),
BindPort: 6443,
},
ServerCert: options.GeneratableKeyCert{
CertKey: options.CertKey{
CertFile: serverCertBundleFile,
KeyFile: serverKeyFile,
},
},
SNICertKeys: namedCertKeys,
})
if err != nil {
t.Errorf("%q - failed applying the SecureServingOptions: %v", title, err)
continue NextTest
}
config.InsecureServingInfo = nil
s, err := config.Complete().New()
if err != nil {
t.Errorf("%q - failed creating the server: %v", title, err)
continue NextTest
}
// patch in a 0-port to enable auto port allocation
s.SecureServingInfo.BindAddress = "127.0.0.1:0"
if err := s.serveSecurely(stopCh); err != nil {
t.Errorf("%q - failed running the server: %v", title, err)
continue NextTest
}
// load ca certificates into a pool
roots := x509.NewCertPool()
for _, caCert := range caCerts {
roots.AddCert(caCert)
}
// try to dial
addr := fmt.Sprintf("localhost:%d", s.effectiveSecurePort)
t.Logf("Dialing %s as %q", addr, test.ServerName)
conn, err := tls.Dial("tcp", addr, &tls.Config{
RootCAs: roots,
ServerName: test.ServerName, // used for SNI in the client HELLO packet
})
if err != nil {
t.Errorf("%q - failed to connect: %v", title, err)
continue NextTest
}
// check returned server certificate
sig := x509CertSignature(conn.ConnectionState().PeerCertificates[0])
gotCertIndex, found := signatures[sig]
if !found {
t.Errorf("%q - unknown signature returned from server: %s", title, sig)
}
if gotCertIndex != test.ExpectedCertIndex {
t.Errorf("%q - expected cert index %d, got cert index %d", title, test.ExpectedCertIndex, gotCertIndex)
}
conn.Close()
// check that the loopback client can connect
host := "127.0.0.1"
if len(test.SelfClientBindAddressOverride) != 0 {
host = test.SelfClientBindAddressOverride
}
config.SecureServingInfo.ServingInfo.BindAddress = net.JoinHostPort(host, strconv.Itoa(s.effectiveSecurePort))
cfg, err := config.SecureServingInfo.NewSelfClientConfig("some-token")
if test.ExpectSelfClientError {
if err == nil {
t.Errorf("%q - expected error creating loopback client config", title)
}
continue NextTest
}
if err != nil {
t.Errorf("%q - failed creating loopback client config: %v", title, err)
continue NextTest
}
client, err := clientset.NewForConfig(cfg)
if err != nil {
t.Errorf("%q - failed to create loopback client: %v", title, err)
continue NextTest
}
got, err := client.ServerVersion()
if err != nil {
t.Errorf("%q - failed to connect with loopback client: %v", title, err)
continue NextTest
}
if expected := &v; !reflect.DeepEqual(got, expected) {
t.Errorf("%q - loopback client didn't get correct version info: expected=%v got=%v", title, expected, got)
}
}
}
func main() {
flag.Parse()
glog.Infof("Starting serve_hostnames soak test with queries=%d and podsPerNode=%d upTo=%d",
*queriesAverage, *podsPerNode, *upTo)
var spec string
if *gke != "" {
spec = filepath.Join(os.Getenv("HOME"), ".config", "gcloud", "kubernetes", "kubeconfig")
} else {
spec = filepath.Join(os.Getenv("HOME"), ".kube", "config")
}
settings, err := clientcmd.LoadFromFile(spec)
if err != nil {
glog.Fatalf("Error loading configuration: %v", err.Error())
}
if *gke != "" {
settings.CurrentContext = *gke
}
config, err := clientcmd.NewDefaultClientConfig(*settings, &clientcmd.ConfigOverrides{}).ClientConfig()
if err != nil {
glog.Fatalf("Failed to construct config: %v", err)
}
client, err := clientset.NewForConfig(config)
if err != nil {
glog.Fatalf("Failed to make client: %v", err)
}
var nodes *v1.NodeList
for start := time.Now(); time.Since(start) < nodeListTimeout; time.Sleep(2 * time.Second) {
nodes, err = client.Nodes().List(v1.ListOptions{})
if err == nil {
break
}
glog.Warningf("Failed to list nodes: %v", err)
}
if err != nil {
glog.Fatalf("Giving up trying to list nodes: %v", err)
}
if len(nodes.Items) == 0 {
glog.Fatalf("Failed to find any nodes.")
}
glog.Infof("Found %d nodes on this cluster:", len(nodes.Items))
for i, node := range nodes.Items {
glog.Infof("%d: %s", i, node.Name)
}
queries := *queriesAverage * len(nodes.Items) * *podsPerNode
// Create the namespace
got, err := client.Namespaces().Create(&v1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "serve-hostnames-"}})
if err != nil {
glog.Fatalf("Failed to create namespace: %v", err)
}
ns := got.Name
defer func(ns string) {
if err := client.Core().Namespaces().Delete(ns, nil); err != nil {
glog.Warningf("Failed to delete namespace ns: %e", ns, err)
} else {
// wait until the namespace disappears
for i := 0; i < int(namespaceDeleteTimeout/time.Second); i++ {
if _, err := client.Namespaces().Get(ns, metav1.GetOptions{}); err != nil {
if errors.IsNotFound(err) {
return
}
}
time.Sleep(time.Second)
}
}
}(ns)
glog.Infof("Created namespace %s", ns)
// Create a service for these pods.
glog.Infof("Creating service %s/serve-hostnames", ns)
// Make several attempts to create a service.
var svc *v1.Service
for start := time.Now(); time.Since(start) < serviceCreateTimeout; time.Sleep(2 * time.Second) {
t := time.Now()
svc, err = client.Services(ns).Create(&v1.Service{
ObjectMeta: v1.ObjectMeta{
Name: "serve-hostnames",
Labels: map[string]string{
"name": "serve-hostname",
},
},
Spec: v1.ServiceSpec{
Ports: []v1.ServicePort{{
Protocol: "TCP",
Port: 9376,
TargetPort: intstr.FromInt(9376),
}},
Selector: map[string]string{
"name": "serve-hostname",
},
},
})
glog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t))
if err == nil {
break
}
glog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err)
}
if err != nil {
glog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err)
return
}
// Clean up service
defer func() {
glog.Infof("Cleaning up service %s/serve-hostnames", ns)
// Make several attempts to delete the service.
for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) {
if err := client.Services(ns).Delete(svc.Name, nil); err == nil {
return
}
glog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err)
}
}()
// Put serve-hostname pods on each node.
podNames := []string{}
for i, node := range nodes.Items {
for j := 0; j < *podsPerNode; j++ {
podName := fmt.Sprintf("serve-hostname-%d-%d", i, j)
podNames = append(podNames, podName)
// Make several attempts
for start := time.Now(); time.Since(start) < podCreateTimeout; time.Sleep(2 * time.Second) {
glog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name)
t := time.Now()
_, err = client.Pods(ns).Create(&v1.Pod{
ObjectMeta: v1.ObjectMeta{
Name: podName,
Labels: map[string]string{
"name": "serve-hostname",
},
},
Spec: v1.PodSpec{
Containers: []v1.Container{
{
Name: "serve-hostname",
Image: "gcr.io/google_containers/serve_hostname:v1.4",
Ports: []v1.ContainerPort{{ContainerPort: 9376}},
},
},
NodeName: node.Name,
},
})
glog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t))
if err == nil {
break
}
glog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err)
}
if err != nil {
glog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err)
return
}
}
}
// Clean up the pods
defer func() {
glog.Info("Cleaning up pods")
// Make several attempts to delete the pods.
for _, podName := range podNames {
for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) {
if err = client.Pods(ns).Delete(podName, nil); err == nil {
break
}
glog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err)
}
}
}()
glog.Info("Waiting for the serve-hostname pods to be ready")
for _, podName := range podNames {
var pod *v1.Pod
for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) {
pod, err = client.Pods(ns).Get(podName, metav1.GetOptions{})
if err != nil {
glog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout)
continue
}
if pod.Status.Phase == v1.PodRunning {
break
}
}
if pod.Status.Phase != v1.PodRunning {
glog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase)
} else {
glog.Infof("%s/%s is running", ns, podName)
}
}
rclient, err := restclient.RESTClientFor(config)
if err != nil {
glog.Warningf("Failed to build restclient: %v", err)
return
}
proxyRequest, errProxy := e2e.GetServicesProxyRequest(client, rclient.Get())
if errProxy != nil {
glog.Warningf("Get services proxy request failed: %v", errProxy)
return
}
// Wait for the endpoints to propagate.
for start := time.Now(); time.Since(start) < endpointTimeout; time.Sleep(10 * time.Second) {
hostname, err := proxyRequest.
Namespace(ns).
Name("serve-hostnames").
DoRaw()
if err != nil {
glog.Infof("After %v while making a proxy call got error %v", time.Since(start), err)
continue
}
var r metav1.Status
if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), hostname, &r); err != nil {
break
}
if r.Status == metav1.StatusFailure {
glog.Infof("After %v got status %v", time.Since(start), string(hostname))
continue
}
break
}
// Repeatedly make requests.
for iteration := 0; iteration != *upTo; iteration++ {
responseChan := make(chan string, queries)
// Use a channel of size *maxPar to throttle the number
// of in-flight requests to avoid overloading the service.
inFlight := make(chan struct{}, *maxPar)
start := time.Now()
for q := 0; q < queries; q++ {
go func(i int, query int) {
inFlight <- struct{}{}
t := time.Now()
hostname, err := proxyRequest.
Namespace(ns).
Name("serve-hostnames").
DoRaw()
glog.V(4).Infof("Proxy call in namespace %s took %v", ns, time.Since(t))
if err != nil {
glog.Warningf("Call failed during iteration %d query %d : %v", i, query, err)
// If the query failed return a string which starts with a character
// that can't be part of a hostname.
responseChan <- fmt.Sprintf("!failed in iteration %d to issue query %d: %v", i, query, err)
} else {
responseChan <- string(hostname)
}
<-inFlight
}(iteration, q)
}
responses := make(map[string]int, *podsPerNode*len(nodes.Items))
missing := 0
for q := 0; q < queries; q++ {
r := <-responseChan
glog.V(4).Infof("Got response from %s", r)
responses[r]++
// If the returned hostname starts with '!' then it indicates
// an error response.
if len(r) > 0 && r[0] == '!' {
glog.V(3).Infof("Got response %s", r)
missing++
}
}
if missing > 0 {
glog.Warningf("Missing %d responses out of %d", missing, queries)
}
// Report any nodes that did not respond.
for n, node := range nodes.Items {
for i := 0; i < *podsPerNode; i++ {
name := fmt.Sprintf("serve-hostname-%d-%d", n, i)
if _, ok := responses[name]; !ok {
glog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration)
}
}
}
glog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing",
iteration, time.Since(start), queries-missing, float64(queries-missing)/time.Since(start).Seconds(), missing)
}
}
func (o DiscoveryServerOptions) RunDiscoveryServer() error {
// if we don't have an etcd to back the server, we must be a legacy server
if len(o.Etcd.StorageConfig.ServerList) == 0 {
return o.RunLegacyDiscoveryServer()
}
// TODO have a "real" external address
if err := o.SecureServing.MaybeDefaultWithSelfSignedCerts("localhost"); err != nil {
return fmt.Errorf("error creating self-signed certificates: %v", err)
}
genericAPIServerConfig := genericapiserver.NewConfig()
if _, err := genericAPIServerConfig.ApplySecureServingOptions(o.SecureServing); err != nil {
return err
}
if _, err := genericAPIServerConfig.ApplyDelegatingAuthenticationOptions(o.Authentication); err != nil {
return err
}
if _, err := genericAPIServerConfig.ApplyDelegatingAuthorizationOptions(o.Authorization); err != nil {
return err
}
genericAPIServerConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
var err error
privilegedLoopbackToken := uuid.NewRandom().String()
if genericAPIServerConfig.LoopbackClientConfig, err = genericAPIServerConfig.SecureServingInfo.NewSelfClientConfig(privilegedLoopbackToken); err != nil {
return err
}
kubeconfig, err := restclient.InClusterConfig()
if err != nil {
return err
}
coreAPIServerClient, err := kubeclientset.NewForConfig(kubeconfig)
if err != nil {
return err
}
config := apiserver.Config{
GenericConfig: genericAPIServerConfig,
RESTOptionsGetter: &restOptionsFactory{storageConfig: &o.Etcd.StorageConfig},
CoreAPIServerClient: coreAPIServerClient,
}
config.ProxyClientCert, err = ioutil.ReadFile(o.ProxyClientCertFile)
if err != nil {
return err
}
config.ProxyClientKey, err = ioutil.ReadFile(o.ProxyClientKeyFile)
if err != nil {
return err
}
server, err := config.Complete().New()
if err != nil {
return err
}
server.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
return nil
}
func main() {
config := HollowNodeConfig{}
config.addFlags(pflag.CommandLine)
flag.InitFlags()
if !knownMorphs.Has(config.Morph) {
glog.Fatalf("Unknown morph: %v. Allowed values: %v", config.Morph, knownMorphs.List())
}
// create a client to communicate with API server.
clientConfig, err := config.createClientConfigFromFile()
if err != nil {
glog.Fatalf("Failed to create a ClientConfig: %v. Exiting.", err)
}
clientset, err := clientset.NewForConfig(clientConfig)
if err != nil {
glog.Fatalf("Failed to create a ClientSet: %v. Exiting.", err)
}
internalClientset, err := internalclientset.NewForConfig(clientConfig)
if err != nil {
glog.Fatalf("Failed to create an internal ClientSet: %v. Exiting.", err)
}
if config.Morph == "kubelet" {
cadvisorInterface := new(cadvisortest.Fake)
containerManager := cm.NewStubContainerManager()
fakeDockerClient := dockertools.NewFakeDockerClient().WithTraceDisabled()
fakeDockerClient.EnableSleep = true
hollowKubelet := kubemark.NewHollowKubelet(
config.NodeName,
clientset,
cadvisorInterface,
fakeDockerClient,
config.KubeletPort,
config.KubeletReadOnlyPort,
containerManager,
maxPods,
podsPerCore,
)
hollowKubelet.Run()
}
if config.Morph == "proxy" {
eventBroadcaster := record.NewBroadcaster()
recorder := eventBroadcaster.NewRecorder(v1.EventSource{Component: "kube-proxy", Host: config.NodeName})
iptInterface := fakeiptables.NewFake()
serviceConfig := proxyconfig.NewServiceConfig()
serviceConfig.RegisterHandler(&kubemark.FakeProxyHandler{})
endpointsConfig := proxyconfig.NewEndpointsConfig()
endpointsConfig.RegisterHandler(&kubemark.FakeProxyHandler{})
hollowProxy := kubemark.NewHollowProxyOrDie(config.NodeName, internalClientset, endpointsConfig, serviceConfig, iptInterface, eventBroadcaster, recorder)
hollowProxy.Run()
}
}
// BeforeEach gets a client and makes a namespace.
func (f *Framework) BeforeEach() {
// The fact that we need this feels like a bug in ginkgo.
// https://github.com/onsi/ginkgo/issues/222
f.cleanupHandle = AddCleanupAction(f.AfterEach)
if f.ClientSet == nil {
By("Creating a kubernetes client")
config, err := LoadConfig()
Expect(err).NotTo(HaveOccurred())
config.QPS = f.options.ClientQPS
config.Burst = f.options.ClientBurst
if f.options.GroupVersion != nil {
config.GroupVersion = f.options.GroupVersion
}
if TestContext.KubeAPIContentType != "" {
config.ContentType = TestContext.KubeAPIContentType
}
f.ClientSet, err = clientset.NewForConfig(config)
Expect(err).NotTo(HaveOccurred())
f.InternalClientset, err = internalclientset.NewForConfig(config)
Expect(err).NotTo(HaveOccurred())
clientRepoConfig := getClientRepoConfig(config)
f.StagingClient, err = staging.NewForConfig(clientRepoConfig)
Expect(err).NotTo(HaveOccurred())
f.ClientPool = dynamic.NewClientPool(config, registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc)
}
if f.federated {
if f.FederationClientset_1_5 == nil {
By("Creating a release 1.4 federation Clientset")
var err error
f.FederationClientset_1_5, err = LoadFederationClientset_1_5()
Expect(err).NotTo(HaveOccurred())
}
By("Waiting for federation-apiserver to be ready")
err := WaitForFederationApiserverReady(f.FederationClientset_1_5)
Expect(err).NotTo(HaveOccurred())
By("federation-apiserver is ready")
By("Creating a federation namespace")
ns, err := f.createFederationNamespace(f.BaseName)
Expect(err).NotTo(HaveOccurred())
f.FederationNamespace = ns
By(fmt.Sprintf("Created federation namespace %s", ns.Name))
}
By("Building a namespace api object")
namespace, err := f.CreateNamespace(f.BaseName, map[string]string{
"e2e-framework": f.BaseName,
})
Expect(err).NotTo(HaveOccurred())
f.Namespace = namespace
if TestContext.VerifyServiceAccount {
By("Waiting for a default service account to be provisioned in namespace")
err = WaitForDefaultServiceAccountInNamespace(f.ClientSet, namespace.Name)
Expect(err).NotTo(HaveOccurred())
} else {
Logf("Skipping waiting for service account")
}
if TestContext.GatherKubeSystemResourceUsageData != "false" && TestContext.GatherKubeSystemResourceUsageData != "none" {
f.gatherer, err = NewResourceUsageGatherer(f.ClientSet, ResourceGathererOptions{
inKubemark: ProviderIs("kubemark"),
masterOnly: TestContext.GatherKubeSystemResourceUsageData == "master",
})
if err != nil {
Logf("Error while creating NewResourceUsageGatherer: %v", err)
} else {
go f.gatherer.startGatheringData()
}
}
if TestContext.GatherLogsSizes {
f.logsSizeWaitGroup = sync.WaitGroup{}
f.logsSizeWaitGroup.Add(1)
f.logsSizeCloseChannel = make(chan bool)
f.logsSizeVerifier = NewLogsVerifier(f.ClientSet, f.logsSizeCloseChannel)
go func() {
f.logsSizeVerifier.Run()
f.logsSizeWaitGroup.Done()
}()
}
}
