func setup(t *testing.T) (*httptest.Server, *garbagecollector.GarbageCollector, clientset.Interface) { masterConfig := framework.NewIntegrationTestMasterConfig() masterConfig.EnableCoreControllers = false masterConfig.GenericConfig.EnableGarbageCollection = true _, s := framework.RunAMaster(masterConfig) clientSet, err := clientset.NewForConfig(&restclient.Config{Host: s.URL}) if err != nil { t.Fatalf("Error in create clientset: %v", err) } preferredResources, err := clientSet.Discovery().ServerPreferredResources() if err != nil { t.Fatalf("Failed to get supported resources from server: %v", err) } deletableResources := discovery.FilteredBy(discovery.SupportsAllVerbs{Verbs: []string{"delete"}}, preferredResources) deletableGroupVersionResources, err := discovery.GroupVersionResources(deletableResources) if err != nil { t.Fatalf("Failed to parse supported resources from server: %v", err) } config := &restclient.Config{Host: s.URL} config.ContentConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: metaonly.NewMetadataCodecFactory()} metaOnlyClientPool := dynamic.NewClientPool(config, registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc) config.ContentConfig.NegotiatedSerializer = nil clientPool := dynamic.NewClientPool(config, registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc) gc, err := garbagecollector.NewGarbageCollector(metaOnlyClientPool, clientPool, registered.RESTMapper(), deletableGroupVersionResources) if err != nil { t.Fatalf("Failed to create garbage collector") } return s, gc, clientSet }
func rmSetup(t *testing.T, enableGarbageCollector bool) (*httptest.Server, *replication.ReplicationManager, cache.SharedIndexInformer, clientset.Interface) { masterConfig := framework.NewIntegrationTestMasterConfig() _, s := framework.RunAMaster(masterConfig) config := restclient.Config{Host: s.URL} clientSet, err := clientset.NewForConfig(&config) if err != nil { t.Fatalf("Error in create clientset: %v", err) } resyncPeriod := 12 * time.Hour resyncPeriodFunc := func() time.Duration { return resyncPeriod } podInformer := informers.NewPodInformer(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pod-informer")), resyncPeriod) rm := replication.NewReplicationManager( podInformer, clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "replication-controller")), resyncPeriodFunc, replication.BurstReplicas, 4096, enableGarbageCollector, ) if err != nil { t.Fatalf("Failed to create replication manager") } return s, rm, podInformer, clientSet }
// TODO: evaluate using pkg/client/clientcmd func newKubeClient(dnsConfig *options.KubeDNSConfig) (clientset.Interface, error) { var ( config *restclient.Config err error ) if dnsConfig.KubeMasterURL != "" && dnsConfig.KubeConfigFile == "" { // Only --kube-master-url was provided. config = &restclient.Config{ Host: dnsConfig.KubeMasterURL, ContentConfig: restclient.ContentConfig{GroupVersion: &schema.GroupVersion{Version: "v1"}}, } } else { // We either have: // 1) --kube-master-url and --kubecfg-file // 2) just --kubecfg-file // 3) neither flag // In any case, the logic is the same. If (3), this will automatically // fall back on the service account token. overrides := &kclientcmd.ConfigOverrides{} overrides.ClusterInfo.Server = dnsConfig.KubeMasterURL // might be "", but that is OK rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: dnsConfig.KubeConfigFile} // might be "", but that is OK if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil { return nil, err } } glog.V(0).Infof("Using %v for kubernetes master, kubernetes API: %v", config.Host, config.GroupVersion) return clientset.NewForConfig(config) }
func rmSetup(t *testing.T, enableGarbageCollector bool) (*httptest.Server, *replicaset.ReplicaSetController, cache.SharedIndexInformer, cache.SharedIndexInformer, clientset.Interface) { masterConfig := framework.NewIntegrationTestMasterConfig() _, s := framework.RunAMaster(masterConfig) config := restclient.Config{Host: s.URL} clientSet, err := clientset.NewForConfig(&config) if err != nil { t.Fatalf("Error in create clientset: %v", err) } resyncPeriod := 12 * time.Hour informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "rs-informers")), nil, resyncPeriod) rm := replicaset.NewReplicaSetController( informers.ReplicaSets(), informers.Pods(), clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "replicaset-controller")), replicaset.BurstReplicas, 4096, enableGarbageCollector, ) if err != nil { t.Fatalf("Failed to create replicaset controller") } return s, rm, informers.ReplicaSets().Informer(), informers.Pods().Informer(), clientSet }
func (b SimpleControllerClientBuilder) Client(name string) (clientset.Interface, error) { clientConfig, err := b.Config(name) if err != nil { return nil, err } return clientset.NewForConfig(clientConfig) }
// getAPIServerClient gets a apiserver client. func getAPIServerClient() (*clientset.Clientset, error) { config, err := framework.LoadConfig() if err != nil { return nil, fmt.Errorf("failed to load config: %v", err) } client, err := clientset.NewForConfig(config) if err != nil { return nil, fmt.Errorf("failed to create client: %v", err) } return client, nil }
func getKubeClient(s *options.KubeletServer) (*clientset.Clientset, error) { clientConfig, err := CreateAPIServerClientConfig(s) if err == nil { kubeClient, err := clientset.NewForConfig(clientConfig) if err != nil { return nil, err } return kubeClient, nil } return nil, err }
// Start starts the namespace controller. func (n *NamespaceController) Start() error { // Use the default QPS config := restclient.AddUserAgent(&restclient.Config{Host: framework.TestContext.Host}, ncName) client, err := clientset.NewForConfig(config) if err != nil { return err } clientPool := dynamic.NewClientPool(config, registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc) discoverResourcesFn := client.Discovery().ServerPreferredNamespacedResources nc := namespacecontroller.NewNamespaceController(client, clientPool, discoverResourcesFn, ncResyncPeriod, v1.FinalizerKubernetes) go nc.Run(ncConcurrency, n.stopCh) return nil }
func createClients(numberOfClients int) ([]*clientset.Clientset, []*internalclientset.Clientset, error) { clients := make([]*clientset.Clientset, numberOfClients) internalClients := make([]*internalclientset.Clientset, numberOfClients) for i := 0; i < numberOfClients; i++ { config, err := framework.LoadConfig() Expect(err).NotTo(HaveOccurred()) config.QPS = 100 config.Burst = 200 if framework.TestContext.KubeAPIContentType != "" { config.ContentType = framework.TestContext.KubeAPIContentType } // For the purpose of this test, we want to force that clients // do not share underlying transport (which is a default behavior // in Kubernetes). Thus, we are explicitly creating transport for // each client here. transportConfig, err := config.TransportConfig() if err != nil { return nil, nil, err } tlsConfig, err := transport.TLSConfigFor(transportConfig) if err != nil { return nil, nil, err } config.Transport = utilnet.SetTransportDefaults(&http.Transport{ Proxy: http.ProxyFromEnvironment, TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig: tlsConfig, MaxIdleConnsPerHost: 100, Dial: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).Dial, }) // Overwrite TLS-related fields from config to avoid collision with // Transport field. config.TLSClientConfig = restclient.TLSClientConfig{} c, err := clientset.NewForConfig(config) if err != nil { return nil, nil, err } clients[i] = c internalClient, err := internalclientset.NewForConfig(config) if err != nil { return nil, nil, err } internalClients[i] = internalClient } return clients, internalClients, nil }
func createClient(s *options.SchedulerServer) (*clientset.Clientset, error) { kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig) if err != nil { return nil, fmt.Errorf("unable to build config from flags: %v", err) } kubeconfig.ContentType = s.ContentType // Override kubeconfig qps/burst settings from flags kubeconfig.QPS = s.KubeAPIQPS kubeconfig.Burst = int(s.KubeAPIBurst) cli, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "leader-election")) if err != nil { return nil, fmt.Errorf("invalid API configuration: %v", err) } return cli, nil }
func rmSetup(t *testing.T) (*httptest.Server, *disruption.DisruptionController, cache.SharedIndexInformer, clientset.Interface) { masterConfig := framework.NewIntegrationTestMasterConfig() _, s := framework.RunAMaster(masterConfig) config := restclient.Config{Host: s.URL} clientSet, err := clientset.NewForConfig(&config) if err != nil { t.Fatalf("Error in create clientset: %v", err) } resyncPeriod := 12 * time.Hour informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "pdb-informers")), nil, resyncPeriod) rm := disruption.NewDisruptionController( informers.Pods().Informer(), clientset.NewForConfigOrDie(restclient.AddUserAgent(&config, "disruption-controller")), ) return s, rm, informers.Pods().Informer(), clientSet }
// Run runs the specified SchedulerServer. This should never exit. func Run(s *options.SchedulerServer) error { if c, err := configz.New("componentconfig"); err == nil { c.Set(s.KubeSchedulerConfiguration) } else { glog.Errorf("unable to register configz: %s", err) } kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig) if err != nil { glog.Errorf("unable to build config from flags: %v", err) return err } kubeconfig.ContentType = s.ContentType // Override kubeconfig qps/burst settings from flags kubeconfig.QPS = s.KubeAPIQPS kubeconfig.Burst = int(s.KubeAPIBurst) if err != nil { glog.Fatalf("Invalid API configuration: %v", err) } leaderElectionClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "leader-election")) if err != nil { glog.Fatalf("Invalid API configuration: %v", err) } go func() { mux := http.NewServeMux() healthz.InstallHandler(mux) if s.EnableProfiling { mux.HandleFunc("/debug/pprof/", pprof.Index) mux.HandleFunc("/debug/pprof/profile", pprof.Profile) mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) } configz.InstallHandler(mux) mux.Handle("/metrics", prometheus.Handler()) server := &http.Server{ Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))), Handler: mux, } glog.Fatal(server.ListenAndServe()) }() configFactory := factory.NewConfigFactory(leaderElectionClient, s.SchedulerName, s.HardPodAffinitySymmetricWeight, s.FailureDomains) config, err := createConfig(s, configFactory) if err != nil { glog.Fatalf("Failed to create scheduler configuration: %v", err) } eventBroadcaster := record.NewBroadcaster() config.Recorder = eventBroadcaster.NewRecorder(v1.EventSource{Component: s.SchedulerName}) eventBroadcaster.StartLogging(glog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: leaderElectionClient.Core().Events("")}) sched := scheduler.New(config) run := func(_ <-chan struct{}) { sched.Run() select {} } if !s.LeaderElection.LeaderElect { run(nil) glog.Fatal("this statement is unreachable") panic("unreachable") } id, err := os.Hostname() if err != nil { glog.Errorf("unable to get hostname: %v", err) return err } // TODO: enable other lock types rl := resourcelock.EndpointsLock{ EndpointsMeta: v1.ObjectMeta{ Namespace: "kube-system", Name: "kube-scheduler", }, Client: leaderElectionClient, LockConfig: resourcelock.ResourceLockConfig{ Identity: id, EventRecorder: config.Recorder, }, } leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ Lock: &rl, LeaseDuration: s.LeaderElection.LeaseDuration.Duration, RenewDeadline: s.LeaderElection.RenewDeadline.Duration, RetryPeriod: s.LeaderElection.RetryPeriod.Duration, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { glog.Fatalf("lost master") }, }, }) glog.Fatal("this statement is unreachable") panic("unreachable") }
func run(s *options.KubeletServer, kubeDeps *kubelet.KubeletDeps) (err error) { // TODO: this should be replaced by a --standalone flag standaloneMode := (len(s.APIServerList) == 0 && !s.RequireKubeConfig) if s.ExitOnLockContention && s.LockFilePath == "" { return errors.New("cannot exit on lock file contention: no lock file specified") } done := make(chan struct{}) if s.LockFilePath != "" { glog.Infof("acquiring file lock on %q", s.LockFilePath) if err := flock.Acquire(s.LockFilePath); err != nil { return fmt.Errorf("unable to acquire file lock on %q: %v", s.LockFilePath, err) } if s.ExitOnLockContention { glog.Infof("watching for inotify events for: %v", s.LockFilePath) if err := watchForLockfileContention(s.LockFilePath, done); err != nil { return err } } } // Set feature gates based on the value in KubeletConfiguration err = utilconfig.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates) if err != nil { return err } // Register current configuration with /configz endpoint cfgz, cfgzErr := initConfigz(&s.KubeletConfiguration) if utilconfig.DefaultFeatureGate.DynamicKubeletConfig() { // Look for config on the API server. If it exists, replace s.KubeletConfiguration // with it and continue. initKubeletConfigSync also starts the background thread that checks for new config. // Don't do dynamic Kubelet configuration in runonce mode if s.RunOnce == false { remoteKC, err := initKubeletConfigSync(s) if err == nil { // Update s (KubeletServer) with new config from API server s.KubeletConfiguration = *remoteKC // Ensure that /configz is up to date with the new config if cfgzErr != nil { glog.Errorf("was unable to register configz before due to %s, will not be able to set now", cfgzErr) } else { setConfigz(cfgz, &s.KubeletConfiguration) } // Update feature gates from the new config err = utilconfig.DefaultFeatureGate.Set(s.KubeletConfiguration.FeatureGates) if err != nil { return err } } } } if kubeDeps == nil { var kubeClient, eventClient *clientset.Clientset var cloud cloudprovider.Interface if s.CloudProvider != componentconfigv1alpha1.AutoDetectCloudProvider { cloud, err = cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile) if err != nil { return err } if cloud == nil { glog.V(2).Infof("No cloud provider specified: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) } else { glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", s.CloudProvider, s.CloudConfigFile) } } if s.BootstrapKubeconfig != "" { nodeName, err := getNodeName(cloud, nodeutil.GetHostname(s.HostnameOverride)) if err != nil { return err } if err := bootstrapClientCert(s.KubeConfig.Value(), s.BootstrapKubeconfig, s.CertDirectory, nodeName); err != nil { return err } } clientConfig, err := CreateAPIServerClientConfig(s) if err == nil { kubeClient, err = clientset.NewForConfig(clientConfig) if err != nil { glog.Warningf("New kubeClient from clientConfig error: %v", err) } // make a separate client for events eventClientConfig := *clientConfig eventClientConfig.QPS = float32(s.EventRecordQPS) eventClientConfig.Burst = int(s.EventBurst) eventClient, err = clientset.NewForConfig(&eventClientConfig) } else { if s.RequireKubeConfig { return fmt.Errorf("invalid kubeconfig: %v", err) } if standaloneMode { glog.Warningf("No API client: %v", err) } } kubeDeps, err = UnsecuredKubeletDeps(s) if err != nil { return err } kubeDeps.Cloud = cloud kubeDeps.KubeClient = kubeClient kubeDeps.EventClient = eventClient } if kubeDeps.Auth == nil { nodeName, err := getNodeName(kubeDeps.Cloud, nodeutil.GetHostname(s.HostnameOverride)) if err != nil { return err } auth, err := buildAuth(nodeName, kubeDeps.KubeClient, s.KubeletConfiguration) if err != nil { return err } kubeDeps.Auth = auth } if kubeDeps.CAdvisorInterface == nil { kubeDeps.CAdvisorInterface, err = cadvisor.New(uint(s.CAdvisorPort), s.ContainerRuntime, s.RootDirectory) if err != nil { return err } } if kubeDeps.ContainerManager == nil { if s.SystemCgroups != "" && s.CgroupRoot == "" { return fmt.Errorf("invalid configuration: system container was specified and cgroup root was not specified") } kubeDeps.ContainerManager, err = cm.NewContainerManager( kubeDeps.Mounter, kubeDeps.CAdvisorInterface, cm.NodeConfig{ RuntimeCgroupsName: s.RuntimeCgroups, SystemCgroupsName: s.SystemCgroups, KubeletCgroupsName: s.KubeletCgroups, ContainerRuntime: s.ContainerRuntime, CgroupsPerQOS: s.ExperimentalCgroupsPerQOS, CgroupRoot: s.CgroupRoot, CgroupDriver: s.CgroupDriver, ProtectKernelDefaults: s.ProtectKernelDefaults, EnableCRI: s.EnableCRI, }, s.ExperimentalFailSwapOn) if err != nil { return err } } if err := checkPermissions(); err != nil { glog.Error(err) } utilruntime.ReallyCrash = s.ReallyCrashForTesting rand.Seed(time.Now().UTC().UnixNano()) // TODO(vmarmol): Do this through container config. oomAdjuster := kubeDeps.OOMAdjuster if err := oomAdjuster.ApplyOOMScoreAdj(0, int(s.OOMScoreAdj)); err != nil { glog.Warning(err) } if err := RunKubelet(&s.KubeletConfiguration, kubeDeps, s.RunOnce, standaloneMode); err != nil { return err } if s.HealthzPort > 0 { healthz.DefaultHealthz() go wait.Until(func() { err := http.ListenAndServe(net.JoinHostPort(s.HealthzBindAddress, strconv.Itoa(int(s.HealthzPort))), nil) if err != nil { glog.Errorf("Starting health server failed: %v", err) } }, 5*time.Second, wait.NeverStop) } if s.RunOnce { return nil } <-done return nil }
func main() { flag.Parse() glog.Infof("Starting serve_hostnames soak test with queries=%d and podsPerNode=%d upTo=%d", *queriesAverage, *podsPerNode, *upTo) var spec string if *gke != "" { spec = filepath.Join(os.Getenv("HOME"), ".config", "gcloud", "kubernetes", "kubeconfig") } else { spec = filepath.Join(os.Getenv("HOME"), ".kube", "config") } settings, err := clientcmd.LoadFromFile(spec) if err != nil { glog.Fatalf("Error loading configuration: %v", err.Error()) } if *gke != "" { settings.CurrentContext = *gke } config, err := clientcmd.NewDefaultClientConfig(*settings, &clientcmd.ConfigOverrides{}).ClientConfig() if err != nil { glog.Fatalf("Failed to construct config: %v", err) } client, err := clientset.NewForConfig(config) if err != nil { glog.Fatalf("Failed to make client: %v", err) } var nodes *v1.NodeList for start := time.Now(); time.Since(start) < nodeListTimeout; time.Sleep(2 * time.Second) { nodes, err = client.Nodes().List(v1.ListOptions{}) if err == nil { break } glog.Warningf("Failed to list nodes: %v", err) } if err != nil { glog.Fatalf("Giving up trying to list nodes: %v", err) } if len(nodes.Items) == 0 { glog.Fatalf("Failed to find any nodes.") } glog.Infof("Found %d nodes on this cluster:", len(nodes.Items)) for i, node := range nodes.Items { glog.Infof("%d: %s", i, node.Name) } queries := *queriesAverage * len(nodes.Items) * *podsPerNode // Create the namespace got, err := client.Namespaces().Create(&v1.Namespace{ObjectMeta: v1.ObjectMeta{GenerateName: "serve-hostnames-"}}) if err != nil { glog.Fatalf("Failed to create namespace: %v", err) } ns := got.Name defer func(ns string) { if err := client.Core().Namespaces().Delete(ns, nil); err != nil { glog.Warningf("Failed to delete namespace ns: %e", ns, err) } else { // wait until the namespace disappears for i := 0; i < int(namespaceDeleteTimeout/time.Second); i++ { if _, err := client.Namespaces().Get(ns, metav1.GetOptions{}); err != nil { if errors.IsNotFound(err) { return } } time.Sleep(time.Second) } } }(ns) glog.Infof("Created namespace %s", ns) // Create a service for these pods. glog.Infof("Creating service %s/serve-hostnames", ns) // Make several attempts to create a service. var svc *v1.Service for start := time.Now(); time.Since(start) < serviceCreateTimeout; time.Sleep(2 * time.Second) { t := time.Now() svc, err = client.Services(ns).Create(&v1.Service{ ObjectMeta: v1.ObjectMeta{ Name: "serve-hostnames", Labels: map[string]string{ "name": "serve-hostname", }, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{{ Protocol: "TCP", Port: 9376, TargetPort: intstr.FromInt(9376), }}, Selector: map[string]string{ "name": "serve-hostname", }, }, }) glog.V(4).Infof("Service create %s/server-hostnames took %v", ns, time.Since(t)) if err == nil { break } glog.Warningf("After %v failed to create service %s/serve-hostnames: %v", time.Since(start), ns, err) } if err != nil { glog.Warningf("Unable to create service %s/%s: %v", ns, svc.Name, err) return } // Clean up service defer func() { glog.Infof("Cleaning up service %s/serve-hostnames", ns) // Make several attempts to delete the service. for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err := client.Services(ns).Delete(svc.Name, nil); err == nil { return } glog.Warningf("After %v unable to delete service %s/%s: %v", time.Since(start), ns, svc.Name, err) } }() // Put serve-hostname pods on each node. podNames := []string{} for i, node := range nodes.Items { for j := 0; j < *podsPerNode; j++ { podName := fmt.Sprintf("serve-hostname-%d-%d", i, j) podNames = append(podNames, podName) // Make several attempts for start := time.Now(); time.Since(start) < podCreateTimeout; time.Sleep(2 * time.Second) { glog.Infof("Creating pod %s/%s on node %s", ns, podName, node.Name) t := time.Now() _, err = client.Pods(ns).Create(&v1.Pod{ ObjectMeta: v1.ObjectMeta{ Name: podName, Labels: map[string]string{ "name": "serve-hostname", }, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name: "serve-hostname", Image: "gcr.io/google_containers/serve_hostname:v1.4", Ports: []v1.ContainerPort{{ContainerPort: 9376}}, }, }, NodeName: node.Name, }, }) glog.V(4).Infof("Pod create %s/%s request took %v", ns, podName, time.Since(t)) if err == nil { break } glog.Warningf("After %s failed to create pod %s/%s: %v", time.Since(start), ns, podName, err) } if err != nil { glog.Warningf("Failed to create pod %s/%s: %v", ns, podName, err) return } } } // Clean up the pods defer func() { glog.Info("Cleaning up pods") // Make several attempts to delete the pods. for _, podName := range podNames { for start := time.Now(); time.Since(start) < deleteTimeout; time.Sleep(1 * time.Second) { if err = client.Pods(ns).Delete(podName, nil); err == nil { break } glog.Warningf("After %v failed to delete pod %s/%s: %v", time.Since(start), ns, podName, err) } } }() glog.Info("Waiting for the serve-hostname pods to be ready") for _, podName := range podNames { var pod *v1.Pod for start := time.Now(); time.Since(start) < podStartTimeout; time.Sleep(5 * time.Second) { pod, err = client.Pods(ns).Get(podName, metav1.GetOptions{}) if err != nil { glog.Warningf("Get pod %s/%s failed, ignoring for %v: %v", ns, podName, err, podStartTimeout) continue } if pod.Status.Phase == v1.PodRunning { break } } if pod.Status.Phase != v1.PodRunning { glog.Warningf("Gave up waiting on pod %s/%s to be running (saw %v)", ns, podName, pod.Status.Phase) } else { glog.Infof("%s/%s is running", ns, podName) } } rclient, err := restclient.RESTClientFor(config) if err != nil { glog.Warningf("Failed to build restclient: %v", err) return } proxyRequest, errProxy := e2e.GetServicesProxyRequest(client, rclient.Get()) if errProxy != nil { glog.Warningf("Get services proxy request failed: %v", errProxy) return } // Wait for the endpoints to propagate. for start := time.Now(); time.Since(start) < endpointTimeout; time.Sleep(10 * time.Second) { hostname, err := proxyRequest. Namespace(ns). Name("serve-hostnames"). DoRaw() if err != nil { glog.Infof("After %v while making a proxy call got error %v", time.Since(start), err) continue } var r metav1.Status if err := runtime.DecodeInto(api.Codecs.UniversalDecoder(), hostname, &r); err != nil { break } if r.Status == metav1.StatusFailure { glog.Infof("After %v got status %v", time.Since(start), string(hostname)) continue } break } // Repeatedly make requests. for iteration := 0; iteration != *upTo; iteration++ { responseChan := make(chan string, queries) // Use a channel of size *maxPar to throttle the number // of in-flight requests to avoid overloading the service. inFlight := make(chan struct{}, *maxPar) start := time.Now() for q := 0; q < queries; q++ { go func(i int, query int) { inFlight <- struct{}{} t := time.Now() hostname, err := proxyRequest. Namespace(ns). Name("serve-hostnames"). DoRaw() glog.V(4).Infof("Proxy call in namespace %s took %v", ns, time.Since(t)) if err != nil { glog.Warningf("Call failed during iteration %d query %d : %v", i, query, err) // If the query failed return a string which starts with a character // that can't be part of a hostname. responseChan <- fmt.Sprintf("!failed in iteration %d to issue query %d: %v", i, query, err) } else { responseChan <- string(hostname) } <-inFlight }(iteration, q) } responses := make(map[string]int, *podsPerNode*len(nodes.Items)) missing := 0 for q := 0; q < queries; q++ { r := <-responseChan glog.V(4).Infof("Got response from %s", r) responses[r]++ // If the returned hostname starts with '!' then it indicates // an error response. if len(r) > 0 && r[0] == '!' { glog.V(3).Infof("Got response %s", r) missing++ } } if missing > 0 { glog.Warningf("Missing %d responses out of %d", missing, queries) } // Report any nodes that did not respond. for n, node := range nodes.Items { for i := 0; i < *podsPerNode; i++ { name := fmt.Sprintf("serve-hostname-%d-%d", n, i) if _, ok := responses[name]; !ok { glog.Warningf("No response from pod %s on node %s at iteration %d", name, node.Name, iteration) } } } glog.Infof("Iteration %d took %v for %d queries (%.2f QPS) with %d missing", iteration, time.Since(start), queries-missing, float64(queries-missing)/time.Since(start).Seconds(), missing) } }
// BeforeEach gets a client and makes a namespace. func (f *Framework) BeforeEach() { // The fact that we need this feels like a bug in ginkgo. // https://github.com/onsi/ginkgo/issues/222 f.cleanupHandle = AddCleanupAction(f.AfterEach) if f.ClientSet == nil { By("Creating a kubernetes client") config, err := LoadConfig() Expect(err).NotTo(HaveOccurred()) config.QPS = f.options.ClientQPS config.Burst = f.options.ClientBurst if f.options.GroupVersion != nil { config.GroupVersion = f.options.GroupVersion } if TestContext.KubeAPIContentType != "" { config.ContentType = TestContext.KubeAPIContentType } f.ClientSet, err = internalclientset.NewForConfig(config) Expect(err).NotTo(HaveOccurred()) f.ClientSet_1_5, err = release_1_5.NewForConfig(config) Expect(err).NotTo(HaveOccurred()) clientRepoConfig := getClientRepoConfig(config) f.StagingClient, err = staging.NewForConfig(clientRepoConfig) Expect(err).NotTo(HaveOccurred()) f.ClientPool = dynamic.NewClientPool(config, registered.RESTMapper(), dynamic.LegacyAPIPathResolverFunc) } if f.federated { if f.FederationClientset_1_5 == nil { By("Creating a release 1.4 federation Clientset") var err error f.FederationClientset_1_5, err = LoadFederationClientset_1_5() Expect(err).NotTo(HaveOccurred()) } By("Waiting for federation-apiserver to be ready") err := WaitForFederationApiserverReady(f.FederationClientset_1_5) Expect(err).NotTo(HaveOccurred()) By("federation-apiserver is ready") By("Creating a federation namespace") ns, err := f.createFederationNamespace(f.BaseName) Expect(err).NotTo(HaveOccurred()) f.FederationNamespace = ns By(fmt.Sprintf("Created federation namespace %s", ns.Name)) } By("Building a namespace api object") namespace, err := f.CreateNamespace(f.BaseName, map[string]string{ "e2e-framework": f.BaseName, }) Expect(err).NotTo(HaveOccurred()) f.Namespace = namespace if TestContext.VerifyServiceAccount { By("Waiting for a default service account to be provisioned in namespace") err = WaitForDefaultServiceAccountInNamespace(f.ClientSet, namespace.Name) Expect(err).NotTo(HaveOccurred()) } else { Logf("Skipping waiting for service account") } if TestContext.GatherKubeSystemResourceUsageData != "false" && TestContext.GatherKubeSystemResourceUsageData != "none" { f.gatherer, err = NewResourceUsageGatherer(f.ClientSet, ResourceGathererOptions{ inKubemark: ProviderIs("kubemark"), masterOnly: TestContext.GatherKubeSystemResourceUsageData == "master", }) if err != nil { Logf("Error while creating NewResourceUsageGatherer: %v", err) } else { go f.gatherer.startGatheringData() } } if TestContext.GatherLogsSizes { f.logsSizeWaitGroup = sync.WaitGroup{} f.logsSizeWaitGroup.Add(1) f.logsSizeCloseChannel = make(chan bool) f.logsSizeVerifier = NewLogsVerifier(f.ClientSet, f.logsSizeCloseChannel) go func() { f.logsSizeVerifier.Run() f.logsSizeWaitGroup.Done() }() } }
func CreateClientAndWaitForAPI(adminConfig *clientcmdapi.Config) (*clientset.Clientset, error) { adminClientConfig, err := clientcmd.NewDefaultClientConfig( *adminConfig, &clientcmd.ConfigOverrides{}, ).ClientConfig() if err != nil { return nil, fmt.Errorf("failed to create API client configuration [%v]", err) } client, err := clientset.NewForConfig(adminClientConfig) if err != nil { return nil, fmt.Errorf("failed to create API client [%v]", err) } fmt.Println("[apiclient] Created API client, waiting for the control plane to become ready") start := time.Now() wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { cs, err := client.ComponentStatuses().List(v1.ListOptions{}) if err != nil { return false, nil } // TODO(phase2) must revisit this when we implement HA if len(cs.Items) < 3 { fmt.Println("[apiclient] Not all control plane components are ready yet") return false, nil } for _, item := range cs.Items { for _, condition := range item.Conditions { if condition.Type != v1.ComponentHealthy { fmt.Printf("[apiclient] Control plane component %q is still unhealthy: %#v\n", item.ObjectMeta.Name, item.Conditions) return false, nil } } } fmt.Printf("[apiclient] All control plane components are healthy after %f seconds\n", time.Since(start).Seconds()) return true, nil }) fmt.Println("[apiclient] Waiting for at least one node to register and become ready") start = time.Now() wait.PollInfinite(apiCallRetryInterval, func() (bool, error) { nodeList, err := client.Nodes().List(v1.ListOptions{}) if err != nil { fmt.Println("[apiclient] Temporarily unable to list nodes (will retry)") return false, nil } if len(nodeList.Items) < 1 { return false, nil } n := &nodeList.Items[0] if !v1.IsNodeReady(n) { fmt.Println("[apiclient] First node has registered, but is not ready yet") return false, nil } fmt.Printf("[apiclient] First node is ready after %f seconds\n", time.Since(start).Seconds()) return true, nil }) createDummyDeployment(client) return client, nil }
func TestConcurrentEvictionRequests(t *testing.T) { podNameFormat := "test-pod-%d" s, rm, podInformer, clientSet := rmSetup(t) defer s.Close() ns := framework.CreateTestingNamespace("concurrent-eviction-requests", s, t) defer framework.DeleteTestingNamespace(ns, s, t) stopCh := make(chan struct{}) go podInformer.Run(stopCh) go rm.Run(stopCh) config := restclient.Config{Host: s.URL} clientSet, err := clientset.NewForConfig(&config) var gracePeriodSeconds int64 = 30 deleteOption := &v1.DeleteOptions{ GracePeriodSeconds: &gracePeriodSeconds, } // Generate 10 pods to evict for i := 0; i < 10; i++ { podName := fmt.Sprintf(podNameFormat, i) pod := newPod(podName) if _, err := clientSet.Core().Pods(ns.Name).Create(pod); err != nil { t.Errorf("Failed to create pod: %v", err) } addPodConditionReady(pod) if _, err := clientSet.Core().Pods(ns.Name).UpdateStatus(pod); err != nil { t.Fatal(err) } } waitToObservePods(t, podInformer, 10) pdb := newPDB() if _, err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Create(pdb); err != nil { t.Errorf("Failed to create PodDisruptionBudget: %v", err) } waitPDBStable(t, clientSet, 10, ns.Name, pdb.Name) doneCh := make(chan bool, 10) errCh := make(chan error, 1) // spawn 10 goroutine to concurrently evict the pods for i := 0; i < 10; i++ { go func(id int, doneCh chan bool, errCh chan error) { evictionName := fmt.Sprintf(podNameFormat, id) eviction := newEviction(ns.Name, evictionName, deleteOption) var e error for { e = clientSet.Policy().Evictions(ns.Name).Evict(eviction) if errors.IsTooManyRequests(e) { time.Sleep(5 * time.Second) } else { break } } if e != nil { if errors.IsConflict(err) { fmt.Errorf("Unexpected Conflict (409) error caused by failing to handle concurrent PDB updates: %v", e) } else { errCh <- e } return } doneCh <- true }(i, doneCh, errCh) } doneCount := 0 for { select { case err := <-errCh: t.Errorf("%v", err) return case <-doneCh: doneCount++ if doneCount == 10 { return } case <-time.After(defaultTimeout): t.Errorf("Eviction did not complete within %v", defaultTimeout) } } for i := 0; i < 10; i++ { podName := fmt.Sprintf(podNameFormat, i) _, err := clientSet.Core().Pods(ns.Name).Get(podName, metav1.GetOptions{}) if !errors.IsNotFound(err) { t.Errorf("Pod %q is expected to be evicted", podName) } } if err := clientSet.Policy().PodDisruptionBudgets(ns.Name).Delete(pdb.Name, deleteOption); err != nil { t.Errorf("Failed to delete PodDisruptionBudget: %v", err) } close(stopCh) }
// Run runs the CMServer. This should never exit. func Run(s *options.CMServer) error { if c, err := configz.New("componentconfig"); err == nil { c.Set(s.KubeControllerManagerConfiguration) } else { glog.Errorf("unable to register configz: %s", err) } kubeconfig, err := clientcmd.BuildConfigFromFlags(s.Master, s.Kubeconfig) if err != nil { return err } kubeconfig.ContentConfig.ContentType = s.ContentType // Override kubeconfig qps/burst settings from flags kubeconfig.QPS = s.KubeAPIQPS kubeconfig.Burst = int(s.KubeAPIBurst) kubeClient, err := clientset.NewForConfig(restclient.AddUserAgent(kubeconfig, "controller-manager")) if err != nil { glog.Fatalf("Invalid API configuration: %v", err) } leaderElectionClient := clientset.NewForConfigOrDie(restclient.AddUserAgent(kubeconfig, "leader-election")) go func() { mux := http.NewServeMux() healthz.InstallHandler(mux) if s.EnableProfiling { mux.HandleFunc("/debug/pprof/", pprof.Index) mux.HandleFunc("/debug/pprof/profile", pprof.Profile) mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol) } configz.InstallHandler(mux) mux.Handle("/metrics", prometheus.Handler()) server := &http.Server{ Addr: net.JoinHostPort(s.Address, strconv.Itoa(int(s.Port))), Handler: mux, } glog.Fatal(server.ListenAndServe()) }() eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.Infof) eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.Core().Events("")}) recorder := eventBroadcaster.NewRecorder(v1.EventSource{Component: "controller-manager"}) run := func(stop <-chan struct{}) { rootClientBuilder := controller.SimpleControllerClientBuilder{ ClientConfig: kubeconfig, } var clientBuilder controller.ControllerClientBuilder if len(s.ServiceAccountKeyFile) > 0 && s.UseServiceAccountCredentials { clientBuilder = controller.SAControllerClientBuilder{ ClientConfig: restclient.AnonymousClientConfig(kubeconfig), CoreClient: kubeClient.Core(), Namespace: "kube-system", } } else { clientBuilder = rootClientBuilder } err := StartControllers(newControllerInitializers(), s, rootClientBuilder, clientBuilder, stop) glog.Fatalf("error running controllers: %v", err) panic("unreachable") } if !s.LeaderElection.LeaderElect { run(nil) panic("unreachable") } id, err := os.Hostname() if err != nil { return err } // TODO: enable other lock types rl := resourcelock.EndpointsLock{ EndpointsMeta: v1.ObjectMeta{ Namespace: "kube-system", Name: "kube-controller-manager", }, Client: leaderElectionClient, LockConfig: resourcelock.ResourceLockConfig{ Identity: id, EventRecorder: recorder, }, } leaderelection.RunOrDie(leaderelection.LeaderElectionConfig{ Lock: &rl, LeaseDuration: s.LeaderElection.LeaseDuration.Duration, RenewDeadline: s.LeaderElection.RenewDeadline.Duration, RetryPeriod: s.LeaderElection.RetryPeriod.Duration, Callbacks: leaderelection.LeaderCallbacks{ OnStartedLeading: run, OnStoppedLeading: func() { glog.Fatalf("leaderelection lost") }, }, }) panic("unreachable") }