func TestGenerateStorageVersionMap(t *testing.T) { testCases := []struct { legacyVersion string storageVersions string defaultVersions string expectedMap map[string]unversioned.GroupVersion }{ { legacyVersion: "v1", storageVersions: "v1,extensions/v1beta1", expectedMap: map[string]unversioned.GroupVersion{ api.GroupName: {Version: "v1"}, extensions.GroupName: {Group: "extensions", Version: "v1beta1"}, }, }, { legacyVersion: "", storageVersions: "extensions/v1beta1,v1", expectedMap: map[string]unversioned.GroupVersion{ api.GroupName: {Version: "v1"}, extensions.GroupName: {Group: "extensions", Version: "v1beta1"}, }, }, { legacyVersion: "", storageVersions: "autoscaling=extensions/v1beta1,v1", defaultVersions: "extensions/v1beta1,v1,autoscaling/v1", expectedMap: map[string]unversioned.GroupVersion{ api.GroupName: {Version: "v1"}, autoscaling.GroupName: {Group: "extensions", Version: "v1beta1"}, extensions.GroupName: {Group: "extensions", Version: "v1beta1"}, }, }, { legacyVersion: "", storageVersions: "", expectedMap: map[string]unversioned.GroupVersion{}, }, } for i, test := range testCases { s := options.ServerRunOptions{ DeprecatedStorageVersion: test.legacyVersion, StorageVersions: test.storageVersions, DefaultStorageVersions: test.defaultVersions, } output, err := s.StorageGroupsToEncodingVersion() if err != nil { t.Errorf("%v: unexpected error: %v", i, err) } if !reflect.DeepEqual(test.expectedMap, output) { t.Errorf("%v: unexpected error. expect: %v, got: %v", i, test.expectedMap, output) } } }
func DefaultAndValidateRunOptions(options *options.ServerRunOptions) { genericvalidation.ValidateRunOptions(options) // If advertise-address is not specified, use bind-address. If bind-address // is not usable (unset, 0.0.0.0, or loopback), we will use the host's default // interface as valid public addr for master (see: util/net#ValidPublicAddrForMaster) if options.AdvertiseAddress == nil || options.AdvertiseAddress.IsUnspecified() { hostIP, err := utilnet.ChooseBindAddress(options.BindAddress) if err != nil { glog.Fatalf("Unable to find suitable network address.error='%v' . "+ "Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err) } options.AdvertiseAddress = hostIP } glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress) // Set default value for ExternalHost if not specified. if len(options.ExternalHost) == 0 { // TODO: extend for other providers if options.CloudProvider == "gce" { cloud, err := cloudprovider.InitCloudProvider(options.CloudProvider, options.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } instances, supported := cloud.Instances() if !supported { glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.") } hostname, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } nodeName, err := instances.CurrentNodeName(hostname) if err != nil { glog.Fatalf("Failed to get NodeName: %v", err) } addrs, err := instances.NodeAddresses(nodeName) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == api.NodeExternalIP { options.ExternalHost = addr.Address } } } } } }
func Run(serverOptions *genericoptions.ServerRunOptions, stopCh <-chan struct{}) error { // Set ServiceClusterIPRange _, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") serverOptions.ServiceClusterIPRange = *serviceClusterIPRange serverOptions.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"} genericvalidation.ValidateRunOptions(serverOptions) genericvalidation.VerifyEtcdServersList(serverOptions) config := genericapiserver.NewConfig().ApplyOptions(serverOptions).Complete() if err := config.MaybeGenerateServingCerts(); err != nil { // this wasn't treated as fatal for this process before fmt.Printf("Error creating cert: %v", err) } config.Authorizer = authorizer.NewAlwaysAllowAuthorizer() config.SwaggerConfig = genericapiserver.DefaultSwaggerConfig() s, err := config.New() if err != nil { return fmt.Errorf("Error in bringing up the server: %v", err) } groupVersion := v1.SchemeGroupVersion groupName := groupVersion.Group groupMeta, err := registered.Group(groupName) if err != nil { return fmt.Errorf("%v", err) } storageFactory := newStorageFactory() storageConfig, err := storageFactory.NewConfig(unversioned.GroupResource{Group: groupName, Resource: "testtype"}) if err != nil { return fmt.Errorf("Unable to get storage config: %v", err) } restStorageMap := map[string]rest.Storage{ "testtypes": testgroupetcd.NewREST(storageConfig, generic.UndecoratedStorage), } apiGroupInfo := genericapiserver.APIGroupInfo{ GroupMeta: *groupMeta, VersionedResourcesStorageMap: map[string]map[string]rest.Storage{ groupVersion.Version: restStorageMap, }, Scheme: api.Scheme, NegotiatedSerializer: api.Codecs, } if err := s.InstallAPIGroup(&apiGroupInfo); err != nil { return fmt.Errorf("Error in installing API: %v", err) } s.PrepareRun().Run(stopCh) return nil }
func Run(serverOptions *genericoptions.ServerRunOptions) error { // Set ServiceClusterIPRange _, serviceClusterIPRange, _ := net.ParseCIDR("10.0.0.0/24") serverOptions.ServiceClusterIPRange = *serviceClusterIPRange serverOptions.StorageConfig.ServerList = []string{"http://127.0.0.1:2379"} genericvalidation.ValidateRunOptions(serverOptions) genericvalidation.VerifyEtcdServersList(serverOptions) config := genericapiserver.NewConfig(serverOptions) config.Authorizer = authorizer.NewAlwaysAllowAuthorizer() config.Serializer = api.Codecs s, err := config.New() if err != nil { return fmt.Errorf("Error in bringing up the server: %v", err) } groupVersion := v1.SchemeGroupVersion groupName := groupVersion.Group groupMeta, err := registered.Group(groupName) if err != nil { return fmt.Errorf("%v", err) } storageFactory := newStorageFactory() storageConfig, err := storageFactory.NewConfig(unversioned.GroupResource{Group: groupName, Resource: "testtype"}) if err != nil { return fmt.Errorf("Unable to get storage config: %v", err) } restStorageMap := map[string]rest.Storage{ "testtypes": testgroupetcd.NewREST(storageConfig, generic.UndecoratedStorage), } apiGroupInfo := genericapiserver.APIGroupInfo{ GroupMeta: *groupMeta, VersionedResourcesStorageMap: map[string]map[string]rest.Storage{ groupVersion.Version: restStorageMap, }, Scheme: api.Scheme, NegotiatedSerializer: api.Codecs, } if err := s.InstallAPIGroup(&apiGroupInfo); err != nil { return fmt.Errorf("Error in installing API: %v", err) } s.Run(serverOptions) return nil }
func DefaultAndValidateRunOptions(options *options.ServerRunOptions) { genericvalidation.ValidateRunOptions(options) glog.Infof("Will report %v as public IP address.", options.AdvertiseAddress) // Set default value for ExternalAddress if not specified. if len(options.ExternalHost) == 0 { // TODO: extend for other providers if options.CloudProvider == "gce" || options.CloudProvider == "aws" { cloud, err := cloudprovider.InitCloudProvider(options.CloudProvider, options.CloudConfigFile) if err != nil { glog.Fatalf("Cloud provider could not be initialized: %v", err) } instances, supported := cloud.Instances() if !supported { glog.Fatalf("%q cloud provider has no instances. this shouldn't happen. exiting.", options.CloudProvider) } hostname, err := os.Hostname() if err != nil { glog.Fatalf("Failed to get hostname: %v", err) } nodeName, err := instances.CurrentNodeName(hostname) if err != nil { glog.Fatalf("Failed to get NodeName: %v", err) } addrs, err := instances.NodeAddresses(nodeName) if err != nil { glog.Warningf("Unable to obtain external host address from cloud provider: %v", err) } else { for _, addr := range addrs { if addr.Type == v1.NodeExternalIP { options.ExternalHost = addr.Address } } } } } }
func (s *GenericAPIServer) Run(options *options.ServerRunOptions) { if s.enableSwaggerSupport { s.InstallSwaggerAPI() } // We serve on 2 ports. See docs/admin/accessing-the-api.md secureLocation := "" if options.SecurePort != 0 { secureLocation = net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)) } insecureLocation := net.JoinHostPort(options.InsecureBindAddress.String(), strconv.Itoa(options.InsecurePort)) var sem chan bool if options.MaxRequestsInFlight > 0 { sem = make(chan bool, options.MaxRequestsInFlight) } longRunningRE := regexp.MustCompile(options.LongRunningRequestRE) longRunningRequestCheck := apiserver.BasicLongRunningRequestCheck(longRunningRE, map[string]string{"watch": "true"}) longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) { // TODO unify this with apiserver.MaxInFlightLimit if longRunningRequestCheck(req) { return nil, "" } return time.After(globalTimeout), "" } if secureLocation != "" { handler := apiserver.TimeoutHandler(s.Handler, longRunningTimeout) secureServer := &http.Server{ Addr: secureLocation, Handler: apiserver.MaxInFlightLimit(sem, longRunningRequestCheck, apiserver.RecoverPanics(handler)), MaxHeaderBytes: 1 << 20, TLSConfig: &tls.Config{ // Can't use SSLv3 because of POODLE and BEAST // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher // Can't use TLSv1.1 because of RC4 cipher usage MinVersion: tls.VersionTLS12, }, } if len(options.ClientCAFile) > 0 { clientCAs, err := crypto.CertPoolFromFile(options.ClientCAFile) if err != nil { glog.Fatalf("Unable to load client CA file: %v", err) } // Populate PeerCertificates in requests, but don't reject connections without certificates // This allows certificates to be validated by authenticators, while still allowing other auth types secureServer.TLSConfig.ClientAuth = tls.RequestClientCert // Specify allowed CAs for client certificates secureServer.TLSConfig.ClientCAs = clientCAs } glog.Infof("Serving securely on %s", secureLocation) if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" { options.TLSCertFile = path.Join(options.CertDirectory, "apiserver.crt") options.TLSPrivateKeyFile = path.Join(options.CertDirectory, "apiserver.key") // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? alternateIPs := []net.IP{s.ServiceReadWriteIP} alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"} // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") if crypto.ShouldGenSelfSignedCerts(options.TLSCertFile, options.TLSPrivateKeyFile) { if err := crypto.GenerateSelfSignedCert(s.ClusterIP.String(), options.TLSCertFile, options.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { glog.Errorf("Unable to generate self signed cert: %v", err) } else { glog.Infof("Using self-signed cert (%s, %s)", options.TLSCertFile, options.TLSPrivateKeyFile) } } } go func() { defer utilruntime.HandleCrash() for { // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } if err := secureServer.ListenAndServeTLS(options.TLSCertFile, options.TLSPrivateKeyFile); err != nil { glog.Errorf("Unable to listen for secure (%v); will try again.", err) } time.Sleep(15 * time.Second) } }() } else { // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } } handler := apiserver.TimeoutHandler(s.InsecureHandler, longRunningTimeout) http := &http.Server{ Addr: insecureLocation, Handler: apiserver.RecoverPanics(handler), MaxHeaderBytes: 1 << 20, } glog.Infof("Serving insecurely on %s", insecureLocation) go func() { defer utilruntime.HandleCrash() for { if err := http.ListenAndServe(); err != nil { glog.Errorf("Unable to listen for insecure (%v); will try again.", err) } time.Sleep(15 * time.Second) } }() select {} }
// Run runs the specified APIServer. This should never exit. func Run(s *genericoptions.ServerRunOptions) error { genericapiserver.DefaultAndValidateRunOptions(s) // TODO: register cluster federation resources here. resourceConfig := genericapiserver.NewResourceConfig() storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( s.StorageConfig, s.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, resourceConfig, s.RuntimeConfig) if err != nil { glog.Fatalf("error in initializing storage factory: %s", err) } for _, override := range s.EtcdServersOverrides { tokens := strings.Split(override, "#") if len(tokens) != 2 { glog.Errorf("invalid value of etcd server overrides: %s", override) continue } apiresource := strings.Split(tokens[0], "/") if len(apiresource) != 2 { glog.Errorf("invalid resource definition: %s", tokens[0]) continue } group := apiresource[0] resource := apiresource[1] groupResource := unversioned.GroupResource{Group: group, Resource: resource} servers := strings.Split(tokens[1], ";") storageFactory.SetEtcdLocation(groupResource, servers) } authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") client, err := s.NewSelfClient() if err != nil { glog.Errorf("Failed to create clientset: %v", err) } sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute) pluginInitializer := admission.NewPluginInitializer(sharedInformers) admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer) if err != nil { glog.Fatalf("Failed to initialize plugins: %v", err) } genericConfig := genericapiserver.NewConfig(s) // TODO: Move the following to generic api server as well. genericConfig.StorageFactory = storageFactory genericConfig.Authenticator = authenticator genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0 genericConfig.Authorizer = authorizer genericConfig.AdmissionControl = admissionController genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource genericConfig.MasterServiceNamespace = s.MasterServiceNamespace genericConfig.Serializer = api.Codecs // TODO: Move this to generic api server (Need to move the command line flag). if s.EnableWatchCache { cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } m, err := genericapiserver.New(genericConfig) if err != nil { return err } installFederationAPIs(s, m, storageFactory) installCoreAPIs(s, m, storageFactory) installExtensionsAPIs(s, m, storageFactory) sharedInformers.Start(wait.NeverStop) m.Run(s) return nil }
func (s *GenericAPIServer) Run(options *options.ServerRunOptions) { // install APIs which depend on other APIs to be installed if s.enableSwaggerSupport { s.InstallSwaggerAPI() } if s.enableOpenAPISupport { s.InstallOpenAPI() } secureStartedCh := make(chan struct{}) if options.SecurePort != 0 { secureLocation := net.JoinHostPort(options.BindAddress.String(), strconv.Itoa(options.SecurePort)) secureServer := &http.Server{ Addr: secureLocation, Handler: s.Handler, MaxHeaderBytes: 1 << 20, TLSConfig: &tls.Config{ // Can't use SSLv3 because of POODLE and BEAST // Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher // Can't use TLSv1.1 because of RC4 cipher usage MinVersion: tls.VersionTLS12, }, } if len(options.ClientCAFile) > 0 { clientCAs, err := certutil.NewPool(options.ClientCAFile) if err != nil { glog.Fatalf("Unable to load client CA file: %v", err) } // Populate PeerCertificates in requests, but don't reject connections without certificates // This allows certificates to be validated by authenticators, while still allowing other auth types secureServer.TLSConfig.ClientAuth = tls.RequestClientCert // Specify allowed CAs for client certificates secureServer.TLSConfig.ClientCAs = clientCAs // "h2" NextProtos is necessary for enabling HTTP2 for go's 1.7 HTTP Server secureServer.TLSConfig.NextProtos = []string{"h2"} } glog.Infof("Serving securely on %s", secureLocation) if options.TLSCertFile == "" && options.TLSPrivateKeyFile == "" { options.TLSCertFile = path.Join(options.CertDirectory, "apiserver.crt") options.TLSPrivateKeyFile = path.Join(options.CertDirectory, "apiserver.key") // TODO (cjcullen): Is ClusterIP the right address to sign a cert with? alternateIPs := []net.IP{s.ServiceReadWriteIP} alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes", "localhost"} // It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless // alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME") if !certutil.CanReadCertOrKey(options.TLSCertFile, options.TLSPrivateKeyFile) { if err := certutil.GenerateSelfSignedCert(s.ClusterIP.String(), options.TLSCertFile, options.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil { glog.Errorf("Unable to generate self signed cert: %v", err) } else { glog.Infof("Using self-signed cert (%s, %s)", options.TLSCertFile, options.TLSPrivateKeyFile) } } } go func() { defer utilruntime.HandleCrash() notifyStarted := sync.Once{} for { if err := secureServer.ListenAndServeTLS(options.TLSCertFile, options.TLSPrivateKeyFile); err != nil { glog.Errorf("Unable to listen for secure (%v); will try again.", err) } else { notifyStarted.Do(func() { close(secureStartedCh) }) } time.Sleep(15 * time.Second) } }() } else { close(secureStartedCh) } insecureLocation := net.JoinHostPort(options.InsecureBindAddress.String(), strconv.Itoa(options.InsecurePort)) insecureServer := &http.Server{ Addr: insecureLocation, Handler: s.InsecureHandler, MaxHeaderBytes: 1 << 20, } insecureStartedCh := make(chan struct{}) glog.Infof("Serving insecurely on %s", insecureLocation) go func() { defer utilruntime.HandleCrash() notifyStarted := sync.Once{} for { if err := insecureServer.ListenAndServe(); err != nil { glog.Errorf("Unable to listen for insecure (%v); will try again.", err) } else { notifyStarted.Do(func() { close(insecureStartedCh) }) } time.Sleep(15 * time.Second) } }() <-secureStartedCh <-insecureStartedCh s.RunPostStartHooks(PostStartHookContext{}) // err == systemd.SdNotifyNoSocket when not running on a systemd system if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket { glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err) } select {} }
func newAPIServer(s *genericoptions.ServerRunOptions) (*genericapiserver.GenericAPIServer, error) { genericapiserver.DefaultAndValidateRunOptions(s) resourceConfig := genericapiserver.NewResourceConfig() storageGroupsToEncodingVersion, err := s.StorageGroupsToEncodingVersion() if err != nil { glog.Fatalf("error generating storage version map: %s", err) } storageFactory, err := genericapiserver.BuildDefaultStorageFactory( s.StorageConfig, s.DefaultStorageMediaType, api.Codecs, genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion, []unversioned.GroupVersionResource{}, resourceConfig, s.RuntimeConfig) if err != nil { glog.Fatalf("error in initializing storage factory: %s", err) } authn, err := authenticator.New(authenticator.AuthenticatorConfig{ BasicAuthFile: s.BasicAuthFile, ClientCAFile: s.ClientCAFile, TokenAuthFile: s.TokenAuthFile, OIDCIssuerURL: s.OIDCIssuerURL, OIDCClientID: s.OIDCClientID, OIDCCAFile: s.OIDCCAFile, OIDCUsernameClaim: s.OIDCUsernameClaim, OIDCGroupsClaim: s.OIDCGroupsClaim, KeystoneURL: s.KeystoneURL, }) if err != nil { glog.Fatalf("Invalid Authentication Config: %v", err) } authorizationModeNames := strings.Split(s.AuthorizationMode, ",") authorizationConfig := genericauthorizer.AuthorizationConfig{ PolicyFile: s.AuthorizationPolicyFile, WebhookConfigFile: s.AuthorizationWebhookConfigFile, WebhookCacheAuthorizedTTL: s.AuthorizationWebhookCacheAuthorizedTTL, WebhookCacheUnauthorizedTTL: s.AuthorizationWebhookCacheUnauthorizedTTL, RBACSuperUser: s.AuthorizationRBACSuperUser, } authorizer, err := genericauthorizer.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, authorizationConfig) if err != nil { glog.Fatalf("Invalid Authorization Config: %v", err) } admissionControlPluginNames := strings.Split(s.AdmissionControl, ",") privilegedLoopbackToken := uuid.NewRandom().String() client, err := s.NewSelfClient(privilegedLoopbackToken) if err != nil { glog.Errorf("Failed to create clientset: %v", err) } sharedInformers := informers.NewSharedInformerFactory(client, 10*time.Minute) pluginInitializer := admission.NewPluginInitializer(sharedInformers) admissionController, err := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile, pluginInitializer) genericConfig := genericapiserver.NewConfig(s) // TODO: Move the following to generic api server as well. genericConfig.Authenticator = authn genericConfig.SupportsBasicAuth = len(s.BasicAuthFile) > 0 genericConfig.Authorizer = authorizer genericConfig.AdmissionControl = admissionController genericConfig.APIResourceConfigSource = storageFactory.APIResourceConfigSource genericConfig.MasterServiceNamespace = s.MasterServiceNamespace genericConfig.Serializer = api.Codecs // TODO: Move this to generic api server (Need to move the command line flag). if s.EnableWatchCache { cachesize.SetWatchCacheSizes(s.WatchCacheSizes) } return genericConfig.New() }