// NewAPIServerCommand provides a CLI handler for the 'apiserver' command
func NewAPIServerCommand(name, fullName string, out io.Writer) *cobra.Command {
apiServerOptions := apiserveroptions.NewAPIServer()
cmd := &cobra.Command{
Use: name,
Short: "Launch Kubernetes apiserver (kube-apiserver)",
Long: apiserverLong,
Run: func(c *cobra.Command, args []string) {
startProfiler()
util.InitLogs()
defer util.FlushLogs()
if err := apiserverapp.Run(apiServerOptions); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
},
}
cmd.SetOutput(out)
flags := cmd.Flags()
flags.SetNormalizeFunc(util.WordSepNormalizeFunc)
flags.AddGoFlagSet(flag.CommandLine)
apiServerOptions.AddFlags(flags)
return cmd
}
func (g *configRESTOptionsGetter) loadWatchCacheSettings() error {
if g.masterOptions.KubernetesMasterConfig == nil {
return nil
}
server := apiserveroptions.NewAPIServer()
if errs := cmdflags.Resolve(g.masterOptions.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(errs) > 0 {
return kerrors.NewAggregate(errs)
}
g.cacheEnabled = server.EnableWatchCache
errs := []error{}
for _, c := range server.WatchCacheSizes {
tokens := strings.Split(c, "#")
if len(tokens) != 2 {
errs = append(errs, fmt.Errorf("invalid watch cache size value '%s', expecting <resource>#<size> format (e.g. builds#100)", c))
continue
}
resource := unversioned.ParseGroupResource(tokens[0])
size, err := strconv.Atoi(tokens[1])
if err != nil {
errs = append(errs, fmt.Errorf("invalid watch cache size value '%s': %v", c, err))
continue
}
g.cacheSizes[resource] = size
}
return kerrors.NewAggregate(errs)
}
func StartAPIServer(lk LocalkubeServer) func() error {
config := options.NewAPIServer()
config.BindAddress = lk.APIServerAddress
config.SecurePort = lk.APIServerPort
config.InsecureBindAddress = lk.APIServerInsecureAddress
config.InsecurePort = lk.APIServerInsecurePort
config.ClientCAFile = lk.GetPublicKeyCertPath()
config.TLSCertFile = lk.GetPublicKeyCertPath()
config.TLSPrivateKeyFile = lk.GetPrivateKeyCertPath()
config.AdmissionControl = "NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota"
// use localkube etcd
config.StorageConfig = storagebackend.Config{ServerList: KubeEtcdClientURLs}
// set Service IP range
config.ServiceClusterIPRange = lk.ServiceClusterIPRange
// defaults from apiserver command
config.EnableProfiling = true
config.EnableWatchCache = true
config.MinRequestTimeout = 1800
config.AllowPrivileged = true
return func() error {
return apiserver.Run(config)
}
}
func TestAPIServerDefaults(t *testing.T) {
defaults := apiserveroptions.NewAPIServer()
// This is a snapshot of the default config
// If the default changes (new fields are added, or default values change), we want to know
// Once we've reacted to the changes appropriately in BuildKubernetesMasterConfig(), update this expected default to match the new upstream defaults
expectedDefaults := &apiserveroptions.APIServer{
ServerRunOptions: &genericapiserveroptions.ServerRunOptions{
BindAddress: net.ParseIP("0.0.0.0"),
CertDirectory: "/var/run/kubernetes",
InsecureBindAddress: net.ParseIP("127.0.0.1"),
InsecurePort: 8080,
LongRunningRequestRE: "(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)",
MaxRequestsInFlight: 400,
SecurePort: 6443,
APIGroupPrefix: "/apis",
APIPrefix: "/api",
EnableLogsSupport: true,
EnableProfiling: true,
EnableWatchCache: true,
MinRequestTimeout: 1800,
RuntimeConfig: utilconfig.ConfigurationMap{},
StorageVersions: registered.AllPreferredGroupVersions(),
MasterCount: 1,
DefaultStorageVersions: registered.AllPreferredGroupVersions(),
StorageConfig: storagebackend.Config{
Prefix: "/registry",
DeserializationCacheSize: genericapiserveroptions.DefaultDeserializationCacheSize,
},
DefaultStorageMediaType: "application/json",
AdmissionControl: "AlwaysAdmit",
AuthorizationMode: "AlwaysAllow",
DeleteCollectionWorkers: 1,
MasterServiceNamespace: "default",
AuthorizationConfig: genericapiserveroptions.AuthorizationConfig{
WebhookCacheAuthorizedTTL: 5 * time.Minute,
WebhookCacheUnauthorizedTTL: 30 * time.Second,
},
},
EventTTL: 1 * time.Hour,
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: 10250,
EnableHttps: true,
HTTPTimeout: time.Duration(5) * time.Second,
},
WebhookTokenAuthnCacheTTL: 2 * time.Minute,
}
if !reflect.DeepEqual(defaults, expectedDefaults) {
t.Logf("expected defaults, actual defaults: \n%s", diff.ObjectReflectDiff(expectedDefaults, defaults))
t.Errorf("Got different defaults than expected, adjust in BuildKubernetesMasterConfig and update expectedDefaults")
}
}
func (g *configRESTOptionsGetter) loadSettings() error {
server := apiserveroptions.NewAPIServer()
if g.masterOptions.KubernetesMasterConfig != nil {
if errs := cmdflags.Resolve(g.masterOptions.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(errs) > 0 {
return kerrors.NewAggregate(errs)
}
}
storageGroupsToEncodingVersion, err := server.StorageGroupsToEncodingVersion()
if err != nil {
return err
}
storageConfig := server.StorageConfig
storageConfig.Prefix = g.masterOptions.EtcdStorageConfig.OpenShiftStoragePrefix
storageConfig.ServerList = g.masterOptions.EtcdClientInfo.URLs
storageConfig.KeyFile = g.masterOptions.EtcdClientInfo.ClientCert.KeyFile
storageConfig.CertFile = g.masterOptions.EtcdClientInfo.ClientCert.CertFile
storageConfig.CAFile = g.masterOptions.EtcdClientInfo.CA
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
storageConfig, server.DefaultStorageMediaType, kapi.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
nil,
g.defaultResourceConfig, server.RuntimeConfig)
if err != nil {
return err
}
storageFactory.DefaultResourcePrefixes = g.defaultResourcePrefixes
g.storageFactory = storageFactory
g.cacheEnabled = server.EnableWatchCache
errs := []error{}
for _, c := range server.WatchCacheSizes {
tokens := strings.Split(c, "#")
if len(tokens) != 2 {
errs = append(errs, fmt.Errorf("invalid watch cache size value '%s', expecting <resource>#<size> format (e.g. builds#100)", c))
continue
}
resource := unversioned.ParseGroupResource(tokens[0])
size, err := strconv.Atoi(tokens[1])
if err != nil {
errs = append(errs, fmt.Errorf("invalid watch cache size value '%s': %v", c, err))
continue
}
g.cacheSizes[resource] = size
}
return kerrors.NewAggregate(errs)
}
// NewKubeAPIServer creates a new hyperkube Server object that includes the
// description and flags.
func NewKubeAPIServer() *Server {
s := options.NewAPIServer()
hks := Server{
SimpleUsage: hyperkube.CommandApiserver,
Long: "The main API entrypoint and interface to the storage system. The API server is also the focal point for all authorization decisions.",
Run: func(_ *Server, _ []string) error {
return app.Run(s)
},
}
s.AddFlags(hks.Flags())
return &hks
}
func ValidateAPIServerExtendedArguments(config api.ExtendedArguments, fldPath *field.Path) ValidationResults {
validationResults := ValidationResults{}
validationResults.AddErrors(ValidateExtendedArguments(config, apiserveroptions.NewAPIServer().AddFlags, fldPath)...)
if len(config["admission-control"]) > 0 {
validationResults.AddWarnings(field.Invalid(fldPath.Key("admission-control"), config["admission-control"], "specified admission ordering is being phased out. Convert to DefaultAdmissionConfig in admissionConfig.pluginConfig."))
}
if len(config["admission-control-config-file"]) > 0 {
validationResults.AddWarnings(field.Invalid(fldPath.Key("admission-control-config-file"), config["admission-control-config-file"], "specify a single admission control config file is being phased out. Convert to admissionConfig.pluginConfig, one file per plugin."))
}
return validationResults
}
// NewAPIServerCommand creates a *cobra.Command object with default parameters
func NewAPIServerCommand() *cobra.Command {
s := options.NewAPIServer()
s.AddFlags(pflag.CommandLine)
cmd := &cobra.Command{
Use: "kube-apiserver",
Long: `The Kubernetes API server validates and configures data
for the api objects which include pods, services, replicationcontrollers, and
others. The API Server services REST operations and provides the frontend to the
cluster's shared state through which all other components interact.`,
Run: func(cmd *cobra.Command, args []string) {
},
}
return cmd
}
func main() {
rand.Seed(time.Now().UTC().UnixNano())
s := options.NewAPIServer()
s.AddFlags(pflag.CommandLine)
flag.InitFlags()
logs.InitLogs()
defer logs.FlushLogs()
verflag.PrintAndExitIfRequested()
if err := app.Run(s); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
}
func NewBootkube(config Config) (*bootkube, error) {
apiServer := apiserver.NewAPIServer()
fs := pflag.NewFlagSet("apiserver", pflag.ExitOnError)
apiServer.AddFlags(fs)
fs.Parse([]string{
"--bind-address=0.0.0.0",
"--secure-port=443",
"--insecure-port=8081", // NOTE: temp hack for single-apiserver
"--allow-privileged=true",
"--tls-private-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerKey),
"--tls-cert-file=" + filepath.Join(config.AssetDir, asset.AssetPathAPIServerCert),
"--client-ca-file=" + filepath.Join(config.AssetDir, asset.AssetPathCACert),
"--etcd-servers=" + config.EtcdServer.String(),
"--service-cluster-ip-range=10.3.0.0/24",
"--service-account-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathServiceAccountPubKey),
"--admission-control=ServiceAccount",
"--runtime-config=extensions/v1beta1/deployments=true,extensions/v1beta1/daemonsets=true",
})
cmServer := controller.NewCMServer()
fs = pflag.NewFlagSet("controllermanager", pflag.ExitOnError)
cmServer.AddFlags(fs)
fs.Parse([]string{
"--master=" + insecureAPIAddr,
"--service-account-private-key-file=" + filepath.Join(config.AssetDir, asset.AssetPathServiceAccountPrivKey),
"--root-ca-file=" + filepath.Join(config.AssetDir, asset.AssetPathCACert),
"--leader-elect=true",
})
schedServer := scheduler.NewSchedulerServer()
fs = pflag.NewFlagSet("scheduler", pflag.ExitOnError)
schedServer.AddFlags(fs)
fs.Parse([]string{
"--master=" + insecureAPIAddr,
"--leader-elect=true",
})
return &bootkube{
apiServer: apiServer,
controller: cmServer,
scheduler: schedServer,
assetDir: config.AssetDir,
}, nil
}
func TestAPIServerDefaults(t *testing.T) {
defaults := apiserveroptions.NewAPIServer()
// This is a snapshot of the default config
// If the default changes (new fields are added, or default values change), we want to know
// Once we've reacted to the changes appropriately in BuildKubernetesMasterConfig(), update this expected default to match the new upstream defaults
expectedDefaults := &apiserveroptions.APIServer{
ServerRunOptions: &genericapiserver.ServerRunOptions{
BindAddress: net.ParseIP("0.0.0.0"),
CertDirectory: "/var/run/kubernetes",
InsecureBindAddress: net.ParseIP("127.0.0.1"),
InsecurePort: 8080,
LongRunningRequestRE: "(/|^)((watch|proxy)(/|$)|(logs?|portforward|exec|attach)/?$)",
SecurePort: 6443,
},
APIGroupPrefix: "/apis",
APIPrefix: "/api",
AdmissionControl: "AlwaysAdmit",
AuthorizationMode: "AlwaysAllow",
DeleteCollectionWorkers: 1,
EnableLogsSupport: true,
EtcdConfig: etcdstorage.EtcdConfig{
Prefix: "/registry",
},
EventTTL: 1 * time.Hour,
MasterCount: 1,
MasterServiceNamespace: "default",
RuntimeConfig: util.ConfigurationMap{},
StorageVersions: registered.AllPreferredGroupVersions(),
DefaultStorageVersions: registered.AllPreferredGroupVersions(),
KubeletConfig: kubeletclient.KubeletClientConfig{
Port: 10250,
EnableHttps: true,
HTTPTimeout: time.Duration(5) * time.Second,
},
}
if !reflect.DeepEqual(defaults, expectedDefaults) {
t.Logf("expected defaults, actual defaults: \n%s", util.ObjectGoPrintDiff(expectedDefaults, defaults))
t.Errorf("Got different defaults than expected, adjust in BuildKubernetesMasterConfig and update expectedDefaults")
}
}
func TestLongRunningRequestRegexp(t *testing.T) {
regexp := regexp.MustCompile(options.NewAPIServer().LongRunningRequestRE)
dontMatch := []string{
"/api/v1/watch-namespace/",
"/api/v1/namespace-proxy/",
"/api/v1/namespace-watch",
"/api/v1/namespace-proxy",
"/api/v1/namespace-portforward/pods",
"/api/v1/portforward/pods",
". anything",
"/ that",
}
doMatch := []string{
"/api/v1/pods/watch",
"/api/v1/watch/stuff",
"/api/v1/default/service/proxy",
"/api/v1/pods/proxy/path/to/thing",
"/api/v1/namespaces/myns/pods/mypod/log",
"/api/v1/namespaces/myns/pods/mypod/logs",
"/api/v1/namespaces/myns/pods/mypod/portforward",
"/api/v1/namespaces/myns/pods/mypod/exec",
"/api/v1/namespaces/myns/pods/mypod/attach",
"/api/v1/namespaces/myns/pods/mypod/log/",
"/api/v1/namespaces/myns/pods/mypod/logs/",
"/api/v1/namespaces/myns/pods/mypod/portforward/",
"/api/v1/namespaces/myns/pods/mypod/exec/",
"/api/v1/namespaces/myns/pods/mypod/attach/",
"/api/v1/watch/namespaces/myns/pods",
}
for _, path := range dontMatch {
if regexp.MatchString(path) {
t.Errorf("path should not have match regexp but did: %s", path)
}
}
for _, path := range doMatch {
if !regexp.MatchString(path) {
t.Errorf("path should have match regexp did not: %s", path)
}
}
}
// Start starts the apiserver, returns when apiserver is ready.
func (a *APIServer) Start() error {
config := options.NewAPIServer()
config.StorageConfig.ServerList = []string{getEtcdClientURL()}
_, ipnet, err := net.ParseCIDR(clusterIPRange)
if err != nil {
return err
}
config.ServiceClusterIPRange = *ipnet
config.AllowPrivileged = true
errCh := make(chan error)
go func() {
defer close(errCh)
err := apiserver.Run(config)
if err != nil {
errCh <- fmt.Errorf("run apiserver error: %v", err)
}
}()
err = readinessCheck([]string{apiserverHealthCheckURL}, errCh)
if err != nil {
return err
}
return nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, informers shared.InformerFactory, admissionControl admission.Interface, originAuthenticator authenticator.Request) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.EnableLogsSupport = false // don't expose server logs
server.EnableProfiling = false
server.APIPrefix = KubeAPIPrefix
server.APIGroupPrefix = KubeAPIGroupPrefix
server.SecurePort = port
server.MasterCount = options.KubernetesMasterConfig.MasterCount
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
cmserver.VolumeConfiguration.EnableDynamicProvisioning = options.VolumeConfig.DynamicProvisioningEnabled
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
resourceEncodingConfig := genericapiserver.NewDefaultResourceEncodingConfig()
resourceEncodingConfig.SetVersionEncoding(
kapi.GroupName,
unversioned.GroupVersion{Group: kapi.GroupName, Version: options.EtcdStorageConfig.KubernetesStorageVersion},
kapi.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
extensions.GroupName,
unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"},
extensions.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
batch.GroupName,
unversioned.GroupVersion{Group: batch.GroupName, Version: "v1"},
batch.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
autoscaling.GroupName,
unversioned.GroupVersion{Group: autoscaling.GroupName, Version: "v1"},
autoscaling.SchemeGroupVersion,
)
etcdConfig := storagebackend.Config{
Prefix: options.EtcdStorageConfig.KubernetesStoragePrefix,
ServerList: options.EtcdClientInfo.URLs,
KeyFile: options.EtcdClientInfo.ClientCert.KeyFile,
CertFile: options.EtcdClientInfo.ClientCert.CertFile,
CAFile: options.EtcdClientInfo.CA,
DeserializationCacheSize: genericapiserveroptions.DefaultDeserializationCacheSize,
}
storageFactory := genericapiserver.NewDefaultStorageFactory(etcdConfig, "", kapi.Codecs, resourceEncodingConfig, master.DefaultAPIResourceConfigSource())
// the order here is important, it defines which version will be used for storage
storageFactory.AddCohabitatingResources(extensions.Resource("jobs"), batch.Resource("jobs"))
storageFactory.AddCohabitatingResources(extensions.Resource("horizontalpodautoscalers"), autoscaling.Resource("horizontalpodautoscalers"))
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
Authenticator: originAuthenticator, // this is used to fulfill the tokenreviews endpoint which is used by node authentication
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionControl,
StorageFactory: storageFactory,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIResourceConfigSource: getAPIResourceConfig(options),
APIPrefix: server.APIPrefix,
APIGroupPrefix: server.APIGroupPrefix,
MasterCount: server.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
EnableLogsSupport: server.EnableLogsSupport,
EnableProfiling: server.EnableProfiling,
EnableWatchCache: server.EnableWatchCache,
MasterServiceNamespace: server.MasterServiceNamespace,
ExternalHost: server.ExternalHost,
MinRequestTimeout: server.MinRequestTimeout,
KubernetesServiceNodePort: server.KubernetesServiceNodePort,
},
EventTTL: server.EventTTL,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
DeleteCollectionWorkers: server.DeleteCollectionWorkers,
}
if server.EnableWatchCache {
cachesize.SetWatchCacheSizes(server.WatchCacheSizes)
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: int32(dnsPort), Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: int32(dnsPort), Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
Informers: informers,
}
return kmaster, nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, pluginInitializer oadmission.PluginInitializer) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
// Connect and setup etcd interfaces
etcdClient, err := etcd.MakeNewEtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
server.EnableLogsSupport = false // don't expose server logs
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 {
server.AdmissionControl = strings.Join(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride, ",")
}
// Defaults are tested in TestCMServerDefaults
cmserver := cmapp.NewCMServer()
// Adjust defaults
cmserver.Address = "" // no healthz endpoint
cmserver.Port = 0 // no healthz endpoint
cmserver.PodEvictionTimeout = unversioned.Duration{Duration: podEvictionTimeout}
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case serviceadmit.ExternalIPPluginName:
// this needs to be moved upstream to be part of core config
reject, admit, err := serviceadmit.ParseCIDRRules(options.NetworkConfig.ExternalIPNetworkCIDRs)
if err != nil {
// should have been caught with validation
return nil, err
}
plugins = append(plugins, serviceadmit.NewExternalIPRanger(reject, admit))
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(internalclientset.FromUnversionedClient(kubeClient))
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
configFile, err := pluginconfig.GetPluginConfigFile(options.KubernetesMasterConfig.AdmissionConfig.PluginConfig, pluginName, server.AdmissionControlConfigFile)
if err != nil {
return nil, err
}
plugin := admission.InitPlugin(pluginName, internalclientset.FromUnversionedClient(kubeClient), configFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
pluginInitializer.Initialize(plugins)
// ensure that plugins have been properly initialized
if err := oadmission.Validate(plugins); err != nil {
return nil, err
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
// TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling
// It needs a refactor to return errors
storageDestinations := genericapiserver.NewStorageDestinations()
// storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks.
// We need to fix the upstream to stop using the storage version as a preferred api version.
storageVersions := map[string]string{}
enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube)
if len(enabledKubeVersions) > 0 {
kubeStorageVersion := unversioned.GroupVersion{Group: configapi.APIGroupKube, Version: options.EtcdStorageConfig.KubernetesStorageVersion}
databaseStorage, err := NewEtcdStorage(etcdClient, kubeStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage)
storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion
}
// enable this if extensions API is enabled (or batch or autoscaling, since they persist to extensions/v1beta1 for now)
// TODO: replace this with a loop over configured storage versions
extensionsEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)) > 0
batchEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupBatch)) > 0
autoscalingEnabled := len(configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupAutoscaling)) > 0
if extensionsEnabled || autoscalingEnabled || batchEnabled {
// TODO: replace this with a configured storage version for extensions once configuration exposes this
extensionsStorageVersion := unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"}
databaseStorage, err := NewEtcdStorage(etcdClient, extensionsStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage)
storageVersions[configapi.APIGroupExtensions] = extensionsStorageVersion.String()
}
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := knet.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
Config: &genericapiserver.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
APIGroupVersionOverrides: getAPIGroupVersionOverrides(options),
APIPrefix: KubeAPIPrefix,
APIGroupPrefix: KubeAPIGroupPrefix,
MasterCount: options.KubernetesMasterConfig.MasterCount,
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
Serializer: kapi.Codecs,
},
EventTTL: server.EventTTL,
//MinRequestTimeout: server.MinRequestTimeout,
KubeletClient: kubeletClient,
EnableCoreControllers: true,
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
}
return kmaster, nil
}
// BuildDefaultAPIServer constructs the appropriate APIServer and StorageFactory for the kubernetes server.
// It returns an error if no KubernetesMasterConfig was defined.
func BuildDefaultAPIServer(options configapi.MasterConfig) (*apiserveroptions.APIServer, genericapiserver.StorageFactory, error) {
if options.KubernetesMasterConfig == nil {
return nil, nil, fmt.Errorf("no kubernetesMasterConfig defined, unable to load settings")
}
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, nil, err
}
portRange, err := knet.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, nil, err
}
// Defaults are tested in TestAPIServerDefaults
server := apiserveroptions.NewAPIServer()
// Adjust defaults
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.EnableLogsSupport = false // don't expose server logs
server.EnableProfiling = false
server.APIPrefix = KubeAPIPrefix
server.APIGroupPrefix = KubeAPIGroupPrefix
server.SecurePort = port
server.MasterCount = options.KubernetesMasterConfig.MasterCount
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, nil, kerrors.NewAggregate(err)
}
resourceEncodingConfig := genericapiserver.NewDefaultResourceEncodingConfig()
resourceEncodingConfig.SetVersionEncoding(
kapi.GroupName,
unversioned.GroupVersion{Group: kapi.GroupName, Version: options.EtcdStorageConfig.KubernetesStorageVersion},
kapi.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
extensions.GroupName,
unversioned.GroupVersion{Group: extensions.GroupName, Version: "v1beta1"},
extensions.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
batch.GroupName,
unversioned.GroupVersion{Group: batch.GroupName, Version: "v1"},
batch.SchemeGroupVersion,
)
resourceEncodingConfig.SetVersionEncoding(
autoscaling.GroupName,
unversioned.GroupVersion{Group: autoscaling.GroupName, Version: "v1"},
autoscaling.SchemeGroupVersion,
)
storageGroupsToEncodingVersion, err := server.StorageGroupsToEncodingVersion()
if err != nil {
return nil, nil, err
}
// use the stock storage config based on args, but override bits from our config where appropriate
etcdConfig := server.StorageConfig
etcdConfig.Prefix = options.EtcdStorageConfig.KubernetesStoragePrefix
etcdConfig.ServerList = options.EtcdClientInfo.URLs
etcdConfig.KeyFile = options.EtcdClientInfo.ClientCert.KeyFile
etcdConfig.CertFile = options.EtcdClientInfo.ClientCert.CertFile
etcdConfig.CAFile = options.EtcdClientInfo.CA
storageFactory, err := genericapiserver.BuildDefaultStorageFactory(
etcdConfig,
server.DefaultStorageMediaType,
kapi.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(),
storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]unversioned.GroupVersionResource{batch.Resource("scheduledjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), server.RuntimeConfig,
)
if err != nil {
return nil, nil, err
}
/*storageFactory := genericapiserver.NewDefaultStorageFactory(
etcdConfig,
server.DefaultStorageMediaType,
kapi.Codecs,
resourceEncodingConfig,
master.DefaultAPIResourceConfigSource(),
)*/
// the order here is important, it defines which version will be used for storage
storageFactory.AddCohabitatingResources(extensions.Resource("jobs"), batch.Resource("jobs"))
storageFactory.AddCohabitatingResources(extensions.Resource("horizontalpodautoscalers"), autoscaling.Resource("horizontalpodautoscalers"))
return server, storageFactory, nil
}
func ValidateAPIServerExtendedArguments(config api.ExtendedArguments, fldPath *field.Path) field.ErrorList {
return ValidateExtendedArguments(config, apiserveroptions.NewAPIServer().AddFlags, fldPath)
}
