func setConfigDefaults(config *unversioned.Config) error {
// if legacy group is not registered, return an error
g, err := latest.Group("")
if err != nil {
return err
}
config.Prefix = "/api"
if config.UserAgent == "" {
config.UserAgent = unversioned.DefaultKubernetesUserAgent()
}
// TODO: Unconditionally set the config.Version, until we fix the config.
//if config.Version == "" {
copyGroupVersion := g.GroupVersion
config.GroupVersion = ©GroupVersion
//}
versionInterfaces, err := g.InterfacesFor(*config.GroupVersion)
if err != nil {
return fmt.Errorf("Legacy API version '%s' is not recognized (valid values: %s)",
config.GroupVersion, g.GroupVersions)
}
config.Codec = versionInterfaces.Codec
if config.QPS == 0 {
config.QPS = 5
}
if config.Burst == 0 {
config.Burst = 10
}
return nil
}
func newServiceAccountTokenGetter(options configapi.MasterConfig, client *etcdclient.Client) (serviceaccount.ServiceAccountTokenGetter, error) {
var tokenGetter serviceaccount.ServiceAccountTokenGetter
if options.KubernetesMasterConfig == nil {
// When we're running against an external Kubernetes, use the external kubernetes client to validate service account tokens
// This prevents infinite auth loops if the privilegedLoopbackKubeClient authenticates using a service account token
kubeClient, _, err := configapi.GetKubeClient(options.MasterClients.ExternalKubernetesKubeConfig)
if err != nil {
return nil, err
}
tokenGetter = serviceaccount.NewGetterFromClient(kubeClient)
} else {
// When we're running in-process, go straight to etcd (using the KubernetesStorageVersion/KubernetesStoragePrefix, since service accounts are kubernetes objects)
legacyGroup, err := kapilatest.Group(kapi.SchemeGroupVersion.Group)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
versionedInterface, err := legacyGroup.InterfacesFor(unversioned.GroupVersion{Group: kapi.SchemeGroupVersion.Group, Version: options.EtcdStorageConfig.KubernetesStorageVersion})
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
ketcdHelper := etcdstorage.NewEtcdStorage(client, versionedInterface.Codec, options.EtcdStorageConfig.KubernetesStoragePrefix)
tokenGetter = serviceaccount.NewGetterFromStorageInterface(ketcdHelper)
}
return tokenGetter, nil
}
func setExperimentalDefaults(config *Config) error {
// if experimental group is not registered, return an error
g, err := latest.Group("experimental")
if err != nil {
return err
}
config.Prefix = "apis/" + g.Group
if config.UserAgent == "" {
config.UserAgent = DefaultKubernetesUserAgent()
}
if config.Version == "" {
config.Version = g.Version
}
versionInterfaces, err := g.InterfacesFor(config.Version)
if err != nil {
return fmt.Errorf("Experimental API version '%s' is not recognized (valid values: %s)",
config.Version, strings.Join(latest.GroupOrDie("experimental").Versions, ", "))
}
if config.Codec == nil {
config.Codec = versionInterfaces.Codec
}
if config.QPS == 0 {
config.QPS = 5
}
if config.Burst == 0 {
config.Burst = 10
}
return nil
}
func (c *clientSwaggerSchema) ValidateBytes(data []byte) error {
version, _, err := runtime.UnstructuredJSONScheme.DataVersionAndKind(data)
if err != nil {
return err
}
if ok := registered.IsRegisteredAPIVersion(version); !ok {
return fmt.Errorf("API version %q isn't supported, only supports API versions %q", version, registered.RegisteredVersions)
}
// First try stable api, if we can't validate using that, try experimental.
// If experimental fails, return error from stable api.
// TODO: Figure out which group to try once multiple group support is merged
// instead of trying everything.
err = getSchemaAndValidate(c.c.RESTClient, data, "api", version, c.cacheDir)
if err != nil && c.ec != nil {
g, err := latest.Group("experimental")
if err != nil {
return err
}
errExp := getSchemaAndValidate(c.ec.RESTClient, data, "apis"+"/"+g.Group, version, c.cacheDir)
if errExp == nil {
return nil
}
}
return err
}
func setExtensionsDefaults(config *Config) error {
// if experimental group is not registered, return an error
g, err := latest.Group(extensions.GroupName)
if err != nil {
return err
}
config.APIPath = defaultAPIPath
if config.UserAgent == "" {
config.UserAgent = DefaultKubernetesUserAgent()
}
// TODO: Unconditionally set the config.Version, until we fix the config.
//if config.Version == "" {
copyGroupVersion := g.GroupVersion
config.GroupVersion = ©GroupVersion
//}
versionInterfaces, err := g.InterfacesFor(*config.GroupVersion)
if err != nil {
return fmt.Errorf("Extensions API group/version '%v' is not recognized (valid values: %v)",
config.GroupVersion, latest.GroupOrDie(extensions.GroupName).GroupVersions)
}
config.Codec = versionInterfaces.Codec
if config.QPS == 0 {
config.QPS = 5
}
if config.Burst == 0 {
config.Burst = 10
}
return nil
}
// SetKubernetesDefaults sets default values on the provided client config for accessing the
// Kubernetes API or returns an error if any of the defaults are impossible or invalid.
// TODO: this method needs to be split into one that sets defaults per group, expected to be fix in PR "Refactoring clientcache.go and helper.go #14592"
func SetKubernetesDefaults(config *Config) error {
if config.Prefix == "" {
config.Prefix = "/api"
}
if len(config.UserAgent) == 0 {
config.UserAgent = DefaultKubernetesUserAgent()
}
g, err := latest.Group(api.GroupName)
if err != nil {
return err
}
// TODO: Unconditionally set the config.Version, until we fix the config.
copyGroupVersion := g.GroupVersion
config.GroupVersion = ©GroupVersion
versionInterfaces, err := g.InterfacesFor(*config.GroupVersion)
if err != nil {
return fmt.Errorf("API version '%v' is not recognized (valid values: %v)", *config.GroupVersion, latest.GroupOrDie(api.GroupName).GroupVersions)
}
if config.Codec == nil {
config.Codec = versionInterfaces.Codec
}
if config.QPS == 0.0 {
config.QPS = 5.0
}
if config.Burst == 0 {
config.Burst = 10
}
return nil
}
func (c *clientSwaggerSchema) ValidateBytes(data []byte) error {
version, kind, err := runtime.UnstructuredJSONScheme.DataVersionAndKind(data)
if err != nil {
return err
}
if ok := registered.IsRegisteredAPIVersion(version); !ok {
return fmt.Errorf("API version %q isn't supported, only supports API versions %q", version, registered.RegisteredVersions)
}
resource, _ := meta.KindToResource(kind, false)
group, err := c.mapper.GroupForResource(resource)
if err != nil {
return fmt.Errorf("could not find api group for %s: %v", kind, err)
}
if group == "experimental" {
g, err := latest.Group(group)
if err != nil {
return err
}
if c.c.ExperimentalClient == nil {
return errors.New("unable to validate: no experimental client")
}
return getSchemaAndValidate(c.c.ExperimentalClient.RESTClient, data, "apis/"+g.Group, version, c.cacheDir)
}
return getSchemaAndValidate(c.c.RESTClient, data, "api", version, c.cacheDir)
}
func setExtensionsDefaults(config *Config) error {
// if experimental group is not registered, return an error
g, err := latest.Group("extensions")
if err != nil {
return err
}
config.Prefix = "apis/"
if config.UserAgent == "" {
config.UserAgent = DefaultKubernetesUserAgent()
}
// TODO: Unconditionally set the config.Version, until we fix the config.
//if config.Version == "" {
gv, err := unversioned.ParseGroupVersion(g.GroupVersion)
if err != nil {
return err
}
config.GroupVersion = &gv
//}
versionInterfaces, err := g.InterfacesFor(config.GroupVersion.String())
if err != nil {
return fmt.Errorf("Extensions API group/version '%v' is not recognized (valid values: %s)",
config.GroupVersion, strings.Join(latest.GroupOrDie("extensions").Versions, ", "))
}
config.Codec = versionInterfaces.Codec
if config.QPS == 0 {
config.QPS = 5
}
if config.Burst == 0 {
config.Burst = 10
}
return nil
}
// New creates a Kubernetes client for the given config. This client works with pods,
// replication controllers, daemons, and services. It allows operations such as list, get, update
// and delete on these objects. An error is returned if the provided configuration
// is not valid.
func New(c *Config) (*Client, error) {
config := *c
if err := SetKubernetesDefaults(&config); err != nil {
return nil, err
}
client, err := RESTClientFor(&config)
if err != nil {
return nil, err
}
discoveryConfig := *c
discoveryClient, err := NewDiscoveryClient(&discoveryConfig)
if err != nil {
return nil, err
}
if _, err := latest.Group("extensions"); err != nil {
return &Client{RESTClient: client, ExtensionsClient: nil, DiscoveryClient: discoveryClient}, nil
}
experimentalConfig := *c
experimentalClient, err := NewExtensions(&experimentalConfig)
if err != nil {
return nil, err
}
return &Client{RESTClient: client, ExtensionsClient: experimentalClient, DiscoveryClient: discoveryClient}, nil
}
// NewEtcdStorage returns a storage interface for the provided storage version.
func NewEtcdStorage(client *etcdclient.Client, version unversioned.GroupVersion, prefix string) (helper storage.Interface, err error) {
group, err := kapilatest.Group(version.Group)
if err != nil {
return nil, err
}
interfaces, err := group.InterfacesFor(version)
if err != nil {
return nil, err
}
return etcdstorage.NewEtcdStorage(client, interfaces.Codec, prefix), nil
}
// InterfacesFor returns the default Codec and ResourceVersioner for a given version
// string, or an error if the version is not known.
func interfacesFor(version string) (*meta.VersionInterfaces, error) {
switch version {
case "metrics/v1alpha1":
return &meta.VersionInterfaces{
Codec: v1alpha1.Codec,
ObjectConvertor: api.Scheme,
MetadataAccessor: accessor,
}, nil
default:
g, _ := latest.Group("metrics")
return nil, fmt.Errorf("unsupported storage version: %s (valid: %v)", version, g.GroupVersions)
}
}
// interfacesFor returns the default Codec and ResourceVersioner for a given version
// string, or an error if the version is not known.
func interfacesFor(version unversioned.GroupVersion) (*meta.VersionInterfaces, error) {
switch version {
case v1beta1.SchemeGroupVersion:
return &meta.VersionInterfaces{
Codec: v1beta1.Codec,
ObjectConvertor: api.Scheme,
MetadataAccessor: accessor,
}, nil
default:
g, _ := latest.Group(extensions.GroupName)
return nil, fmt.Errorf("unsupported storage version: %s (valid: %v)", version, g.GroupVersions)
}
}
// interfacesFor returns the default Codec and ResourceVersioner for a given version
// string, or an error if the version is not known.
func interfacesFor(version string) (*meta.VersionInterfaces, error) {
switch version {
case "componentconfig/v1alpha1":
return &meta.VersionInterfaces{
Codec: v1alpha1.Codec,
ObjectConvertor: api.Scheme,
MetadataAccessor: accessor,
}, nil
default:
g, _ := latest.Group("componentconfig")
return nil, fmt.Errorf("unsupported storage version: %s (valid: %s)", version, strings.Join(g.Versions, ", "))
}
}
// RunExplain executes the appropriate steps to print a model's documentation
func RunExplain(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string) error {
if len(args) != 1 {
return cmdutil.UsageError(cmd, "We accept only this format: explain RESOURCE")
}
client, err := f.Client()
if err != nil {
return err
}
recursive := cmdutil.GetFlagBool(cmd, "recursive")
apiVersionString := cmdutil.GetFlagString(cmd, "api-version")
apiVersion := unversioned.GroupVersion{}
mapper, _ := f.Object()
// TODO: After we figured out the new syntax to separate group and resource, allow
// the users to use it in explain (kubectl explain <group><syntax><resource>).
// Refer to issue #16039 for why we do this. Refer to PR #15808 that used "/" syntax.
inModel, fieldsPath, err := kubectl.SplitAndParseResourceRequest(args[0], mapper)
if err != nil {
return err
}
// TODO: We should deduce the group for a resource by discovering the supported resources at server.
gvk, err := mapper.KindFor(inModel)
if err != nil {
return err
}
if len(apiVersionString) == 0 {
groupMeta, err := latest.Group(gvk.Group)
if err != nil {
return err
}
apiVersion = groupMeta.GroupVersion
} else {
apiVersion, err = unversioned.ParseGroupVersion(apiVersionString)
if err != nil {
return nil
}
}
swagSchema, err := kubectl.GetSwaggerSchema(apiVersion, client)
if err != nil {
return err
}
return kubectl.PrintModelDescription(inModel, fieldsPath, out, swagSchema, recursive)
}
// Check whether the kind in groupVersion is scoped at the root of the api hierarchy
func isNamespacedKind(kind, groupVersion string) (bool, error) {
group := apiutil.GetGroup(groupVersion)
g, err := latest.Group(group)
if err != nil {
return false, err
}
restMapping, err := g.RESTMapper.RESTMapping(unversioned.GroupKind{Group: group, Kind: kind}, apiutil.GetVersion(groupVersion))
if err != nil {
return false, err
}
scopeName := restMapping.Scope.Name()
if scopeName == meta.RESTScopeNameNamespace {
return true, nil
}
return false, nil
}
// forbidden renders a simple forbidden error
func forbidden(reason string, attributes authorizer.AuthorizationAttributes, w http.ResponseWriter, req *http.Request) {
kind := ""
name := ""
apiVersion := klatest.ExternalVersions[0]
// the attributes can be empty for two basic reasons:
// 1. malformed API request
// 2. not an API request at all
// In these cases, just assume default that will work better than nothing
if attributes != nil {
apiVersion = unversioned.GroupVersion{Group: attributes.GetAPIGroup(), Version: attributes.GetAPIVersion()}
kind = attributes.GetResource()
if len(attributes.GetAPIGroup()) > 0 {
kind = attributes.GetAPIGroup() + "." + kind
}
name = attributes.GetResourceName()
}
// Reason is an opaque string that describes why access is allowed or forbidden (forbidden by the time we reach here).
// We don't have direct access to kind or name (not that those apply either in the general case)
// We create a NewForbidden to stay close the API, but then we override the message to get a serialization
// that makes sense when a human reads it.
forbiddenError, _ := kapierrors.NewForbidden(kind, name, errors.New("") /*discarded*/).(*kapierrors.StatusError)
forbiddenError.ErrStatus.Message = reason
// Not all API versions in valid API requests will have a matching codec in kubernetes. If we can't find one,
// just default to the latest kube codec.
codec := klatest.GroupOrDie(kapi.SchemeGroupVersion.Group).Codec
if requestedGroup, err := klatest.Group(apiVersion.Group); err == nil {
if requestedCodec, err := requestedGroup.InterfacesFor(apiVersion); err == nil {
codec = requestedCodec
}
}
formatted := &bytes.Buffer{}
output, err := codec.Encode(&forbiddenError.ErrStatus)
if err != nil {
fmt.Fprintf(formatted, "%s", forbiddenError.Error())
} else {
json.Indent(formatted, output, "", " ")
}
w.Header().Set("Content-Type", restful.MIME_JSON)
w.WriteHeader(http.StatusForbidden)
w.Write(formatted.Bytes())
}
// New creates a Kubernetes client for the given config. This client works with pods,
// replication controllers, daemons, and services. It allows operations such as list, get, update
// and delete on these objects. An error is returned if the provided configuration
// is not valid.
func New(c *Config) (*Client, error) {
config := *c
if err := SetKubernetesDefaults(&config); err != nil {
return nil, err
}
client, err := RESTClientFor(&config)
if err != nil {
return nil, err
}
if _, err := latest.Group("experimental"); err != nil {
return &Client{RESTClient: client, ExperimentalClient: nil}, nil
}
experimentalConfig := *c
experimentalClient, err := NewExperimental(&experimentalConfig)
if err != nil {
return nil, err
}
return &Client{RESTClient: client, ExperimentalClient: experimentalClient}, nil
}
// RunExplain executes the appropriate steps to print a model's documentation
func RunExplain(f *cmdutil.Factory, out io.Writer, cmd *cobra.Command, args []string) error {
if len(args) != 1 {
return cmdutil.UsageError(cmd, "We accept only this format: explain RESOURCE")
}
client, err := f.Client()
if err != nil {
return err
}
recursive := cmdutil.GetFlagBool(cmd, "recursive")
apiV := cmdutil.GetFlagString(cmd, "api-version")
mapper, _ := f.Object()
group, inModel, fieldsPath, err := kubectl.SplitAndParseResourceRequest(args[0], mapper)
if err != nil {
return err
}
if len(group) == 0 {
// TODO: We should deduce the group for a resource by discovering the supported resources at server.
group, err = mapper.GroupForResource(inModel)
if err != nil {
return err
}
}
if len(apiV) == 0 {
groupMeta, err := latest.Group(group)
if err != nil {
return err
}
apiV = groupMeta.GroupVersion
}
swagSchema, err := kubectl.GetSwaggerSchema(apiV, client)
if err != nil {
return err
}
return kubectl.PrintModelDescription(inModel, fieldsPath, out, swagSchema, recursive)
}
// parse the value of --etcd-servers-overrides and update given storageDestinations.
func updateEtcdOverrides(overrides []string, storageVersions map[string]string, prefix string, storageDestinations *master.StorageDestinations, newEtcdFn newEtcdFunc) {
if len(overrides) == 0 {
return
}
for _, override := range overrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
}
group := apiresource[0]
resource := apiresource[1]
apigroup, err := latest.Group(group)
if err != nil {
glog.Errorf("invalid api group %s: %v", group, err)
continue
}
if _, found := storageVersions[apigroup.Group]; !found {
glog.Errorf("Couldn't find the storage version for group %s", apigroup.Group)
continue
}
servers := strings.Split(tokens[1], ";")
etcdOverrideStorage, err := newEtcdFn("", servers, apigroup.InterfacesFor, storageVersions[apigroup.Group], prefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd for %s: %v", tokens[0], err)
}
storageDestinations.AddStorageOverride(group, resource, etcdOverrideStorage)
}
}
func init() {
// this keeps us consistent with old code. We can decide if we want to expand our RESTMapper to cover
// api.RESTMapper, which is different than what you'd get from latest.
kubeAPIGroup, err := klatest.Group("")
if err != nil {
panic(err)
}
kubeMapper := kubeAPIGroup.RESTMapper
// list of versions we support on the server, in preferred order
versions := []string{"v1", "v1beta3"}
originMapper := kmeta.NewDefaultRESTMapper(
"",
versions,
func(version string) (*kmeta.VersionInterfaces, error) {
interfaces, err := InterfacesFor(version)
if err != nil {
return nil, err
}
return interfaces, nil
},
)
// the list of kinds that are scoped at the root of the api hierarchy
// if a kind is not enumerated here, it is assumed to have a namespace scope
kindToRootScope := map[string]bool{
"Status": true,
"Project": true,
"ProjectRequest": true,
"Image": true,
"User": true,
"Identity": true,
"UserIdentityMapping": true,
"Group": true,
"OAuthAccessToken": true,
"OAuthAuthorizeToken": true,
"OAuthClient": true,
"OAuthClientAuthorization": true,
"ClusterRole": true,
"ClusterRoleBinding": true,
"ClusterPolicy": true,
"ClusterPolicyBinding": true,
"ClusterNetwork": true,
"HostSubnet": true,
"NetNamespace": true,
}
// enumerate all supported versions, get the kinds, and register with the mapper how to address our resources
for _, version := range versions {
for kind, t := range api.Scheme.KnownTypes(version) {
if !strings.Contains(t.PkgPath(), "openshift/origin") {
if _, ok := kindToRootScope[kind]; !ok {
continue
}
}
originTypes.Insert(kind)
scope := kmeta.RESTScopeNamespace
_, found := kindToRootScope[kind]
if found || (strings.HasSuffix(kind, "List") && kindToRootScope[strings.TrimSuffix(kind, "List")]) {
scope = kmeta.RESTScopeRoot
}
glog.V(6).Infof("Registering %s %s %s", kind, version, scope.Name())
originMapper.Add(scope, kind, version, false)
}
}
// For Origin we use MultiRESTMapper that handles both Origin and Kubernetes
// objects
RESTMapper = kmeta.MultiRESTMapper{originMapper, kubeMapper}
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
// Connect and setup etcd interfaces
etcdClient, err := etcd.EtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kclient.NewKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := util.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
server := app.NewAPIServer()
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
server.AdmissionControl = strings.Join(AdmissionPlugins, ",")
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cmserver := cmapp.NewCMServer()
cmserver.PodEvictionTimeout = podEvictionTimeout
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(kubeClient)
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
plugin := admission.InitPlugin(pluginName, kubeClient, server.AdmissionControlConfigFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
// TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling
// It needs a refactor to return errors
storageDestinations := master.NewStorageDestinations()
// storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks.
// We need to fix the upstream to stop using the storage version as a preferred api version.
storageVersions := map[string]string{}
enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube)
enabledKubeVersionSet := sets.NewString(enabledKubeVersions...)
if len(enabledKubeVersions) > 0 {
databaseStorage, err := master.NewEtcdStorage(etcdClient, kapilatest.InterfacesForLegacyGroup, options.EtcdStorageConfig.KubernetesStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage)
storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion
}
enabledExtensionsVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)
if len(enabledExtensionsVersions) > 0 {
groupMeta, err := kapilatest.Group(configapi.APIGroupExtensions)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
// TODO expose storage version options for api groups
databaseStorage, err := master.NewEtcdStorage(etcdClient, groupMeta.InterfacesFor, groupMeta.GroupVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage)
storageVersions[configapi.APIGroupExtensions] = enabledExtensionsVersions[0]
}
m := &master.Config{
PublicAddress: net.ParseIP(options.KubernetesMasterConfig.MasterIP),
ReadWritePort: port,
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
EventTTL: server.EventTTL,
//MinRequestTimeout: server.MinRequestTimeout,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
KubeletClient: kubeletClient,
APIPrefix: KubeAPIPrefix,
APIGroupPrefix: KubeAPIGroupPrefix,
EnableCoreControllers: true,
MasterCount: options.KubernetesMasterConfig.MasterCount,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
EnableExp: len(enabledExtensionsVersions) > 0,
DisableV1: !enabledKubeVersionSet.Has("v1"),
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
}
// set for consistency -- Origin only used m.EnableExp
cmserver.EnableExperimental = m.EnableExp
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP, TargetPort: util.NewIntOrStringFromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP, TargetPort: util.NewIntOrStringFromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
}
return kmaster, nil
}
// Run runs the specified APIServer. This should never exit.
func (s *APIServer) Run(_ []string) error {
s.verifyClusterIPFlags()
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), setDefaults() in
// pkg/master/master.go will do the right thing and use the host's default
// interface.
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
s.AdvertiseAddress = s.BindAddress
}
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
glog.Fatalf("Specify either --etcd-servers or --etcd-config")
}
if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) {
glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort))
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
// Setup tunneler if needed
var tunneler master.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
// Set up the tunneler
tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
apiGroupVersionOverrides, err := s.parseRuntimeConfig()
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &client.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
Version: s.DeprecatedStorageVersion,
}
client, err := client.New(clientConfig)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
legacyV1Group, err := latest.Group("")
if err != nil {
return err
}
storageDestinations := master.NewStorageDestinations()
storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions)
if _, found := storageVersions[legacyV1Group.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.Group, storageVersions)
}
etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, legacyV1Group.InterfacesFor, storageVersions[legacyV1Group.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("", etcdStorage)
if !apiGroupVersionOverrides["extensions/v1beta1"].Disable {
expGroup, err := latest.Group("extensions")
if err != nil {
glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
if _, found := storageVersions[expGroup.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.Group, storageVersions)
}
expEtcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("extensions", expEtcdStorage)
}
updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd)
n := s.ServiceClusterIPRange
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if apiserver.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
authenticator, err := apiserver.NewAuthenticator(apiserver.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
Storage: etcdStorage,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
config := &master.Config{
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
ServiceClusterIPRange: &n,
EnableCoreControllers: true,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableProfiling: s.EnableProfiling,
EnableWatchCache: s.EnableWatchCache,
EnableIndex: true,
APIPrefix: s.APIPrefix,
APIGroupPrefix: s.APIGroupPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
APIGroupVersionOverrides: apiGroupVersionOverrides,
MasterServiceNamespace: s.MasterServiceNamespace,
ClusterName: s.ClusterName,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
ProxyDialer: proxyDialerFn,
ProxyTLSClientConfig: proxyTLSClientConfig,
Tunneler: tunneler,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
}
m := master.New(config)
// We serve on 2 ports. See docs/accessing_the_api.md
secureLocation := ""
if s.SecurePort != 0 {
secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
}
insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
var sem chan bool
if s.MaxRequestsInFlight > 0 {
sem = make(chan bool, s.MaxRequestsInFlight)
}
longRunningRE := regexp.MustCompile(s.LongRunningRequestRE)
longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) {
// TODO unify this with apiserver.MaxInFlightLimit
if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" {
return nil, ""
}
return time.After(time.Minute), ""
}
if secureLocation != "" {
handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout)
secureServer := &http.Server{
Addr: secureLocation,
Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)),
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
},
}
if len(s.ClientCAFile) > 0 {
clientCAs, err := util.CertPoolFromFile(s.ClientCAFile)
if err != nil {
glog.Fatalf("Unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
}
glog.Infof("Serving securely on %s", secureLocation)
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt")
s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key")
// TODO (cjcullen): Is PublicAddress the right address to sign a cert with?
alternateIPs := []net.IP{config.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
}
}
go func() {
defer util.HandleCrash()
for {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout)
http := &http.Server{
Addr: insecureLocation,
Handler: apiserver.RecoverPanics(handler),
MaxHeaderBytes: 1 << 20,
}
if secureLocation == "" {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
}
glog.Infof("Serving insecurely on %s", insecureLocation)
glog.Fatal(http.ListenAndServe())
return nil
}
// init initializes master.
func (m *Master) init(c *Config) {
healthzChecks := []healthz.HealthzChecker{}
m.clock = util.RealClock{}
dbClient := func(resource string) storage.Interface { return c.StorageDestinations.get("", resource) }
podStorage := podetcd.NewStorage(dbClient("pods"), c.EnableWatchCache, c.KubeletClient)
podTemplateStorage := podtemplateetcd.NewREST(dbClient("podTemplates"))
eventStorage := eventetcd.NewREST(dbClient("events"), uint64(c.EventTTL.Seconds()))
limitRangeStorage := limitrangeetcd.NewREST(dbClient("limitRanges"))
resourceQuotaStorage, resourceQuotaStatusStorage := resourcequotaetcd.NewREST(dbClient("resourceQuotas"))
secretStorage := secretetcd.NewREST(dbClient("secrets"))
serviceAccountStorage := serviceaccountetcd.NewREST(dbClient("serviceAccounts"))
persistentVolumeStorage, persistentVolumeStatusStorage := pvetcd.NewREST(dbClient("persistentVolumes"))
persistentVolumeClaimStorage, persistentVolumeClaimStatusStorage := pvcetcd.NewREST(dbClient("persistentVolumeClaims"))
namespaceStorage, namespaceStatusStorage, namespaceFinalizeStorage := namespaceetcd.NewREST(dbClient("namespaces"))
m.namespaceRegistry = namespace.NewRegistry(namespaceStorage)
endpointsStorage := endpointsetcd.NewREST(dbClient("endpoints"), c.EnableWatchCache)
m.endpointRegistry = endpoint.NewRegistry(endpointsStorage)
nodeStorage, nodeStatusStorage := nodeetcd.NewREST(dbClient("nodes"), c.EnableWatchCache, c.KubeletClient)
m.nodeRegistry = node.NewRegistry(nodeStorage)
serviceStorage := serviceetcd.NewREST(dbClient("services"))
m.serviceRegistry = service.NewRegistry(serviceStorage)
var serviceClusterIPRegistry service.RangeRegistry
serviceClusterIPAllocator := ipallocator.NewAllocatorCIDRRange(m.serviceClusterIPRange, func(max int, rangeSpec string) allocator.Interface {
mem := allocator.NewAllocationMap(max, rangeSpec)
etcd := etcdallocator.NewEtcd(mem, "/ranges/serviceips", "serviceipallocation", dbClient("services"))
serviceClusterIPRegistry = etcd
return etcd
})
m.serviceClusterIPAllocator = serviceClusterIPRegistry
var serviceNodePortRegistry service.RangeRegistry
serviceNodePortAllocator := portallocator.NewPortAllocatorCustom(m.serviceNodePortRange, func(max int, rangeSpec string) allocator.Interface {
mem := allocator.NewAllocationMap(max, rangeSpec)
etcd := etcdallocator.NewEtcd(mem, "/ranges/servicenodeports", "servicenodeportallocation", dbClient("services"))
serviceNodePortRegistry = etcd
return etcd
})
m.serviceNodePortAllocator = serviceNodePortRegistry
controllerStorage := controlleretcd.NewREST(dbClient("replicationControllers"))
// TODO: Factor out the core API registration
m.storage = map[string]rest.Storage{
"pods": podStorage.Pod,
"pods/attach": podStorage.Attach,
"pods/status": podStorage.Status,
"pods/log": podStorage.Log,
"pods/exec": podStorage.Exec,
"pods/portforward": podStorage.PortForward,
"pods/proxy": podStorage.Proxy,
"pods/binding": podStorage.Binding,
"bindings": podStorage.Binding,
"podTemplates": podTemplateStorage,
"replicationControllers": controllerStorage,
"services": service.NewStorage(m.serviceRegistry, m.endpointRegistry, serviceClusterIPAllocator, serviceNodePortAllocator),
"endpoints": endpointsStorage,
"nodes": nodeStorage,
"nodes/status": nodeStatusStorage,
"events": eventStorage,
"limitRanges": limitRangeStorage,
"resourceQuotas": resourceQuotaStorage,
"resourceQuotas/status": resourceQuotaStatusStorage,
"namespaces": namespaceStorage,
"namespaces/status": namespaceStatusStorage,
"namespaces/finalize": namespaceFinalizeStorage,
"secrets": secretStorage,
"serviceAccounts": serviceAccountStorage,
"persistentVolumes": persistentVolumeStorage,
"persistentVolumes/status": persistentVolumeStatusStorage,
"persistentVolumeClaims": persistentVolumeClaimStorage,
"persistentVolumeClaims/status": persistentVolumeClaimStatusStorage,
"componentStatuses": componentstatus.NewStorage(func() map[string]apiserver.Server { return m.getServersToValidate(c) }),
}
// establish the node proxy dialer
if len(c.SSHUser) > 0 {
// Usernames are capped @ 32
if len(c.SSHUser) > 32 {
glog.Warning("SSH User is too long, truncating to 32 chars")
c.SSHUser = c.SSHUser[0:32]
}
glog.Infof("Setting up proxy: %s %s", c.SSHUser, c.SSHKeyfile)
// public keyfile is written last, so check for that.
publicKeyFile := c.SSHKeyfile + ".pub"
exists, err := util.FileExists(publicKeyFile)
if err != nil {
glog.Errorf("Error detecting if key exists: %v", err)
} else if !exists {
glog.Infof("Key doesn't exist, attempting to create")
err := m.generateSSHKey(c.SSHUser, c.SSHKeyfile, publicKeyFile)
if err != nil {
glog.Errorf("Failed to create key pair: %v", err)
}
}
m.tunnels = &util.SSHTunnelList{}
m.dialer = m.Dial
m.setupSecureProxy(c.SSHUser, c.SSHKeyfile, publicKeyFile)
m.lastSync = m.clock.Now().Unix()
// This is pretty ugly. A better solution would be to pull this all the way up into the
// server.go file.
httpKubeletClient, ok := c.KubeletClient.(*client.HTTPKubeletClient)
if ok {
httpKubeletClient.Config.Dial = m.dialer
transport, err := client.MakeTransport(httpKubeletClient.Config)
if err != nil {
glog.Errorf("Error setting up transport over SSH: %v", err)
} else {
httpKubeletClient.Client.Transport = transport
}
} else {
glog.Errorf("Failed to cast %v to HTTPKubeletClient, skipping SSH tunnel.", c.KubeletClient)
}
healthzChecks = append(healthzChecks, healthz.NamedCheck("SSH Tunnel Check", m.IsTunnelSyncHealthy))
m.lastSyncMetric = prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "apiserver_proxy_tunnel_sync_latency_secs",
Help: "The time since the last successful synchronization of the SSH tunnels for proxy requests.",
}, func() float64 { return float64(m.secondsSinceSync()) })
}
apiVersions := []string{}
if m.v1 {
if err := m.api_v1().InstallREST(m.handlerContainer); err != nil {
glog.Fatalf("Unable to setup API v1: %v", err)
}
apiVersions = append(apiVersions, "v1")
}
apiserver.InstallSupport(m.muxHelper, m.rootWebService, c.EnableProfiling, healthzChecks...)
apiserver.AddApiWebService(m.handlerContainer, c.APIPrefix, apiVersions)
apiserver.InstallServiceErrorHandler(m.handlerContainer, m.newAPIRequestInfoResolver(), apiVersions)
// allGroups records all supported groups at /apis
allGroups := []api.APIGroup{}
if m.exp {
m.thirdPartyStorage = c.StorageDestinations.APIGroups["experimental"].Default
m.thirdPartyResources = map[string]*thirdpartyresourcedataetcd.REST{}
expVersion := m.experimental(c)
if err := expVersion.InstallREST(m.handlerContainer); err != nil {
glog.Fatalf("Unable to setup experimental api: %v", err)
}
g, err := latest.Group("experimental")
if err != nil {
glog.Fatalf("Unable to setup experimental api: %v", err)
}
expAPIVersions := []api.GroupVersion{
{
GroupVersion: expVersion.Version,
Version: apiutil.GetVersion(expVersion.Version),
},
}
storageVersion, found := c.StorageVersions[g.Group]
if !found {
glog.Fatalf("Couldn't find storage version of group %v", g.Group)
}
group := api.APIGroup{
Name: g.Group,
Versions: expAPIVersions,
PreferredVersion: api.GroupVersion{GroupVersion: storageVersion, Version: apiutil.GetVersion(storageVersion)},
}
apiserver.AddGroupWebService(m.handlerContainer, c.APIGroupPrefix+"/"+latest.GroupOrDie("experimental").Group+"/", group)
allGroups = append(allGroups, group)
apiserver.InstallServiceErrorHandler(m.handlerContainer, m.newAPIRequestInfoResolver(), []string{expVersion.Version})
}
// This should be done after all groups are registered
// TODO: replace the hardcoded "apis".
apiserver.AddApisWebService(m.handlerContainer, "/apis", allGroups)
// Register root handler.
// We do not register this using restful Webservice since we do not want to surface this in api docs.
// Allow master to be embedded in contexts which already have something registered at the root
if c.EnableIndex {
m.mux.HandleFunc("/", apiserver.IndexHandler(m.handlerContainer, m.muxHelper))
}
if c.EnableLogsSupport {
apiserver.InstallLogsSupport(m.muxHelper)
}
if c.EnableUISupport {
ui.InstallSupport(m.muxHelper, m.enableSwaggerSupport)
}
if c.EnableProfiling {
m.mux.HandleFunc("/debug/pprof/", pprof.Index)
m.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
m.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
}
handler := http.Handler(m.mux.(*http.ServeMux))
// TODO: handle CORS and auth using go-restful
// See github.com/emicklei/go-restful/blob/master/examples/restful-CORS-filter.go, and
// github.com/emicklei/go-restful/blob/master/examples/restful-basic-authentication.go
if len(c.CorsAllowedOriginList) > 0 {
allowedOriginRegexps, err := util.CompileRegexps(c.CorsAllowedOriginList)
if err != nil {
glog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(c.CorsAllowedOriginList, ","), err)
}
handler = apiserver.CORS(handler, allowedOriginRegexps, nil, nil, "true")
}
m.InsecureHandler = handler
attributeGetter := apiserver.NewRequestAttributeGetter(m.requestContextMapper, m.newAPIRequestInfoResolver())
handler = apiserver.WithAuthorizationCheck(handler, attributeGetter, m.authorizer)
// Install Authenticator
if c.Authenticator != nil {
authenticatedHandler, err := handlers.NewRequestAuthenticator(m.requestContextMapper, c.Authenticator, handlers.Unauthorized(c.SupportsBasicAuth), handler)
if err != nil {
glog.Fatalf("Could not initialize authenticator: %v", err)
}
handler = authenticatedHandler
}
// Install root web services
m.handlerContainer.Add(m.rootWebService)
// TODO: Make this optional? Consumers of master depend on this currently.
m.Handler = handler
if m.enableSwaggerSupport {
m.InstallSwaggerAPI()
}
// After all wrapping is done, put a context filter around both handlers
if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.Handler); err != nil {
glog.Fatalf("Could not initialize request context filter: %v", err)
} else {
m.Handler = handler
}
if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.InsecureHandler); err != nil {
glog.Fatalf("Could not initialize request context filter: %v", err)
} else {
m.InsecureHandler = handler
}
// TODO: Attempt clean shutdown?
if m.enableCoreControllers {
m.NewBootstrapController().Start()
}
}
// init initializes master.
func (m *Master) init(c *Config) {
if c.ProxyDialer != nil || c.ProxyTLSClientConfig != nil {
m.proxyTransport = util.SetTransportDefaults(&http.Transport{
Dial: c.ProxyDialer,
TLSClientConfig: c.ProxyTLSClientConfig,
})
}
healthzChecks := []healthz.HealthzChecker{}
storageDecorator := c.storageDecorator()
dbClient := func(resource string) storage.Interface { return c.StorageDestinations.get("", resource) }
podStorage := podetcd.NewStorage(dbClient("pods"), storageDecorator, c.KubeletClient, m.proxyTransport)
podTemplateStorage := podtemplateetcd.NewREST(dbClient("podTemplates"), storageDecorator)
eventStorage := eventetcd.NewREST(dbClient("events"), storageDecorator, uint64(c.EventTTL.Seconds()))
limitRangeStorage := limitrangeetcd.NewREST(dbClient("limitRanges"), storageDecorator)
resourceQuotaStorage, resourceQuotaStatusStorage := resourcequotaetcd.NewREST(dbClient("resourceQuotas"), storageDecorator)
secretStorage := secretetcd.NewREST(dbClient("secrets"), storageDecorator)
serviceAccountStorage := serviceaccountetcd.NewREST(dbClient("serviceAccounts"), storageDecorator)
persistentVolumeStorage, persistentVolumeStatusStorage := pvetcd.NewREST(dbClient("persistentVolumes"), storageDecorator)
persistentVolumeClaimStorage, persistentVolumeClaimStatusStorage := pvcetcd.NewREST(dbClient("persistentVolumeClaims"), storageDecorator)
namespaceStorage, namespaceStatusStorage, namespaceFinalizeStorage := namespaceetcd.NewREST(dbClient("namespaces"), storageDecorator)
m.namespaceRegistry = namespace.NewRegistry(namespaceStorage)
endpointsStorage := endpointsetcd.NewREST(dbClient("endpoints"), storageDecorator)
m.endpointRegistry = endpoint.NewRegistry(endpointsStorage)
nodeStorage, nodeStatusStorage := nodeetcd.NewREST(dbClient("nodes"), storageDecorator, c.KubeletClient, m.proxyTransport)
m.nodeRegistry = node.NewRegistry(nodeStorage)
serviceStorage := serviceetcd.NewREST(dbClient("services"), storageDecorator)
m.serviceRegistry = service.NewRegistry(serviceStorage)
var serviceClusterIPRegistry service.RangeRegistry
serviceClusterIPAllocator := ipallocator.NewAllocatorCIDRRange(m.serviceClusterIPRange, func(max int, rangeSpec string) allocator.Interface {
mem := allocator.NewAllocationMap(max, rangeSpec)
etcd := etcdallocator.NewEtcd(mem, "/ranges/serviceips", "serviceipallocation", dbClient("services"))
serviceClusterIPRegistry = etcd
return etcd
})
m.serviceClusterIPAllocator = serviceClusterIPRegistry
var serviceNodePortRegistry service.RangeRegistry
serviceNodePortAllocator := portallocator.NewPortAllocatorCustom(m.serviceNodePortRange, func(max int, rangeSpec string) allocator.Interface {
mem := allocator.NewAllocationMap(max, rangeSpec)
etcd := etcdallocator.NewEtcd(mem, "/ranges/servicenodeports", "servicenodeportallocation", dbClient("services"))
serviceNodePortRegistry = etcd
return etcd
})
m.serviceNodePortAllocator = serviceNodePortRegistry
controllerStorage, controllerStatusStorage := controlleretcd.NewREST(dbClient("replicationControllers"), storageDecorator)
// TODO: Factor out the core API registration
m.storage = map[string]rest.Storage{
"pods": podStorage.Pod,
"pods/attach": podStorage.Attach,
"pods/status": podStorage.Status,
"pods/log": podStorage.Log,
"pods/exec": podStorage.Exec,
"pods/portforward": podStorage.PortForward,
"pods/proxy": podStorage.Proxy,
"pods/binding": podStorage.Binding,
"bindings": podStorage.Binding,
"podTemplates": podTemplateStorage,
"replicationControllers": controllerStorage,
"replicationControllers/status": controllerStatusStorage,
"services": service.NewStorage(m.serviceRegistry, m.endpointRegistry, serviceClusterIPAllocator, serviceNodePortAllocator, m.proxyTransport),
"endpoints": endpointsStorage,
"nodes": nodeStorage,
"nodes/status": nodeStatusStorage,
"events": eventStorage,
"limitRanges": limitRangeStorage,
"resourceQuotas": resourceQuotaStorage,
"resourceQuotas/status": resourceQuotaStatusStorage,
"namespaces": namespaceStorage,
"namespaces/status": namespaceStatusStorage,
"namespaces/finalize": namespaceFinalizeStorage,
"secrets": secretStorage,
"serviceAccounts": serviceAccountStorage,
"persistentVolumes": persistentVolumeStorage,
"persistentVolumes/status": persistentVolumeStatusStorage,
"persistentVolumeClaims": persistentVolumeClaimStorage,
"persistentVolumeClaims/status": persistentVolumeClaimStatusStorage,
"componentStatuses": componentstatus.NewStorage(func() map[string]apiserver.Server { return m.getServersToValidate(c) }),
}
if m.tunneler != nil {
m.tunneler.Run(m.getNodeAddresses)
healthzChecks = append(healthzChecks, healthz.NamedCheck("SSH Tunnel Check", m.IsTunnelSyncHealthy))
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "apiserver_proxy_tunnel_sync_latency_secs",
Help: "The time since the last successful synchronization of the SSH tunnels for proxy requests.",
}, func() float64 { return float64(m.tunneler.SecondsSinceSync()) })
}
apiVersions := []string{}
// Install v1 unless disabled.
if !m.apiGroupVersionOverrides["api/v1"].Disable {
if err := m.api_v1().InstallREST(m.handlerContainer); err != nil {
glog.Fatalf("Unable to setup API v1: %v", err)
}
apiVersions = append(apiVersions, "v1")
}
apiserver.InstallSupport(m.muxHelper, m.rootWebService, c.EnableProfiling, healthzChecks...)
apiserver.AddApiWebService(m.handlerContainer, c.APIPrefix, apiVersions)
apiserver.InstallServiceErrorHandler(m.handlerContainer, m.newRequestInfoResolver(), apiVersions)
// allGroups records all supported groups at /apis
allGroups := []unversioned.APIGroup{}
// Install extensions unless disabled.
if !m.apiGroupVersionOverrides["extensions/v1beta1"].Disable {
m.thirdPartyStorage = c.StorageDestinations.APIGroups["extensions"].Default
m.thirdPartyResources = map[string]*thirdpartyresourcedataetcd.REST{}
expVersion := m.experimental(c)
if err := expVersion.InstallREST(m.handlerContainer); err != nil {
glog.Fatalf("Unable to setup experimental api: %v", err)
}
g, err := latest.Group("extensions")
if err != nil {
glog.Fatalf("Unable to setup experimental api: %v", err)
}
expAPIVersions := []unversioned.GroupVersionForDiscovery{
{
GroupVersion: expVersion.GroupVersion.String(),
Version: expVersion.GroupVersion.Version,
},
}
storageVersion, found := c.StorageVersions[g.Group]
if !found {
glog.Fatalf("Couldn't find storage version of group %v", g.Group)
}
group := unversioned.APIGroup{
Name: g.Group,
Versions: expAPIVersions,
PreferredVersion: unversioned.GroupVersionForDiscovery{GroupVersion: storageVersion, Version: apiutil.GetVersion(storageVersion)},
}
apiserver.AddGroupWebService(m.handlerContainer, c.APIGroupPrefix+"/"+latest.GroupOrDie("extensions").Group, group)
allGroups = append(allGroups, group)
apiserver.InstallServiceErrorHandler(m.handlerContainer, m.newRequestInfoResolver(), []string{expVersion.GroupVersion.String()})
}
// This should be done after all groups are registered
// TODO: replace the hardcoded "apis".
apiserver.AddApisWebService(m.handlerContainer, "/apis", allGroups)
// Register root handler.
// We do not register this using restful Webservice since we do not want to surface this in api docs.
// Allow master to be embedded in contexts which already have something registered at the root
if c.EnableIndex {
m.mux.HandleFunc("/", apiserver.IndexHandler(m.handlerContainer, m.muxHelper))
}
if c.EnableLogsSupport {
apiserver.InstallLogsSupport(m.muxHelper)
}
if c.EnableUISupport {
ui.InstallSupport(m.muxHelper, m.enableSwaggerSupport)
}
if c.EnableProfiling {
m.mux.HandleFunc("/debug/pprof/", pprof.Index)
m.mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
m.mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
}
handler := http.Handler(m.mux.(*http.ServeMux))
insecureHandler := handler
// TODO: handle CORS and auth using go-restful
// See github.com/emicklei/go-restful/blob/master/examples/restful-CORS-filter.go, and
// github.com/emicklei/go-restful/blob/master/examples/restful-basic-authentication.go
attributeGetter := apiserver.NewRequestAttributeGetter(m.requestContextMapper, m.newRequestInfoResolver())
handler = apiserver.WithAuthorizationCheck(handler, attributeGetter, m.authorizer)
// Install Authenticator
if c.Authenticator != nil {
authenticatedHandler, err := handlers.NewRequestAuthenticator(m.requestContextMapper, c.Authenticator, handlers.Unauthorized(c.SupportsBasicAuth), handler)
if err != nil {
glog.Fatalf("Could not initialize authenticator: %v", err)
}
handler = authenticatedHandler
}
// Since OPTIONS request cannot carry authn headers (by w3c standards), we are doing CORS check
// before auth check. Otherwise all the CORS request will be rejected.
if len(c.CorsAllowedOriginList) > 0 {
allowedOriginRegexps, err := util.CompileRegexps(c.CorsAllowedOriginList)
if err != nil {
glog.Fatalf("Invalid CORS allowed origin, --cors-allowed-origins flag was set to %v - %v", strings.Join(c.CorsAllowedOriginList, ","), err)
}
handler = apiserver.CORS(handler, allowedOriginRegexps, nil, nil, "true")
insecureHandler = apiserver.CORS(insecureHandler, allowedOriginRegexps, nil, nil, "true")
}
m.InsecureHandler = insecureHandler
// Install root web services
m.handlerContainer.Add(m.rootWebService)
// TODO: Make this optional? Consumers of master depend on this currently.
m.Handler = handler
if m.enableSwaggerSupport {
m.InstallSwaggerAPI()
}
// After all wrapping is done, put a context filter around both handlers
if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.Handler); err != nil {
glog.Fatalf("Could not initialize request context filter: %v", err)
} else {
m.Handler = handler
}
if handler, err := api.NewRequestContextFilter(m.requestContextMapper, m.InsecureHandler); err != nil {
glog.Fatalf("Could not initialize request context filter: %v", err)
} else {
m.InsecureHandler = handler
}
// TODO: Attempt clean shutdown?
if m.enableCoreControllers {
m.NewBootstrapController().Start()
}
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.APIServer) error {
verifyClusterIPFlags(s)
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), we will use the host's default
// interface as valid public addr for master (see: util#ValidPublicAddrForMaster)
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
hostIP, err := util.ValidPublicAddrForMaster(s.BindAddress)
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v' . "+
"Try to set the AdvertiseAddress directly or provide a valid BindAddress to fix this.", err)
}
s.AdvertiseAddress = hostIP
}
glog.Infof("Will report %v as public IP address.", s.AdvertiseAddress)
if len(s.EtcdServerList) == 0 {
glog.Fatalf("--etcd-servers must be specified")
}
if s.KubernetesServiceNodePort > 0 && !s.ServiceNodePortRange.Contains(s.KubernetesServiceNodePort) {
glog.Fatalf("Kubernetes service port range %v doesn't contain %v", s.ServiceNodePortRange, (s.KubernetesServiceNodePort))
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
// Setup tunneler if needed
var tunneler master.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
// Set up the tunneler
tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := kubeletclient.NewStaticKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
apiGroupVersionOverrides, err := parseRuntimeConfig(s)
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &client.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
}
if len(s.DeprecatedStorageVersion) != 0 {
gv, err := unversioned.ParseGroupVersion(s.DeprecatedStorageVersion)
if err != nil {
glog.Fatalf("error in parsing group version: %s", err)
}
clientConfig.GroupVersion = &gv
}
client, err := client.New(clientConfig)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
legacyV1Group, err := latest.Group(api.GroupName)
if err != nil {
return err
}
storageDestinations := genericapiserver.NewStorageDestinations()
storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions)
if _, found := storageVersions[legacyV1Group.GroupVersion.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.GroupVersion.Group, storageVersions)
}
etcdStorage, err := newEtcd(s.EtcdServerList, legacyV1Group.InterfacesFor, storageVersions[legacyV1Group.GroupVersion.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("", etcdStorage)
if !apiGroupVersionOverrides["extensions/v1beta1"].Disable {
expGroup, err := latest.Group(extensions.GroupName)
if err != nil {
glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
if _, found := storageVersions[expGroup.GroupVersion.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.GroupVersion.Group, storageVersions)
}
expEtcdStorage, err := newEtcd(s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.GroupVersion.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup(extensions.GroupName, expEtcdStorage)
}
updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd)
n := s.ServiceClusterIPRange
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if authenticator.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("No RSA key provided, service account token authentication disabled")
}
}
var serviceAccountGetter serviceaccount.ServiceAccountTokenGetter
if s.ServiceAccountLookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
serviceAccountGetter = serviceaccountcontroller.NewGetterFromStorageInterface(etcdStorage)
}
authenticator, err := authenticator.New(authenticator.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
ServiceAccountTokenGetter: serviceAccountGetter,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("GCE cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("Failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("Unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
config := &master.Config{
Config: &genericapiserver.Config{
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
ServiceClusterIPRange: &n,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableProfiling: s.EnableProfiling,
EnableWatchCache: s.EnableWatchCache,
EnableIndex: true,
APIPrefix: s.APIPrefix,
APIGroupPrefix: s.APIGroupPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
APIGroupVersionOverrides: apiGroupVersionOverrides,
MasterServiceNamespace: s.MasterServiceNamespace,
MasterCount: s.MasterCount,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
ProxyDialer: proxyDialerFn,
ProxyTLSClientConfig: proxyTLSClientConfig,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
},
EnableCoreControllers: true,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
Tunneler: tunneler,
}
m := master.New(config)
m.Run(s.ServerRunOptions)
return nil
}
func BuildKubernetesMasterConfig(options configapi.MasterConfig, requestContextMapper kapi.RequestContextMapper, kubeClient *kclient.Client, projectCache *projectcache.ProjectCache) (*MasterConfig, error) {
if options.KubernetesMasterConfig == nil {
return nil, errors.New("insufficient information to build KubernetesMasterConfig")
}
// Connect and setup etcd interfaces
etcdClient, err := etcd.EtcdClient(options.EtcdClientInfo)
if err != nil {
return nil, err
}
kubeletClientConfig := configapi.GetKubeletClientConfig(options)
kubeletClient, err := kubeletclient.NewStaticKubeletClient(kubeletClientConfig)
if err != nil {
return nil, fmt.Errorf("unable to configure Kubelet client: %v", err)
}
// in-order list of plug-ins that should intercept admission decisions
// TODO: Push node environment support to upstream in future
_, portString, err := net.SplitHostPort(options.ServingInfo.BindAddress)
if err != nil {
return nil, err
}
port, err := strconv.Atoi(portString)
if err != nil {
return nil, err
}
portRange, err := util.ParsePortRange(options.KubernetesMasterConfig.ServicesNodePortRange)
if err != nil {
return nil, err
}
podEvictionTimeout, err := time.ParseDuration(options.KubernetesMasterConfig.PodEvictionTimeout)
if err != nil {
return nil, fmt.Errorf("unable to parse PodEvictionTimeout: %v", err)
}
server := app.NewAPIServer()
server.EventTTL = 2 * time.Hour
server.ServiceClusterIPRange = net.IPNet(flagtypes.DefaultIPNet(options.KubernetesMasterConfig.ServicesSubnet))
server.ServiceNodePortRange = *portRange
admissionPlugins := AdmissionPlugins
server.AdmissionControl = strings.Join(admissionPlugins, ",")
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.APIServerArguments, server.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
if len(options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride) > 0 {
admissionPlugins := options.KubernetesMasterConfig.AdmissionConfig.PluginOrderOverride
server.AdmissionControl = strings.Join(admissionPlugins, ",")
}
cmserver := cmapp.NewCMServer()
cmserver.PodEvictionTimeout = podEvictionTimeout
// resolve extended arguments
// TODO: this should be done in config validation (along with the above) so we can provide
// proper errors
if err := cmdflags.Resolve(options.KubernetesMasterConfig.ControllerArguments, cmserver.AddFlags); len(err) > 0 {
return nil, kerrors.NewAggregate(err)
}
cloud, err := cloudprovider.InitCloudProvider(cmserver.CloudProvider, cmserver.CloudConfigFile)
if err != nil {
return nil, err
}
if cloud != nil {
glog.V(2).Infof("Successfully initialized cloud provider: %q from the config file: %q\n", server.CloudProvider, server.CloudConfigFile)
}
// This is a placeholder to provide additional initialization
// objects to plugins
pluginInitializer := oadmission.PluginInitializer{
ProjectCache: projectCache,
}
plugins := []admission.Interface{}
for _, pluginName := range strings.Split(server.AdmissionControl, ",") {
switch pluginName {
case saadmit.PluginName:
// we need to set some custom parameters on the service account admission controller, so create that one by hand
saAdmitter := saadmit.NewServiceAccount(kubeClient)
saAdmitter.LimitSecretReferences = options.ServiceAccountConfig.LimitSecretReferences
saAdmitter.Run()
plugins = append(plugins, saAdmitter)
default:
configFile := server.AdmissionControlConfigFile
pluginConfig := options.KubernetesMasterConfig.AdmissionConfig.PluginConfig
// Check whether a config is specified for this plugin. If not, default to the
// global plugin config file specifiedd in the server config.
if cfg, hasConfig := pluginConfig[pluginName]; hasConfig {
configFile, err = pluginconfig.GetPluginConfig(cfg)
if err != nil {
return nil, err
}
}
plugin := admission.InitPlugin(pluginName, kubeClient, configFile)
if plugin != nil {
plugins = append(plugins, plugin)
}
}
}
pluginInitializer.Initialize(plugins)
// ensure that plugins have been properly initialized
if err := oadmission.Validate(plugins); err != nil {
return nil, err
}
admissionController := admission.NewChainHandler(plugins...)
var proxyClientCerts []tls.Certificate
if len(options.KubernetesMasterConfig.ProxyClientInfo.CertFile) > 0 {
clientCert, err := tls.LoadX509KeyPair(
options.KubernetesMasterConfig.ProxyClientInfo.CertFile,
options.KubernetesMasterConfig.ProxyClientInfo.KeyFile,
)
if err != nil {
return nil, err
}
proxyClientCerts = append(proxyClientCerts, clientCert)
}
// TODO you have to know every APIGroup you're enabling or upstream will panic. It's alternative to panicing is Fataling
// It needs a refactor to return errors
storageDestinations := master.NewStorageDestinations()
// storageVersions is a map from API group to allowed versions that must be a version exposed by the REST API or it breaks.
// We need to fix the upstream to stop using the storage version as a preferred api version.
storageVersions := map[string]string{}
enabledKubeVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupKube)
if len(enabledKubeVersions) > 0 {
kubeStorageVersion := unversioned.GroupVersion{Group: configapi.APIGroupKube, Version: options.EtcdStorageConfig.KubernetesStorageVersion}
databaseStorage, err := NewEtcdStorage(etcdClient, kubeStorageVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupKube, databaseStorage)
storageVersions[configapi.APIGroupKube] = options.EtcdStorageConfig.KubernetesStorageVersion
}
enabledExtensionsVersions := configapi.GetEnabledAPIVersionsForGroup(*options.KubernetesMasterConfig, configapi.APIGroupExtensions)
if len(enabledExtensionsVersions) > 0 {
groupMeta, err := kapilatest.Group(configapi.APIGroupExtensions)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
// TODO expose storage version options for api groups
databaseStorage, err := NewEtcdStorage(etcdClient, groupMeta.GroupVersion, options.EtcdStorageConfig.KubernetesStoragePrefix)
if err != nil {
return nil, fmt.Errorf("Error setting up Kubernetes extensions server storage: %v", err)
}
storageDestinations.AddAPIGroup(configapi.APIGroupExtensions, databaseStorage)
storageVersions[configapi.APIGroupExtensions] = enabledExtensionsVersions[0]
}
// Preserve previous behavior of using the first non-loopback address
// TODO: Deprecate this behavior and just require a valid value to be passed in
publicAddress := net.ParseIP(options.KubernetesMasterConfig.MasterIP)
if publicAddress == nil || publicAddress.IsUnspecified() || publicAddress.IsLoopback() {
hostIP, err := util.ChooseHostInterface()
if err != nil {
glog.Fatalf("Unable to find suitable network address.error='%v'. Set the masterIP directly to avoid this error.", err)
}
publicAddress = hostIP
glog.Infof("Will report %v as public IP address.", publicAddress)
}
m := &master.Config{
PublicAddress: publicAddress,
ReadWritePort: port,
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
EventTTL: server.EventTTL,
//MinRequestTimeout: server.MinRequestTimeout,
ServiceClusterIPRange: (*net.IPNet)(&server.ServiceClusterIPRange),
ServiceNodePortRange: server.ServiceNodePortRange,
RequestContextMapper: requestContextMapper,
KubeletClient: kubeletClient,
APIPrefix: KubeAPIPrefix,
APIGroupPrefix: KubeAPIGroupPrefix,
EnableCoreControllers: true,
MasterCount: options.KubernetesMasterConfig.MasterCount,
Authorizer: apiserver.NewAlwaysAllowAuthorizer(),
AdmissionControl: admissionController,
APIGroupVersionOverrides: getAPIGroupVersionOverrides(options),
// Set the TLS options for proxying to pods and services
// Proxying to nodes uses the kubeletClient TLS config (so can provide a different cert, and verify the node hostname)
ProxyTLSClientConfig: &tls.Config{
// Proxying to pods and services cannot verify hostnames, since they are contacted on randomly allocated IPs
InsecureSkipVerify: true,
Certificates: proxyClientCerts,
},
}
if options.DNSConfig != nil {
_, dnsPortStr, err := net.SplitHostPort(options.DNSConfig.BindAddress)
if err != nil {
return nil, fmt.Errorf("unable to parse DNS bind address %s: %v", options.DNSConfig.BindAddress, err)
}
dnsPort, err := strconv.Atoi(dnsPortStr)
if err != nil {
return nil, fmt.Errorf("invalid DNS port: %v", err)
}
m.ExtraServicePorts = append(m.ExtraServicePorts,
kapi.ServicePort{Name: "dns", Port: 53, Protocol: kapi.ProtocolUDP, TargetPort: intstr.FromInt(dnsPort)},
kapi.ServicePort{Name: "dns-tcp", Port: 53, Protocol: kapi.ProtocolTCP, TargetPort: intstr.FromInt(dnsPort)},
)
m.ExtraEndpointPorts = append(m.ExtraEndpointPorts,
kapi.EndpointPort{Name: "dns", Port: dnsPort, Protocol: kapi.ProtocolUDP},
kapi.EndpointPort{Name: "dns-tcp", Port: dnsPort, Protocol: kapi.ProtocolTCP},
)
}
kmaster := &MasterConfig{
Options: *options.KubernetesMasterConfig,
KubeClient: kubeClient,
Master: m,
ControllerManager: cmserver,
CloudProvider: cloud,
}
return kmaster, nil
}
func (m *Master) InstallAPIs(c *Config) {
apiVersions := []string{}
// Install v1 unless disabled.
if !m.ApiGroupVersionOverrides["api/v1"].Disable {
if err := m.api_v1(c).InstallREST(m.HandlerContainer); err != nil {
glog.Fatalf("Unable to setup API v1: %v", err)
}
apiVersions = append(apiVersions, "v1")
}
// Run the tunnel.
healthzChecks := []healthz.HealthzChecker{}
if m.tunneler != nil {
m.tunneler.Run(m.getNodeAddresses)
healthzChecks = append(healthzChecks, healthz.NamedCheck("SSH Tunnel Check", m.IsTunnelSyncHealthy))
prometheus.NewGaugeFunc(prometheus.GaugeOpts{
Name: "apiserver_proxy_tunnel_sync_latency_secs",
Help: "The time since the last successful synchronization of the SSH tunnels for proxy requests.",
}, func() float64 { return float64(m.tunneler.SecondsSinceSync()) })
}
// TODO(nikhiljindal): Refactor generic parts of support services (like /versions) to genericapiserver.
apiserver.InstallSupport(m.MuxHelper, m.RootWebService, c.EnableProfiling, healthzChecks...)
// Install root web services
m.HandlerContainer.Add(m.RootWebService)
apiserver.AddApiWebService(m.HandlerContainer, c.APIPrefix, apiVersions)
apiserver.InstallServiceErrorHandler(m.HandlerContainer, m.NewRequestInfoResolver(), apiVersions)
// allGroups records all supported groups at /apis
allGroups := []unversioned.APIGroup{}
// Install extensions unless disabled.
if !m.ApiGroupVersionOverrides["extensions/v1beta1"].Disable {
m.thirdPartyStorage = c.StorageDestinations.APIGroups[extensions.GroupName].Default
m.thirdPartyResources = map[string]thirdPartyEntry{}
expVersion := m.experimental(c)
if err := expVersion.InstallREST(m.HandlerContainer); err != nil {
glog.Fatalf("Unable to setup experimental api: %v", err)
}
g, err := latest.Group(extensions.GroupName)
if err != nil {
glog.Fatalf("Unable to setup experimental api: %v", err)
}
expAPIVersions := []unversioned.GroupVersionForDiscovery{
{
GroupVersion: expVersion.GroupVersion.String(),
Version: expVersion.GroupVersion.Version,
},
}
storageVersion, found := c.StorageVersions[g.GroupVersion.Group]
if !found {
glog.Fatalf("Couldn't find storage version of group %v", g.GroupVersion.Group)
}
group := unversioned.APIGroup{
Name: g.GroupVersion.Group,
Versions: expAPIVersions,
PreferredVersion: unversioned.GroupVersionForDiscovery{GroupVersion: storageVersion, Version: apiutil.GetVersion(storageVersion)},
}
apiserver.AddGroupWebService(m.HandlerContainer, c.APIGroupPrefix+"/"+latest.GroupOrDie(extensions.GroupName).GroupVersion.Group, group)
allGroups = append(allGroups, group)
apiserver.InstallServiceErrorHandler(m.HandlerContainer, m.NewRequestInfoResolver(), []string{expVersion.GroupVersion.String()})
}
// This should be done after all groups are registered
// TODO: replace the hardcoded "apis".
apiserver.AddApisWebService(m.HandlerContainer, "/apis", func() []unversioned.APIGroup {
groups := []unversioned.APIGroup{}
for ix := range allGroups {
groups = append(groups, allGroups[ix])
}
m.thirdPartyResourcesLock.Lock()
defer m.thirdPartyResourcesLock.Unlock()
if m.thirdPartyResources != nil {
for key := range m.thirdPartyResources {
groups = append(groups, m.thirdPartyResources[key].group)
}
}
return groups
})
}
