// newAuthenticatorFromClientCAFile returns an authenticator.Request or an error
func newAuthenticatorFromClientCAFile(clientCAFile string) (authenticator.Request, error) {
roots, err := util.CertPoolFromFile(clientCAFile)
if err != nil {
return nil, err
}
opts := x509.DefaultVerifyOptions()
opts.Roots = roots
return x509.New(opts, x509.CommonNameUserConversion), nil
}
// InClusterConfig returns a config object which uses the service account
// kubernetes gives to pods. It's intended for clients that expect to be
// running inside a pod running on kuberenetes. It will return an error if
// called from a process not running in a kubernetes environment.
func InClusterConfig() (*Config, error) {
token, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/" + api.ServiceAccountTokenKey)
if err != nil {
return nil, err
}
tlsClientConfig := TLSClientConfig{}
rootCAFile := "/var/run/secrets/kubernetes.io/serviceaccount/" + api.ServiceAccountRootCAKey
if _, err := util.CertPoolFromFile(rootCAFile); err != nil {
glog.Errorf("Expected to load root CA config from %s, but got err: %v", rootCAFile, err)
} else {
tlsClientConfig.CAFile = rootCAFile
}
return &Config{
// TODO: switch to using cluster DNS.
Host: "https://" + net.JoinHostPort(os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")),
BearerToken: string(token),
TLSClientConfig: tlsClientConfig,
}, nil
}
// New creates a new OpenID Connect client with the given issuerURL and clientID.
// NOTE(yifan): For now we assume the server provides the "jwks_uri" so we don't
// need to manager the key sets by ourselves.
func New(issuerURL, clientID, caFile, usernameClaim string) (*OIDCAuthenticator, error) {
var cfg oidc.ProviderConfig
var err error
var roots *x509.CertPool
url, err := url.Parse(issuerURL)
if err != nil {
return nil, err
}
if url.Scheme != "https" {
return nil, fmt.Errorf("'oidc-issuer-url' (%q) has invalid scheme (%q), require 'https'", issuerURL, url.Scheme)
}
if caFile != "" {
roots, err = util.CertPoolFromFile(caFile)
if err != nil {
glog.Errorf("Failed to read the CA file: %v", err)
}
}
if roots == nil {
glog.Info("No x509 certificates provided, will use host's root CA set")
}
// Copied from http.DefaultTransport.
tr := &http.Transport{
// According to golang's doc, if RootCAs is nil,
// TLS uses the host's root CA set.
TLSClientConfig: &tls.Config{RootCAs: roots},
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
}
hc := &http.Client{}
hc.Transport = tr
for i := 0; i <= maxRetries; i++ {
if i == maxRetries {
return nil, fmt.Errorf("failed to fetch provider config after %v retries", maxRetries)
}
cfg, err = oidc.FetchProviderConfig(hc, issuerURL)
if err == nil {
break
}
glog.Errorf("Failed to fetch provider config, trying again in %v: %v", retryBackoff, err)
time.Sleep(retryBackoff)
}
glog.Infof("Fetched provider config from %s: %#v", issuerURL, cfg)
if cfg.KeysEndpoint == "" {
return nil, fmt.Errorf("OIDC provider must provide 'jwks_uri' for public key discovery")
}
ccfg := oidc.ClientConfig{
HTTPClient: hc,
Credentials: oidc.ClientCredentials{ID: clientID},
ProviderConfig: cfg,
}
client, err := oidc.NewClient(ccfg)
if err != nil {
return nil, err
}
// SyncProviderConfig will start a goroutine to periodically synchronize the provider config.
// The synchronization interval is set by the expiration length of the config, and has a mininum
// and maximum threshold.
client.SyncProviderConfig(issuerURL)
return &OIDCAuthenticator{ccfg, client, usernameClaim}, nil
}
// Run runs the specified APIServer. This should never exit.
func (s *APIServer) Run(_ []string) error {
s.verifyClusterIPFlags()
// If advertise-address is not specified, use bind-address. If bind-address
// is not usable (unset, 0.0.0.0, or loopback), setDefaults() in
// pkg/master/master.go will do the right thing and use the host's default
// interface.
if s.AdvertiseAddress == nil || s.AdvertiseAddress.IsUnspecified() {
s.AdvertiseAddress = s.BindAddress
}
if (s.EtcdConfigFile != "" && len(s.EtcdServerList) != 0) || (s.EtcdConfigFile == "" && len(s.EtcdServerList) == 0) {
glog.Fatalf("specify either --etcd-servers or --etcd-config")
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider, s.CloudConfigFile)
if err != nil {
glog.Fatalf("Cloud provider could not be initialized: %v", err)
}
// Setup tunneler if needed
var tunneler master.Tunneler
var proxyDialerFn apiserver.ProxyDialerFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSH master.InstallSSHKey
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSH = instances.AddSSHKeyToAllInstances
}
}
// Set up the tunneler
tunneler = master.NewSSHTunneler(s.SSHUser, s.SSHKeyfile, installSSH)
// Use the tunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = tunneler.Dial
// Use the tunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = tunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
kubeletClient, err := client.NewKubeletClient(&s.KubeletConfig)
if err != nil {
glog.Fatalf("Failure to start kubelet client: %v", err)
}
apiGroupVersionOverrides, err := s.parseRuntimeConfig()
if err != nil {
glog.Fatalf("error in parsing runtime-config: %s", err)
}
clientConfig := &client.Config{
Host: net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort)),
Version: s.DeprecatedStorageVersion,
}
client, err := client.New(clientConfig)
if err != nil {
glog.Fatalf("Invalid server address: %v", err)
}
legacyV1Group, err := latest.Group("")
if err != nil {
return err
}
storageDestinations := master.NewStorageDestinations()
storageVersions := generateStorageVersionMap(s.DeprecatedStorageVersion, s.StorageVersions)
if _, found := storageVersions[legacyV1Group.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", legacyV1Group.Group, storageVersions)
}
etcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, legacyV1Group.InterfacesFor, storageVersions[legacyV1Group.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("", etcdStorage)
if !apiGroupVersionOverrides["extensions/v1beta1"].Disable {
expGroup, err := latest.Group("extensions")
if err != nil {
glog.Fatalf("Extensions API is enabled in runtime config, but not enabled in the environment variable KUBE_API_VERSIONS. Error: %v", err)
}
if _, found := storageVersions[expGroup.Group]; !found {
glog.Fatalf("Couldn't find the storage version for group: %q in storageVersions: %v", expGroup.Group, storageVersions)
}
expEtcdStorage, err := newEtcd(s.EtcdConfigFile, s.EtcdServerList, expGroup.InterfacesFor, storageVersions[expGroup.Group], s.EtcdPathPrefix)
if err != nil {
glog.Fatalf("Invalid extensions storage version or misconfigured etcd: %v", err)
}
storageDestinations.AddAPIGroup("extensions", expEtcdStorage)
}
updateEtcdOverrides(s.EtcdServersOverrides, storageVersions, s.EtcdPathPrefix, &storageDestinations, newEtcd)
n := s.ServiceClusterIPRange
// Default to the private server key for service account token signing
if s.ServiceAccountKeyFile == "" && s.TLSPrivateKeyFile != "" {
if apiserver.IsValidServiceAccountKeyFile(s.TLSPrivateKeyFile) {
s.ServiceAccountKeyFile = s.TLSPrivateKeyFile
} else {
glog.Warning("no RSA key provided, service account token authentication disabled")
}
}
authenticator, err := apiserver.NewAuthenticator(apiserver.AuthenticatorConfig{
BasicAuthFile: s.BasicAuthFile,
ClientCAFile: s.ClientCAFile,
TokenAuthFile: s.TokenAuthFile,
OIDCIssuerURL: s.OIDCIssuerURL,
OIDCClientID: s.OIDCClientID,
OIDCCAFile: s.OIDCCAFile,
OIDCUsernameClaim: s.OIDCUsernameClaim,
ServiceAccountKeyFile: s.ServiceAccountKeyFile,
ServiceAccountLookup: s.ServiceAccountLookup,
Storage: etcdStorage,
KeystoneURL: s.KeystoneURL,
})
if err != nil {
glog.Fatalf("Invalid Authentication Config: %v", err)
}
authorizationModeNames := strings.Split(s.AuthorizationMode, ",")
authorizer, err := apiserver.NewAuthorizerFromAuthorizationConfig(authorizationModeNames, s.AuthorizationPolicyFile)
if err != nil {
glog.Fatalf("Invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.AdmissionControl, ",")
admissionController := admission.NewFromPlugins(client, admissionControlPluginNames, s.AdmissionControlConfigFile)
if len(s.ExternalHost) == 0 {
// TODO: extend for other providers
if s.CloudProvider == "gce" {
instances, supported := cloud.Instances()
if !supported {
glog.Fatalf("gce cloud provider has no instances. this shouldn't happen. exiting.")
}
name, err := os.Hostname()
if err != nil {
glog.Fatalf("failed to get hostname: %v", err)
}
addrs, err := instances.NodeAddresses(name)
if err != nil {
glog.Warningf("unable to obtain external host address from cloud provider: %v", err)
} else {
for _, addr := range addrs {
if addr.Type == api.NodeExternalIP {
s.ExternalHost = addr.Address
}
}
}
}
}
config := &master.Config{
StorageDestinations: storageDestinations,
StorageVersions: storageVersions,
EventTTL: s.EventTTL,
KubeletClient: kubeletClient,
ServiceClusterIPRange: &n,
EnableCoreControllers: true,
EnableLogsSupport: s.EnableLogsSupport,
EnableUISupport: true,
EnableSwaggerSupport: true,
EnableProfiling: s.EnableProfiling,
EnableWatchCache: s.EnableWatchCache,
EnableIndex: true,
APIPrefix: s.APIPrefix,
APIGroupPrefix: s.APIGroupPrefix,
CorsAllowedOriginList: s.CorsAllowedOriginList,
ReadWritePort: s.SecurePort,
PublicAddress: s.AdvertiseAddress,
Authenticator: authenticator,
SupportsBasicAuth: len(s.BasicAuthFile) > 0,
Authorizer: authorizer,
AdmissionControl: admissionController,
APIGroupVersionOverrides: apiGroupVersionOverrides,
MasterServiceNamespace: s.MasterServiceNamespace,
ClusterName: s.ClusterName,
ExternalHost: s.ExternalHost,
MinRequestTimeout: s.MinRequestTimeout,
ProxyDialer: proxyDialerFn,
ProxyTLSClientConfig: proxyTLSClientConfig,
Tunneler: tunneler,
ServiceNodePortRange: s.ServiceNodePortRange,
}
m := master.New(config)
// We serve on 2 ports. See docs/accessing_the_api.md
secureLocation := ""
if s.SecurePort != 0 {
secureLocation = net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.SecurePort))
}
insecureLocation := net.JoinHostPort(s.InsecureBindAddress.String(), strconv.Itoa(s.InsecurePort))
// See the flag commentary to understand our assumptions when opening the read-only and read-write ports.
var sem chan bool
if s.MaxRequestsInFlight > 0 {
sem = make(chan bool, s.MaxRequestsInFlight)
}
longRunningRE := regexp.MustCompile(s.LongRunningRequestRE)
longRunningTimeout := func(req *http.Request) (<-chan time.Time, string) {
// TODO unify this with apiserver.MaxInFlightLimit
if longRunningRE.MatchString(req.URL.Path) || req.URL.Query().Get("watch") == "true" {
return nil, ""
}
return time.After(time.Minute), ""
}
if secureLocation != "" {
handler := apiserver.TimeoutHandler(m.Handler, longRunningTimeout)
secureServer := &http.Server{
Addr: secureLocation,
Handler: apiserver.MaxInFlightLimit(sem, longRunningRE, apiserver.RecoverPanics(handler)),
MaxHeaderBytes: 1 << 20,
TLSConfig: &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
},
}
if len(s.ClientCAFile) > 0 {
clientCAs, err := util.CertPoolFromFile(s.ClientCAFile)
if err != nil {
glog.Fatalf("unable to load client CA file: %v", err)
}
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
// Specify allowed CAs for client certificates
secureServer.TLSConfig.ClientCAs = clientCAs
}
glog.Infof("Serving securely on %s", secureLocation)
if s.TLSCertFile == "" && s.TLSPrivateKeyFile == "" {
s.TLSCertFile = path.Join(s.CertDirectory, "apiserver.crt")
s.TLSPrivateKeyFile = path.Join(s.CertDirectory, "apiserver.key")
// TODO (cjcullen): Is PublicAddress the right address to sign a cert with?
alternateIPs := []net.IP{config.ServiceReadWriteIP}
alternateDNS := []string{"kubernetes.default.svc", "kubernetes.default", "kubernetes"}
// It would be nice to set a fqdn subject alt name, but only the kubelets know, the apiserver is clueless
// alternateDNS = append(alternateDNS, "kubernetes.default.svc.CLUSTER.DNS.NAME")
if err := util.GenerateSelfSignedCert(config.PublicAddress.String(), s.TLSCertFile, s.TLSPrivateKeyFile, alternateIPs, alternateDNS); err != nil {
glog.Errorf("Unable to generate self signed cert: %v", err)
} else {
glog.Infof("Using self-signed cert (%s, %s)", s.TLSCertFile, s.TLSPrivateKeyFile)
}
}
go func() {
defer util.HandleCrash()
for {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
if err := secureServer.ListenAndServeTLS(s.TLSCertFile, s.TLSPrivateKeyFile); err != nil {
glog.Errorf("Unable to listen for secure (%v); will try again.", err)
}
time.Sleep(15 * time.Second)
}
}()
}
handler := apiserver.TimeoutHandler(m.InsecureHandler, longRunningTimeout)
http := &http.Server{
Addr: insecureLocation,
Handler: apiserver.RecoverPanics(handler),
MaxHeaderBytes: 1 << 20,
}
if secureLocation == "" {
// err == systemd.SdNotifyNoSocket when not running on a systemd system
if err := systemd.SdNotify("READY=1\n"); err != nil && err != systemd.SdNotifyNoSocket {
glog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
}
glog.Infof("Serving insecurely on %s", insecureLocation)
glog.Fatal(http.ListenAndServe())
return nil
}