// init registers the various means by which credentials may
// be resolved on GCP.
func init() {
tr := utilnet.SetTransportDefaults(&http.Transport{})
metadataHTTPClientTimeout := time.Second * 10
httpClient := &http.Client{
Transport: tr,
Timeout: metadataHTTPClientTimeout,
}
credentialprovider.RegisterCredentialProvider("google-dockercfg",
&credentialprovider.CachingDockerConfigProvider{
Provider: &dockerConfigKeyProvider{
metadataProvider{Client: httpClient},
},
Lifetime: 60 * time.Second,
})
credentialprovider.RegisterCredentialProvider("google-dockercfg-url",
&credentialprovider.CachingDockerConfigProvider{
Provider: &dockerConfigUrlKeyProvider{
metadataProvider{Client: httpClient},
},
Lifetime: 60 * time.Second,
})
credentialprovider.RegisterCredentialProvider("google-container-registry",
// Never cache this. The access token is already
// cached by the metadata service.
&containerRegistryProvider{
metadataProvider{Client: httpClient},
})
}
func TestAllProvidersNoMetadata(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusNotFound)
}))
defer server.Close()
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL + req.URL.Path)
},
})
providers := []credentialprovider.DockerConfigProvider{
&dockerConfigKeyProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
},
&dockerConfigUrlKeyProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
},
&containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
},
}
for _, provider := range providers {
if provider.Enabled() {
t.Errorf("Provider %s is unexpectedly enabled", reflect.TypeOf(provider).String())
}
}
}
func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Response, error) {
tr := utilnet.SetTransportDefaults(&http.Transport{
DisableKeepAlives: true,
})
client := &http.Client{
Transport: tr,
Timeout: timeout,
}
return client.Get(url)
}
func newMesosClient(
md detector.Master,
mesosHttpClientTimeout, stateCacheTTL time.Duration) (*mesosClient, error) {
tr := utilnet.SetTransportDefaults(&http.Transport{})
httpClient := &http.Client{
Transport: tr,
Timeout: mesosHttpClientTimeout,
}
return createMesosClient(md, httpClient, tr, stateCacheTTL)
}
func makeTransport(config *schedulerapi.ExtenderConfig) (http.RoundTripper, error) {
var cfg restclient.Config
if config.TLSConfig != nil {
cfg.TLSClientConfig = *config.TLSConfig
}
if config.EnableHttps {
hasCA := len(cfg.CAFile) > 0 || len(cfg.CAData) > 0
if !hasCA {
cfg.Insecure = true
}
}
tlsConfig, err := restclient.TLSConfigFor(&cfg)
if err != nil {
return nil, err
}
if tlsConfig != nil {
return utilnet.SetTransportDefaults(&http.Transport{
TLSClientConfig: tlsConfig,
}), nil
}
return utilnet.SetTransportDefaults(&http.Transport{}), nil
}
func createClients(numberOfClients int) ([]*clientset.Clientset, []*internalclientset.Clientset, error) {
clients := make([]*clientset.Clientset, numberOfClients)
internalClients := make([]*internalclientset.Clientset, numberOfClients)
for i := 0; i < numberOfClients; i++ {
config, err := framework.LoadConfig()
Expect(err).NotTo(HaveOccurred())
config.QPS = 100
config.Burst = 200
if framework.TestContext.KubeAPIContentType != "" {
config.ContentType = framework.TestContext.KubeAPIContentType
}
// For the purpose of this test, we want to force that clients
// do not share underlying transport (which is a default behavior
// in Kubernetes). Thus, we are explicitly creating transport for
// each client here.
transportConfig, err := config.TransportConfig()
if err != nil {
return nil, nil, err
}
tlsConfig, err := transport.TLSConfigFor(transportConfig)
if err != nil {
return nil, nil, err
}
config.Transport = utilnet.SetTransportDefaults(&http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: tlsConfig,
MaxIdleConnsPerHost: 100,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
})
// Overwrite TLS-related fields from config to avoid collision with
// Transport field.
config.TLSClientConfig = restclient.TLSClientConfig{}
c, err := clientset.NewForConfig(config)
if err != nil {
return nil, nil, err
}
clients[i] = c
internalClient, err := internalclientset.NewForConfig(config)
if err != nil {
return nil, nil, err
}
internalClients[i] = internalClient
}
return clients, internalClients, nil
}
// New creates a token authenticator which validates OpenID Connect ID Tokens.
func New(opts OIDCOptions) (*OIDCAuthenticator, error) {
url, err := url.Parse(opts.IssuerURL)
if err != nil {
return nil, err
}
if url.Scheme != "https" {
return nil, fmt.Errorf("'oidc-issuer-url' (%q) has invalid scheme (%q), require 'https'", opts.IssuerURL, url.Scheme)
}
if opts.UsernameClaim == "" {
return nil, errors.New("no username claim provided")
}
var roots *x509.CertPool
if opts.CAFile != "" {
roots, err = certutil.NewPool(opts.CAFile)
if err != nil {
return nil, fmt.Errorf("Failed to read the CA file: %v", err)
}
} else {
glog.Info("OIDC: No x509 certificates provided, will use host's root CA set")
}
// Copied from http.DefaultTransport.
tr := net.SetTransportDefaults(&http.Transport{
// According to golang's doc, if RootCAs is nil,
// TLS uses the host's root CA set.
TLSClientConfig: &tls.Config{RootCAs: roots},
})
authenticator := &OIDCAuthenticator{
issuerURL: opts.IssuerURL,
trustedClientID: opts.ClientID,
usernameClaim: opts.UsernameClaim,
groupsClaim: opts.GroupsClaim,
httpClient: &http.Client{Transport: tr},
}
// Attempt to initialize the authenticator asynchronously.
//
// Ignore errors instead of returning it since the OpenID Connect provider might not be
// available yet, for instance if it's running on the cluster and needs the API server
// to come up first. Errors will be logged within the client() method.
go func() {
defer runtime.HandleCrash()
authenticator.client()
}()
return authenticator, nil
}
// buildTransport creates a transport for use in executing HTTPS requests with
// the given certs. Note that the given rootCA must be configured with isCA=true.
func buildTransport(serverName string, rootCA []byte) (*http.Transport, error) {
pool := x509.NewCertPool()
ok := pool.AppendCertsFromPEM(rootCA)
if !ok {
return nil, fmt.Errorf("Unable to load serverCA.")
}
return utilnet.SetTransportDefaults(&http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: false,
ServerName: serverName,
RootCAs: pool,
},
}), nil
}
func dialHTTP(request, hostPort string) (string, error) {
transport := utilnet.SetTransportDefaults(&http.Transport{})
httpClient := createHTTPClient(transport)
resp, err := httpClient.Get(fmt.Sprintf("http://%s/%s", hostPort, request))
defer transport.CloseIdleConnections()
if err == nil {
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err == nil {
return string(body), nil
}
}
return "", err
}
func (l *SSHTunnelList) healthCheck(e sshTunnelEntry) error {
// GET the healthcheck path using the provided tunnel's dial function.
transport := utilnet.SetTransportDefaults(&http.Transport{
Dial: e.Tunnel.Dial,
// TODO(cjcullen): Plumb real TLS options through.
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
// We don't reuse the clients, so disable the keep-alive to properly
// close the connection.
DisableKeepAlives: true,
})
client := &http.Client{Transport: transport}
resp, err := client.Get(l.healthCheckURL.String())
if err != nil {
return err
}
resp.Body.Close()
return nil
}
func TestComputePlatformScopeSubstitutesStorageScope(t *testing.T) {
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if scopeEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, `["https://www.googleapis.com/auth/compute.read_write","https://www.googleapis.com/auth/cloud-platform.read-only"]`)
} else if serviceAccountsEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintln(w, "default/\ncustom")
} else {
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL + req.URL.Path)
},
})
provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
}
if !provider.Enabled() {
t.Errorf("Provider is unexpectedly disabled")
}
}
// setUp is a convience function for setting up for (most) tests.
func setUp(t *testing.T) (*Master, *etcdtesting.EtcdTestServer, Config, *assert.Assertions) {
server, storageConfig := etcdtesting.NewUnsecuredEtcd3TestClientServer(t)
config := &Config{
GenericConfig: genericapiserver.NewConfig(),
APIResourceConfigSource: DefaultAPIResourceConfigSource(),
APIServerServicePort: 443,
MasterCount: 1,
}
resourceEncoding := genericapiserver.NewDefaultResourceEncodingConfig()
resourceEncoding.SetVersionEncoding(api.GroupName, api.Registry.GroupOrDie(api.GroupName).GroupVersion, schema.GroupVersion{Group: api.GroupName, Version: runtime.APIVersionInternal})
resourceEncoding.SetVersionEncoding(autoscaling.GroupName, *testapi.Autoscaling.GroupVersion(), schema.GroupVersion{Group: autoscaling.GroupName, Version: runtime.APIVersionInternal})
resourceEncoding.SetVersionEncoding(batch.GroupName, *testapi.Batch.GroupVersion(), schema.GroupVersion{Group: batch.GroupName, Version: runtime.APIVersionInternal})
resourceEncoding.SetVersionEncoding(apps.GroupName, *testapi.Apps.GroupVersion(), schema.GroupVersion{Group: apps.GroupName, Version: runtime.APIVersionInternal})
resourceEncoding.SetVersionEncoding(extensions.GroupName, *testapi.Extensions.GroupVersion(), schema.GroupVersion{Group: extensions.GroupName, Version: runtime.APIVersionInternal})
resourceEncoding.SetVersionEncoding(rbac.GroupName, *testapi.Rbac.GroupVersion(), schema.GroupVersion{Group: rbac.GroupName, Version: runtime.APIVersionInternal})
resourceEncoding.SetVersionEncoding(certificates.GroupName, *testapi.Certificates.GroupVersion(), schema.GroupVersion{Group: certificates.GroupName, Version: runtime.APIVersionInternal})
storageFactory := genericapiserver.NewDefaultStorageFactory(*storageConfig, testapi.StorageMediaType(), api.Codecs, resourceEncoding, DefaultAPIResourceConfigSource())
kubeVersion := kubeversion.Get()
config.GenericConfig.Version = &kubeVersion
config.StorageFactory = storageFactory
config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: api.Codecs}}
config.GenericConfig.PublicAddress = net.ParseIP("192.168.10.4")
config.GenericConfig.LegacyAPIGroupPrefixes = sets.NewString("/api")
config.GenericConfig.RequestContextMapper = genericapirequest.NewRequestContextMapper()
config.GenericConfig.LoopbackClientConfig = &restclient.Config{APIPath: "/api", ContentConfig: restclient.ContentConfig{NegotiatedSerializer: api.Codecs}}
config.GenericConfig.EnableMetrics = true
config.EnableCoreControllers = false
config.KubeletClientConfig = kubeletclient.KubeletClientConfig{Port: 10250}
config.ProxyTransport = utilnet.SetTransportDefaults(&http.Transport{
Dial: func(network, addr string) (net.Conn, error) { return nil, nil },
TLSClientConfig: &tls.Config{},
})
master, err := config.Complete().New()
if err != nil {
t.Fatal(err)
}
return master, server, *config, assert.New(t)
}
func TestContainerRegistryNoServiceAccount(t *testing.T) {
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if serviceAccountsEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
bytes, err := json.Marshal([]string{})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
fmt.Fprintln(w, string(bytes))
} else {
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL + req.URL.Path)
},
})
provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
}
if provider.Enabled() {
t.Errorf("Provider is unexpectedly enabled")
}
}
func (c *tlsTransportCache) get(config *Config) (http.RoundTripper, error) {
key, err := tlsConfigKey(config)
if err != nil {
return nil, err
}
// Ensure we only create a single transport for the given TLS options
c.mu.Lock()
defer c.mu.Unlock()
// See if we already have a custom transport for this config
if t, ok := c.transports[key]; ok {
return t, nil
}
// Get the TLS options for this client config
tlsConfig, err := TLSConfigFor(config)
if err != nil {
return nil, err
}
// The options didn't require a custom TLS config
if tlsConfig == nil {
return http.DefaultTransport, nil
}
// Cache a single transport for these options
c.transports[key] = utilnet.SetTransportDefaults(&http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: tlsConfig,
MaxIdleConnsPerHost: idleConnsPerHost,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
})
return c.transports[key], nil
}
func newTransportForETCD2(certFile, keyFile, caFile string) (*http.Transport, error) {
info := transport.TLSInfo{
CertFile: certFile,
KeyFile: keyFile,
CAFile: caFile,
}
cfg, err := info.ClientConfig()
if err != nil {
return nil, err
}
// Copied from etcd.DefaultTransport declaration.
// TODO: Determine if transport needs optimization
tr := utilnet.SetTransportDefaults(&http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
MaxIdleConnsPerHost: 500,
TLSClientConfig: cfg,
})
return tr, nil
}
func makeHttpMocks() (*httptest.Server, *http.Client, *http.Transport) {
httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.V(4).Infof("Mocking response for HTTP request: %#v", r)
if r.URL.Path == "/state.json" {
w.WriteHeader(200) // OK
w.Header().Set("Content-Type", "application/json")
fmt.Fprintln(w, TEST_STATE_JSON)
} else {
w.WriteHeader(400)
fmt.Fprintln(w, "Bad Request")
}
}))
// Intercept all client requests and feed them to the test server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(httpServer.URL)
},
})
httpClient := &http.Client{Transport: transport}
return httpServer, httpClient, transport
}
func TestProxyUpgrade(t *testing.T) {
localhostPool := x509.NewCertPool()
if !localhostPool.AppendCertsFromPEM(localhostCert) {
t.Errorf("error setting up localhostCert pool")
}
testcases := map[string]struct {
ServerFunc func(http.Handler) *httptest.Server
ProxyTransport http.RoundTripper
}{
"http": {
ServerFunc: httptest.NewServer,
ProxyTransport: nil,
},
"https (invalid hostname + InsecureSkipVerify)": {
ServerFunc: func(h http.Handler) *httptest.Server {
cert, err := tls.X509KeyPair(exampleCert, exampleKey)
if err != nil {
t.Errorf("https (invalid hostname): proxy_test: %v", err)
}
ts := httptest.NewUnstartedServer(h)
ts.TLS = &tls.Config{
Certificates: []tls.Certificate{cert},
}
ts.StartTLS()
return ts
},
ProxyTransport: utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}),
},
"https (valid hostname + RootCAs)": {
ServerFunc: func(h http.Handler) *httptest.Server {
cert, err := tls.X509KeyPair(localhostCert, localhostKey)
if err != nil {
t.Errorf("https (valid hostname): proxy_test: %v", err)
}
ts := httptest.NewUnstartedServer(h)
ts.TLS = &tls.Config{
Certificates: []tls.Certificate{cert},
}
ts.StartTLS()
return ts
},
ProxyTransport: utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: &tls.Config{RootCAs: localhostPool}}),
},
"https (valid hostname + RootCAs + custom dialer)": {
ServerFunc: func(h http.Handler) *httptest.Server {
cert, err := tls.X509KeyPair(localhostCert, localhostKey)
if err != nil {
t.Errorf("https (valid hostname): proxy_test: %v", err)
}
ts := httptest.NewUnstartedServer(h)
ts.TLS = &tls.Config{
Certificates: []tls.Certificate{cert},
}
ts.StartTLS()
return ts
},
ProxyTransport: utilnet.SetTransportDefaults(&http.Transport{Dial: net.Dial, TLSClientConfig: &tls.Config{RootCAs: localhostPool}}),
},
}
// Enable StreamingProxyRedirects for test.
utilconfig.DefaultFeatureGate.Set("StreamingProxyRedirects=true")
for k, tc := range testcases {
for _, redirect := range []bool{false, true} {
tcName := k
backendPath := "/hello"
if redirect {
tcName += " with redirect"
backendPath = "/redirect"
}
func() { // Cleanup after each test case.
backend := http.NewServeMux()
backend.Handle("/hello", websocket.Handler(func(ws *websocket.Conn) {
defer ws.Close()
body := make([]byte, 5)
ws.Read(body)
ws.Write([]byte("hello " + string(body)))
}))
backend.Handle("/redirect", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
http.Redirect(w, r, "/hello", http.StatusFound)
}))
backendServer := tc.ServerFunc(backend)
defer backendServer.Close()
serverURL, _ := url.Parse(backendServer.URL)
serverURL.Path = backendPath
proxyHandler := &UpgradeAwareProxyHandler{
Location: serverURL,
Transport: tc.ProxyTransport,
InterceptRedirects: redirect,
}
proxy := httptest.NewServer(proxyHandler)
defer proxy.Close()
ws, err := websocket.Dial("ws://"+proxy.Listener.Addr().String()+"/some/path", "", "http://127.0.0.1/")
if err != nil {
t.Fatalf("%s: websocket dial err: %s", tcName, err)
}
defer ws.Close()
if _, err := ws.Write([]byte("world")); err != nil {
t.Fatalf("%s: write err: %s", tcName, err)
}
response := make([]byte, 20)
n, err := ws.Read(response)
if err != nil {
t.Fatalf("%s: read err: %s", tcName, err)
}
if e, a := "hello world", string(response[0:n]); e != a {
t.Fatalf("%s: expected '%#v', got '%#v'", tcName, e, a)
}
}()
}
}
}
// buildInsecureClient returns an insecure http client. Can be used for "curl -k".
func buildInsecureClient(timeout time.Duration) *http.Client {
t := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
return &http.Client{Timeout: timeout, Transport: utilnet.SetTransportDefaults(t)}
}
func TestContainerRegistryBasics(t *testing.T) {
registryUrl := "container.cloud.google.com"
email := "*****@*****.**"
token := &tokenBlob{AccessToken: "ya26.lots-of-indiscernible-garbage"}
const (
serviceAccountsEndpoint = "/computeMetadata/v1/instance/service-accounts/"
defaultEndpoint = "/computeMetadata/v1/instance/service-accounts/default/"
scopeEndpoint = defaultEndpoint + "scopes"
emailEndpoint = defaultEndpoint + "email"
tokenEndpoint = defaultEndpoint + "token"
)
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the URL key and the value endpoint
if scopeEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintf(w, `["%s.read_write"]`, storageScopePrefix)
} else if emailEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
fmt.Fprint(w, email)
} else if tokenEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
bytes, err := json.Marshal(token)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
fmt.Fprintln(w, string(bytes))
} else if serviceAccountsEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
fmt.Fprintln(w, "default/\ncustom")
} else {
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL + req.URL.Path)
},
})
keyring := &credentialprovider.BasicDockerKeyring{}
provider := &containerRegistryProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
}
if !provider.Enabled() {
t.Errorf("Provider is unexpectedly disabled")
}
keyring.Add(provider.Provide())
creds, ok := keyring.Lookup(registryUrl)
if !ok {
t.Errorf("Didn't find expected URL: %s", registryUrl)
return
}
if len(creds) > 1 {
t.Errorf("Got more hits than expected: %s", creds)
}
val := creds[0]
if "_token" != val.Username {
t.Errorf("Unexpected username value, want: %s, got: %s", "_token", val.Username)
}
if token.AccessToken != val.Password {
t.Errorf("Unexpected password value, want: %s, got: %s", token.AccessToken, val.Password)
}
if email != val.Email {
t.Errorf("Unexpected email value, want: %s, got: %s", email, val.Email)
}
}
func TestDockerKeyringFromGoogleDockerConfigMetadata(t *testing.T) {
registryUrl := "hello.kubernetes.io"
email := "*****@*****.**"
username := "******"
password := "******"
auth := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password)))
sampleDockerConfig := fmt.Sprintf(`{
"https://%s": {
"email": %q,
"auth": %q
}
}`, registryUrl, email, auth)
var err error
gceProductNameFile, err = createProductNameFile()
if err != nil {
t.Errorf("failed to create gce product name file: %v", err)
}
defer os.Remove(gceProductNameFile)
const probeEndpoint = "/computeMetadata/v1/"
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Only serve the one metadata key.
if probeEndpoint == r.URL.Path {
w.WriteHeader(http.StatusOK)
} else if strings.HasSuffix(dockerConfigKey, r.URL.Path) {
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
fmt.Fprintln(w, sampleDockerConfig)
} else {
w.WriteHeader(http.StatusNotFound)
}
}))
defer server.Close()
// Make a transport that reroutes all traffic to the example server
transport := utilnet.SetTransportDefaults(&http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(server.URL + req.URL.Path)
},
})
keyring := &credentialprovider.BasicDockerKeyring{}
provider := &dockerConfigKeyProvider{
metadataProvider{Client: &http.Client{Transport: transport}},
}
if !provider.Enabled() {
t.Errorf("Provider is unexpectedly disabled")
}
keyring.Add(provider.Provide())
creds, ok := keyring.Lookup(registryUrl)
if !ok {
t.Errorf("Didn't find expected URL: %s", registryUrl)
return
}
if len(creds) > 1 {
t.Errorf("Got more hits than expected: %s", creds)
}
val := creds[0]
if username != val.Username {
t.Errorf("Unexpected username value, want: %s, got: %s", username, val.Username)
}
if password != val.Password {
t.Errorf("Unexpected password value, want: %s, got: %s", password, val.Password)
}
if email != val.Email {
t.Errorf("Unexpected email value, want: %s, got: %s", email, val.Email)
}
}
func New() HTTPProber {
tlsConfig := &tls.Config{InsecureSkipVerify: true}
transport := utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: tlsConfig, DisableKeepAlives: true})
return httpProber{transport}
}
// Run runs the specified APIServer. This should never exit.
func Run(s *options.ServerRunOptions) error {
// set defaults
if err := s.GenericServerRunOptions.DefaultAdvertiseAddress(s.SecureServing, s.InsecureServing); err != nil {
return err
}
serviceIPRange, apiServerServiceIP, err := master.DefaultServiceIPRange(s.ServiceClusterIPRange)
if err != nil {
return fmt.Errorf("error determining service IP ranges: %v", err)
}
if err := s.SecureServing.MaybeDefaultWithSelfSignedCerts(s.GenericServerRunOptions.AdvertiseAddress.String(), apiServerServiceIP); err != nil {
return fmt.Errorf("error creating self-signed certificates: %v", err)
}
if err := s.CloudProvider.DefaultExternalHost(s.GenericServerRunOptions); err != nil {
return fmt.Errorf("error setting the external host value: %v", err)
}
s.Authentication.ApplyAuthorization(s.Authorization)
// validate options
if errs := s.Validate(); len(errs) != 0 {
return utilerrors.NewAggregate(errs)
}
// create config from options
genericConfig := genericapiserver.NewConfig().
ApplyOptions(s.GenericServerRunOptions).
ApplyInsecureServingOptions(s.InsecureServing)
if _, err := genericConfig.ApplySecureServingOptions(s.SecureServing); err != nil {
return fmt.Errorf("failed to configure https: %s", err)
}
if err = s.Authentication.Apply(genericConfig); err != nil {
return fmt.Errorf("failed to configure authentication: %s", err)
}
capabilities.Initialize(capabilities.Capabilities{
AllowPrivileged: s.AllowPrivileged,
// TODO(vmarmol): Implement support for HostNetworkSources.
PrivilegedSources: capabilities.PrivilegedSources{
HostNetworkSources: []string{},
HostPIDSources: []string{},
HostIPCSources: []string{},
},
PerConnectionBandwidthLimitBytesPerSec: s.MaxConnectionBytesPerSec,
})
// Setup nodeTunneler if needed
var nodeTunneler tunneler.Tunneler
var proxyDialerFn utilnet.DialFunc
if len(s.SSHUser) > 0 {
// Get ssh key distribution func, if supported
var installSSHKey tunneler.InstallSSHKey
cloud, err := cloudprovider.InitCloudProvider(s.CloudProvider.CloudProvider, s.CloudProvider.CloudConfigFile)
if err != nil {
return fmt.Errorf("cloud provider could not be initialized: %v", err)
}
if cloud != nil {
if instances, supported := cloud.Instances(); supported {
installSSHKey = instances.AddSSHKeyToAllInstances
}
}
if s.KubeletConfig.Port == 0 {
return fmt.Errorf("must enable kubelet port if proxy ssh-tunneling is specified")
}
if s.KubeletConfig.ReadOnlyPort == 0 {
return fmt.Errorf("must enable kubelet readonly port if proxy ssh-tunneling is specified")
}
// Set up the nodeTunneler
// TODO(cjcullen): If we want this to handle per-kubelet ports or other
// kubelet listen-addresses, we need to plumb through options.
healthCheckPath := &url.URL{
Scheme: "http",
Host: net.JoinHostPort("127.0.0.1", strconv.FormatUint(uint64(s.KubeletConfig.ReadOnlyPort), 10)),
Path: "healthz",
}
nodeTunneler = tunneler.New(s.SSHUser, s.SSHKeyfile, healthCheckPath, installSSHKey)
// Use the nodeTunneler's dialer to connect to the kubelet
s.KubeletConfig.Dial = nodeTunneler.Dial
// Use the nodeTunneler's dialer when proxying to pods, services, and nodes
proxyDialerFn = nodeTunneler.Dial
}
// Proxying to pods and services is IP-based... don't expect to be able to verify the hostname
proxyTLSClientConfig := &tls.Config{InsecureSkipVerify: true}
if s.Etcd.StorageConfig.DeserializationCacheSize == 0 {
// When size of cache is not explicitly set, estimate its size based on
// target memory usage.
glog.V(2).Infof("Initializing deserialization cache size based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
// This is the heuristics that from memory capacity is trying to infer
// the maximum number of nodes in the cluster and set cache sizes based
// on that value.
// From our documentation, we officially recomment 120GB machines for
// 2000 nodes, and we scale from that point. Thus we assume ~60MB of
// capacity per node.
// TODO: We may consider deciding that some percentage of memory will
// be used for the deserialization cache and divide it by the max object
// size to compute its size. We may even go further and measure
// collective sizes of the objects in the cache.
clusterSize := s.GenericServerRunOptions.TargetRAMMB / 60
s.Etcd.StorageConfig.DeserializationCacheSize = 25 * clusterSize
if s.Etcd.StorageConfig.DeserializationCacheSize < 1000 {
s.Etcd.StorageConfig.DeserializationCacheSize = 1000
}
}
storageGroupsToEncodingVersion, err := s.GenericServerRunOptions.StorageGroupsToEncodingVersion()
if err != nil {
return fmt.Errorf("error generating storage version map: %s", err)
}
storageFactory, err := kubeapiserver.BuildDefaultStorageFactory(
s.Etcd.StorageConfig, s.GenericServerRunOptions.DefaultStorageMediaType, api.Codecs,
genericapiserver.NewDefaultResourceEncodingConfig(), storageGroupsToEncodingVersion,
// FIXME: this GroupVersionResource override should be configurable
[]schema.GroupVersionResource{batch.Resource("cronjobs").WithVersion("v2alpha1")},
master.DefaultAPIResourceConfigSource(), s.GenericServerRunOptions.RuntimeConfig)
if err != nil {
return fmt.Errorf("error in initializing storage factory: %s", err)
}
storageFactory.AddCohabitatingResources(autoscaling.Resource("horizontalpodautoscalers"), extensions.Resource("horizontalpodautoscalers"))
for _, override := range s.Etcd.EtcdServersOverrides {
tokens := strings.Split(override, "#")
if len(tokens) != 2 {
glog.Errorf("invalid value of etcd server overrides: %s", override)
continue
}
apiresource := strings.Split(tokens[0], "/")
if len(apiresource) != 2 {
glog.Errorf("invalid resource definition: %s", tokens[0])
continue
}
group := apiresource[0]
resource := apiresource[1]
groupResource := schema.GroupResource{Group: group, Resource: resource}
servers := strings.Split(tokens[1], ";")
storageFactory.SetEtcdLocation(groupResource, servers)
}
// Default to the private server key for service account token signing
if len(s.Authentication.ServiceAccounts.KeyFiles) == 0 && s.SecureServing.ServerCert.CertKey.KeyFile != "" {
if kubeauthenticator.IsValidServiceAccountKeyFile(s.SecureServing.ServerCert.CertKey.KeyFile) {
s.Authentication.ServiceAccounts.KeyFiles = []string{s.SecureServing.ServerCert.CertKey.KeyFile}
} else {
glog.Warning("No TLS key provided, service account token authentication disabled")
}
}
authenticatorConfig := s.Authentication.ToAuthenticationConfig()
if s.Authentication.ServiceAccounts.Lookup {
// If we need to look up service accounts and tokens,
// go directly to etcd to avoid recursive auth insanity
storageConfig, err := storageFactory.NewConfig(api.Resource("serviceaccounts"))
if err != nil {
return fmt.Errorf("unable to get serviceaccounts storage: %v", err)
}
authenticatorConfig.ServiceAccountTokenGetter = serviceaccountcontroller.NewGetterFromStorageInterface(storageConfig, storageFactory.ResourcePrefix(api.Resource("serviceaccounts")), storageFactory.ResourcePrefix(api.Resource("secrets")))
}
apiAuthenticator, securityDefinitions, err := authenticatorConfig.New()
if err != nil {
return fmt.Errorf("invalid Authentication Config: %v", err)
}
privilegedLoopbackToken := uuid.NewRandom().String()
selfClientConfig, err := genericapiserver.NewSelfClientConfig(genericConfig.SecureServingInfo, genericConfig.InsecureServingInfo, privilegedLoopbackToken)
if err != nil {
return fmt.Errorf("failed to create clientset: %v", err)
}
client, err := internalclientset.NewForConfig(selfClientConfig)
if err != nil {
kubeAPIVersions := os.Getenv("KUBE_API_VERSIONS")
if len(kubeAPIVersions) == 0 {
return fmt.Errorf("failed to create clientset: %v", err)
}
// KUBE_API_VERSIONS is used in test-update-storage-objects.sh, disabling a number of API
// groups. This leads to a nil client above and undefined behaviour further down.
// TODO: get rid of KUBE_API_VERSIONS or define sane behaviour if set
glog.Errorf("Failed to create clientset with KUBE_API_VERSIONS=%q. KUBE_API_VERSIONS is only for testing. Things will break.", kubeAPIVersions)
}
sharedInformers := informers.NewSharedInformerFactory(nil, client, 10*time.Minute)
authorizationConfig := s.Authorization.ToAuthorizationConfig(sharedInformers)
apiAuthorizer, err := authorizationConfig.New()
if err != nil {
return fmt.Errorf("invalid Authorization Config: %v", err)
}
admissionControlPluginNames := strings.Split(s.GenericServerRunOptions.AdmissionControl, ",")
pluginInitializer := kubeadmission.NewPluginInitializer(client, sharedInformers, apiAuthorizer)
admissionConfigProvider, err := kubeadmission.ReadAdmissionConfiguration(admissionControlPluginNames, s.GenericServerRunOptions.AdmissionControlConfigFile)
if err != nil {
return fmt.Errorf("failed to read plugin config: %v", err)
}
admissionController, err := admission.NewFromPlugins(admissionControlPluginNames, admissionConfigProvider, pluginInitializer)
if err != nil {
return fmt.Errorf("failed to initialize plugins: %v", err)
}
proxyTransport := utilnet.SetTransportDefaults(&http.Transport{
Dial: proxyDialerFn,
TLSClientConfig: proxyTLSClientConfig,
})
kubeVersion := version.Get()
genericConfig.Version = &kubeVersion
genericConfig.LoopbackClientConfig = selfClientConfig
genericConfig.Authenticator = apiAuthenticator
genericConfig.Authorizer = apiAuthorizer
genericConfig.AdmissionControl = admissionController
genericConfig.OpenAPIConfig = genericapiserver.DefaultOpenAPIConfig(generatedopenapi.OpenAPIDefinitions)
genericConfig.OpenAPIConfig.SecurityDefinitions = securityDefinitions
genericConfig.OpenAPIConfig.Info.Title = "Kubernetes"
genericConfig.SwaggerConfig = genericapiserver.DefaultSwaggerConfig()
genericConfig.EnableMetrics = true
genericConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
config := &master.Config{
GenericConfig: genericConfig,
APIResourceConfigSource: storageFactory.APIResourceConfigSource,
StorageFactory: storageFactory,
EnableWatchCache: s.GenericServerRunOptions.EnableWatchCache,
EnableCoreControllers: true,
DeleteCollectionWorkers: s.GenericServerRunOptions.DeleteCollectionWorkers,
EventTTL: s.EventTTL,
KubeletClientConfig: s.KubeletConfig,
EnableUISupport: true,
EnableLogsSupport: true,
ProxyTransport: proxyTransport,
Tunneler: nodeTunneler,
ServiceIPRange: serviceIPRange,
APIServerServiceIP: apiServerServiceIP,
APIServerServicePort: 443,
ServiceNodePortRange: s.ServiceNodePortRange,
KubernetesServiceNodePort: s.KubernetesServiceNodePort,
MasterCount: s.MasterCount,
}
if s.GenericServerRunOptions.EnableWatchCache {
glog.V(2).Infof("Initializing cache sizes based on %dMB limit", s.GenericServerRunOptions.TargetRAMMB)
cachesize.InitializeWatchCacheSizes(s.GenericServerRunOptions.TargetRAMMB)
cachesize.SetWatchCacheSizes(s.GenericServerRunOptions.WatchCacheSizes)
}
m, err := config.Complete().New()
if err != nil {
return err
}
sharedInformers.Start(wait.NeverStop)
m.GenericAPIServer.PrepareRun().Run(wait.NeverStop)
return nil
}
func TestProxyUpgrade(t *testing.T) {
localhostPool := x509.NewCertPool()
if !localhostPool.AppendCertsFromPEM(localhostCert) {
t.Errorf("error setting up localhostCert pool")
}
testcases := map[string]struct {
ServerFunc func(http.Handler) *httptest.Server
ProxyTransport http.RoundTripper
}{
"http": {
ServerFunc: httptest.NewServer,
ProxyTransport: nil,
},
"https (invalid hostname + InsecureSkipVerify)": {
ServerFunc: func(h http.Handler) *httptest.Server {
cert, err := tls.X509KeyPair(exampleCert, exampleKey)
if err != nil {
t.Errorf("https (invalid hostname): proxy_test: %v", err)
}
ts := httptest.NewUnstartedServer(h)
ts.TLS = &tls.Config{
Certificates: []tls.Certificate{cert},
}
ts.StartTLS()
return ts
},
ProxyTransport: utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}),
},
"https (valid hostname + RootCAs)": {
ServerFunc: func(h http.Handler) *httptest.Server {
cert, err := tls.X509KeyPair(localhostCert, localhostKey)
if err != nil {
t.Errorf("https (valid hostname): proxy_test: %v", err)
}
ts := httptest.NewUnstartedServer(h)
ts.TLS = &tls.Config{
Certificates: []tls.Certificate{cert},
}
ts.StartTLS()
return ts
},
ProxyTransport: utilnet.SetTransportDefaults(&http.Transport{TLSClientConfig: &tls.Config{RootCAs: localhostPool}}),
},
"https (valid hostname + RootCAs + custom dialer)": {
ServerFunc: func(h http.Handler) *httptest.Server {
cert, err := tls.X509KeyPair(localhostCert, localhostKey)
if err != nil {
t.Errorf("https (valid hostname): proxy_test: %v", err)
}
ts := httptest.NewUnstartedServer(h)
ts.TLS = &tls.Config{
Certificates: []tls.Certificate{cert},
}
ts.StartTLS()
return ts
},
ProxyTransport: utilnet.SetTransportDefaults(&http.Transport{Dial: net.Dial, TLSClientConfig: &tls.Config{RootCAs: localhostPool}}),
},
}
for k, tc := range testcases {
backendServer := tc.ServerFunc(websocket.Handler(func(ws *websocket.Conn) {
defer ws.Close()
body := make([]byte, 5)
ws.Read(body)
ws.Write([]byte("hello " + string(body)))
}))
defer backendServer.Close()
serverURL, _ := url.Parse(backendServer.URL)
simpleStorage := &SimpleRESTStorage{
errors: map[string]error{},
resourceLocation: serverURL,
resourceLocationTransport: tc.ProxyTransport,
expectedResourceNamespace: "myns",
}
namespaceHandler := handleNamespaced(map[string]rest.Storage{"foo": simpleStorage})
server := newTestServer(namespaceHandler)
defer server.Close()
ws, err := websocket.Dial("ws://"+server.Listener.Addr().String()+"/"+prefix+"/"+newGroupVersion.Group+"/"+newGroupVersion.Version+"/proxy/namespaces/myns/foo/123", "", "http://127.0.0.1/")
if err != nil {
t.Errorf("%s: websocket dial err: %s", k, err)
continue
}
defer ws.Close()
if _, err := ws.Write([]byte("world")); err != nil {
t.Errorf("%s: write err: %s", k, err)
continue
}
response := make([]byte, 20)
n, err := ws.Read(response)
if err != nil {
t.Errorf("%s: read err: %s", k, err)
continue
}
if e, a := "hello world", string(response[0:n]); e != a {
t.Errorf("%s: expected '%#v', got '%#v'", k, e, a)
continue
}
}
}
