func StartConfiguredMasterWithOptions(masterConfig *configapi.MasterConfig, testOptions TestOptions) (string, error) {
if testOptions.DeleteAllEtcdKeys {
util.DeleteAllEtcdKeys()
}
if err := start.NewMaster(masterConfig, true, true).Start(); err != nil {
return "", err
}
adminKubeConfigFile := util.KubeConfigPath()
clientConfig, err := util.GetClusterAdminClientConfig(adminKubeConfigFile)
if err != nil {
return "", err
}
masterURL, err := url.Parse(clientConfig.Host)
if err != nil {
return "", err
}
// wait for the server to come up: 35 seconds
if err := cmdutil.WaitForSuccessfulDial(true, "tcp", masterURL.Host, 100*time.Millisecond, 1*time.Second, 35); err != nil {
return "", err
}
for {
// confirm that we can actually query from the api server
if client, err := util.GetClusterAdminClient(adminKubeConfigFile); err == nil {
if _, err := client.ClusterPolicies().List(labels.Everything(), fields.Everything()); err == nil {
break
}
}
time.Sleep(100 * time.Millisecond)
}
return adminKubeConfigFile, nil
}
// Run launches the OpenShift master by creating a kubernetes master, installing
// OpenShift APIs into it and then running it.
func (c *MasterConfig) Run(kc *kubernetes.MasterConfig, assetConfig *AssetConfig) {
var (
messages []string
err error
)
kc.Master.GenericConfig.BuildHandlerChainsFunc, messages, err = c.buildHandlerChain(assetConfig)
if err != nil {
glog.Fatalf("Failed to launch master: %v", err)
}
kmaster, err := kc.Master.Complete().New()
if err != nil {
glog.Fatalf("Failed to launch master: %v", err)
}
c.InstallProtectedAPI(kmaster.GenericAPIServer.HandlerContainer)
messages = append(messages, c.kubernetesAPIMessages(kc)...)
for _, s := range messages {
glog.Infof(s, c.Options.ServingInfo.BindAddress)
}
go kmaster.GenericAPIServer.PrepareRun().Run(utilwait.NeverStop)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
}
func (h *Helper) TestIP(ip string) error {
// Start test server on host
id, err := h.runHelper.New().Image(h.image).
Privileged().
HostNetwork().
Entrypoint("socat").
Command("TCP-LISTEN:8443,crlf,reuseaddr,fork", "SYSTEM:\"echo 'hello world'\"").Start()
if err != nil {
return errors.NewError("cannnot start simple server on Docker host").WithCause(err)
}
defer func() {
errors.LogError(h.dockerHelper.StopAndRemoveContainer(id))
}()
// Attempt to connect to test container
testHost := fmt.Sprintf("%s:8443", ip)
glog.V(4).Infof("Attempting to dial %s", testHost)
if err = cmdutil.WaitForSuccessfulDial(false, "tcp", testHost, 200*time.Millisecond, 1*time.Second, 10); err != nil {
glog.V(2).Infof("Dial error: %v", err)
return err
}
glog.V(4).Infof("Successfully dialed %s", testHost)
return nil
}
// RunKubelet starts the Kubelet.
func (c *NodeConfig) RunKubelet() {
if c.KubeletConfig.ClusterDNS == nil {
if service, err := c.Client.Services(kapi.NamespaceDefault).Get("kubernetes"); err == nil {
if includesServicePort(service.Spec.Ports, 53, "dns") {
// Use master service if service includes "dns" port 53.
c.KubeletConfig.ClusterDNS = net.ParseIP(service.Spec.ClusterIP)
}
}
}
if c.KubeletConfig.ClusterDNS == nil {
if endpoint, err := c.Client.Endpoints(kapi.NamespaceDefault).Get("kubernetes"); err == nil {
if endpointIP, ok := firstEndpointIPWithNamedPort(endpoint, 53, "dns"); ok {
// Use first endpoint if endpoint includes "dns" port 53.
c.KubeletConfig.ClusterDNS = net.ParseIP(endpointIP)
} else if endpointIP, ok := firstEndpointIP(endpoint, 53); ok {
// Test and use first endpoint if endpoint includes any port 53.
if err := cmdutil.WaitForSuccessfulDial(false, "tcp", fmt.Sprintf("%s:%d", endpointIP, 53), 50*time.Millisecond, 0, 2); err == nil {
c.KubeletConfig.ClusterDNS = net.ParseIP(endpointIP)
}
}
}
}
c.KubeletConfig.DockerClient = c.DockerClient
// updated by NodeConfig.EnsureVolumeDir
c.KubeletConfig.RootDirectory = c.VolumeDir
// hook for overriding the cadvisor interface for integration tests
c.KubeletConfig.CAdvisorInterface = defaultCadvisorInterface
go func() {
glog.Fatal(c.KubeletServer.Run(c.KubeletConfig))
}()
}
func runRegistry() error {
config := `version: 0.1
log:
level: debug
http:
addr: 127.0.0.1:5000
storage:
inmemory: {}
auth:
openshift:
middleware:
registry:
- name: openshift
repository:
- name: openshift
options:
acceptschema2: false
pullthrough: true
enforcequota: false
projectcachettl: 1m
blobrepositorycachettl: 10m
storage:
- name: openshift
`
os.Setenv("DOCKER_REGISTRY_URL", "127.0.0.1:5000")
go dockerregistry.Execute(strings.NewReader(config))
if err := cmdutil.WaitForSuccessfulDial(false, "tcp", "127.0.0.1:5000", 100*time.Millisecond, 1*time.Second, 35); err != nil {
return err
}
return nil
}
// RunDNSServer starts the DNS server
func (c *MasterConfig) RunDNSServer() {
config, err := dns.NewServerDefaults()
if err != nil {
glog.Fatalf("Could not start DNS: %v", err)
}
config.DnsAddr = c.Options.DNSConfig.BindAddress
config.NoRec = true // do not want to deploy an open resolver
_, port, err := net.SplitHostPort(c.Options.DNSConfig.BindAddress)
if err != nil {
glog.Fatalf("Could not start DNS: %v", err)
}
if port != "53" {
glog.Warningf("Binding DNS on port %v instead of 53 (you may need to run as root and update your config), using %s which will not resolve from all locations", port, c.Options.DNSConfig.BindAddress)
}
if ok, err := cmdutil.TryListen(c.Options.DNSConfig.BindAddress); !ok {
glog.Warningf("Could not start DNS: %v", err)
return
}
go func() {
err := dns.ListenAndServe(config, c.DNSServerClient(), c.EtcdHelper.Client.(*etcdclient.Client))
glog.Fatalf("Could not start DNS: %v", err)
}()
cmdutil.WaitForSuccessfulDial(false, "tcp", c.Options.DNSConfig.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
glog.Infof("DNS listening at %s", c.Options.DNSConfig.BindAddress)
}
// Run starts an http server for the static assets listening on the configured
// bind address
func (c *AssetConfig) Run() {
publicURL, err := url.Parse(c.Options.PublicURL)
if err != nil {
glog.Fatal(err)
}
mux := http.NewServeMux()
err = c.addHandlers(mux)
if err != nil {
glog.Fatal(err)
}
if publicURL.Path != "/" {
mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
http.Redirect(w, req, publicURL.Path, http.StatusFound)
})
}
timeout := c.Options.ServingInfo.RequestTimeoutSeconds
if timeout == -1 {
timeout = 0
}
server := &http.Server{
Addr: c.Options.ServingInfo.BindAddress,
Handler: mux,
ReadTimeout: time.Duration(timeout) * time.Second,
WriteTimeout: time.Duration(timeout) * time.Second,
MaxHeaderBytes: 1 << 20,
}
isTLS := configapi.UseTLS(c.Options.ServingInfo.ServingInfo)
go util.Forever(func() {
if isTLS {
extraCerts, err := configapi.GetNamedCertificateMap(c.Options.ServingInfo.NamedCertificates)
if err != nil {
glog.Fatal(err)
}
server.TLSConfig = &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
// Set SNI certificate func
GetCertificate: cmdutil.GetCertificateFunc(extraCerts),
}
glog.Infof("Web console listening at https://%s", c.Options.ServingInfo.BindAddress)
glog.Fatal(cmdutil.ListenAndServeTLS(server, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile))
} else {
glog.Infof("Web console listening at http://%s", c.Options.ServingInfo.BindAddress)
glog.Fatal(server.ListenAndServe())
}
}, 0)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(isTLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
glog.Infof("Web console available at %s", c.Options.PublicURL)
}
func testIPDial(ip string) error {
// Attempt to connect to test container
testHost := fmt.Sprintf("%s:8443", ip)
glog.V(4).Infof("Attempting to dial %s", testHost)
if err := cmdutil.WaitForSuccessfulDial(false, "tcp", testHost, 200*time.Millisecond, 1*time.Second, 10); err != nil {
glog.V(2).Infof("Dial error: %v", err)
return err
}
glog.V(4).Infof("Successfully dialed %s", testHost)
return nil
}
// Run starts an http server for the static assets listening on the configured
// bind address
func (c *AssetConfig) Run() {
mux, err := c.addHandlers(nil)
if err != nil {
glog.Fatal(err)
}
timeout := c.Options.ServingInfo.RequestTimeoutSeconds
if timeout == -1 {
timeout = 0
}
server := &http.Server{
Addr: c.Options.ServingInfo.BindAddress,
Handler: mux,
ReadTimeout: time.Duration(timeout) * time.Second,
WriteTimeout: time.Duration(timeout) * time.Second,
MaxHeaderBytes: 1 << 20,
}
isTLS := configapi.UseTLS(c.Options.ServingInfo.ServingInfo)
go utilwait.Forever(func() {
if isTLS {
extraCerts, err := configapi.GetNamedCertificateMap(c.Options.ServingInfo.NamedCertificates)
if err != nil {
glog.Fatal(err)
}
server.TLSConfig = crypto.SecureTLSConfig(&tls.Config{
// Set SNI certificate func
GetCertificate: cmdutil.GetCertificateFunc(extraCerts),
})
glog.Infof("Web console listening at https://%s", c.Options.ServingInfo.BindAddress)
glog.Fatal(cmdutil.ListenAndServeTLS(server, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile))
} else {
glog.Infof("Web console listening at http://%s", c.Options.ServingInfo.BindAddress)
glog.Fatal(server.ListenAndServe())
}
}, 0)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(isTLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
glog.Infof("Web console available at %s", c.Options.PublicURL)
}
func (c *MasterConfig) RunInProxyMode(proxy *kubernetes.ProxyConfig, assetConfig *AssetConfig) {
handlerChain, messages, err := c.buildHandlerChain(assetConfig)
if err != nil {
glog.Fatalf("Failed to launch master: %v", err)
}
// TODO(sttts): create a genericapiserver here
container := genericmux.NewAPIContainer(http.NewServeMux(), kapi.Codecs)
// install /api proxy forwarder
proxyMessages, err := proxy.InstallAPI(container.Container)
if err != nil {
glog.Fatalf("Failed to launch master: %v", err)
}
messages = append(messages, proxyMessages...)
// install GenericAPIServer handlers manually, usually done by GenericAPIServer.PrepareRun()
healthz.InstallHandler(&container.NonSwaggerRoutes, healthz.PingHealthz)
swaggerConfig := genericapiserver.DefaultSwaggerConfig()
swaggerConfig.WebServicesUrl = c.Options.MasterPublicURL
genericroutes.Swagger{Config: swaggerConfig}.Install(container)
messages = append(messages, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerConfig.ApiPath))
genericroutes.OpenAPI{Config: kubernetes.DefaultOpenAPIConfig()}.Install(container)
messages = append(messages, fmt.Sprintf("Started OpenAPI Schema at %%s%s", openAPIServePath))
// install origin handlers
c.InstallProtectedAPI(container)
// TODO(sttts): split cmd/server/kubernetes config generation into generic and master-specific
// until then: create ad-hoc config
genericConfig := genericapiserver.NewConfig()
genericConfig.RequestContextMapper = c.RequestContextMapper
genericConfig.LegacyAPIGroupPrefixes = kubernetes.LegacyAPIGroupPrefixes
genericConfig.MaxRequestsInFlight = c.Options.ServingInfo.MaxRequestsInFlight
secureHandler, _ := handlerChain(container.ServeMux, genericConfig)
c.serve(secureHandler, messages)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
}
// RunDNSServer starts the DNS server
func (c *MasterConfig) RunDNSServer() {
config, err := dns.NewServerDefaults()
if err != nil {
glog.Fatalf("Could not start DNS: %v", err)
}
switch c.Options.DNSConfig.BindNetwork {
case "tcp":
config.BindNetwork = "ip"
case "tcp4":
config.BindNetwork = "ipv4"
case "tcp6":
config.BindNetwork = "ipv6"
}
config.DnsAddr = c.Options.DNSConfig.BindAddress
config.NoRec = !c.Options.DNSConfig.AllowRecursiveQueries
_, port, err := net.SplitHostPort(c.Options.DNSConfig.BindAddress)
if err != nil {
glog.Fatalf("Could not start DNS: %v", err)
}
if port != "53" {
glog.Warningf("Binding DNS on port %v instead of 53, which may not be resolvable from all clients", port)
}
if ok, err := cmdutil.TryListen(c.Options.DNSConfig.BindNetwork, c.Options.DNSConfig.BindAddress); !ok {
glog.Warningf("Could not start DNS: %v", err)
return
}
go func() {
etcdClient, err := etcd.GetAndTestEtcdClient(c.Options.EtcdClientInfo)
if err != nil {
glog.Fatalf("Could not get etcd client: %v", err)
return
}
err = dns.ListenAndServe(config, c.DNSServerClient(), etcdClient)
glog.Fatalf("Could not start DNS: %v", err)
}()
cmdutil.WaitForSuccessfulDial(false, "tcp", c.Options.DNSConfig.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
glog.Infof("DNS listening at %s", c.Options.DNSConfig.BindAddress)
}
// RunKubelet starts the Kubelet.
func (c *NodeConfig) RunKubelet() {
// TODO: clean this up and make it more formal (service named 'dns'?). Use multiple ports.
clusterDNS := c.KubeletConfig.ClusterDNS
if clusterDNS == nil {
if service, err := c.Client.Endpoints(kapi.NamespaceDefault).Get("kubernetes"); err == nil {
if ip, ok := firstIP(service, 53); ok {
if err := cmdutil.WaitForSuccessfulDial(false, "tcp", fmt.Sprintf("%s:%d", ip, 53), 50*time.Millisecond, 0, 2); err == nil {
c.KubeletConfig.ClusterDNS = util.IP(net.ParseIP(ip))
}
}
}
}
c.KubeletConfig.DockerClient = c.DockerClient
// updated by NodeConfig.EnsureVolumeDir
c.KubeletConfig.RootDirectory = c.VolumeDir
go func() {
glog.Fatal(c.KubeletServer.Run(c.KubeletConfig))
}()
}
func StartConfiguredNode(nodeConfig *configapi.NodeConfig) error {
kubernetes.SetFakeCadvisorInterfaceForIntegrationTest()
_, nodePort, err := net.SplitHostPort(nodeConfig.ServingInfo.BindAddress)
if err != nil {
return err
}
nodeTLS := configapi.UseTLS(nodeConfig.ServingInfo)
if err := start.StartNode(*nodeConfig); err != nil {
return err
}
// wait for the server to come up for 30 seconds (average time on desktop is 2 seconds, but Jenkins timed out at 10 seconds)
if err := cmdutil.WaitForSuccessfulDial(nodeTLS, "tcp", net.JoinHostPort(nodeConfig.NodeName, nodePort), 100*time.Millisecond, 1*time.Second, 30); err != nil {
return err
}
return nil
}
// RunDNSServer starts the DNS server
func (c *MasterConfig) RunDNSServer() {
config, err := dns.NewServerDefaults()
if err != nil {
glog.Fatalf("Could not start DNS: %v", err)
}
switch c.Options.DNSConfig.BindNetwork {
case "tcp":
config.BindNetwork = "ip"
case "tcp4":
config.BindNetwork = "ipv4"
case "tcp6":
config.BindNetwork = "ipv6"
}
config.DnsAddr = c.Options.DNSConfig.BindAddress
config.NoRec = !c.Options.DNSConfig.AllowRecursiveQueries
_, port, err := net.SplitHostPort(c.Options.DNSConfig.BindAddress)
if err != nil {
glog.Fatalf("Could not start DNS: %v", err)
}
if port != "53" {
glog.Warningf("Binding DNS on port %v instead of 53 (you may need to run as root and update your config), using %s which will not resolve from all locations", port, c.Options.DNSConfig.BindAddress)
}
if ok, err := cmdutil.TryListen(c.Options.DNSConfig.BindNetwork, c.Options.DNSConfig.BindAddress); !ok {
glog.Warningf("Could not start DNS: %v", err)
return
}
go func() {
err := dns.ListenAndServe(config, c.DNSServerClient(), c.EtcdClient)
glog.Fatalf("Could not start DNS: %v", err)
}()
cmdutil.WaitForSuccessfulDial(false, "tcp", c.Options.DNSConfig.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
glog.Infof("DNS listening at %s", c.Options.DNSConfig.BindAddress)
}
// Run launches the OpenShift master. It takes optional installers that may install additional endpoints into the server.
// All endpoints get configured CORS behavior
// Protected installers' endpoints are protected by API authentication and authorization.
// Unprotected installers' endpoints do not have any additional protection added.
func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) {
var extra []string
safe := kmaster.NewHandlerContainer(http.NewServeMux())
open := kmaster.NewHandlerContainer(http.NewServeMux())
// enforce authentication on protected endpoints
protected = append(protected, APIInstallFunc(c.InstallProtectedAPI))
for _, i := range protected {
extra = append(extra, i.InstallAPI(safe)...)
}
handler := c.authorizationFilter(safe)
handler = authenticationHandlerFilter(handler, c.Authenticator, c.getRequestContextMapper())
handler = namespacingFilter(handler, c.getRequestContextMapper())
handler = cacheControlFilter(handler, "no-store") // protected endpoints should not be cached
// unprotected resources
unprotected = append(unprotected, APIInstallFunc(c.InstallUnprotectedAPI))
for _, i := range unprotected {
extra = append(extra, i.InstallAPI(open)...)
}
handler = indexAPIPaths(handler)
open.Handle("/", handler)
// install swagger
swaggerConfig := swagger.Config{
WebServicesUrl: c.Options.MasterPublicURL,
WebServices: append(safe.RegisteredWebServices(), open.RegisteredWebServices()...),
ApiPath: swaggerAPIPrefix,
PostBuildHandler: customizeSwaggerDefinition,
}
// log nothing from swagger
swagger.LogInfo = func(format string, v ...interface{}) {}
swagger.RegisterSwaggerService(swaggerConfig, open)
extra = append(extra, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerAPIPrefix))
handler = open
// add CORS support
if origins := c.ensureCORSAllowedOrigins(); len(origins) != 0 {
handler = apiserver.CORS(handler, origins, nil, nil, "true")
}
if c.WebConsoleEnabled() {
handler = assetServerRedirect(handler, c.Options.AssetConfig.PublicURL)
}
// Make the outermost filter the requestContextMapper to ensure all components share the same context
if contextHandler, err := kapi.NewRequestContextFilter(c.getRequestContextMapper(), handler); err != nil {
glog.Fatalf("Error setting up request context filter: %v", err)
} else {
handler = contextHandler
}
// TODO: MaxRequestsInFlight should be subdivided by intent, type of behavior, and speed of
// execution - updates vs reads, long reads vs short reads, fat reads vs skinny reads.
if c.Options.ServingInfo.MaxRequestsInFlight > 0 {
sem := make(chan bool, c.Options.ServingInfo.MaxRequestsInFlight)
handler = apiserver.MaxInFlightLimit(sem, longRunningRE, handler)
}
c.serve(handler, extra)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
}
// Run launches the OpenShift master. It takes optional installers that may install additional endpoints into the server.
// All endpoints get configured CORS behavior
// Protected installers' endpoints are protected by API authentication and authorization.
// Unprotected installers' endpoints do not have any additional protection added.
func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) {
var extra []string
safe := kmaster.NewHandlerContainer(http.NewServeMux())
open := kmaster.NewHandlerContainer(http.NewServeMux())
// enforce authentication on protected endpoints
protected = append(protected, APIInstallFunc(c.InstallProtectedAPI))
for _, i := range protected {
extra = append(extra, i.InstallAPI(safe)...)
}
handler := c.authorizationFilter(safe)
handler = authenticationHandlerFilter(handler, c.Authenticator, c.getRequestContextMapper())
handler = namespacingFilter(handler, c.getRequestContextMapper())
handler = cacheControlFilter(handler, "no-store") // protected endpoints should not be cached
// unprotected resources
unprotected = append(unprotected, APIInstallFunc(c.InstallUnprotectedAPI))
for _, i := range unprotected {
extra = append(extra, i.InstallAPI(open)...)
}
handler = indexAPIPaths(handler)
open.Handle("/", handler)
// install swagger
swaggerConfig := swagger.Config{
WebServicesUrl: c.Options.MasterPublicURL,
WebServices: append(safe.RegisteredWebServices(), open.RegisteredWebServices()...),
ApiPath: swaggerAPIPrefix,
PostBuildHandler: customizeSwaggerDefinition,
}
// log nothing from swagger
swagger.LogInfo = func(format string, v ...interface{}) {}
swagger.RegisterSwaggerService(swaggerConfig, open)
extra = append(extra, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerAPIPrefix))
handler = open
// add CORS support
if origins := c.ensureCORSAllowedOrigins(); len(origins) != 0 {
handler = apiserver.CORS(handler, origins, nil, nil, "true")
}
if c.WebConsoleEnabled() {
handler = assetServerRedirect(handler, c.Options.AssetConfig.PublicURL)
}
// Make the outermost filter the requestContextMapper to ensure all components share the same context
if contextHandler, err := kapi.NewRequestContextFilter(c.getRequestContextMapper(), handler); err != nil {
glog.Fatalf("Error setting up request context filter: %v", err)
} else {
handler = contextHandler
}
// TODO: MaxRequestsInFlight should be subdivided by intent, type of behavior, and speed of
// execution - updates vs reads, long reads vs short reads, fat reads vs skinny reads.
if c.Options.ServingInfo.MaxRequestsInFlight > 0 {
sem := make(chan bool, c.Options.ServingInfo.MaxRequestsInFlight)
handler = apiserver.MaxInFlightLimit(sem, longRunningRE, handler)
}
timeout := c.Options.ServingInfo.RequestTimeoutSeconds
if timeout == -1 {
timeout = 0
}
server := &http.Server{
Addr: c.Options.ServingInfo.BindAddress,
Handler: handler,
ReadTimeout: time.Duration(timeout) * time.Second,
WriteTimeout: time.Duration(timeout) * time.Second,
MaxHeaderBytes: 1 << 20,
}
go util.Forever(func() {
for _, s := range extra {
glog.Infof(s, c.Options.ServingInfo.BindAddress)
}
if c.TLS {
server.TLSConfig = &tls.Config{
// Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability)
MinVersion: tls.VersionTLS10,
// Populate PeerCertificates in requests, but don't reject connections without certificates
// This allows certificates to be validated by authenticators, while still allowing other auth types
ClientAuth: tls.RequestClientCert,
ClientCAs: c.ClientCAs,
}
glog.Fatal(cmdutil.ListenAndServeTLS(server, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile))
} else {
glog.Fatal(server.ListenAndServe())
}
}, 0)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
// Create required policy rules if needed
c.ensureComponentAuthorizationRules()
// Ensure the default SCCs are created
c.ensureDefaultSecurityContextConstraints()
// Bind default roles for service accounts in the default namespace if needed
c.ensureDefaultNamespaceServiceAccountRoles()
// Create the infra namespace
c.ensureOpenShiftInfraNamespace()
// Create the shared resource namespace
c.ensureOpenShiftSharedResourcesNamespace()
}
func TestV2RegistryGetTags(t *testing.T) {
testutil.RequireEtcd(t)
defer testutil.DumpEtcdOnFailure(t)
_, clusterAdminKubeConfig, err := testserver.StartTestMasterAPI()
if err != nil {
t.Fatalf("error starting master: %v", err)
}
clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("error getting cluster admin client: %v", err)
}
clusterAdminClientConfig, err := testutil.GetClusterAdminClientConfig(clusterAdminKubeConfig)
if err != nil {
t.Fatalf("error getting cluster admin client config: %v", err)
}
user := "******"
adminClient, err := testserver.CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, testutil.Namespace(), user)
if err != nil {
t.Fatalf("error creating project: %v", err)
}
token, err := tokencmd.RequestToken(clusterAdminClientConfig, nil, user, "password")
if err != nil {
t.Fatalf("error requesting token: %v", err)
}
config := `version: 0.1
log:
level: debug
http:
addr: 127.0.0.1:5000
storage:
inmemory: {}
auth:
openshift:
middleware:
registry:
- name: openshift
repository:
- name: openshift
storage:
- name: openshift
`
os.Setenv("OPENSHIFT_CA_DATA", string(clusterAdminClientConfig.CAData))
os.Setenv("OPENSHIFT_CERT_DATA", string(clusterAdminClientConfig.CertData))
os.Setenv("OPENSHIFT_KEY_DATA", string(clusterAdminClientConfig.KeyData))
os.Setenv("OPENSHIFT_MASTER", clusterAdminClientConfig.Host)
os.Setenv("DOCKER_REGISTRY_URL", "127.0.0.1:5000")
go dockerregistry.Execute(strings.NewReader(config))
if err := cmdutil.WaitForSuccessfulDial(false, "tcp", "127.0.0.1:5000", 100*time.Millisecond, 1*time.Second, 35); err != nil {
t.Fatal(err)
}
stream := imageapi.ImageStream{
ObjectMeta: kapi.ObjectMeta{
Namespace: testutil.Namespace(),
Name: "test",
},
}
if _, err := adminClient.ImageStreams(testutil.Namespace()).Create(&stream); err != nil {
t.Fatalf("error creating image stream: %s", err)
}
tags, err := getTags(stream.Name, user, token)
if err != nil {
t.Fatal(err)
}
if len(tags) > 0 {
t.Fatalf("expected 0 tags, got: %#v", tags)
}
dgst, err := putManifest(stream.Name, user, token)
if err != nil {
t.Fatal(err)
}
tags, err = getTags(stream.Name, user, token)
if err != nil {
t.Fatal(err)
}
if len(tags) != 1 {
t.Fatalf("expected 1 tag, got %d: %v", len(tags), tags)
}
if tags[0] != imageapi.DefaultImageTag {
t.Fatalf("expected latest, got %q", tags[0])
}
// test get by tag
url := fmt.Sprintf("http://127.0.0.1:5000/v2/%s/%s/manifests/%s", testutil.Namespace(), stream.Name, imageapi.DefaultImageTag)
req, err := http.NewRequest("GET", url, nil)
if err != nil {
t.Fatalf("error creating request: %v", err)
}
req.SetBasicAuth(user, token)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("error retrieving manifest from registry: %s", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
body, err := ioutil.ReadAll(resp.Body)
var retrievedManifest schema1.Manifest
if err := json.Unmarshal(body, &retrievedManifest); err != nil {
t.Fatalf("error unmarshaling retrieved manifest")
}
if retrievedManifest.Name != fmt.Sprintf("%s/%s", testutil.Namespace(), stream.Name) {
t.Fatalf("unexpected manifest name: %s", retrievedManifest.Name)
}
if retrievedManifest.Tag != imageapi.DefaultImageTag {
t.Fatalf("unexpected manifest tag: %s", retrievedManifest.Tag)
}
// test get by digest
url = fmt.Sprintf("http://127.0.0.1:5000/v2/%s/%s/manifests/%s", testutil.Namespace(), stream.Name, dgst.String())
req, err = http.NewRequest("GET", url, nil)
if err != nil {
t.Fatalf("error creating request: %v", err)
}
req.SetBasicAuth(user, token)
resp, err = http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("error retrieving manifest from registry: %s", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected status code: %d", resp.StatusCode)
}
body, err = ioutil.ReadAll(resp.Body)
if err := json.Unmarshal(body, &retrievedManifest); err != nil {
t.Fatalf("error unmarshaling retrieved manifest")
}
if retrievedManifest.Name != fmt.Sprintf("%s/%s", testutil.Namespace(), stream.Name) {
t.Fatalf("unexpected manifest name: %s", retrievedManifest.Name)
}
if retrievedManifest.Tag != imageapi.DefaultImageTag {
t.Fatalf("unexpected manifest tag: %s", retrievedManifest.Tag)
}
image, err := adminClient.ImageStreamImages(testutil.Namespace()).Get(stream.Name, dgst.String())
if err != nil {
t.Fatalf("error getting imageStreamImage: %s", err)
}
if e, a := fmt.Sprintf("test@%s", dgst.String()), image.Name; e != a {
t.Errorf("image name: expected %q, got %q", e, a)
}
if e, a := dgst.String(), image.Image.Name; e != a {
t.Errorf("image name: expected %q, got %q", e, a)
}
if e, a := fmt.Sprintf("127.0.0.1:5000/%s/%s@%s", testutil.Namespace(), stream.Name, dgst.String()), image.Image.DockerImageReference; e != a {
t.Errorf("image dockerImageReference: expected %q, got %q", e, a)
}
if e, a := "foo", image.Image.DockerImageMetadata.ID; e != a {
t.Errorf("image dockerImageMetadata.ID: expected %q, got %q", e, a)
}
// test auto provisioning
otherStream, err := adminClient.ImageStreams(testutil.Namespace()).Get("otherrepo")
t.Logf("otherStream=%#v, err=%v", otherStream, err)
if err == nil {
t.Fatalf("expected error getting otherrepo")
}
otherDigest, err := putManifest("otherrepo", user, token)
if err != nil {
t.Fatal(err)
}
otherStream, err = adminClient.ImageStreams(testutil.Namespace()).Get("otherrepo")
if err != nil {
t.Fatalf("unexpected error getting otherrepo: %s", err)
}
if otherStream == nil {
t.Fatalf("unexpected nil otherrepo")
}
if len(otherStream.Status.Tags) != 1 {
t.Errorf("expected 1 tag, got %#v", otherStream.Status.Tags)
}
history, ok := otherStream.Status.Tags[imageapi.DefaultImageTag]
if !ok {
t.Fatal("unable to find 'latest' tag")
}
if len(history.Items) != 1 {
t.Errorf("expected 1 tag event, got %#v", history.Items)
}
if e, a := otherDigest.String(), history.Items[0].Image; e != a {
t.Errorf("digest: expected %q, got %q", e, a)
}
}
// Start starts the OpenShift master as a Docker container
// and returns a directory in the local file system where
// the OpenShift configuration has been copied
func (h *Helper) Start(opt *StartOptions, out io.Writer) (string, error) {
// Ensure that socat is available locally
if opt.PortForwarding {
err := CheckSocat()
if err != nil {
return "", err
}
}
binds := openShiftContainerBinds
env := []string{}
if opt.UseSharedVolume {
binds = append(binds, fmt.Sprintf("%[1]s:%[1]s:shared", opt.HostVolumesDir))
env = append(env, "OPENSHIFT_CONTAINERIZED=false")
} else {
binds = append(binds, "/:/rootfs:ro")
propagationMode := ""
if opt.SetPropagationMode {
propagationMode = ":rslave"
}
binds = append(binds, fmt.Sprintf("%[1]s:%[1]s%[2]s", opt.HostVolumesDir, propagationMode))
}
env = append(env, opt.Environment...)
binds = append(binds, fmt.Sprintf("%s:/var/lib/origin/openshift.local.config:z", opt.HostConfigDir))
// Check if a configuration exists before creating one if UseExistingConfig
// was specified
var configDir string
cleanupConfig := func() {
errors.LogError(os.RemoveAll(configDir))
}
skipCreateConfig := false
if opt.UseExistingConfig {
var err error
configDir, err = h.copyConfig(opt.HostConfigDir)
if err == nil {
_, err = os.Stat(filepath.Join(configDir, "master", "master-config.yaml"))
if err == nil {
skipCreateConfig = true
}
}
}
// Create configuration if needed
var nodeHost string
if !skipCreateConfig {
glog.V(1).Infof("Creating openshift configuration at %s on Docker host", opt.HostConfigDir)
fmt.Fprintf(out, "Creating initial OpenShift configuration\n")
createConfigCmd := []string{
"start",
fmt.Sprintf("--images=%s", opt.Images),
fmt.Sprintf("--volume-dir=%s", opt.HostVolumesDir),
fmt.Sprintf("--dns=0.0.0.0:%d", opt.DNSPort),
"--write-config=/var/lib/origin/openshift.local.config",
}
if opt.PortForwarding {
internalIP, err := h.ServerIP()
if err != nil {
return "", err
}
nodeHost = internalIP
createConfigCmd = append(createConfigCmd, fmt.Sprintf("--master=%s", internalIP))
createConfigCmd = append(createConfigCmd, fmt.Sprintf("--public-master=https://%s:8443", opt.ServerIP))
} else {
nodeHost = opt.ServerIP
createConfigCmd = append(createConfigCmd, fmt.Sprintf("--master=%s", opt.ServerIP))
if len(h.publicHost) > 0 {
createConfigCmd = append(createConfigCmd, fmt.Sprintf("--public-master=https://%s:8443", h.publicHost))
}
}
createConfigCmd = append(createConfigCmd, fmt.Sprintf("--hostname=%s", nodeHost))
_, err := h.runHelper.New().Image(h.image).
Privileged().
DiscardContainer().
HostNetwork().
HostPid().
Bind(binds...).
Env(env...).
Command(createConfigCmd...).Run()
if err != nil {
return "", errors.NewError("could not create OpenShift configuration").WithCause(err)
}
configDir, err = h.copyConfig(opt.HostConfigDir)
if err != nil {
return "", errors.NewError("could not copy OpenShift configuration").WithCause(err)
}
err = h.updateConfig(configDir, opt.HostConfigDir, opt.RouterIP, opt.MetricsHost, opt.LoggingHost)
if err != nil {
cleanupConfig()
return "", errors.NewError("could not update OpenShift configuration").WithCause(err)
}
}
if nodeHost == "" {
if opt.PortForwarding {
var err error
nodeHost, err = h.ServerIP()
if err != nil {
return "", err
}
} else {
var err error
hostName, err := h.hostHelper.Hostname()
if err != nil {
return "", err
}
nodeHost, err = h.DetermineNodeHost(opt.HostConfigDir, opt.ServerIP, hostName)
if err != nil {
return "", err
}
}
}
masterConfig, nodeConfig, err := h.getOpenShiftConfigFiles(nodeHost)
if err != nil {
cleanupConfig()
return "", errors.NewError("could not get OpenShift configuration file paths").WithCause(err)
}
fmt.Fprintf(out, "Starting OpenShift using container '%s'\n", h.containerName)
startCmd := []string{
"start",
fmt.Sprintf("--master-config=%s", masterConfig),
fmt.Sprintf("--node-config=%s", nodeConfig),
}
if opt.LogLevel > 0 {
startCmd = append(startCmd, fmt.Sprintf("--loglevel=%d", opt.LogLevel))
}
if opt.PortForwarding {
err = h.startSocatTunnel()
if err != nil {
return "", err
}
}
if len(opt.HostDataDir) > 0 {
binds = append(binds, fmt.Sprintf("%s:/var/lib/origin/openshift.local.etcd:z", opt.HostDataDir))
}
_, err = h.runHelper.New().Image(h.image).
Name(h.containerName).
Privileged().
HostNetwork().
HostPid().
Bind(binds...).
Env(env...).
Command(startCmd...).
Start()
if err != nil {
return "", errors.NewError("cannot start OpenShift daemon").WithCause(err)
}
// Wait a minimum amount of time and check whether we're still running. If not, we know the daemon didn't start
time.Sleep(initialStatusCheckWait)
_, running, err := h.dockerHelper.GetContainerState(h.containerName)
if err != nil {
return "", errors.NewError("cannot get state of OpenShift container %s", h.containerName).WithCause(err)
}
if !running {
return "", ErrOpenShiftFailedToStart(h.containerName).WithDetails(h.OriginLog())
}
// Wait until the API server is listening
fmt.Fprintf(out, "Waiting for API server to start listening\n")
masterHost := fmt.Sprintf("%s:8443", opt.ServerIP)
if err = cmdutil.WaitForSuccessfulDial(true, "tcp", masterHost, 200*time.Millisecond, 1*time.Second, serverUpTimeout); err != nil {
return "", ErrTimedOutWaitingForStart(h.containerName).WithDetails(h.OriginLog())
}
// Check for healthz endpoint to be ready
client, err := masterHTTPClient(configDir)
if err != nil {
return "", err
}
for {
resp, ierr := client.Get(h.healthzReadyURL(opt.ServerIP))
if ierr != nil {
return "", errors.NewError("cannot access master readiness URL %s", h.healthzReadyURL(opt.ServerIP)).WithCause(err).WithDetails(h.OriginLog())
}
if resp.StatusCode == http.StatusOK {
break
}
if resp.StatusCode == http.StatusServiceUnavailable ||
resp.StatusCode == http.StatusForbidden {
time.Sleep(500 * time.Millisecond)
continue
}
var responseBody string
body, rerr := ioutil.ReadAll(resp.Body)
if rerr == nil {
responseBody = string(body)
}
return "", errors.NewError("server is not ready. Response (%d): %s", resp.StatusCode, responseBody).WithCause(ierr).WithDetails(h.OriginLog())
}
fmt.Fprintf(out, "OpenShift server started\n")
return configDir, nil
}
// Run launches the OpenShift master. It takes optional installers that may install additional endpoints into the server.
// All endpoints get configured CORS behavior
// Protected installers' endpoints are protected by API authentication and authorization.
// Unprotected installers' endpoints do not have any additional protection added.
func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) {
var extra []string
safe := genericapiserver.NewHandlerContainer(http.NewServeMux(), kapi.Codecs)
open := genericapiserver.NewHandlerContainer(http.NewServeMux(), kapi.Codecs)
// enforce authentication on protected endpoints
protected = append(protected, APIInstallFunc(c.InstallProtectedAPI))
for _, i := range protected {
msgs, err := i.InstallAPI(safe)
if err != nil {
glog.Fatalf("error installing api %v", err)
}
extra = append(extra, msgs...)
}
handler := c.versionSkewFilter(safe)
handler = c.authorizationFilter(handler)
handler = c.impersonationFilter(handler)
// audit handler must comes before the impersonationFilter to read the original user
handler = c.auditHandler(handler)
handler = authenticationHandlerFilter(handler, c.Authenticator, c.getRequestContextMapper())
handler = namespacingFilter(handler, c.getRequestContextMapper())
handler = cacheControlFilter(handler, "no-store") // protected endpoints should not be cached
// unprotected resources
unprotected = append(unprotected, APIInstallFunc(c.InstallUnprotectedAPI))
for _, i := range unprotected {
msgs, err := i.InstallAPI(open)
if err != nil {
glog.Fatalf("error installing api %v", err)
}
extra = append(extra, msgs...)
}
var kubeAPILevels []string
if c.Options.KubernetesMasterConfig != nil {
kubeAPILevels = configapi.GetEnabledAPIVersionsForGroup(*c.Options.KubernetesMasterConfig, kapi.GroupName)
}
handler = indexAPIPaths(c.Options.APILevels, kubeAPILevels, handler)
open.Handle("/", handler)
// install swagger
swaggerConfig := swagger.Config{
WebServicesUrl: c.Options.MasterPublicURL,
WebServices: append(safe.RegisteredWebServices(), open.RegisteredWebServices()...),
ApiPath: swaggerAPIPrefix,
PostBuildHandler: customizeSwaggerDefinition,
}
// log nothing from swagger
swagger.LogInfo = func(format string, v ...interface{}) {}
swagger.RegisterSwaggerService(swaggerConfig, open)
extra = append(extra, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerAPIPrefix))
openAPIConfig := openapi.Config{
SwaggerConfig: &swaggerConfig,
IgnorePrefixes: []string{"/swaggerapi"},
Info: &spec.Info{
InfoProps: spec.InfoProps{
Title: "OpenShift API (with Kubernetes)",
Version: version.Get().String(),
License: &spec.License{
Name: "Apache 2.0 (ASL2.0)",
URL: "http://www.apache.org/licenses/LICENSE-2.0",
},
Description: heredoc.Doc(`
OpenShift provides builds, application lifecycle, image content management,
and administrative policy on top of Kubernetes. The API allows consistent
management of those objects.
All API operations are authenticated via an Authorization bearer token that
is provided for service accounts as a generated secret (in JWT form) or via
the native OAuth endpoint located at /oauth/authorize. Core infrastructure
components may use client certificates that require no authentication.
All API operations return a 'resourceVersion' string that represents the
version of the object in the underlying storage. The standard LIST operation
performs a snapshot read of the underlying objects, returning a resourceVersion
representing a consistent version of the listed objects. The WATCH operation
allows all updates to a set of objects after the provided resourceVersion to
be observed by a client. By listing and beginning a watch from the returned
resourceVersion, clients may observe a consistent view of the state of one
or more objects. Note that WATCH always returns the update after the provided
resourceVersion. Watch may be extended a limited time in the past - using
etcd 2 the watch window is 1000 events (which on a large cluster may only
be a few tens of seconds) so clients must explicitly handle the "watch
to old error" by re-listing.
Objects are divided into two rough categories - those that have a lifecycle
and must reflect the state of the cluster, and those that have no state.
Objects with lifecycle typically have three main sections:
* 'metadata' common to all objects
* a 'spec' that represents the desired state
* a 'status' that represents how much of the desired state is reflected on
the cluster at the current time
Objects that have no state have 'metadata' but may lack a 'spec' or 'status'
section.
Objects are divided into those that are namespace scoped (only exist inside
of a namespace) and those that are cluster scoped (exist outside of
a namespace). A namespace scoped resource will be deleted when the namespace
is deleted and cannot be created if the namespace has not yet been created
or is in the process of deletion. Cluster scoped resources are typically
only accessible to admins - resources like nodes, persistent volumes, and
cluster policy.
All objects have a schema that is a combination of the 'kind' and
'apiVersion' fields. This schema is additive only for any given version -
no backwards incompatible changes are allowed without incrementing the
apiVersion. The server will return and accept a number of standard
responses that share a common schema - for instance, the common
error type is 'unversioned.Status' (described below) and will be returned
on any error from the API server.
The API is available in multiple serialization formats - the default is
JSON (Accept: application/json and Content-Type: application/json) but
clients may also use YAML (application/yaml) or the native Protobuf
schema (application/vnd.kubernetes.protobuf). Note that the format
of the WATCH API call is slightly different - for JSON it returns newline
delimited objects while for Protobuf it returns length-delimited frames
(4 bytes in network-order) that contain a 'versioned.Watch' Protobuf
object.
See the OpenShift documentation at https://docs.openshift.org for more
information.
`),
},
},
DefaultResponse: &spec.Response{
ResponseProps: spec.ResponseProps{
Description: "Default Response.",
},
},
}
err := openapi.RegisterOpenAPIService(&openAPIConfig, open)
if err != nil {
glog.Fatalf("Failed to generate open api spec: %v", err)
}
extra = append(extra, fmt.Sprintf("Started OpenAPI Schema at %%s%s", openapi.OpenAPIServePath))
handler = open
// add CORS support
if origins := c.ensureCORSAllowedOrigins(); len(origins) != 0 {
handler = apiserver.CORS(handler, origins, nil, nil, "true")
}
if c.WebConsoleEnabled() {
handler = assetServerRedirect(handler, c.Options.AssetConfig.PublicURL)
}
// Make the outermost filter the requestContextMapper to ensure all components share the same context
if contextHandler, err := kapi.NewRequestContextFilter(c.getRequestContextMapper(), handler); err != nil {
glog.Fatalf("Error setting up request context filter: %v", err)
} else {
handler = contextHandler
}
longRunningRequestCheck := apiserver.BasicLongRunningRequestCheck(longRunningRE, map[string]string{"watch": "true"})
// TODO: MaxRequestsInFlight should be subdivided by intent, type of behavior, and speed of
// execution - updates vs reads, long reads vs short reads, fat reads vs skinny reads.
if c.Options.ServingInfo.MaxRequestsInFlight > 0 {
sem := make(chan bool, c.Options.ServingInfo.MaxRequestsInFlight)
handler = apiserver.MaxInFlightLimit(sem, longRunningRequestCheck, handler)
}
c.serve(handler, extra)
// Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try)
cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100)
}
