func StartConfiguredMasterWithOptions(masterConfig *configapi.MasterConfig, testOptions TestOptions) (string, error) { if testOptions.DeleteAllEtcdKeys { util.DeleteAllEtcdKeys() } if err := start.NewMaster(masterConfig, true, true).Start(); err != nil { return "", err } adminKubeConfigFile := util.KubeConfigPath() clientConfig, err := util.GetClusterAdminClientConfig(adminKubeConfigFile) if err != nil { return "", err } masterURL, err := url.Parse(clientConfig.Host) if err != nil { return "", err } // wait for the server to come up: 35 seconds if err := cmdutil.WaitForSuccessfulDial(true, "tcp", masterURL.Host, 100*time.Millisecond, 1*time.Second, 35); err != nil { return "", err } for { // confirm that we can actually query from the api server if client, err := util.GetClusterAdminClient(adminKubeConfigFile); err == nil { if _, err := client.ClusterPolicies().List(labels.Everything(), fields.Everything()); err == nil { break } } time.Sleep(100 * time.Millisecond) } return adminKubeConfigFile, nil }
// Run launches the OpenShift master by creating a kubernetes master, installing // OpenShift APIs into it and then running it. func (c *MasterConfig) Run(kc *kubernetes.MasterConfig, assetConfig *AssetConfig) { var ( messages []string err error ) kc.Master.GenericConfig.BuildHandlerChainsFunc, messages, err = c.buildHandlerChain(assetConfig) if err != nil { glog.Fatalf("Failed to launch master: %v", err) } kmaster, err := kc.Master.Complete().New() if err != nil { glog.Fatalf("Failed to launch master: %v", err) } c.InstallProtectedAPI(kmaster.GenericAPIServer.HandlerContainer) messages = append(messages, c.kubernetesAPIMessages(kc)...) for _, s := range messages { glog.Infof(s, c.Options.ServingInfo.BindAddress) } go kmaster.GenericAPIServer.PrepareRun().Run(utilwait.NeverStop) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) }
func (h *Helper) TestIP(ip string) error { // Start test server on host id, err := h.runHelper.New().Image(h.image). Privileged(). HostNetwork(). Entrypoint("socat"). Command("TCP-LISTEN:8443,crlf,reuseaddr,fork", "SYSTEM:\"echo 'hello world'\"").Start() if err != nil { return errors.NewError("cannnot start simple server on Docker host").WithCause(err) } defer func() { errors.LogError(h.dockerHelper.StopAndRemoveContainer(id)) }() // Attempt to connect to test container testHost := fmt.Sprintf("%s:8443", ip) glog.V(4).Infof("Attempting to dial %s", testHost) if err = cmdutil.WaitForSuccessfulDial(false, "tcp", testHost, 200*time.Millisecond, 1*time.Second, 10); err != nil { glog.V(2).Infof("Dial error: %v", err) return err } glog.V(4).Infof("Successfully dialed %s", testHost) return nil }
// RunKubelet starts the Kubelet. func (c *NodeConfig) RunKubelet() { if c.KubeletConfig.ClusterDNS == nil { if service, err := c.Client.Services(kapi.NamespaceDefault).Get("kubernetes"); err == nil { if includesServicePort(service.Spec.Ports, 53, "dns") { // Use master service if service includes "dns" port 53. c.KubeletConfig.ClusterDNS = net.ParseIP(service.Spec.ClusterIP) } } } if c.KubeletConfig.ClusterDNS == nil { if endpoint, err := c.Client.Endpoints(kapi.NamespaceDefault).Get("kubernetes"); err == nil { if endpointIP, ok := firstEndpointIPWithNamedPort(endpoint, 53, "dns"); ok { // Use first endpoint if endpoint includes "dns" port 53. c.KubeletConfig.ClusterDNS = net.ParseIP(endpointIP) } else if endpointIP, ok := firstEndpointIP(endpoint, 53); ok { // Test and use first endpoint if endpoint includes any port 53. if err := cmdutil.WaitForSuccessfulDial(false, "tcp", fmt.Sprintf("%s:%d", endpointIP, 53), 50*time.Millisecond, 0, 2); err == nil { c.KubeletConfig.ClusterDNS = net.ParseIP(endpointIP) } } } } c.KubeletConfig.DockerClient = c.DockerClient // updated by NodeConfig.EnsureVolumeDir c.KubeletConfig.RootDirectory = c.VolumeDir // hook for overriding the cadvisor interface for integration tests c.KubeletConfig.CAdvisorInterface = defaultCadvisorInterface go func() { glog.Fatal(c.KubeletServer.Run(c.KubeletConfig)) }() }
func runRegistry() error { config := `version: 0.1 log: level: debug http: addr: 127.0.0.1:5000 storage: inmemory: {} auth: openshift: middleware: registry: - name: openshift repository: - name: openshift options: acceptschema2: false pullthrough: true enforcequota: false projectcachettl: 1m blobrepositorycachettl: 10m storage: - name: openshift ` os.Setenv("DOCKER_REGISTRY_URL", "127.0.0.1:5000") go dockerregistry.Execute(strings.NewReader(config)) if err := cmdutil.WaitForSuccessfulDial(false, "tcp", "127.0.0.1:5000", 100*time.Millisecond, 1*time.Second, 35); err != nil { return err } return nil }
// RunDNSServer starts the DNS server func (c *MasterConfig) RunDNSServer() { config, err := dns.NewServerDefaults() if err != nil { glog.Fatalf("Could not start DNS: %v", err) } config.DnsAddr = c.Options.DNSConfig.BindAddress config.NoRec = true // do not want to deploy an open resolver _, port, err := net.SplitHostPort(c.Options.DNSConfig.BindAddress) if err != nil { glog.Fatalf("Could not start DNS: %v", err) } if port != "53" { glog.Warningf("Binding DNS on port %v instead of 53 (you may need to run as root and update your config), using %s which will not resolve from all locations", port, c.Options.DNSConfig.BindAddress) } if ok, err := cmdutil.TryListen(c.Options.DNSConfig.BindAddress); !ok { glog.Warningf("Could not start DNS: %v", err) return } go func() { err := dns.ListenAndServe(config, c.DNSServerClient(), c.EtcdHelper.Client.(*etcdclient.Client)) glog.Fatalf("Could not start DNS: %v", err) }() cmdutil.WaitForSuccessfulDial(false, "tcp", c.Options.DNSConfig.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) glog.Infof("DNS listening at %s", c.Options.DNSConfig.BindAddress) }
// Run starts an http server for the static assets listening on the configured // bind address func (c *AssetConfig) Run() { publicURL, err := url.Parse(c.Options.PublicURL) if err != nil { glog.Fatal(err) } mux := http.NewServeMux() err = c.addHandlers(mux) if err != nil { glog.Fatal(err) } if publicURL.Path != "/" { mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { http.Redirect(w, req, publicURL.Path, http.StatusFound) }) } timeout := c.Options.ServingInfo.RequestTimeoutSeconds if timeout == -1 { timeout = 0 } server := &http.Server{ Addr: c.Options.ServingInfo.BindAddress, Handler: mux, ReadTimeout: time.Duration(timeout) * time.Second, WriteTimeout: time.Duration(timeout) * time.Second, MaxHeaderBytes: 1 << 20, } isTLS := configapi.UseTLS(c.Options.ServingInfo.ServingInfo) go util.Forever(func() { if isTLS { extraCerts, err := configapi.GetNamedCertificateMap(c.Options.ServingInfo.NamedCertificates) if err != nil { glog.Fatal(err) } server.TLSConfig = &tls.Config{ // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) MinVersion: tls.VersionTLS10, // Set SNI certificate func GetCertificate: cmdutil.GetCertificateFunc(extraCerts), } glog.Infof("Web console listening at https://%s", c.Options.ServingInfo.BindAddress) glog.Fatal(cmdutil.ListenAndServeTLS(server, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile)) } else { glog.Infof("Web console listening at http://%s", c.Options.ServingInfo.BindAddress) glog.Fatal(server.ListenAndServe()) } }, 0) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) cmdutil.WaitForSuccessfulDial(isTLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) glog.Infof("Web console available at %s", c.Options.PublicURL) }
func testIPDial(ip string) error { // Attempt to connect to test container testHost := fmt.Sprintf("%s:8443", ip) glog.V(4).Infof("Attempting to dial %s", testHost) if err := cmdutil.WaitForSuccessfulDial(false, "tcp", testHost, 200*time.Millisecond, 1*time.Second, 10); err != nil { glog.V(2).Infof("Dial error: %v", err) return err } glog.V(4).Infof("Successfully dialed %s", testHost) return nil }
// Run starts an http server for the static assets listening on the configured // bind address func (c *AssetConfig) Run() { mux, err := c.addHandlers(nil) if err != nil { glog.Fatal(err) } timeout := c.Options.ServingInfo.RequestTimeoutSeconds if timeout == -1 { timeout = 0 } server := &http.Server{ Addr: c.Options.ServingInfo.BindAddress, Handler: mux, ReadTimeout: time.Duration(timeout) * time.Second, WriteTimeout: time.Duration(timeout) * time.Second, MaxHeaderBytes: 1 << 20, } isTLS := configapi.UseTLS(c.Options.ServingInfo.ServingInfo) go utilwait.Forever(func() { if isTLS { extraCerts, err := configapi.GetNamedCertificateMap(c.Options.ServingInfo.NamedCertificates) if err != nil { glog.Fatal(err) } server.TLSConfig = crypto.SecureTLSConfig(&tls.Config{ // Set SNI certificate func GetCertificate: cmdutil.GetCertificateFunc(extraCerts), }) glog.Infof("Web console listening at https://%s", c.Options.ServingInfo.BindAddress) glog.Fatal(cmdutil.ListenAndServeTLS(server, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile)) } else { glog.Infof("Web console listening at http://%s", c.Options.ServingInfo.BindAddress) glog.Fatal(server.ListenAndServe()) } }, 0) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) cmdutil.WaitForSuccessfulDial(isTLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) glog.Infof("Web console available at %s", c.Options.PublicURL) }
func (c *MasterConfig) RunInProxyMode(proxy *kubernetes.ProxyConfig, assetConfig *AssetConfig) { handlerChain, messages, err := c.buildHandlerChain(assetConfig) if err != nil { glog.Fatalf("Failed to launch master: %v", err) } // TODO(sttts): create a genericapiserver here container := genericmux.NewAPIContainer(http.NewServeMux(), kapi.Codecs) // install /api proxy forwarder proxyMessages, err := proxy.InstallAPI(container.Container) if err != nil { glog.Fatalf("Failed to launch master: %v", err) } messages = append(messages, proxyMessages...) // install GenericAPIServer handlers manually, usually done by GenericAPIServer.PrepareRun() healthz.InstallHandler(&container.NonSwaggerRoutes, healthz.PingHealthz) swaggerConfig := genericapiserver.DefaultSwaggerConfig() swaggerConfig.WebServicesUrl = c.Options.MasterPublicURL genericroutes.Swagger{Config: swaggerConfig}.Install(container) messages = append(messages, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerConfig.ApiPath)) genericroutes.OpenAPI{Config: kubernetes.DefaultOpenAPIConfig()}.Install(container) messages = append(messages, fmt.Sprintf("Started OpenAPI Schema at %%s%s", openAPIServePath)) // install origin handlers c.InstallProtectedAPI(container) // TODO(sttts): split cmd/server/kubernetes config generation into generic and master-specific // until then: create ad-hoc config genericConfig := genericapiserver.NewConfig() genericConfig.RequestContextMapper = c.RequestContextMapper genericConfig.LegacyAPIGroupPrefixes = kubernetes.LegacyAPIGroupPrefixes genericConfig.MaxRequestsInFlight = c.Options.ServingInfo.MaxRequestsInFlight secureHandler, _ := handlerChain(container.ServeMux, genericConfig) c.serve(secureHandler, messages) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) }
// RunDNSServer starts the DNS server func (c *MasterConfig) RunDNSServer() { config, err := dns.NewServerDefaults() if err != nil { glog.Fatalf("Could not start DNS: %v", err) } switch c.Options.DNSConfig.BindNetwork { case "tcp": config.BindNetwork = "ip" case "tcp4": config.BindNetwork = "ipv4" case "tcp6": config.BindNetwork = "ipv6" } config.DnsAddr = c.Options.DNSConfig.BindAddress config.NoRec = !c.Options.DNSConfig.AllowRecursiveQueries _, port, err := net.SplitHostPort(c.Options.DNSConfig.BindAddress) if err != nil { glog.Fatalf("Could not start DNS: %v", err) } if port != "53" { glog.Warningf("Binding DNS on port %v instead of 53, which may not be resolvable from all clients", port) } if ok, err := cmdutil.TryListen(c.Options.DNSConfig.BindNetwork, c.Options.DNSConfig.BindAddress); !ok { glog.Warningf("Could not start DNS: %v", err) return } go func() { etcdClient, err := etcd.GetAndTestEtcdClient(c.Options.EtcdClientInfo) if err != nil { glog.Fatalf("Could not get etcd client: %v", err) return } err = dns.ListenAndServe(config, c.DNSServerClient(), etcdClient) glog.Fatalf("Could not start DNS: %v", err) }() cmdutil.WaitForSuccessfulDial(false, "tcp", c.Options.DNSConfig.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) glog.Infof("DNS listening at %s", c.Options.DNSConfig.BindAddress) }
// RunKubelet starts the Kubelet. func (c *NodeConfig) RunKubelet() { // TODO: clean this up and make it more formal (service named 'dns'?). Use multiple ports. clusterDNS := c.KubeletConfig.ClusterDNS if clusterDNS == nil { if service, err := c.Client.Endpoints(kapi.NamespaceDefault).Get("kubernetes"); err == nil { if ip, ok := firstIP(service, 53); ok { if err := cmdutil.WaitForSuccessfulDial(false, "tcp", fmt.Sprintf("%s:%d", ip, 53), 50*time.Millisecond, 0, 2); err == nil { c.KubeletConfig.ClusterDNS = util.IP(net.ParseIP(ip)) } } } } c.KubeletConfig.DockerClient = c.DockerClient // updated by NodeConfig.EnsureVolumeDir c.KubeletConfig.RootDirectory = c.VolumeDir go func() { glog.Fatal(c.KubeletServer.Run(c.KubeletConfig)) }() }
func StartConfiguredNode(nodeConfig *configapi.NodeConfig) error { kubernetes.SetFakeCadvisorInterfaceForIntegrationTest() _, nodePort, err := net.SplitHostPort(nodeConfig.ServingInfo.BindAddress) if err != nil { return err } nodeTLS := configapi.UseTLS(nodeConfig.ServingInfo) if err := start.StartNode(*nodeConfig); err != nil { return err } // wait for the server to come up for 30 seconds (average time on desktop is 2 seconds, but Jenkins timed out at 10 seconds) if err := cmdutil.WaitForSuccessfulDial(nodeTLS, "tcp", net.JoinHostPort(nodeConfig.NodeName, nodePort), 100*time.Millisecond, 1*time.Second, 30); err != nil { return err } return nil }
// RunDNSServer starts the DNS server func (c *MasterConfig) RunDNSServer() { config, err := dns.NewServerDefaults() if err != nil { glog.Fatalf("Could not start DNS: %v", err) } switch c.Options.DNSConfig.BindNetwork { case "tcp": config.BindNetwork = "ip" case "tcp4": config.BindNetwork = "ipv4" case "tcp6": config.BindNetwork = "ipv6" } config.DnsAddr = c.Options.DNSConfig.BindAddress config.NoRec = !c.Options.DNSConfig.AllowRecursiveQueries _, port, err := net.SplitHostPort(c.Options.DNSConfig.BindAddress) if err != nil { glog.Fatalf("Could not start DNS: %v", err) } if port != "53" { glog.Warningf("Binding DNS on port %v instead of 53 (you may need to run as root and update your config), using %s which will not resolve from all locations", port, c.Options.DNSConfig.BindAddress) } if ok, err := cmdutil.TryListen(c.Options.DNSConfig.BindNetwork, c.Options.DNSConfig.BindAddress); !ok { glog.Warningf("Could not start DNS: %v", err) return } go func() { err := dns.ListenAndServe(config, c.DNSServerClient(), c.EtcdClient) glog.Fatalf("Could not start DNS: %v", err) }() cmdutil.WaitForSuccessfulDial(false, "tcp", c.Options.DNSConfig.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) glog.Infof("DNS listening at %s", c.Options.DNSConfig.BindAddress) }
// Run launches the OpenShift master. It takes optional installers that may install additional endpoints into the server. // All endpoints get configured CORS behavior // Protected installers' endpoints are protected by API authentication and authorization. // Unprotected installers' endpoints do not have any additional protection added. func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) { var extra []string safe := kmaster.NewHandlerContainer(http.NewServeMux()) open := kmaster.NewHandlerContainer(http.NewServeMux()) // enforce authentication on protected endpoints protected = append(protected, APIInstallFunc(c.InstallProtectedAPI)) for _, i := range protected { extra = append(extra, i.InstallAPI(safe)...) } handler := c.authorizationFilter(safe) handler = authenticationHandlerFilter(handler, c.Authenticator, c.getRequestContextMapper()) handler = namespacingFilter(handler, c.getRequestContextMapper()) handler = cacheControlFilter(handler, "no-store") // protected endpoints should not be cached // unprotected resources unprotected = append(unprotected, APIInstallFunc(c.InstallUnprotectedAPI)) for _, i := range unprotected { extra = append(extra, i.InstallAPI(open)...) } handler = indexAPIPaths(handler) open.Handle("/", handler) // install swagger swaggerConfig := swagger.Config{ WebServicesUrl: c.Options.MasterPublicURL, WebServices: append(safe.RegisteredWebServices(), open.RegisteredWebServices()...), ApiPath: swaggerAPIPrefix, PostBuildHandler: customizeSwaggerDefinition, } // log nothing from swagger swagger.LogInfo = func(format string, v ...interface{}) {} swagger.RegisterSwaggerService(swaggerConfig, open) extra = append(extra, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerAPIPrefix)) handler = open // add CORS support if origins := c.ensureCORSAllowedOrigins(); len(origins) != 0 { handler = apiserver.CORS(handler, origins, nil, nil, "true") } if c.WebConsoleEnabled() { handler = assetServerRedirect(handler, c.Options.AssetConfig.PublicURL) } // Make the outermost filter the requestContextMapper to ensure all components share the same context if contextHandler, err := kapi.NewRequestContextFilter(c.getRequestContextMapper(), handler); err != nil { glog.Fatalf("Error setting up request context filter: %v", err) } else { handler = contextHandler } // TODO: MaxRequestsInFlight should be subdivided by intent, type of behavior, and speed of // execution - updates vs reads, long reads vs short reads, fat reads vs skinny reads. if c.Options.ServingInfo.MaxRequestsInFlight > 0 { sem := make(chan bool, c.Options.ServingInfo.MaxRequestsInFlight) handler = apiserver.MaxInFlightLimit(sem, longRunningRE, handler) } c.serve(handler, extra) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) }
// Run launches the OpenShift master. It takes optional installers that may install additional endpoints into the server. // All endpoints get configured CORS behavior // Protected installers' endpoints are protected by API authentication and authorization. // Unprotected installers' endpoints do not have any additional protection added. func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) { var extra []string safe := kmaster.NewHandlerContainer(http.NewServeMux()) open := kmaster.NewHandlerContainer(http.NewServeMux()) // enforce authentication on protected endpoints protected = append(protected, APIInstallFunc(c.InstallProtectedAPI)) for _, i := range protected { extra = append(extra, i.InstallAPI(safe)...) } handler := c.authorizationFilter(safe) handler = authenticationHandlerFilter(handler, c.Authenticator, c.getRequestContextMapper()) handler = namespacingFilter(handler, c.getRequestContextMapper()) handler = cacheControlFilter(handler, "no-store") // protected endpoints should not be cached // unprotected resources unprotected = append(unprotected, APIInstallFunc(c.InstallUnprotectedAPI)) for _, i := range unprotected { extra = append(extra, i.InstallAPI(open)...) } handler = indexAPIPaths(handler) open.Handle("/", handler) // install swagger swaggerConfig := swagger.Config{ WebServicesUrl: c.Options.MasterPublicURL, WebServices: append(safe.RegisteredWebServices(), open.RegisteredWebServices()...), ApiPath: swaggerAPIPrefix, PostBuildHandler: customizeSwaggerDefinition, } // log nothing from swagger swagger.LogInfo = func(format string, v ...interface{}) {} swagger.RegisterSwaggerService(swaggerConfig, open) extra = append(extra, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerAPIPrefix)) handler = open // add CORS support if origins := c.ensureCORSAllowedOrigins(); len(origins) != 0 { handler = apiserver.CORS(handler, origins, nil, nil, "true") } if c.WebConsoleEnabled() { handler = assetServerRedirect(handler, c.Options.AssetConfig.PublicURL) } // Make the outermost filter the requestContextMapper to ensure all components share the same context if contextHandler, err := kapi.NewRequestContextFilter(c.getRequestContextMapper(), handler); err != nil { glog.Fatalf("Error setting up request context filter: %v", err) } else { handler = contextHandler } // TODO: MaxRequestsInFlight should be subdivided by intent, type of behavior, and speed of // execution - updates vs reads, long reads vs short reads, fat reads vs skinny reads. if c.Options.ServingInfo.MaxRequestsInFlight > 0 { sem := make(chan bool, c.Options.ServingInfo.MaxRequestsInFlight) handler = apiserver.MaxInFlightLimit(sem, longRunningRE, handler) } timeout := c.Options.ServingInfo.RequestTimeoutSeconds if timeout == -1 { timeout = 0 } server := &http.Server{ Addr: c.Options.ServingInfo.BindAddress, Handler: handler, ReadTimeout: time.Duration(timeout) * time.Second, WriteTimeout: time.Duration(timeout) * time.Second, MaxHeaderBytes: 1 << 20, } go util.Forever(func() { for _, s := range extra { glog.Infof(s, c.Options.ServingInfo.BindAddress) } if c.TLS { server.TLSConfig = &tls.Config{ // Change default from SSLv3 to TLSv1.0 (because of POODLE vulnerability) MinVersion: tls.VersionTLS10, // Populate PeerCertificates in requests, but don't reject connections without certificates // This allows certificates to be validated by authenticators, while still allowing other auth types ClientAuth: tls.RequestClientCert, ClientCAs: c.ClientCAs, } glog.Fatal(cmdutil.ListenAndServeTLS(server, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.ServerCert.CertFile, c.Options.ServingInfo.ServerCert.KeyFile)) } else { glog.Fatal(server.ListenAndServe()) } }, 0) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) // Create required policy rules if needed c.ensureComponentAuthorizationRules() // Ensure the default SCCs are created c.ensureDefaultSecurityContextConstraints() // Bind default roles for service accounts in the default namespace if needed c.ensureDefaultNamespaceServiceAccountRoles() // Create the infra namespace c.ensureOpenShiftInfraNamespace() // Create the shared resource namespace c.ensureOpenShiftSharedResourcesNamespace() }
func TestV2RegistryGetTags(t *testing.T) { testutil.RequireEtcd(t) defer testutil.DumpEtcdOnFailure(t) _, clusterAdminKubeConfig, err := testserver.StartTestMasterAPI() if err != nil { t.Fatalf("error starting master: %v", err) } clusterAdminClient, err := testutil.GetClusterAdminClient(clusterAdminKubeConfig) if err != nil { t.Fatalf("error getting cluster admin client: %v", err) } clusterAdminClientConfig, err := testutil.GetClusterAdminClientConfig(clusterAdminKubeConfig) if err != nil { t.Fatalf("error getting cluster admin client config: %v", err) } user := "******" adminClient, err := testserver.CreateNewProject(clusterAdminClient, *clusterAdminClientConfig, testutil.Namespace(), user) if err != nil { t.Fatalf("error creating project: %v", err) } token, err := tokencmd.RequestToken(clusterAdminClientConfig, nil, user, "password") if err != nil { t.Fatalf("error requesting token: %v", err) } config := `version: 0.1 log: level: debug http: addr: 127.0.0.1:5000 storage: inmemory: {} auth: openshift: middleware: registry: - name: openshift repository: - name: openshift storage: - name: openshift ` os.Setenv("OPENSHIFT_CA_DATA", string(clusterAdminClientConfig.CAData)) os.Setenv("OPENSHIFT_CERT_DATA", string(clusterAdminClientConfig.CertData)) os.Setenv("OPENSHIFT_KEY_DATA", string(clusterAdminClientConfig.KeyData)) os.Setenv("OPENSHIFT_MASTER", clusterAdminClientConfig.Host) os.Setenv("DOCKER_REGISTRY_URL", "127.0.0.1:5000") go dockerregistry.Execute(strings.NewReader(config)) if err := cmdutil.WaitForSuccessfulDial(false, "tcp", "127.0.0.1:5000", 100*time.Millisecond, 1*time.Second, 35); err != nil { t.Fatal(err) } stream := imageapi.ImageStream{ ObjectMeta: kapi.ObjectMeta{ Namespace: testutil.Namespace(), Name: "test", }, } if _, err := adminClient.ImageStreams(testutil.Namespace()).Create(&stream); err != nil { t.Fatalf("error creating image stream: %s", err) } tags, err := getTags(stream.Name, user, token) if err != nil { t.Fatal(err) } if len(tags) > 0 { t.Fatalf("expected 0 tags, got: %#v", tags) } dgst, err := putManifest(stream.Name, user, token) if err != nil { t.Fatal(err) } tags, err = getTags(stream.Name, user, token) if err != nil { t.Fatal(err) } if len(tags) != 1 { t.Fatalf("expected 1 tag, got %d: %v", len(tags), tags) } if tags[0] != imageapi.DefaultImageTag { t.Fatalf("expected latest, got %q", tags[0]) } // test get by tag url := fmt.Sprintf("http://127.0.0.1:5000/v2/%s/%s/manifests/%s", testutil.Namespace(), stream.Name, imageapi.DefaultImageTag) req, err := http.NewRequest("GET", url, nil) if err != nil { t.Fatalf("error creating request: %v", err) } req.SetBasicAuth(user, token) resp, err := http.DefaultClient.Do(req) if err != nil { t.Fatalf("error retrieving manifest from registry: %s", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("unexpected status code: %d", resp.StatusCode) } body, err := ioutil.ReadAll(resp.Body) var retrievedManifest schema1.Manifest if err := json.Unmarshal(body, &retrievedManifest); err != nil { t.Fatalf("error unmarshaling retrieved manifest") } if retrievedManifest.Name != fmt.Sprintf("%s/%s", testutil.Namespace(), stream.Name) { t.Fatalf("unexpected manifest name: %s", retrievedManifest.Name) } if retrievedManifest.Tag != imageapi.DefaultImageTag { t.Fatalf("unexpected manifest tag: %s", retrievedManifest.Tag) } // test get by digest url = fmt.Sprintf("http://127.0.0.1:5000/v2/%s/%s/manifests/%s", testutil.Namespace(), stream.Name, dgst.String()) req, err = http.NewRequest("GET", url, nil) if err != nil { t.Fatalf("error creating request: %v", err) } req.SetBasicAuth(user, token) resp, err = http.DefaultClient.Do(req) if err != nil { t.Fatalf("error retrieving manifest from registry: %s", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { t.Fatalf("unexpected status code: %d", resp.StatusCode) } body, err = ioutil.ReadAll(resp.Body) if err := json.Unmarshal(body, &retrievedManifest); err != nil { t.Fatalf("error unmarshaling retrieved manifest") } if retrievedManifest.Name != fmt.Sprintf("%s/%s", testutil.Namespace(), stream.Name) { t.Fatalf("unexpected manifest name: %s", retrievedManifest.Name) } if retrievedManifest.Tag != imageapi.DefaultImageTag { t.Fatalf("unexpected manifest tag: %s", retrievedManifest.Tag) } image, err := adminClient.ImageStreamImages(testutil.Namespace()).Get(stream.Name, dgst.String()) if err != nil { t.Fatalf("error getting imageStreamImage: %s", err) } if e, a := fmt.Sprintf("test@%s", dgst.String()), image.Name; e != a { t.Errorf("image name: expected %q, got %q", e, a) } if e, a := dgst.String(), image.Image.Name; e != a { t.Errorf("image name: expected %q, got %q", e, a) } if e, a := fmt.Sprintf("127.0.0.1:5000/%s/%s@%s", testutil.Namespace(), stream.Name, dgst.String()), image.Image.DockerImageReference; e != a { t.Errorf("image dockerImageReference: expected %q, got %q", e, a) } if e, a := "foo", image.Image.DockerImageMetadata.ID; e != a { t.Errorf("image dockerImageMetadata.ID: expected %q, got %q", e, a) } // test auto provisioning otherStream, err := adminClient.ImageStreams(testutil.Namespace()).Get("otherrepo") t.Logf("otherStream=%#v, err=%v", otherStream, err) if err == nil { t.Fatalf("expected error getting otherrepo") } otherDigest, err := putManifest("otherrepo", user, token) if err != nil { t.Fatal(err) } otherStream, err = adminClient.ImageStreams(testutil.Namespace()).Get("otherrepo") if err != nil { t.Fatalf("unexpected error getting otherrepo: %s", err) } if otherStream == nil { t.Fatalf("unexpected nil otherrepo") } if len(otherStream.Status.Tags) != 1 { t.Errorf("expected 1 tag, got %#v", otherStream.Status.Tags) } history, ok := otherStream.Status.Tags[imageapi.DefaultImageTag] if !ok { t.Fatal("unable to find 'latest' tag") } if len(history.Items) != 1 { t.Errorf("expected 1 tag event, got %#v", history.Items) } if e, a := otherDigest.String(), history.Items[0].Image; e != a { t.Errorf("digest: expected %q, got %q", e, a) } }
// Start starts the OpenShift master as a Docker container // and returns a directory in the local file system where // the OpenShift configuration has been copied func (h *Helper) Start(opt *StartOptions, out io.Writer) (string, error) { // Ensure that socat is available locally if opt.PortForwarding { err := CheckSocat() if err != nil { return "", err } } binds := openShiftContainerBinds env := []string{} if opt.UseSharedVolume { binds = append(binds, fmt.Sprintf("%[1]s:%[1]s:shared", opt.HostVolumesDir)) env = append(env, "OPENSHIFT_CONTAINERIZED=false") } else { binds = append(binds, "/:/rootfs:ro") propagationMode := "" if opt.SetPropagationMode { propagationMode = ":rslave" } binds = append(binds, fmt.Sprintf("%[1]s:%[1]s%[2]s", opt.HostVolumesDir, propagationMode)) } env = append(env, opt.Environment...) binds = append(binds, fmt.Sprintf("%s:/var/lib/origin/openshift.local.config:z", opt.HostConfigDir)) // Check if a configuration exists before creating one if UseExistingConfig // was specified var configDir string cleanupConfig := func() { errors.LogError(os.RemoveAll(configDir)) } skipCreateConfig := false if opt.UseExistingConfig { var err error configDir, err = h.copyConfig(opt.HostConfigDir) if err == nil { _, err = os.Stat(filepath.Join(configDir, "master", "master-config.yaml")) if err == nil { skipCreateConfig = true } } } // Create configuration if needed var nodeHost string if !skipCreateConfig { glog.V(1).Infof("Creating openshift configuration at %s on Docker host", opt.HostConfigDir) fmt.Fprintf(out, "Creating initial OpenShift configuration\n") createConfigCmd := []string{ "start", fmt.Sprintf("--images=%s", opt.Images), fmt.Sprintf("--volume-dir=%s", opt.HostVolumesDir), fmt.Sprintf("--dns=0.0.0.0:%d", opt.DNSPort), "--write-config=/var/lib/origin/openshift.local.config", } if opt.PortForwarding { internalIP, err := h.ServerIP() if err != nil { return "", err } nodeHost = internalIP createConfigCmd = append(createConfigCmd, fmt.Sprintf("--master=%s", internalIP)) createConfigCmd = append(createConfigCmd, fmt.Sprintf("--public-master=https://%s:8443", opt.ServerIP)) } else { nodeHost = opt.ServerIP createConfigCmd = append(createConfigCmd, fmt.Sprintf("--master=%s", opt.ServerIP)) if len(h.publicHost) > 0 { createConfigCmd = append(createConfigCmd, fmt.Sprintf("--public-master=https://%s:8443", h.publicHost)) } } createConfigCmd = append(createConfigCmd, fmt.Sprintf("--hostname=%s", nodeHost)) _, err := h.runHelper.New().Image(h.image). Privileged(). DiscardContainer(). HostNetwork(). HostPid(). Bind(binds...). Env(env...). Command(createConfigCmd...).Run() if err != nil { return "", errors.NewError("could not create OpenShift configuration").WithCause(err) } configDir, err = h.copyConfig(opt.HostConfigDir) if err != nil { return "", errors.NewError("could not copy OpenShift configuration").WithCause(err) } err = h.updateConfig(configDir, opt.HostConfigDir, opt.RouterIP, opt.MetricsHost, opt.LoggingHost) if err != nil { cleanupConfig() return "", errors.NewError("could not update OpenShift configuration").WithCause(err) } } if nodeHost == "" { if opt.PortForwarding { var err error nodeHost, err = h.ServerIP() if err != nil { return "", err } } else { var err error hostName, err := h.hostHelper.Hostname() if err != nil { return "", err } nodeHost, err = h.DetermineNodeHost(opt.HostConfigDir, opt.ServerIP, hostName) if err != nil { return "", err } } } masterConfig, nodeConfig, err := h.getOpenShiftConfigFiles(nodeHost) if err != nil { cleanupConfig() return "", errors.NewError("could not get OpenShift configuration file paths").WithCause(err) } fmt.Fprintf(out, "Starting OpenShift using container '%s'\n", h.containerName) startCmd := []string{ "start", fmt.Sprintf("--master-config=%s", masterConfig), fmt.Sprintf("--node-config=%s", nodeConfig), } if opt.LogLevel > 0 { startCmd = append(startCmd, fmt.Sprintf("--loglevel=%d", opt.LogLevel)) } if opt.PortForwarding { err = h.startSocatTunnel() if err != nil { return "", err } } if len(opt.HostDataDir) > 0 { binds = append(binds, fmt.Sprintf("%s:/var/lib/origin/openshift.local.etcd:z", opt.HostDataDir)) } _, err = h.runHelper.New().Image(h.image). Name(h.containerName). Privileged(). HostNetwork(). HostPid(). Bind(binds...). Env(env...). Command(startCmd...). Start() if err != nil { return "", errors.NewError("cannot start OpenShift daemon").WithCause(err) } // Wait a minimum amount of time and check whether we're still running. If not, we know the daemon didn't start time.Sleep(initialStatusCheckWait) _, running, err := h.dockerHelper.GetContainerState(h.containerName) if err != nil { return "", errors.NewError("cannot get state of OpenShift container %s", h.containerName).WithCause(err) } if !running { return "", ErrOpenShiftFailedToStart(h.containerName).WithDetails(h.OriginLog()) } // Wait until the API server is listening fmt.Fprintf(out, "Waiting for API server to start listening\n") masterHost := fmt.Sprintf("%s:8443", opt.ServerIP) if err = cmdutil.WaitForSuccessfulDial(true, "tcp", masterHost, 200*time.Millisecond, 1*time.Second, serverUpTimeout); err != nil { return "", ErrTimedOutWaitingForStart(h.containerName).WithDetails(h.OriginLog()) } // Check for healthz endpoint to be ready client, err := masterHTTPClient(configDir) if err != nil { return "", err } for { resp, ierr := client.Get(h.healthzReadyURL(opt.ServerIP)) if ierr != nil { return "", errors.NewError("cannot access master readiness URL %s", h.healthzReadyURL(opt.ServerIP)).WithCause(err).WithDetails(h.OriginLog()) } if resp.StatusCode == http.StatusOK { break } if resp.StatusCode == http.StatusServiceUnavailable || resp.StatusCode == http.StatusForbidden { time.Sleep(500 * time.Millisecond) continue } var responseBody string body, rerr := ioutil.ReadAll(resp.Body) if rerr == nil { responseBody = string(body) } return "", errors.NewError("server is not ready. Response (%d): %s", resp.StatusCode, responseBody).WithCause(ierr).WithDetails(h.OriginLog()) } fmt.Fprintf(out, "OpenShift server started\n") return configDir, nil }
// Run launches the OpenShift master. It takes optional installers that may install additional endpoints into the server. // All endpoints get configured CORS behavior // Protected installers' endpoints are protected by API authentication and authorization. // Unprotected installers' endpoints do not have any additional protection added. func (c *MasterConfig) Run(protected []APIInstaller, unprotected []APIInstaller) { var extra []string safe := genericapiserver.NewHandlerContainer(http.NewServeMux(), kapi.Codecs) open := genericapiserver.NewHandlerContainer(http.NewServeMux(), kapi.Codecs) // enforce authentication on protected endpoints protected = append(protected, APIInstallFunc(c.InstallProtectedAPI)) for _, i := range protected { msgs, err := i.InstallAPI(safe) if err != nil { glog.Fatalf("error installing api %v", err) } extra = append(extra, msgs...) } handler := c.versionSkewFilter(safe) handler = c.authorizationFilter(handler) handler = c.impersonationFilter(handler) // audit handler must comes before the impersonationFilter to read the original user handler = c.auditHandler(handler) handler = authenticationHandlerFilter(handler, c.Authenticator, c.getRequestContextMapper()) handler = namespacingFilter(handler, c.getRequestContextMapper()) handler = cacheControlFilter(handler, "no-store") // protected endpoints should not be cached // unprotected resources unprotected = append(unprotected, APIInstallFunc(c.InstallUnprotectedAPI)) for _, i := range unprotected { msgs, err := i.InstallAPI(open) if err != nil { glog.Fatalf("error installing api %v", err) } extra = append(extra, msgs...) } var kubeAPILevels []string if c.Options.KubernetesMasterConfig != nil { kubeAPILevels = configapi.GetEnabledAPIVersionsForGroup(*c.Options.KubernetesMasterConfig, kapi.GroupName) } handler = indexAPIPaths(c.Options.APILevels, kubeAPILevels, handler) open.Handle("/", handler) // install swagger swaggerConfig := swagger.Config{ WebServicesUrl: c.Options.MasterPublicURL, WebServices: append(safe.RegisteredWebServices(), open.RegisteredWebServices()...), ApiPath: swaggerAPIPrefix, PostBuildHandler: customizeSwaggerDefinition, } // log nothing from swagger swagger.LogInfo = func(format string, v ...interface{}) {} swagger.RegisterSwaggerService(swaggerConfig, open) extra = append(extra, fmt.Sprintf("Started Swagger Schema API at %%s%s", swaggerAPIPrefix)) openAPIConfig := openapi.Config{ SwaggerConfig: &swaggerConfig, IgnorePrefixes: []string{"/swaggerapi"}, Info: &spec.Info{ InfoProps: spec.InfoProps{ Title: "OpenShift API (with Kubernetes)", Version: version.Get().String(), License: &spec.License{ Name: "Apache 2.0 (ASL2.0)", URL: "http://www.apache.org/licenses/LICENSE-2.0", }, Description: heredoc.Doc(` OpenShift provides builds, application lifecycle, image content management, and administrative policy on top of Kubernetes. The API allows consistent management of those objects. All API operations are authenticated via an Authorization bearer token that is provided for service accounts as a generated secret (in JWT form) or via the native OAuth endpoint located at /oauth/authorize. Core infrastructure components may use client certificates that require no authentication. All API operations return a 'resourceVersion' string that represents the version of the object in the underlying storage. The standard LIST operation performs a snapshot read of the underlying objects, returning a resourceVersion representing a consistent version of the listed objects. The WATCH operation allows all updates to a set of objects after the provided resourceVersion to be observed by a client. By listing and beginning a watch from the returned resourceVersion, clients may observe a consistent view of the state of one or more objects. Note that WATCH always returns the update after the provided resourceVersion. Watch may be extended a limited time in the past - using etcd 2 the watch window is 1000 events (which on a large cluster may only be a few tens of seconds) so clients must explicitly handle the "watch to old error" by re-listing. Objects are divided into two rough categories - those that have a lifecycle and must reflect the state of the cluster, and those that have no state. Objects with lifecycle typically have three main sections: * 'metadata' common to all objects * a 'spec' that represents the desired state * a 'status' that represents how much of the desired state is reflected on the cluster at the current time Objects that have no state have 'metadata' but may lack a 'spec' or 'status' section. Objects are divided into those that are namespace scoped (only exist inside of a namespace) and those that are cluster scoped (exist outside of a namespace). A namespace scoped resource will be deleted when the namespace is deleted and cannot be created if the namespace has not yet been created or is in the process of deletion. Cluster scoped resources are typically only accessible to admins - resources like nodes, persistent volumes, and cluster policy. All objects have a schema that is a combination of the 'kind' and 'apiVersion' fields. This schema is additive only for any given version - no backwards incompatible changes are allowed without incrementing the apiVersion. The server will return and accept a number of standard responses that share a common schema - for instance, the common error type is 'unversioned.Status' (described below) and will be returned on any error from the API server. The API is available in multiple serialization formats - the default is JSON (Accept: application/json and Content-Type: application/json) but clients may also use YAML (application/yaml) or the native Protobuf schema (application/vnd.kubernetes.protobuf). Note that the format of the WATCH API call is slightly different - for JSON it returns newline delimited objects while for Protobuf it returns length-delimited frames (4 bytes in network-order) that contain a 'versioned.Watch' Protobuf object. See the OpenShift documentation at https://docs.openshift.org for more information. `), }, }, DefaultResponse: &spec.Response{ ResponseProps: spec.ResponseProps{ Description: "Default Response.", }, }, } err := openapi.RegisterOpenAPIService(&openAPIConfig, open) if err != nil { glog.Fatalf("Failed to generate open api spec: %v", err) } extra = append(extra, fmt.Sprintf("Started OpenAPI Schema at %%s%s", openapi.OpenAPIServePath)) handler = open // add CORS support if origins := c.ensureCORSAllowedOrigins(); len(origins) != 0 { handler = apiserver.CORS(handler, origins, nil, nil, "true") } if c.WebConsoleEnabled() { handler = assetServerRedirect(handler, c.Options.AssetConfig.PublicURL) } // Make the outermost filter the requestContextMapper to ensure all components share the same context if contextHandler, err := kapi.NewRequestContextFilter(c.getRequestContextMapper(), handler); err != nil { glog.Fatalf("Error setting up request context filter: %v", err) } else { handler = contextHandler } longRunningRequestCheck := apiserver.BasicLongRunningRequestCheck(longRunningRE, map[string]string{"watch": "true"}) // TODO: MaxRequestsInFlight should be subdivided by intent, type of behavior, and speed of // execution - updates vs reads, long reads vs short reads, fat reads vs skinny reads. if c.Options.ServingInfo.MaxRequestsInFlight > 0 { sem := make(chan bool, c.Options.ServingInfo.MaxRequestsInFlight) handler = apiserver.MaxInFlightLimit(sem, longRunningRequestCheck, handler) } c.serve(handler, extra) // Attempt to verify the server came up for 20 seconds (100 tries * 100ms, 100ms timeout per try) cmdutil.WaitForSuccessfulDial(c.TLS, c.Options.ServingInfo.BindNetwork, c.Options.ServingInfo.BindAddress, 100*time.Millisecond, 100*time.Millisecond, 100) }