// ListenAndServe creates a listener and serves handler on it, closing the // listener when signalled by the stopper. The handling server implements HTTP1 // and HTTP2, with or without TLS. Note that the "real" server also implements // the postgres wire protocol, and so does not use this function, but the // pattern used is similar; that implementation is in server/server.go. func ListenAndServe(stopper *stop.Stopper, handler http.Handler, addr net.Addr, tlsConfig *tls.Config) (net.Listener, error) { ln, err := net.Listen(addr.Network(), addr.String()) if err != nil { return ln, err } stopper.RunWorker(func() { <-stopper.ShouldDrain() // Some unit tests manually close `ln`, so it may already be closed // when we get here. FatalIfUnexpected(ln.Close()) }) if tlsConfig != nil { // We're in TLS mode. ALPN will be used to automatically handle HTTP1 and // HTTP2 requests. ServeHandler(stopper, handler, tls.NewListener(ln, tlsConfig), tlsConfig) } else { // We're not in TLS mode. We're going to implement h2c (HTTP2 Clear Text) // ourselves. m := cmux.New(ln) // HTTP2 connections are easy to identify because they have a common // preface. h2L := m.Match(cmux.HTTP2()) // All other connections will get the default treatment. anyL := m.Match(cmux.Any()) // Construct our h2c handler function. var h2 http2.Server serveConnOpts := &http2.ServeConnOpts{ Handler: handler, } serveH2 := func(conn net.Conn) { h2.ServeConn(conn, serveConnOpts) } // Start serving HTTP1 on all non-HTTP2 connections. serveConn := ServeHandler(stopper, handler, anyL, tlsConfig) // Start serving h2c on all HTTP2 connections. stopper.RunWorker(func() { FatalIfUnexpected(serveConn(h2L, serveH2)) }) // Finally start the multiplexing listener. stopper.RunWorker(func() { FatalIfUnexpected(m.Serve()) }) } return ln, nil }
// serve accepts incoming connections on the listener l, // creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. func serve(sctx *serveCtx, s *etcdserver.EtcdServer, tlscfg *tls.Config, handler http.Handler) error { logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0) <-s.ReadyNotify() plog.Info("ready to serve client requests") m := cmux.New(sctx.l) if sctx.insecure { gs := v3rpc.Server(s, nil) grpcl := m.Match(cmux.HTTP2()) go func() { plog.Fatal(gs.Serve(grpcl)) }() srvhttp := &http.Server{ Handler: handler, ErrorLog: logger, // do not log user error } httpl := m.Match(cmux.HTTP1()) go func() { plog.Fatal(srvhttp.Serve(httpl)) }() plog.Noticef("serving insecure client requests on %s, this is strongly discouraged!", sctx.host) } if sctx.secure { gs := v3rpc.Server(s, tlscfg) handler = grpcHandlerFunc(gs, handler) tlsl := tls.NewListener(m.Match(cmux.Any()), tlscfg) // TODO: add debug flag; enable logging when debug flag is set srv := &http.Server{ Handler: handler, TLSConfig: tlscfg, ErrorLog: logger, // do not log user error } go func() { plog.Fatal(srv.Serve(tlsl)) }() plog.Infof("serving client requests on %s", sctx.host) } return m.Serve() }
// serve accepts incoming connections on the listener l, // creating a new service goroutine for each. The service goroutines // read requests and then call handler to reply to them. func serve(sctx *serveCtx, s *etcdserver.EtcdServer, tlscfg *tls.Config, handler http.Handler) error { logger := defaultLog.New(ioutil.Discard, "etcdhttp", 0) <-s.ReadyNotify() plog.Info("ready to serve client requests") m := cmux.New(sctx.l) if sctx.insecure { gs := v3rpc.Server(s, nil) grpcl := m.Match(cmux.HTTP2()) go func() { plog.Fatal(gs.Serve(grpcl)) }() opts := []grpc.DialOption{ grpc.WithInsecure(), } gwmux, err := registerGateway(sctx.l.Addr().String(), opts) if err != nil { return err } httpmux := http.NewServeMux() httpmux.Handle("/v3alpha/", gwmux) httpmux.Handle("/", handler) srvhttp := &http.Server{ Handler: httpmux, ErrorLog: logger, // do not log user error } httpl := m.Match(cmux.HTTP1()) go func() { plog.Fatal(srvhttp.Serve(httpl)) }() plog.Noticef("serving insecure client requests on %s, this is strongly discouraged!", sctx.host) } if sctx.secure { gs := v3rpc.Server(s, tlscfg) handler = grpcHandlerFunc(gs, handler) dtls := *tlscfg // trust local server dtls.InsecureSkipVerify = true creds := credentials.NewTLS(&dtls) opts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} gwmux, err := registerGateway(sctx.l.Addr().String(), opts) if err != nil { return err } tlsl := tls.NewListener(m.Match(cmux.Any()), tlscfg) // TODO: add debug flag; enable logging when debug flag is set httpmux := http.NewServeMux() httpmux.Handle("/v3alpha/", gwmux) httpmux.Handle("/", handler) srv := &http.Server{ Handler: httpmux, TLSConfig: tlscfg, ErrorLog: logger, // do not log user error } go func() { plog.Fatal(srv.Serve(tlsl)) }() plog.Infof("serving client requests on %s", sctx.host) } return m.Serve() }
// Start starts the server on the specified port, starts gossip and // initializes the node using the engines from the server's context. func (s *Server) Start() error { s.initHTTP() tlsConfig, err := s.ctx.GetServerTLSConfig() if err != nil { return err } // The following code is a specialization of util/net.go's ListenAndServe // which adds pgwire support. A single port is used to serve all protocols // (pg, http, h2) via the following construction: // // non-TLS case: // net.Listen -> cmux.New -> pgwire.Match -> pgwire.Server.ServeConn // | // - -> cmux.HTTP2 -> http2.(*Server).ServeConn // - -> cmux.Any -> http.(*Server).Serve // // TLS case: // net.Listen -> cmux.New -> pgwire.Match -> pgwire.Server.ServeConn // | // - -> cmux.Any -> tls.NewListener -> http.(*Server).Serve // // Note that the difference between the TLS and non-TLS cases exists due to // Go's lack of an h2c (HTTP2 Clear Text) implementation. See inline comments // in util.ListenAndServe for an explanation of how h2c is implemented there // and here. ln, err := net.Listen("tcp", s.ctx.Addr) if err != nil { return err } unresolvedAddr, err := officialAddr(s.ctx.Addr, ln.Addr()) if err != nil { return err } s.ctx.Addr = unresolvedAddr.String() s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := ln.Close(); err != nil { log.Fatal(err) } }) m := cmux.New(ln) pgL := m.Match(pgwire.Match) var serveConn func(net.Listener, func(net.Conn)) error if tlsConfig != nil { anyL := m.Match(cmux.Any()) serveConn = util.ServeHandler(s.stopper, s, tls.NewListener(anyL, tlsConfig), tlsConfig) } else { h2L := m.Match(cmux.HTTP2()) anyL := m.Match(cmux.Any()) var h2 http2.Server serveConnOpts := &http2.ServeConnOpts{ Handler: s, } serveH2 := func(conn net.Conn) { h2.ServeConn(conn, serveConnOpts) } serveConn = util.ServeHandler(s.stopper, s, anyL, tlsConfig) s.stopper.RunWorker(func() { util.FatalIfUnexpected(serveConn(h2L, serveH2)) }) } s.stopper.RunWorker(func() { util.FatalIfUnexpected(serveConn(pgL, func(conn net.Conn) { if err := s.pgServer.ServeConn(conn); err != nil && !util.IsClosedConnection(err) { log.Error(err) } })) }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(m.Serve()) }) s.rpcContext.SetLocalServer(s.rpc, s.ctx.Addr) s.gossip.Start(s.grpc, unresolvedAddr) if err := s.node.start(s.rpc, unresolvedAddr, s.ctx.Engines, s.ctx.NodeAttributes); err != nil { return err } // Begin recording runtime statistics. runtime := status.NewRuntimeStatRecorder(s.node.Descriptor.NodeID, s.clock) s.tsDB.PollSource(runtime, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper) // Begin recording time series data collected by the status monitor. s.tsDB.PollSource(s.recorder, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper) // Begin recording status summaries. s.node.startWriteSummaries(s.ctx.MetricsFrequency) s.sqlExecutor.SetNodeID(s.node.Descriptor.NodeID) // Create and start the schema change manager only after a NodeID // has been assigned. s.schemaChangeManager = sql.NewSchemaChangeManager(*s.db, s.gossip, s.leaseMgr) s.schemaChangeManager.Start(s.stopper) log.Infof("starting %s/postgres server at %s", s.ctx.HTTPRequestScheme(), unresolvedAddr) return nil }
func startGRPCProxy(cmd *cobra.Command, args []string) { l, err := net.Listen("tcp", grpcProxyListenAddr) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } if l, err = transport.NewKeepAliveListener(l, "tcp", nil); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } plog.Infof("listening for grpc-proxy client requests on %s", grpcProxyListenAddr) defer func() { l.Close() plog.Infof("stopping listening for grpc-proxy client requests on %s", grpcProxyListenAddr) }() m := cmux.New(l) cfg, err := newClientCfg() if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } client, err := clientv3.New(*cfg) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } kvp := grpcproxy.NewKvProxy(client) watchp := grpcproxy.NewWatchProxy(client) clusterp := grpcproxy.NewClusterProxy(client) leasep := grpcproxy.NewLeaseProxy(client) mainp := grpcproxy.NewMaintenanceProxy(client) authp := grpcproxy.NewAuthProxy(client) server := grpc.NewServer( grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), ) pb.RegisterKVServer(server, kvp) pb.RegisterWatchServer(server, watchp) pb.RegisterClusterServer(server, clusterp) pb.RegisterLeaseServer(server, leasep) pb.RegisterMaintenanceServer(server, mainp) pb.RegisterAuthServer(server, authp) errc := make(chan error) grpcl := m.Match(cmux.HTTP2()) go func() { errc <- server.Serve(grpcl) }() httpmux := http.NewServeMux() httpmux.HandleFunc("/", http.NotFound) httpmux.Handle("/metrics", prometheus.Handler()) srvhttp := &http.Server{ Handler: httpmux, } var httpl net.Listener if cfg.TLS != nil { srvhttp.TLSConfig = cfg.TLS httpl = tls.NewListener(m.Match(cmux.Any()), cfg.TLS) } else { httpl = m.Match(cmux.HTTP1()) } go func() { errc <- srvhttp.Serve(httpl) }() go func() { errc <- m.Serve() }() fmt.Fprintln(os.Stderr, <-errc) os.Exit(1) }