// IsClosedConnection returns true if err is an error produced by gRPC on closed connections. func IsClosedConnection(err error) bool { if err == context.Canceled || err == transport.ErrConnClosing || grpc.Code(err) == codes.Canceled { return true } if streamErr, ok := err.(transport.StreamError); ok && streamErr.Code == codes.Canceled { return true } return util.IsClosedConnection(err) }
// IsClosedConnection returns true if err is an error produced by gRPC on closed connections. func IsClosedConnection(err error) bool { if err == context.Canceled || grpc.Code(err) == codes.Canceled || grpc.ErrorDesc(err) == grpc.ErrClientConnClosing.Error() || strings.Contains(err.Error(), "is closing") { return true } if streamErr, ok := err.(transport.StreamError); ok && streamErr.Code == codes.Canceled { return true } return util.IsClosedConnection(err) }
// readRequests synchronously reads a stream of requests from a // connection. Each request is handled in a new background goroutine; // when the handler finishes the response is written to the responses // channel. When the connection is closed (and any pending requests // have finished), we close the responses channel. func (s *Server) readRequests(conn net.Conn, codec rpc.ServerCodec, authHook func(proto.Message, bool) error, responses chan<- serverResponse) { var wg sync.WaitGroup var closed bool defer func() { wg.Wait() if !closed { s.runCloseCallbacks(conn) } close(responses) }() for { req, meth, args, err := s.readRequest(codec) if err != nil { if err == io.EOF || err == io.ErrUnexpectedEOF || util.IsClosedConnection(err) { closed = true s.runCloseCallbacks(conn) return } log.Warningf("rpc: server cannot decode request: %s", err) return } if meth.handler == nil { responses <- serverResponse{ req: req, err: util.Errorf("rpc: couldn't find method: %s", req.ServiceMethod), } continue } if err := authHook(args, meth.public); err != nil { responses <- serverResponse{ req: req, err: err, } // We got an unauthorized request. For now, leave the connection // open. We may want to close it in the future because security. continue } wg.Add(1) meth.handler(args, func(reply proto.Message, err error) { responses <- serverResponse{ req: req, reply: reply, err: err, } wg.Done() }) } }
// ListenAndServeGRPC creates a listener and serves server on it, closing // the listener when signalled by the stopper. func ListenAndServeGRPC(stopper *stop.Stopper, server *grpc.Server, addr net.Addr, config *tls.Config) (net.Listener, error) { ln, err := util.Listen(addr, config) if err != nil { return nil, err } stopper.RunWorker(func() { if err := server.Serve(ln); err != nil && !util.IsClosedConnection(err) { log.Fatal(err) } }) stopper.RunWorker(func() { <-stopper.ShouldDrain() if err := ln.Close(); err != nil { log.Fatal(err) } }) return ln, nil }
// Start starts the server on the specified port, starts gossip and // initializes the node using the engines from the server's context. func (s *Server) Start() error { s.initHTTP() tlsConfig, err := s.ctx.GetServerTLSConfig() if err != nil { return err } // The following code is a specialization of util/net.go's ListenAndServe // which adds pgwire support. A single port is used to serve all protocols // (pg, http, h2) via the following construction: // // non-TLS case: // net.Listen -> cmux.New -> pgwire.Match -> pgwire.Server.ServeConn // | // - -> cmux.HTTP2 -> http2.(*Server).ServeConn // - -> cmux.Any -> http.(*Server).Serve // // TLS case: // net.Listen -> cmux.New -> pgwire.Match -> pgwire.Server.ServeConn // | // - -> cmux.Any -> tls.NewListener -> http.(*Server).Serve // // Note that the difference between the TLS and non-TLS cases exists due to // Go's lack of an h2c (HTTP2 Clear Text) implementation. See inline comments // in util.ListenAndServe for an explanation of how h2c is implemented there // and here. ln, err := net.Listen("tcp", s.ctx.Addr) if err != nil { return err } unresolvedAddr, err := officialAddr(s.ctx.Addr, ln.Addr()) if err != nil { return err } s.ctx.Addr = unresolvedAddr.String() s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := ln.Close(); err != nil { log.Fatal(err) } }) m := cmux.New(ln) pgL := m.Match(pgwire.Match) var serveConn func(net.Listener, func(net.Conn)) error if tlsConfig != nil { anyL := m.Match(cmux.Any()) serveConn = util.ServeHandler(s.stopper, s, tls.NewListener(anyL, tlsConfig), tlsConfig) } else { h2L := m.Match(cmux.HTTP2()) anyL := m.Match(cmux.Any()) var h2 http2.Server serveConnOpts := &http2.ServeConnOpts{ Handler: s, } serveH2 := func(conn net.Conn) { h2.ServeConn(conn, serveConnOpts) } serveConn = util.ServeHandler(s.stopper, s, anyL, tlsConfig) s.stopper.RunWorker(func() { util.FatalIfUnexpected(serveConn(h2L, serveH2)) }) } s.stopper.RunWorker(func() { util.FatalIfUnexpected(serveConn(pgL, func(conn net.Conn) { if err := s.pgServer.ServeConn(conn); err != nil && !util.IsClosedConnection(err) { log.Error(err) } })) }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(m.Serve()) }) s.rpcContext.SetLocalServer(s.rpc, s.ctx.Addr) s.gossip.Start(s.grpc, unresolvedAddr) if err := s.node.start(s.rpc, unresolvedAddr, s.ctx.Engines, s.ctx.NodeAttributes); err != nil { return err } // Begin recording runtime statistics. runtime := status.NewRuntimeStatRecorder(s.node.Descriptor.NodeID, s.clock) s.tsDB.PollSource(runtime, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper) // Begin recording time series data collected by the status monitor. s.tsDB.PollSource(s.recorder, s.ctx.MetricsFrequency, ts.Resolution10s, s.stopper) // Begin recording status summaries. s.node.startWriteSummaries(s.ctx.MetricsFrequency) s.sqlExecutor.SetNodeID(s.node.Descriptor.NodeID) // Create and start the schema change manager only after a NodeID // has been assigned. s.schemaChangeManager = sql.NewSchemaChangeManager(*s.db, s.gossip, s.leaseMgr) s.schemaChangeManager.Start(s.stopper) log.Infof("starting %s/postgres server at %s", s.ctx.HTTPRequestScheme(), unresolvedAddr) return nil }
// Start starts the server on the specified port, starts gossip and // initializes the node using the engines from the server's context. func (s *Server) Start() error { s.initHTTP() tlsConfig, err := s.ctx.GetServerTLSConfig() if err != nil { return err } // The following code is a specialization of util/net.go's ListenAndServe // which adds pgwire support. A single port is used to serve all protocols // (pg, http, h2) via the following construction: // // non-TLS case: // net.Listen -> cmux.New // | // - -> pgwire.Match -> pgwire.Server.ServeConn // - -> cmux.Any -> grpc.(*Server).Serve // // TLS case: // net.Listen -> cmux.New // | // - -> pgwire.Match -> pgwire.Server.ServeConn // - -> cmux.Any -> grpc.(*Server).Serve // // Note that the difference between the TLS and non-TLS cases exists due to // Go's lack of an h2c (HTTP2 Clear Text) implementation. See inline comments // in util.ListenAndServe for an explanation of how h2c is implemented there // and here. ln, err := net.Listen("tcp", s.ctx.Addr) if err != nil { return err } unresolvedAddr, err := officialAddr(s.ctx.Addr, ln.Addr()) if err != nil { return err } s.ctx.Addr = unresolvedAddr.String() s.rpcContext.SetLocalInternalServer(s.node, s.ctx.Addr) s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := ln.Close(); err != nil { log.Fatal(err) } }) m := cmux.New(ln) pgL := m.Match(pgwire.Match) anyL := m.Match(cmux.Any()) httpLn, err := net.Listen("tcp", s.ctx.HTTPAddr) if err != nil { return err } unresolvedHTTPAddr, err := officialAddr(s.ctx.HTTPAddr, httpLn.Addr()) if err != nil { return err } s.ctx.HTTPAddr = unresolvedHTTPAddr.String() s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := httpLn.Close(); err != nil { log.Fatal(err) } }) if tlsConfig != nil { httpMux := cmux.New(httpLn) clearL := httpMux.Match(cmux.HTTP1Fast()) tlsL := httpMux.Match(cmux.Any()) s.stopper.RunWorker(func() { util.FatalIfUnexpected(httpMux.Serve()) }) util.ServeHandler(s.stopper, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // TODO(tamird): s/308/http.StatusPermanentRedirect/ when it exists. http.Redirect(w, r, "https://"+r.Host+r.RequestURI, 308) }), clearL, tlsConfig) httpLn = tls.NewListener(tlsL, tlsConfig) } serveConn := util.ServeHandler(s.stopper, s, httpLn, tlsConfig) s.stopper.RunWorker(func() { util.FatalIfUnexpected(s.grpc.Serve(anyL)) }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(serveConn(pgL, func(conn net.Conn) { if err := s.pgServer.ServeConn(conn); err != nil && !util.IsClosedConnection(err) { log.Error(err) } })) }) if len(s.ctx.SocketFile) != 0 { // Unix socket enabled: postgres protocol only. unixLn, err := net.Listen("unix", s.ctx.SocketFile) if err != nil { return err } s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := unixLn.Close(); err != nil { log.Fatal(err) } }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(serveConn(unixLn, func(conn net.Conn) { if err := s.pgServer.ServeConn(conn); err != nil && !util.IsClosedConnection(err) { log.Error(err) } })) }) } s.gossip.Start(s.grpc, unresolvedAddr) if err := s.node.start(unresolvedAddr, s.ctx.Engines, s.ctx.NodeAttributes); err != nil { return err } // Begin recording runtime statistics. s.startSampleEnvironment(s.ctx.MetricsSampleInterval) // Begin recording time series data collected by the status monitor. s.tsDB.PollSource(s.recorder, s.ctx.MetricsSampleInterval, ts.Resolution10s, s.stopper) // Begin recording status summaries. s.node.startWriteSummaries(s.ctx.MetricsSampleInterval) s.sqlExecutor.SetNodeID(s.node.Descriptor.NodeID) // Create and start the schema change manager only after a NodeID // has been assigned. s.schemaChangeManager = sql.NewSchemaChangeManager(*s.db, s.gossip, s.leaseMgr) s.schemaChangeManager.Start(s.stopper) s.periodicallyCheckForUpdates() log.Infof("starting %s server at %s", s.ctx.HTTPRequestScheme(), unresolvedHTTPAddr) log.Infof("starting grpc/postgres server at %s", unresolvedAddr) if len(s.ctx.SocketFile) != 0 { log.Infof("starting postgres server at unix:%s", s.ctx.SocketFile) } s.stopper.RunWorker(func() { util.FatalIfUnexpected(m.Serve()) }) // Register admin service. Must happen after serving starts. s.stopper.AddCloser(s.admin) RegisterAdminServer(s.grpc, s.admin) return s.admin.RegisterGRPCGateway(s.ctx) }
// Start starts the server on the specified port, starts gossip and // initializes the node using the engines from the server's context. func (s *Server) Start() error { tlsConfig, err := s.ctx.GetServerTLSConfig() if err != nil { return err } httpServer := util.MakeServer(s.stopper, tlsConfig, s) plainRedirectServer := util.MakeServer(s.stopper, tlsConfig, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // TODO(tamird): s/308/http.StatusPermanentRedirect/ when it exists. http.Redirect(w, r, "https://"+r.Host+r.RequestURI, 308) })) // The following code is a specialization of util/net.go's ListenAndServe // which adds pgwire support. A single port is used to serve all protocols // (pg, http, h2) via the following construction: // // non-TLS case: // net.Listen -> cmux.New // | // - -> pgwire.Match -> pgwire.Server.ServeConn // - -> cmux.Any -> grpc.(*Server).Serve // // TLS case: // net.Listen -> cmux.New // | // - -> pgwire.Match -> pgwire.Server.ServeConn // - -> cmux.Any -> grpc.(*Server).Serve // // Note that the difference between the TLS and non-TLS cases exists due to // Go's lack of an h2c (HTTP2 Clear Text) implementation. See inline comments // in util.ListenAndServe for an explanation of how h2c is implemented there // and here. ln, err := net.Listen("tcp", s.ctx.Addr) if err != nil { return err } unresolvedAddr, err := officialAddr(s.ctx.Addr, ln.Addr()) if err != nil { return err } s.ctx.Addr = unresolvedAddr.String() s.rpcContext.SetLocalInternalServer(s.node) s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := ln.Close(); err != nil { log.Fatal(err) } }) m := cmux.New(ln) pgL := m.Match(pgwire.Match) anyL := m.Match(cmux.Any()) httpLn, err := net.Listen("tcp", s.ctx.HTTPAddr) if err != nil { return err } unresolvedHTTPAddr, err := officialAddr(s.ctx.HTTPAddr, httpLn.Addr()) if err != nil { return err } s.ctx.HTTPAddr = unresolvedHTTPAddr.String() s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := httpLn.Close(); err != nil { log.Fatal(err) } }) if tlsConfig != nil { httpMux := cmux.New(httpLn) clearL := httpMux.Match(cmux.HTTP1Fast()) tlsL := httpMux.Match(cmux.Any()) s.stopper.RunWorker(func() { util.FatalIfUnexpected(httpMux.Serve()) }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(plainRedirectServer.Serve(clearL)) }) httpLn = tls.NewListener(tlsL, tlsConfig) } s.stopper.RunWorker(func() { util.FatalIfUnexpected(httpServer.Serve(httpLn)) }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(s.grpc.Serve(anyL)) }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(httpServer.ServeWith(pgL, func(conn net.Conn) { if err := s.pgServer.ServeConn(conn); err != nil && !util.IsClosedConnection(err) { log.Error(err) } })) }) if len(s.ctx.SocketFile) != 0 { // Unix socket enabled: postgres protocol only. unixLn, err := net.Listen("unix", s.ctx.SocketFile) if err != nil { return err } s.stopper.RunWorker(func() { <-s.stopper.ShouldDrain() if err := unixLn.Close(); err != nil { log.Fatal(err) } }) s.stopper.RunWorker(func() { util.FatalIfUnexpected(httpServer.ServeWith(unixLn, func(conn net.Conn) { if err := s.pgServer.ServeConn(conn); err != nil && !util.IsClosedConnection(err) { log.Error(err) } })) }) } s.gossip.Start(s.grpc, unresolvedAddr) if err := s.node.start(unresolvedAddr, s.ctx.Engines, s.ctx.NodeAttributes); err != nil { return err } // Begin recording runtime statistics. s.startSampleEnvironment(s.ctx.MetricsSampleInterval) // Begin recording time series data collected by the status monitor. s.tsDB.PollSource(s.recorder, s.ctx.MetricsSampleInterval, ts.Resolution10s, s.stopper) // Begin recording status summaries. s.node.startWriteSummaries(s.ctx.MetricsSampleInterval) s.sqlExecutor.SetNodeID(s.node.Descriptor.NodeID) // Create and start the schema change manager only after a NodeID // has been assigned. testingKnobs := new(sql.SchemaChangeManagerTestingKnobs) if s.ctx.TestingKnobs.SQLSchemaChangeManager != nil { testingKnobs = s.ctx.TestingKnobs.SQLSchemaChangeManager.(*sql.SchemaChangeManagerTestingKnobs) } sql.NewSchemaChangeManager(testingKnobs, *s.db, s.gossip, s.leaseMgr).Start(s.stopper) s.periodicallyCheckForUpdates() log.Infof("starting %s server at %s", s.ctx.HTTPRequestScheme(), unresolvedHTTPAddr) log.Infof("starting grpc/postgres server at %s", unresolvedAddr) if len(s.ctx.SocketFile) != 0 { log.Infof("starting postgres server at unix:%s", s.ctx.SocketFile) } s.stopper.RunWorker(func() { util.FatalIfUnexpected(m.Serve()) }) // Initialize grpc-gateway mux and context. jsonpb := &util.JSONPb{ EnumsAsInts: true, EmitDefaults: true, Indent: " ", } protopb := new(util.ProtoPb) gwMux := gwruntime.NewServeMux( gwruntime.WithMarshalerOption(gwruntime.MIMEWildcard, jsonpb), gwruntime.WithMarshalerOption(util.JSONContentType, jsonpb), gwruntime.WithMarshalerOption(util.AltJSONContentType, jsonpb), gwruntime.WithMarshalerOption(util.ProtoContentType, protopb), gwruntime.WithMarshalerOption(util.AltProtoContentType, protopb), ) gwCtx, gwCancel := context.WithCancel(context.Background()) s.stopper.AddCloser(stop.CloserFn(gwCancel)) // Setup HTTP<->gRPC handlers. var opts []grpc.DialOption if s.ctx.Insecure { opts = append(opts, grpc.WithInsecure()) } else { tlsConfig, err := s.ctx.GetClientTLSConfig() if err != nil { return err } opts = append( opts, // TODO(tamird): remove this timeout. It is currently necessary because // GRPC will not actually bail on a bad certificate error - it will just // retry indefinitely. See https://github.com/grpc/grpc-go/issues/622. grpc.WithTimeout(base.NetworkTimeout), grpc.WithBlock(), grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)), ) } conn, err := s.rpcContext.GRPCDial(s.ctx.Addr, opts...) if err != nil { return util.Errorf("error constructing grpc-gateway: %s; are your certificates valid?", err) } for _, gw := range []grpcGatewayServer{&s.admin, s.status, &s.tsServer} { if err := gw.RegisterGateway(gwCtx, gwMux, conn); err != nil { return err } } s.mux.Handle("/", http.FileServer(uiFileSystem)) // TODO(marc): when cookie-based authentication exists, // apply it for all web endpoints. s.mux.HandleFunc(debugEndpoint, http.HandlerFunc(handleDebug)) s.mux.Handle(adminEndpoint, gwMux) s.mux.Handle(ts.URLPrefix, gwMux) s.mux.Handle(statusPrefix, s.status) s.mux.Handle(healthEndpoint, s.status) if err := sdnotify.Ready(); err != nil { log.Errorf("failed to signal readiness using systemd protocol: %s", err) } return nil }