// Serve serves stuff.
func Serve(
registerFunc func(*grpc.Server),
options ServeOptions,
serveEnv ServeEnv,
) (retErr error) {
defer func(start time.Time) { logServerFinished(start, retErr) }(time.Now())
if registerFunc == nil {
return ErrMustSpecifyRegisterFunc
}
if serveEnv.GRPCPort == 0 {
serveEnv.GRPCPort = 7070
}
grpcServer := grpc.NewServer(
grpc.MaxConcurrentStreams(math.MaxUint32),
grpc.UnaryInterceptor(protorpclog.LoggingUnaryServerInterceptor),
)
registerFunc(grpcServer)
if options.Version != nil {
protoversion.RegisterAPIServer(grpcServer, protoversion.NewAPIServer(options.Version, protoversion.APIServerOptions{}))
}
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", serveEnv.GRPCPort))
if err != nil {
return err
}
errC := make(chan error)
go func() { errC <- grpcServer.Serve(listener) }()
protolion.Info(
&ServerStarted{
Port: uint32(serveEnv.GRPCPort),
},
)
return <-errC
}
// NewServer loads various TLS certificates and creates a
// gRPC Server that verifies the client certificate was
// issued by the provided issuer certificate and presents a
// a server TLS certificate.
func NewServer(c *cmd.GRPCServerConfig, stats metrics.Scope) (*grpc.Server, net.Listener, error) {
cert, err := tls.LoadX509KeyPair(c.ServerCertificatePath, c.ServerKeyPath)
if err != nil {
return nil, nil, err
}
clientIssuerBytes, err := ioutil.ReadFile(c.ClientIssuerPath)
if err != nil {
return nil, nil, err
}
clientCAs := x509.NewCertPool()
if ok := clientCAs.AppendCertsFromPEM(clientIssuerBytes); !ok {
return nil, nil, errors.New("Failed to parse client issuer certificates")
}
servConf := &tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: clientCAs,
}
creds := credentials.NewTLS(servConf)
l, err := net.Listen("tcp", c.Address)
if err != nil {
return nil, nil, err
}
si := &serverInterceptor{stats, clock.Default()}
return grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(si.intercept)), l, nil
}
func Server(s *etcdserver.EtcdServer, tls *tls.Config) *grpc.Server {
var opts []grpc.ServerOption
opts = append(opts, grpc.CustomCodec(&codec{}))
if tls != nil {
opts = append(opts, grpc.Creds(credentials.NewTLS(tls)))
}
opts = append(opts, grpc.UnaryInterceptor(newUnaryInterceptor(s)))
opts = append(opts, grpc.StreamInterceptor(newStreamInterceptor(s)))
grpcServer := grpc.NewServer(opts...)
pb.RegisterKVServer(grpcServer, NewQuotaKVServer(s))
pb.RegisterWatchServer(grpcServer, NewWatchServer(s))
pb.RegisterLeaseServer(grpcServer, NewQuotaLeaseServer(s))
pb.RegisterClusterServer(grpcServer, NewClusterServer(s))
pb.RegisterAuthServer(grpcServer, NewAuthServer(s))
pb.RegisterMaintenanceServer(grpcServer, NewMaintenanceServer(s))
return grpcServer
}
// NewServer loads various TLS certificates and creates a
// gRPC Server that verifies the client certificate was
// issued by the provided issuer certificate and presents a
// a server TLS certificate.
func NewServer(c *cmd.GRPCServerConfig, stats metrics.Scope) (*grpc.Server, net.Listener, error) {
if stats == nil {
return nil, nil, errNilScope
}
cert, err := tls.LoadX509KeyPair(c.ServerCertificatePath, c.ServerKeyPath)
if err != nil {
return nil, nil, err
}
clientIssuerBytes, err := ioutil.ReadFile(c.ClientIssuerPath)
if err != nil {
return nil, nil, err
}
clientCAs := x509.NewCertPool()
if ok := clientCAs.AppendCertsFromPEM(clientIssuerBytes); !ok {
return nil, nil, errors.New("Failed to parse client issuer certificates")
}
servTLSConfig := &tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: clientCAs,
}
acceptedSANs := make(map[string]struct{})
for _, name := range c.ClientNames {
acceptedSANs[name] = struct{}{}
}
creds, err := bcreds.NewServerCredentials(servTLSConfig, acceptedSANs)
if err != nil {
return nil, nil, err
}
l, err := net.Listen("tcp", c.Address)
if err != nil {
return nil, nil, err
}
grpc_prometheus.EnableHandlingTimeHistogram()
si := &serverInterceptor{stats.NewScope("gRPCServer"), clock.Default()}
return grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(si.intercept)), l, nil
}
// ServeWithHTTP serves stuff.
func ServeWithHTTP(
registerFunc func(*grpc.Server),
httpRegisterFunc func(context.Context, *runtime.ServeMux, *grpc.ClientConn) error,
options ServeWithHTTPOptions,
serveEnv ServeEnv,
handlerEnv pkghttp.HandlerEnv,
) (retErr error) {
defer func(start time.Time) { logServerFinished(start, retErr) }(time.Now())
if registerFunc == nil || httpRegisterFunc == nil {
return ErrMustSpecifyRegisterFunc
}
if serveEnv.GRPCPort == 0 {
serveEnv.GRPCPort = 7070
}
if handlerEnv.Port == 0 {
handlerEnv.Port = 8080
}
grpcServer := grpc.NewServer(
grpc.MaxConcurrentStreams(math.MaxUint32),
grpc.UnaryInterceptor(protorpclog.LoggingUnaryServerInterceptor),
)
registerFunc(grpcServer)
if options.Version != nil {
protoversion.RegisterAPIServer(grpcServer, protoversion.NewAPIServer(options.Version, protoversion.APIServerOptions{}))
}
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", serveEnv.GRPCPort))
if err != nil {
return err
}
grpcErrC := make(chan error)
go func() { grpcErrC <- grpcServer.Serve(listener) }()
time.Sleep(1 * time.Second)
ctx, cancel := context.WithCancel(context.Background())
conn, err := grpc.Dial(fmt.Sprintf("0.0.0.0:%d", serveEnv.GRPCPort), grpc.WithInsecure())
if err != nil {
cancel()
return err
}
go func() {
<-ctx.Done()
_ = conn.Close()
}()
mux := runtime.NewServeMux()
if options.Version != nil {
if err := protoversion.RegisterAPIHandler(ctx, mux, conn); err != nil {
cancel()
return err
}
}
if err := httpRegisterFunc(ctx, mux, conn); err != nil {
cancel()
return err
}
var handler http.Handler
handler = mux
if options.HTTPHandlerModifier != nil {
handler, err = options.HTTPHandlerModifier(mux)
if err != nil {
cancel()
return err
}
}
httpErrC := make(chan error)
go func() { httpErrC <- pkghttp.ListenAndServe(handler, handlerEnv) }()
protolion.Info(
&ServerStarted{
Port: uint32(serveEnv.GRPCPort),
HttpPort: uint32(handlerEnv.Port),
},
)
var errs []error
grpcStopped := false
for i := 0; i < 2; i++ {
select {
case grpcErr := <-grpcErrC:
if grpcErr != nil {
errs = append(errs, fmt.Errorf("grpc error: %s", grpcErr.Error()))
}
grpcStopped = true
case httpErr := <-httpErrC:
if httpErr != nil {
errs = append(errs, fmt.Errorf("http error: %s", httpErr.Error()))
}
if !grpcStopped {
grpcServer.Stop()
_ = listener.Close()
grpcStopped = true
}
}
}
if len(errs) > 0 {
return fmt.Errorf("%v", errs)
}
return nil
}
func main() {
app := cli.NewApp()
app.Name = "containerd"
app.Version = containerd.Version
app.Usage = `
__ _ __
_________ ____ / /_____ _(_)___ ___ _________/ /
/ ___/ __ \/ __ \/ __/ __ ` + "`" + `/ / __ \/ _ \/ ___/ __ /
/ /__/ /_/ / / / / /_/ /_/ / / / / / __/ / / /_/ /
\___/\____/_/ /_/\__/\__,_/_/_/ /_/\___/_/ \__,_/
high performance container runtime
`
app.Flags = []cli.Flag{
cli.BoolFlag{
Name: "debug",
Usage: "enable debug output in logs",
},
cli.StringFlag{
Name: "root",
Usage: "containerd state directory",
Value: "/run/containerd",
},
cli.StringFlag{
Name: "runtime",
Usage: "default runtime for execution",
Value: "runc",
},
cli.StringFlag{
Name: "socket, s",
Usage: "socket path for containerd's GRPC server",
Value: "/run/containerd/containerd.sock",
},
cli.StringFlag{
Name: "metrics-address, m",
Usage: "tcp address to serve metrics on",
Value: "127.0.0.1:7897",
},
cli.StringFlag{
Name: "events-address, e",
Usage: "nats address to serve events on",
Value: nats.DefaultURL,
},
}
app.Before = func(context *cli.Context) error {
if context.GlobalBool("debug") {
logrus.SetLevel(logrus.DebugLevel)
}
return nil
}
app.Action = func(context *cli.Context) error {
signals := make(chan os.Signal, 2048)
signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)
if address := context.GlobalString("metrics-address"); address != "" {
go serveMetrics(address)
}
s, err := startNATSServer(context)
if err != nil {
return nil
}
defer s.Shutdown()
path := context.GlobalString("socket")
if path == "" {
return fmt.Errorf("--socket path cannot be empty")
}
l, err := createUnixSocket(path)
if err != nil {
return err
}
var executor execution.Executor
switch context.GlobalString("runtime") {
case "runc":
executor, err = oci.New(context.GlobalString("root"))
if err != nil {
return err
}
}
// Get events publisher
nec, err := getNATSPublisher(context)
if err != nil {
return err
}
defer nec.Close()
ctx := log.WithModule(gocontext.Background(), "containerd")
ctx = log.WithModule(ctx, "execution")
ctx = events.WithPoster(ctx, events.GetNATSPoster(nec))
execService, err := execution.New(ctx, executor)
if err != nil {
return err
}
// Intercept the GRPC call in order to populate the correct module path
interceptor := func(ctx gocontext.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
ctx = log.WithModule(ctx, "containerd")
switch info.Server.(type) {
case api.ExecutionServiceServer:
ctx = log.WithModule(ctx, "execution")
ctx = events.WithPoster(ctx, events.GetNATSPoster(nec))
default:
fmt.Println("Unknown type: %#v", info.Server)
}
return handler(ctx, req)
}
server := grpc.NewServer(grpc.UnaryInterceptor(interceptor))
api.RegisterExecutionServiceServer(server, execService)
go serveGRPC(server, l)
for s := range signals {
switch s {
case syscall.SIGUSR1:
dumpStacks()
default:
logrus.WithField("signal", s).Info("containerd: stopping GRPC server")
server.Stop()
return nil
}
}
return nil
}
if err := app.Run(os.Args); err != nil {
fmt.Fprintf(os.Stderr, "containerd: %s\n", err)
os.Exit(1)
}
}
func (c *Component) ServerOptions() []grpc.ServerOption {
unary := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
var peerAddr string
peer, ok := peer.FromContext(ctx)
if ok {
peerAddr = peer.Addr.String()
}
var peerID string
meta, ok := metadata.FromContext(ctx)
if ok {
id, ok := meta["id"]
if ok && len(id) > 0 {
peerID = id[0]
}
}
logCtx := c.Ctx.WithFields(log.Fields{
"CallerID": peerID,
"CallerIP": peerAddr,
"Method": info.FullMethod,
})
t := time.Now()
iface, err := handler(ctx, req)
err = errors.BuildGRPCError(err)
logCtx = logCtx.WithField("Duration", time.Now().Sub(t))
if grpc.Code(err) == codes.OK || grpc.Code(err) == codes.Canceled {
logCtx.Debug("Handled request")
} else {
logCtx.WithField("ErrCode", grpc.Code(err)).WithError(err).Debug("Handled request with error")
}
return iface, err
}
stream := func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
var peerAddr string
peer, ok := peer.FromContext(stream.Context())
if ok {
peerAddr = peer.Addr.String()
}
var peerID string
meta, ok := metadata.FromContext(stream.Context())
if ok {
id, ok := meta["id"]
if ok && len(id) > 0 {
peerID = id[0]
}
}
logCtx := c.Ctx.WithFields(log.Fields{
"CallerID": peerID,
"CallerIP": peerAddr,
"Method": info.FullMethod,
})
t := time.Now()
logCtx.Debug("Start stream")
err := handler(srv, stream)
err = errors.BuildGRPCError(err)
logCtx = logCtx.WithField("Duration", time.Now().Sub(t))
if grpc.Code(err) == codes.OK || grpc.Code(err) == codes.Canceled {
logCtx.Debug("End stream")
} else {
logCtx.WithField("ErrCode", grpc.Code(err)).WithError(err).Debug("End stream with error")
}
return err
}
opts := []grpc.ServerOption{
grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(unary)),
grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(stream)),
}
if c.tlsConfig != nil {
opts = append(opts, grpc.Creds(credentials.NewTLS(c.tlsConfig)))
}
return opts
}
func startGRPCProxy(cmd *cobra.Command, args []string) {
l, err := net.Listen("tcp", grpcProxyListenAddr)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
if l, err = transport.NewKeepAliveListener(l, "tcp", nil); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
plog.Infof("listening for grpc-proxy client requests on %s", grpcProxyListenAddr)
defer func() {
l.Close()
plog.Infof("stopping listening for grpc-proxy client requests on %s", grpcProxyListenAddr)
}()
m := cmux.New(l)
cfg, err := newClientCfg()
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
client, err := clientv3.New(*cfg)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
kvp := grpcproxy.NewKvProxy(client)
watchp := grpcproxy.NewWatchProxy(client)
clusterp := grpcproxy.NewClusterProxy(client)
leasep := grpcproxy.NewLeaseProxy(client)
mainp := grpcproxy.NewMaintenanceProxy(client)
authp := grpcproxy.NewAuthProxy(client)
server := grpc.NewServer(
grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
)
pb.RegisterKVServer(server, kvp)
pb.RegisterWatchServer(server, watchp)
pb.RegisterClusterServer(server, clusterp)
pb.RegisterLeaseServer(server, leasep)
pb.RegisterMaintenanceServer(server, mainp)
pb.RegisterAuthServer(server, authp)
errc := make(chan error)
grpcl := m.Match(cmux.HTTP2())
go func() { errc <- server.Serve(grpcl) }()
httpmux := http.NewServeMux()
httpmux.HandleFunc("/", http.NotFound)
httpmux.Handle("/metrics", prometheus.Handler())
srvhttp := &http.Server{
Handler: httpmux,
}
var httpl net.Listener
if cfg.TLS != nil {
srvhttp.TLSConfig = cfg.TLS
httpl = tls.NewListener(m.Match(cmux.Any()), cfg.TLS)
} else {
httpl = m.Match(cmux.HTTP1())
}
go func() { errc <- srvhttp.Serve(httpl) }()
go func() { errc <- m.Serve() }()
fmt.Fprintln(os.Stderr, <-errc)
os.Exit(1)
}
func TestRetryApply(t *testing.T) {
ctx := context.Background()
errCount := 0
code := codes.Unavailable // Will be retried
// Intercept requests and return an error or defer to the underlying handler
errInjector := func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
if strings.HasSuffix(info.FullMethod, "MutateRow") && errCount < 3 {
errCount++
return nil, grpc.Errorf(code, "")
}
return handler(ctx, req)
}
tbl, cleanup, err := setupFakeServer(grpc.UnaryInterceptor(errInjector))
defer cleanup()
if err != nil {
t.Fatalf("fake server setup: %v", err)
}
mut := NewMutation()
mut.Set("cf", "col", 1, []byte("val"))
if err := tbl.Apply(ctx, "row1", mut); err != nil {
t.Errorf("applying single mutation with retries: %v", err)
}
row, err := tbl.ReadRow(ctx, "row1")
if err != nil {
t.Errorf("reading single value with retries: %v", err)
}
if row == nil {
t.Errorf("applying single mutation with retries: could not read back row")
}
code = codes.FailedPrecondition // Won't be retried
errCount = 0
if err := tbl.Apply(ctx, "row", mut); err == nil {
t.Errorf("applying single mutation with no retries: no error")
}
// Check and mutate
mutTrue := NewMutation()
mutTrue.DeleteRow()
mutFalse := NewMutation()
mutFalse.Set("cf", "col", 1, []byte("val"))
condMut := NewCondMutation(ValueFilter("."), mutTrue, mutFalse)
errCount = 0
code = codes.Unavailable // Will be retried
if err := tbl.Apply(ctx, "row1", condMut); err != nil {
t.Errorf("conditionally mutating row with retries: %v", err)
}
row, err = tbl.ReadRow(ctx, "row1") // row1 already in the table
if err != nil {
t.Errorf("reading single value after conditional mutation: %v", err)
}
if row != nil {
t.Errorf("reading single value after conditional mutation: row not deleted")
}
errCount = 0
code = codes.FailedPrecondition // Won't be retried
if err := tbl.Apply(ctx, "row", condMut); err == nil {
t.Errorf("conditionally mutating row with no retries: no error")
}
}
