func NewGRPCClientDialOpt(c *Config) (grpc.DialOption, error) {
var opt grpc.DialOption
if c.InsecureSkipVerify {
return grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})), nil
}
tlsConf, err := NewClientTLSConfig(c)
if err != nil {
return opt, err
}
return grpc.WithTransportCredentials(credentials.NewTLS(tlsConf)), nil
}
func (t *grpcTransportListener) Accept(fn func(transport.Socket)) error {
var opts []grpc.ServerOption
// setup tls if specified
if t.secure || t.tls != nil {
config := t.tls
if config == nil {
var err error
addr := t.listener.Addr().String()
config, err = getTLSConfig(addr)
if err != nil {
return err
}
}
creds := credentials.NewTLS(config)
opts = append(opts, grpc.Creds(creds))
}
// new service
srv := grpc.NewServer(opts...)
// register service
pb.RegisterTransportServer(srv, µTransport{fn: fn})
// start serving
return srv.Serve(t.listener)
}
// CheckFlags checks that the required config values are set.
func (c *Config) CheckFlags(required RequiredFlags) error {
var missing []string
if c.CertFile != "" {
b, err := ioutil.ReadFile(c.CertFile)
if err != nil {
return fmt.Errorf("Failed to load certificates from %s: %v", c.CertFile, err)
}
cp := x509.NewCertPool()
if !cp.AppendCertsFromPEM(b) {
return fmt.Errorf("Failed to append certificates from %s", c.CertFile)
}
c.TLSCreds = credentials.NewTLS(&tls.Config{RootCAs: cp})
}
if required != NoneRequired {
c.SetFromGcloud()
}
if required&ProjectRequired != 0 && c.Project == "" {
missing = append(missing, "-project")
}
if required&InstanceRequired != 0 && c.Instance == "" {
missing = append(missing, "-instance")
}
if len(missing) > 0 {
return fmt.Errorf("Missing %s", strings.Join(missing, " and "))
}
return nil
}
func (s *SRVLoader) getConfig() (*pb.NodeConfig, error) {
nconfig := &pb.NodeConfig{}
var opts []grpc.DialOption
var creds credentials.TransportAuthenticator
creds = credentials.NewTLS(&tls.Config{
InsecureSkipVerify: true,
})
opts = append(opts, grpc.WithTransportCredentials(creds))
conn, err := grpc.Dial(s.SyndicateURL, opts...)
if err != nil {
return nconfig, fmt.Errorf("Failed to dial ring server for config: %s", err)
}
defer conn.Close()
client := pb.NewSyndicateClient(conn)
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
rr := &pb.RegisterRequest{}
rr.Hostname, _ = os.Hostname()
addrs, _ := net.InterfaceAddrs()
for k, _ := range addrs {
rr.Addrs = append(rr.Addrs, addrs[k].String())
}
rr.Hardware, err = GetHardwareProfile()
if err != nil {
return nconfig, err
}
rr.Tiers = []string{rr.Hostname}
nconfig, err = client.RegisterNode(ctx, rr)
return nconfig, err
}
func newClient(cfg *Config) (*Client, error) {
if cfg == nil {
cfg = &Config{RetryDialer: dialEndpointList}
}
var creds *credentials.TransportAuthenticator
if cfg.TLS != nil {
c := credentials.NewTLS(cfg.TLS)
creds = &c
}
// use a temporary skeleton client to bootstrap first connection
ctx, cancel := context.WithCancel(context.TODO())
conn, err := cfg.RetryDialer(&Client{cfg: *cfg, creds: creds, ctx: ctx})
if err != nil {
return nil, err
}
client := &Client{
conn: conn,
cfg: *cfg,
creds: creds,
ctx: ctx,
cancel: cancel,
}
client.Cluster = NewCluster(client)
client.KV = NewKV(client)
client.Lease = NewLease(client)
client.Watcher = NewWatcher(client)
client.Auth = NewAuth(client)
client.Maintenance = &maintenance{c: client}
return client, nil
}
func (cc *CCServer) Serve() error {
l, err := net.Listen("tcp", cc.cfg.ListenAddress)
if err != nil {
return err
}
var opts []grpc.ServerOption
if cc.cfg.UseTLS {
if cc.cfg.MutualTLS {
mtc, err := mutualTLS(*cc.cfg)
if err != nil {
return err
}
creds := credentials.NewTLS(mtc)
opts = []grpc.ServerOption{grpc.Creds(creds)}
} else {
creds, err := credentials.NewServerTLSFromFile(cc.cfg.CertFile, cc.cfg.KeyFile)
if err != nil {
return err
}
opts = []grpc.ServerOption{grpc.Creds(creds)}
}
}
s := grpc.NewServer(opts...)
pb.RegisterCmdCtrlServer(s, cc)
return s.Serve(l)
}
func GroupWrite(c *GroupClientConfig) {
defer c.wg.Done()
var err error
var opts []grpc.DialOption
var creds credentials.TransportAuthenticator
creds = credentials.NewTLS(&tls.Config{
InsecureSkipVerify: true,
})
opts = append(opts, grpc.WithTransportCredentials(creds))
conn, err := grpc.Dial(c.addr, opts...)
if err != nil {
log.Fatalln(fmt.Sprintf("Failed to dial server: %s", err))
}
defer conn.Close()
client := gp.NewGroupStoreClient(conn)
w := &gp.WriteRequest{
Value: *c.value,
}
empty := []byte("")
for i, _ := range c.wm {
ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
c.wm[i].Value = *c.value
w.TimestampMicro = brimtime.TimeToUnixMicro(time.Now())
res, err := client.Write(ctx, c.wm[i])
if err != nil {
log.Println("Client", c.id, ":", err)
}
if res.TimestampMicro > w.TimestampMicro {
log.Printf("TSM is newer than attempted, Key %d-%d Got %s, Sent: %s", c.id, i, brimtime.UnixMicroToTime(res.TimestampMicro), brimtime.UnixMicroToTime(w.TimestampMicro))
}
c.wm[i].Value = empty
}
}
func GroupStreamRead(c *GroupClientConfig) {
defer c.wg.Done()
var err error
var opts []grpc.DialOption
var creds credentials.TransportAuthenticator
creds = credentials.NewTLS(&tls.Config{
InsecureSkipVerify: true,
})
opts = append(opts, grpc.WithTransportCredentials(creds))
conn, err := grpc.Dial(c.addr, opts...)
if err != nil {
log.Fatalln(fmt.Sprintf("Failed to dial server: %s", err))
}
defer conn.Close()
client := gp.NewGroupStoreClient(conn)
stream, err := client.StreamRead(context.Background())
for i, _ := range c.rm {
if err := stream.Send(c.rm[i]); err != nil {
log.Println(err)
continue
}
_, err := stream.Recv()
if err == io.EOF {
return
}
if err != nil {
log.Println(err)
continue
}
}
stream.CloseSend()
}
// serverWithAuth builds a gRPC server, possibly with authentication if key / cert files are given.
func serverWithAuth(keyFile, certFile, caCertFile string) *grpc.Server {
if keyFile == "" {
return grpc.NewServer(grpc.MaxMsgSize(maxMsgSize)) // No auth.
}
log.Debug("Loading x509 key pair from key: %s cert: %s", keyFile, certFile)
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
log.Fatalf("Failed to load x509 key pair: %s", err)
}
config := tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequestClientCert,
}
if caCertFile != "" {
cert, err := ioutil.ReadFile(caCertFile)
if err != nil {
log.Fatalf("Failed to read CA cert file: %s", err)
}
config.ClientCAs = x509.NewCertPool()
if !config.ClientCAs.AppendCertsFromPEM(cert) {
log.Fatalf("Failed to find any PEM certificates in CA cert")
}
}
return grpc.NewServer(grpc.Creds(credentials.NewTLS(&config)), grpc.MaxMsgSize(maxMsgSize))
}
// NewServer loads various TLS certificates and creates a
// gRPC Server that verifies the client certificate was
// issued by the provided issuer certificate and presents a
// a server TLS certificate.
func NewServer(c *cmd.GRPCServerConfig, stats metrics.Scope) (*grpc.Server, net.Listener, error) {
cert, err := tls.LoadX509KeyPair(c.ServerCertificatePath, c.ServerKeyPath)
if err != nil {
return nil, nil, err
}
clientIssuerBytes, err := ioutil.ReadFile(c.ClientIssuerPath)
if err != nil {
return nil, nil, err
}
clientCAs := x509.NewCertPool()
if ok := clientCAs.AppendCertsFromPEM(clientIssuerBytes); !ok {
return nil, nil, errors.New("Failed to parse client issuer certificates")
}
servConf := &tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequireAndVerifyClientCert,
ClientCAs: clientCAs,
}
creds := credentials.NewTLS(servConf)
l, err := net.Listen("tcp", c.Address)
if err != nil {
return nil, nil, err
}
si := &serverInterceptor{stats, clock.Default()}
return grpc.NewServer(grpc.Creds(creds), grpc.UnaryInterceptor(si.intercept)), l, nil
}
// GetRemoteCA returns the remote endpoint's CA certificate
func GetRemoteCA(ctx context.Context, d digest.Digest, picker *picker.Picker) (RootCA, error) {
// We need a valid picker to be able to Dial to a remote CA
if picker == nil {
return RootCA{}, fmt.Errorf("valid remote address picker required")
}
// This TLS Config is intentionally using InsecureSkipVerify. Either we're
// doing TOFU, in which case we don't validate the remote CA, or we're using
// a user supplied hash to check the integrity of the CA certificate.
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
opts := []grpc.DialOption{
grpc.WithTransportCredentials(insecureCreds),
grpc.WithBackoffMaxDelay(10 * time.Second),
grpc.WithPicker(picker)}
firstAddr, err := picker.PickAddr()
if err != nil {
return RootCA{}, err
}
conn, err := grpc.Dial(firstAddr, opts...)
if err != nil {
return RootCA{}, err
}
defer conn.Close()
client := api.NewCAClient(conn)
response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
if err != nil {
return RootCA{}, err
}
if d != "" {
verifier, err := digest.NewDigestVerifier(d)
if err != nil {
return RootCA{}, fmt.Errorf("unexpected error getting digest verifier: %v", err)
}
io.Copy(verifier, bytes.NewReader(response.Certificate))
if !verifier.Verified() {
return RootCA{}, fmt.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())
}
}
// Check the validity of the remote Cert
_, err = helpers.ParseCertificatePEM(response.Certificate)
if err != nil {
return RootCA{}, err
}
// Create a Pool with our RootCACertificate
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(response.Certificate) {
return RootCA{}, fmt.Errorf("failed to append certificate to cert pool")
}
return RootCA{Cert: response.Certificate, Pool: pool}, nil
}
func (s *OortGroupStore) ListenAndServe() {
go func(s *OortGroupStore) {
s.grpcStopping = false
for {
var err error
listenAddr := s.oort.GetListenAddr()
if listenAddr == "" {
log.Fatalln("No listen address specified in ring at address2")
}
l, err := net.Listen("tcp", listenAddr)
if err != nil {
log.Fatalln("Unable to bind to address:", err)
}
log.Println("GroupStore bound to:", listenAddr)
var opts []grpc.ServerOption
creds := credentials.NewTLS(s.serverTLSConfig)
opts = []grpc.ServerOption{grpc.Creds(creds)}
s.grpc = grpc.NewServer(opts...)
groupproto.RegisterGroupStoreServer(s.grpc, s)
err = s.grpc.Serve(l)
if err != nil && !s.grpcStopping {
log.Println("GroupStore Serve encountered error:", err, "will attempt to restart")
} else if err != nil && s.grpcStopping {
log.Println("GroupStore got error but halt is in progress:", err)
l.Close()
break
} else {
log.Println("GroupStore Serve exited without error, quiting")
l.Close()
break
}
}
}(s)
}
// RegisterGRPCGateway starts the gateway (i.e. reverse proxy) that proxies
// HTTP requests to the appropriate gRPC endpoints.
func (s *adminServer) RegisterGRPCGateway(serverCtx *Context) error {
// Setup HTTP<->gRPC handlers.
var opts []grpc.DialOption
if serverCtx.Insecure {
opts = append(opts, grpc.WithInsecure())
} else {
tlsConfig, err := serverCtx.GetClientTLSConfig()
if err != nil {
return err
}
opts = append(
opts,
// TODO(tamird): remove this timeout. It is currently necessary because
// GRPC will not actually bail on a bad certificate error - it will just
// retry indefinitely. See https://github.com/grpc/grpc-go/issues/622.
grpc.WithTimeout(time.Second),
grpc.WithBlock(),
grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig)),
)
}
if err := RegisterAdminHandlerFromEndpoint(s.gwCtx, s.gwMux, serverCtx.Addr, opts); err != nil {
return util.Errorf("error constructing grpc-gateway: %s. are your certificates valid?", err)
}
// Pass all requests for gRPC-based API endpoints to the gateway mux.
s.ServeMux.Handle(apiEndpoint, s.gwMux)
return nil
}
func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
opts := []grpc.DialOption{}
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
opts = append(opts, grpc.WithTransportCredentials(insecureCreds))
// Using listen address instead of advertised address because this is a
// local connection.
addr := n.config.ListenControlAPI
opts = append(opts, grpc.WithDialer(
func(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout("unix", addr, timeout)
}))
conn, err := grpc.Dial(addr, opts...)
if err != nil {
return err
}
client := api.NewHealthClient(conn)
for {
resp, err := client.Check(ctx, &api.HealthCheckRequest{Service: "ControlAPI"})
if err != nil {
return err
}
if resp.Status == api.HealthCheckResponse_SERVING {
break
}
time.Sleep(500 * time.Millisecond)
}
n.setControlSocket(conn)
if ready != nil {
close(ready)
}
return nil
}
func benchmarkEchoGRPC(b *testing.B, size int) {
var conn *grpc.ClientConn
var client EchoClient
benchmarkEcho(b, size, listenAndServeGRPC,
func(addr net.Addr) {
var err error
conn, err = grpc.Dial(addr.String(), grpc.WithTransportCredentials(credentials.NewTLS(clientTLSConfig)))
if err != nil {
b.Fatal(err)
}
client = NewEchoClient(conn)
},
func() {
if err := conn.Close(); err != nil {
b.Fatal(err)
}
},
func(echoMsg string) string {
resp, err := client.Echo(context.Background(), &EchoRequest{Msg: echoMsg})
if err != nil {
b.Fatal(err)
}
return resp.Msg
},
)
}
func New(addr string, certFile string, keyFile string) (_ *Manager, err error) {
c := &Manager{}
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return nil, err
}
cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return nil, err
}
c.Cert = &cert
c.Self, err = NewPeerInfo("", cert.Certificate[0])
c.tlsConfig = &tls.Config{
Certificates: []tls.Certificate{cert},
ClientAuth: tls.RequestClientCert,
InsecureSkipVerify: true,
}
c.Listener, err = net.Listen("tcp", addr)
if err != nil {
return nil, err
}
serverTls := c.tlsConfig
serverTls.ServerName = addr
c.Server = grpc.NewServer(grpc.Creds(credentials.NewTLS(serverTls)))
go c.Server.Serve(c.Listener)
return c, nil
}
func (n *Node) initManagerConnection(ctx context.Context, ready chan<- struct{}) error {
opts := []grpc.DialOption{}
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
opts = append(opts, grpc.WithTransportCredentials(insecureCreds))
// Using listen address instead of advertised address because this is a
// local connection.
addr := n.config.ListenControlAPI
opts = append(opts, grpc.WithDialer(
func(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout("unix", addr, timeout)
}))
conn, err := grpc.Dial(addr, opts...)
if err != nil {
return err
}
state := grpc.Idle
for {
s, err := conn.WaitForStateChange(ctx, state)
if err != nil {
n.setControlSocket(nil)
return err
}
if s == grpc.Ready {
n.setControlSocket(conn)
if ready != nil {
close(ready)
ready = nil
}
} else if state == grpc.Shutdown {
n.setControlSocket(nil)
}
state = s
}
}
func main() {
flag.Parse()
var err error
tls, err := createTLS("127.0.0.1")
if err != nil {
fmt.Printf("Could not create TLS config %s\n", err)
}
creds := credentials.NewTLS(tls)
conn, err = grpc.Dial(*clientAddress, grpc.WithTransportCredentials(creds))
if err != nil {
fmt.Printf("Could not dial client %s\n", err)
}
defer conn.Close()
client = pb.NewTestServiceClient(conn)
router := mux.NewRouter()
router.StrictSlash(true)
router.HandleFunc("/", indexHandler)
router.HandleFunc("/hello/", indexHandler)
router.HandleFunc("/hello/{name}", helloHandler).Methods("GET")
fmt.Printf("Starting server at %s\n", *bind)
err = http.ListenAndServe(*bind, router)
if err != nil {
fmt.Println(err)
}
}
// NewGRPCConnection is a convenience method that returns GRPC Client Connection given a hostname, endpoint, and TLS options
func NewGRPCConnection(hostname string, port string, tlsConfig *tls.Config) (*grpc.ClientConn, error) {
var opts []grpc.DialOption
netAddr := net.JoinHostPort(hostname, port)
creds := credentials.NewTLS(tlsConfig)
opts = append(opts, grpc.WithTransportCredentials(creds))
return grpc.Dial(netAddr, opts...)
}
func (c *Client) dialTarget(endpoint string) (proto string, host string, creds *credentials.TransportCredentials) {
proto = "tcp"
host = endpoint
creds = c.creds
url, uerr := url.Parse(endpoint)
if uerr != nil || !strings.Contains(endpoint, "://") {
return
}
// strip scheme:// prefix since grpc dials by host
host = url.Host
switch url.Scheme {
case "unix":
proto = "unix"
case "http":
creds = nil
case "https":
if creds != nil {
break
}
tlsconfig := &tls.Config{}
emptyCreds := credentials.NewTLS(tlsconfig)
creds = &emptyCreds
default:
return "", "", nil
}
return
}
// set up the GRPC server
func setupGRPCServer(signerConfig signer.Config) (*grpc.Server, net.Listener, error) {
//RPC server setup
kms := &api.KeyManagementServer{
CryptoServices: signerConfig.CryptoServices,
}
ss := &api.SignerServer{
CryptoServices: signerConfig.CryptoServices,
}
hs := ghealth.NewServer()
lis, err := net.Listen("tcp", signerConfig.GRPCAddr)
if err != nil {
return nil, nil, fmt.Errorf("grpc server failed to listen on %s: %v",
signerConfig.GRPCAddr, err)
}
creds := credentials.NewTLS(signerConfig.TLSConfig)
opts := []grpc.ServerOption{grpc.Creds(creds)}
grpcServer := grpc.NewServer(opts...)
pb.RegisterKeyManagementServer(grpcServer, kms)
pb.RegisterSignerServer(grpcServer, ss)
healthpb.RegisterHealthServer(grpcServer, hs)
// Set status for both of the grpc service "KeyManagement" and "Signer", these are
// the only two we have at present, if we add more grpc service in the future,
// we should add a new line for that service here as well.
hs.SetServingStatus(notary.HealthCheckKeyManagement, healthpb.HealthCheckResponse_SERVING)
hs.SetServingStatus(notary.HealthCheckSigner, healthpb.HealthCheckResponse_SERVING)
return grpcServer, lis, nil
}
func createGRPCLayer(port int) (*grpc.Server, net.Listener, grpc.DialOption) {
var s *grpc.Server
var ll net.Listener
var err error
var serverOpts []grpc.ServerOption
var dialOpts grpc.DialOption
keyFileName := fmt.Sprintf("key.%d.pem", time.Now().UnixNano())
certFileName := fmt.Sprintf("cert.%d.pem", time.Now().UnixNano())
defer os.Remove(keyFileName)
defer os.Remove(certFileName)
err = generateCertificates(keyFileName, certFileName)
if err == nil {
var creds credentials.TransportCredentials
creds, err = credentials.NewServerTLSFromFile(certFileName, keyFileName)
serverOpts = append(serverOpts, grpc.Creds(creds))
ta := credentials.NewTLS(&tls.Config{
InsecureSkipVerify: true,
})
dialOpts = grpc.WithTransportCredentials(&authCreds{tlsCreds: ta})
} else {
dialOpts = grpc.WithInsecure()
}
listenAddress := fmt.Sprintf("%s:%d", "", port)
ll, err = net.Listen("tcp", listenAddress)
if err != nil {
panic(err)
}
s = grpc.NewServer(serverOpts...)
return s, ll, dialOpts
}
func newClient(cfg *Config) (*Client, error) {
if cfg == nil {
cfg = &Config{}
}
var creds *credentials.TransportCredentials
if cfg.TLS != nil {
c := credentials.NewTLS(cfg.TLS)
creds = &c
}
// use a temporary skeleton client to bootstrap first connection
ctx, cancel := context.WithCancel(context.TODO())
client := &Client{
conn: nil,
cfg: *cfg,
creds: creds,
ctx: ctx,
cancel: cancel,
}
if cfg.Username != "" && cfg.Password != "" {
client.Username = cfg.Username
client.Password = cfg.Password
}
client.balancer = newSimpleBalancer(cfg.Endpoints)
conn, err := client.dial(cfg.Endpoints[0], grpc.WithBalancer(client.balancer))
if err != nil {
return nil, err
}
client.conn = conn
client.retryWrapper = client.newRetryWrapper()
// wait for a connection
if cfg.DialTimeout > 0 {
hasConn := false
waitc := time.After(cfg.DialTimeout)
select {
case <-client.balancer.readyc:
hasConn = true
case <-ctx.Done():
case <-waitc:
}
if !hasConn {
client.cancel()
conn.Close()
return nil, grpc.ErrClientConnTimeout
}
}
client.Cluster = NewCluster(client)
client.KV = NewKV(client)
client.Lease = NewLease(client)
client.Watcher = NewWatcher(client)
client.Auth = NewAuth(client)
client.Maintenance = NewMaintenance(client)
go client.autoSync()
return client, nil
}
/*
NewGRPCServerFromListener creates a new implementation of a GRPCServer given an existing Listener instance.
In order to enable TLS, serverKey and ServerCertificate are required.
Parameters:
listener: Listener to use
serverKey: PEM-encoded private key to be used by the server for TLS communication
serverCertificate: PEM-encoded X509 public key to be used by the server for TLS communication
serverRootCAs: (optional) Set of PEM-encoded X509 certificate authorities to optionally send as part of
the server handshake
clientRootCAs: (optional) Set of PEM-encoded X509 certificate authorities to use when verifying client
certificates
*/
func NewGRPCServerFromListener(listener net.Listener, serverKey []byte, serverCertificate []byte,
serverRootCAs [][]byte, clientRootCAs [][]byte) (GRPCServer, error) {
grpcServer := &grpcServerImpl{
address: listener.Addr().String(),
listener: listener,
}
//set up our server options
var serverOpts []grpc.ServerOption
//check for TLS parameters
if serverKey != nil || serverCertificate != nil {
//both are required
if serverKey != nil && serverCertificate != nil {
grpcServer.tlsEnabled = true
//load server public and private keys
cert, err := tls.X509KeyPair(serverCertificate, serverKey)
if err != nil {
return nil, err
}
grpcServer.serverCertificate = cert
//set up our TLS config
//base server certificate
certificates := []tls.Certificate{grpcServer.serverCertificate}
/**
//if we have server root CAs append them
if len(serverRootCAs) > 0 {
//certificates = append(certificates, serverRootCAs...)
}
//if we have client root CAs, create a certPool
if len(clientRootCAs) > 0 {
grpcServer.clientRootCAs = x509.NewCertPool()
}
*/
tlsConfig := &tls.Config{
Certificates: certificates,
}
//create credentials
creds := credentials.NewTLS(tlsConfig)
//add to server options
serverOpts = append(serverOpts, grpc.Creds(creds))
} else {
return nil, errors.New("Both serverKey and serverCertificate are required in order to enable TLS")
}
}
grpcServer.server = grpc.NewServer(serverOpts...)
return grpcServer, nil
}
func main() {
var (
caCert = flag.String("ca-cert", withConfigDir("ca.pem"), "Trusted CA certificate.")
debugListenAddr = flag.String("debug-listen-addr", "127.0.0.1:7901", "HTTP listen address.")
listenAddr = flag.String("listen-addr", "0.0.0.0:7900", "HTTP listen address.")
tlsCert = flag.String("tls-cert", withConfigDir("cert.pem"), "TLS server certificate.")
tlsKey = flag.String("tls-key", withConfigDir("key.pem"), "TLS server key.")
jwtPublicKey = flag.String("jwt-public-key", withConfigDir("jwt.pem"), "The RSA public key to use for validating JWTs")
)
flag.Parse()
log.Println("Hello service starting...")
cert, err := tls.LoadX509KeyPair(*tlsCert, *tlsKey)
if err != nil {
log.Fatal(err)
}
rawCaCert, err := ioutil.ReadFile(*caCert)
if err != nil {
log.Fatal(err)
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(rawCaCert)
creds := credentials.NewTLS(&tls.Config{
Certificates: []tls.Certificate{cert},
ClientCAs: caCertPool,
ClientAuth: tls.RequireAndVerifyClientCert,
})
gs := grpc.NewServer(grpc.Creds(creds))
hs, err := NewHelloServer(*jwtPublicKey)
if err != nil {
log.Fatal(err)
}
pb.RegisterHelloServer(gs, hs)
healthServer := health.NewHealthServer()
healthServer.SetServingStatus("grpc.health.v1.helloservice", 1)
healthpb.RegisterHealthServer(gs, healthServer)
ln, err := net.Listen("tcp", *listenAddr)
if err != nil {
log.Fatal(err)
}
go gs.Serve(ln)
trace.AuthRequest = func(req *http.Request) (any, sensitive bool) { return true, true }
log.Println("Hello service started successfully.")
log.Fatal(http.ListenAndServe(*debugListenAddr, nil))
}
// GetRemoteCA returns the remote endpoint's CA certificate
func GetRemoteCA(ctx context.Context, d digest.Digest, r remotes.Remotes) (RootCA, error) {
// This TLS Config is intentionally using InsecureSkipVerify. We use the
// digest instead to check the integrity of the CA certificate.
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
conn, peer, err := getGRPCConnection(insecureCreds, r)
if err != nil {
return RootCA{}, err
}
defer conn.Close()
client := api.NewCAClient(conn)
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
defer func() {
if err != nil {
r.Observe(peer, -remotes.DefaultObservationWeight)
return
}
r.Observe(peer, remotes.DefaultObservationWeight)
}()
response, err := client.GetRootCACertificate(ctx, &api.GetRootCACertificateRequest{})
if err != nil {
return RootCA{}, err
}
if d != "" {
verifier, err := digest.NewDigestVerifier(d)
if err != nil {
return RootCA{}, errors.Wrap(err, "unexpected error getting digest verifier")
}
io.Copy(verifier, bytes.NewReader(response.Certificate))
if !verifier.Verified() {
return RootCA{}, errors.Errorf("remote CA does not match fingerprint. Expected: %s", d.Hex())
}
}
// Check the validity of the remote Cert
_, err = helpers.ParseCertificatePEM(response.Certificate)
if err != nil {
return RootCA{}, err
}
// Create a Pool with our RootCACertificate
pool := x509.NewCertPool()
if !pool.AppendCertsFromPEM(response.Certificate) {
return RootCA{}, errors.New("failed to append certificate to cert pool")
}
return RootCA{Cert: response.Certificate, Digest: digest.FromBytes(response.Certificate), Pool: pool}, nil
}
// setupWS ...
func setupWS(svr string) *grpc.ClientConn {
var opts []grpc.DialOption
creds := credentials.NewTLS(&tls.Config{
InsecureSkipVerify: true,
})
opts = append(opts, grpc.WithTransportCredentials(creds))
conn, err := grpc.Dial(svr, opts...)
if err != nil {
log.Fatalf("failed to dial: %v", err)
}
return conn
}
// GRPCDial calls grpc.Dial with the options appropriate for the context.
func (ctx *Context) GRPCDial(target string, opts ...grpc.DialOption) (*grpc.ClientConn, error) {
ctx.conns.Lock()
meta, ok := ctx.conns.cache[target]
if !ok {
meta = &connMeta{}
ctx.conns.cache[target] = meta
}
ctx.conns.Unlock()
meta.Do(func() {
var dialOpt grpc.DialOption
if ctx.Insecure {
dialOpt = grpc.WithInsecure()
} else {
tlsConfig, err := ctx.GetClientTLSConfig()
if err != nil {
meta.err = err
return
}
dialOpt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
}
dialOpts := make([]grpc.DialOption, 0, 2+len(opts))
dialOpts = append(dialOpts, dialOpt)
dialOpts = append(dialOpts, grpc.WithBackoffMaxDelay(maxBackoff))
dialOpts = append(dialOpts, opts...)
if log.V(1) {
log.Infof(ctx.masterCtx, "dialing %s", target)
}
meta.conn, meta.err = grpc.DialContext(ctx.masterCtx, target, dialOpts...)
if meta.err == nil {
if err := ctx.Stopper.RunTask(func() {
ctx.Stopper.RunWorker(func() {
err := ctx.runHeartbeat(meta.conn, target)
if err != nil && !grpcutil.IsClosedConnection(err) {
log.Error(ctx.masterCtx, err)
}
ctx.removeConn(target, meta)
})
}); err != nil {
meta.err = err
// removeConn and ctx's cleanup worker both lock ctx.conns. However,
// to avoid racing with meta's initialization, the cleanup worker
// blocks on meta.Do while holding ctx.conns. Invoke removeConn
// asynchronously to avoid deadlock.
go ctx.removeConn(target, meta)
}
}
})
return meta.conn, meta.err
}
func (m *Manager) updateKEK(ctx context.Context, cluster *api.Cluster) error {
securityConfig := m.config.SecurityConfig
nodeID := m.config.SecurityConfig.ClientTLSCreds.NodeID()
logger := log.G(ctx).WithFields(logrus.Fields{
"node.id": nodeID,
"node.role": ca.ManagerRole,
})
kekData := ca.KEKData{Version: cluster.Meta.Version.Index}
for _, encryptionKey := range cluster.UnlockKeys {
if encryptionKey.Subsystem == ca.ManagerRole {
kekData.KEK = encryptionKey.Key
break
}
}
updated, unlockedToLocked, err := m.dekRotator.MaybeUpdateKEK(kekData)
if err != nil {
logger.WithError(err).Errorf("failed to re-encrypt TLS key with a new KEK")
return err
}
if updated {
logger.Debug("successfully rotated KEK")
}
if unlockedToLocked {
// a best effort attempt to update the TLS certificate - if it fails, it'll be updated the next time it renews;
// don't wait because it might take a bit
go func() {
insecureCreds := credentials.NewTLS(&tls.Config{InsecureSkipVerify: true})
conn, err := grpc.Dial(
m.config.ControlAPI,
grpc.WithTransportCredentials(insecureCreds),
grpc.WithDialer(
func(addr string, timeout time.Duration) (net.Conn, error) {
return xnet.DialTimeoutLocal(addr, timeout)
}),
)
if err != nil {
logger.WithError(err).Error("failed to connect to local manager socket after locking the cluster")
return
}
defer conn.Close()
connBroker := connectionbroker.New(remotes.NewRemotes())
connBroker.SetLocalConn(conn)
if err := ca.RenewTLSConfigNow(ctx, securityConfig, connBroker); err != nil {
logger.WithError(err).Error("failed to download new TLS certificate after locking the cluster")
}
}()
}
return nil
}
// GRPCDialOption returns the GRPC dialing option appropriate for the context.
func (ctx *Context) GRPCDialOption() (grpc.DialOption, error) {
var dialOpt grpc.DialOption
if ctx.Insecure {
dialOpt = grpc.WithInsecure()
} else {
tlsConfig, err := ctx.GetClientTLSConfig()
if err != nil {
return dialOpt, err
}
dialOpt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
}
return dialOpt, nil
}