// Check the health of a connection
func Check(conn *grpc.ClientConn) (bool, error) {
res, err := healthpb.NewHealthClient(conn).Check(context.Background(), &healthpb.HealthCheckRequest{})
if err != nil {
return false, err
}
return res.Status == healthpb.HealthCheckResponse_SERVING, nil
}
// getClient returns a connection to the Suite containerd
func (cs *ContainerdSuite) getClient(socket string) error {
// Parse proto://address form addresses.
bindParts := strings.SplitN(socket, "://", 2)
if len(bindParts) != 2 {
return fmt.Errorf("bad bind address format %s, expected proto://address", socket)
}
// reset the logger for grpc to log to dev/null so that it does not mess with our stdio
grpclog.SetLogger(log.New(ioutil.Discard, "", log.LstdFlags))
dialOpts := []grpc.DialOption{grpc.WithInsecure()}
dialOpts = append(dialOpts,
grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
return net.DialTimeout(bindParts[0], bindParts[1], timeout)
}),
grpc.WithBlock(),
grpc.WithTimeout(5*time.Second),
)
conn, err := grpc.Dial(socket, dialOpts...)
if err != nil {
return err
}
healthClient := grpc_health_v1.NewHealthClient(conn)
if _, err := healthClient.Check(context.Background(), &grpc_health_v1.HealthCheckRequest{}); err != nil {
return err
}
cs.grpcClient = types.NewAPIClient(conn)
return nil
}
// NewNotarySigner is a convenience method that returns NotarySigner given a GRPC connection
func NewNotarySigner(conn *grpc.ClientConn) *NotarySigner {
kmClient := pb.NewKeyManagementClient(conn)
sClient := pb.NewSignerClient(conn)
hc := healthpb.NewHealthClient(conn)
return &NotarySigner{
kmClient: kmClient,
sClient: sClient,
healthClient: hc,
}
}
func (cache *rpcCache) connect(config *core.Configuration) {
// Change grpc to log using our implementation
grpclog.SetLogger(&grpcLogMabob{})
log.Info("Connecting to RPC cache at %s", config.Cache.RpcUrl)
opts := []grpc.DialOption{grpc.WithTimeout(cache.timeout)}
if config.Cache.RpcPublicKey != "" || config.Cache.RpcCACert != "" || config.Cache.RpcSecure {
auth, err := loadAuth(config.Cache.RpcCACert, config.Cache.RpcPublicKey, config.Cache.RpcPrivateKey)
if err != nil {
log.Warning("Failed to load RPC cache auth keys: %s", err)
return
}
opts = append(opts, auth)
} else {
opts = append(opts, grpc.WithInsecure())
}
connection, err := grpc.Dial(config.Cache.RpcUrl, opts...)
if err != nil {
cache.Connecting = false
log.Warning("Failed to connect to RPC cache: %s", err)
return
}
// Note that we have to actually send it a message here to validate the connection;
// Dial() only seems to return errors for superficial failures like syntactically invalid addresses,
// it will return essentially immediately even if the server doesn't exist.
healthclient := healthpb.NewHealthClient(connection)
ctx, cancel := context.WithTimeout(context.Background(), cache.timeout)
defer cancel()
resp, err := healthclient.Check(ctx, &healthpb.HealthCheckRequest{Service: "plz-rpc-cache"})
if err != nil {
cache.Connecting = false
log.Warning("Failed to contact RPC cache: %s", err)
} else if resp.Status != healthpb.HealthCheckResponse_SERVING {
cache.Connecting = false
log.Warning("RPC cache says it is not serving (%d)", resp.Status)
} else {
cache.connection = connection
cache.client = pb.NewRpcCacheClient(connection)
cache.Connected = true
cache.Connecting = false
log.Info("RPC cache connected after %0.2fs", time.Since(cache.startTime).Seconds())
}
}
func (r *remote) handleConnectionChange() {
var transientFailureCount = 0
ticker := time.NewTicker(500 * time.Millisecond)
defer ticker.Stop()
healthClient := grpc_health_v1.NewHealthClient(r.rpcConn)
for {
<-ticker.C
ctx, cancel := context.WithTimeout(context.Background(), containerdHealthCheckTimeout)
_, err := healthClient.Check(ctx, &grpc_health_v1.HealthCheckRequest{})
cancel()
if err == nil {
continue
}
logrus.Debugf("libcontainerd: containerd health check returned error: %v", err)
if r.daemonPid != -1 {
if strings.Contains(err.Error(), "is closing") {
// Well, we asked for it to stop, just return
return
}
// all other errors are transient
// Reset state to be notified of next failure
transientFailureCount++
if transientFailureCount >= maxConnectionRetryCount {
transientFailureCount = 0
if utils.IsProcessAlive(r.daemonPid) {
utils.KillProcess(r.daemonPid)
}
<-r.daemonWaitCh
if err := r.runContainerdDaemon(); err != nil { //FIXME: Handle error
logrus.Errorf("libcontainerd: error restarting containerd: %v", err)
}
continue
}
}
}
}