func (lt *localRPCTransport) Send(req *RaftMessageRequest) error {
client, err := lt.getClient(req.ToReplica.StoreID)
if err != nil {
return err
}
stream, err := client.RaftMessage(grpcutil.NewContextWithStopper(context.Background(), lt.stopper))
if err != nil {
return err
}
return stream.Send(req)
}
// processQueue creates a client and sends messages from its designated queue
// via that client, exiting when the client fails or when it idles out. All
// messages remaining in the queue at that point are lost and a new instance of
// processQueue should be started by the next message to be sent.
// TODO(tschottdorf) should let raft know if the node is down;
// need a feedback mechanism for that. Potentially easiest is to arrange for
// the next call to Send() to fail appropriately.
func (t *rpcTransport) processQueue(nodeID roachpb.NodeID, storeID roachpb.StoreID) {
t.mu.Lock()
ch, ok := t.queues[storeID]
t.mu.Unlock()
if !ok {
return
}
// Clean-up when the loop below shuts down.
defer func() {
t.mu.Lock()
delete(t.queues, storeID)
t.mu.Unlock()
}()
addr, err := t.gossip.GetNodeIDAddress(nodeID)
if err != nil {
if log.V(1) {
log.Errorf("could not get address for node %d: %s", nodeID, err)
}
return
}
var dialOpt grpc.DialOption
if t.rpcContext.Insecure {
dialOpt = grpc.WithInsecure()
} else {
tlsConfig, err := t.rpcContext.GetClientTLSConfig()
if err != nil {
log.Error(err)
return
}
dialOpt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
}
conn, err := grpc.Dial(addr.String(), dialOpt)
if err != nil {
log.Errorf("failed to dial: %v", err)
return
}
defer func() {
if err := conn.Close(); err != nil {
log.Error(err)
}
}()
client := storage.NewMultiRaftClient(conn)
ctx := grpcutil.NewContextWithStopper(context.Background(), t.rpcContext.Stopper)
stream, err := client.RaftMessage(ctx)
if err != nil {
log.Error(err)
return
}
defer func() {
if err := stream.CloseSend(); err != nil {
log.Error(err)
}
}()
var raftIdleTimer util.Timer
defer raftIdleTimer.Stop()
for {
raftIdleTimer.Reset(raftIdleTimeout)
select {
case <-ctx.Done():
return
case <-raftIdleTimer.C:
raftIdleTimer.Read = true
if log.V(1) {
log.Infof("closing Raft transport to %d due to inactivity", nodeID)
}
return
case req := <-ch:
if err := stream.Send(req); err != nil {
log.Error(err)
return
}
}
}
}
// gossip loops, sending deltas of the infostore and receiving deltas
// in turn. If an alternate is proposed on response, the client addr
// is modified and method returns for forwarding by caller.
func (c *client) gossip(g *Gossip, gossipClient GossipClient, stopper *stop.Stopper) error {
// For un-bootstrapped node, g.is.NodeID is 0 when client start gossip,
// so it's better to get nodeID from g.is every time.
g.mu.Lock()
addr := g.is.NodeAddr
g.mu.Unlock()
ctx := grpcutil.NewContextWithStopper(context.Background(), stopper)
stream, err := gossipClient.Gossip(ctx)
if err != nil {
return err
}
defer func() {
if err := stream.CloseSend(); err != nil {
log.Error(err)
}
}()
if err := c.requestGossip(g, addr, stream); err != nil {
return err
}
sendGossipChan := make(chan struct{}, 1)
// Register a callback for gossip updates.
updateCallback := func(_ string, _ roachpb.Value) {
select {
case sendGossipChan <- struct{}{}:
default:
}
}
// Defer calling "undoer" callback returned from registration.
defer g.RegisterCallback(".*", updateCallback)()
// Loop in worker, sending updates from the info store.
stopper.RunWorker(func() {
for {
select {
case <-sendGossipChan:
if err := c.sendGossip(g, addr, stream); err != nil {
if !grpcutil.IsClosedConnection(err) {
log.Error(err)
}
return
}
case <-stopper.ShouldStop():
return
}
}
})
// Loop until stopper is signalled, or until either the gossip or RPC clients are closed.
// The stopper's signal is propagated through the context attached to the stream.
for {
reply, err := stream.Recv()
if err != nil {
return err
}
if err := c.handleResponse(g, reply); err != nil {
return err
}
}
}
