func (p *peer) resolveAddr(ctx context.Context, id uint64) (string, error) {
resp, err := api.NewRaftClient(p.conn()).ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: id})
if err != nil {
return "", errors.Wrap(err, "failed to resolve address")
}
return resp.Addr, nil
}
// ConnectToMember returns a member object with an initialized
// connection to communicate with other raft members
func (n *Node) ConnectToMember(addr string, timeout time.Duration) (*membership.Member, error) {
conn, err := dial(addr, "tcp", n.tlsCredentials, timeout)
if err != nil {
return nil, err
}
return &membership.Member{
RaftClient: api.NewRaftClient(conn),
Conn: conn,
}, nil
}
func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error {
ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout)
defer cancel()
_, err := api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
if grpc.Code(err) == codes.NotFound && grpc.ErrorDesc(err) == membership.ErrMemberRemoved.Error() {
p.tr.config.NodeRemoved()
}
if m.Type == raftpb.MsgSnap {
if err != nil {
p.tr.config.ReportSnapshot(m.To, raft.SnapshotFailure)
} else {
}
}
p.reportSnapshot(err != nil)
if err != nil {
p.tr.config.ReportUnreachable(m.To)
return err
}
return nil
}
func (n *Node) sendToMember(ctx context.Context, members map[uint64]*membership.Member, m raftpb.Message, lastSend <-chan struct{}, thisSend chan<- struct{}) {
defer n.asyncTasks.Done()
defer close(thisSend)
ctx, cancel := context.WithTimeout(ctx, n.opts.SendTimeout)
defer cancel()
if lastSend != nil {
select {
case <-lastSend:
case <-ctx.Done():
return
}
}
if n.cluster.IsIDRemoved(m.To) {
// Should not send to removed members
return
}
var (
conn *membership.Member
)
if toMember, ok := members[m.To]; ok {
conn = toMember
} else {
// If we are being asked to send to a member that's not in
// our member list, that could indicate that the current leader
// was added while we were offline. Try to resolve its address.
log.G(ctx).Warningf("sending message to an unrecognized member ID %x", m.To)
// Choose a random member
var (
queryMember *membership.Member
id uint64
)
for id, queryMember = range members {
if id != n.Config.ID {
break
}
}
if queryMember == nil || queryMember.RaftID == n.Config.ID {
log.G(ctx).Error("could not find cluster member to query for leader address")
return
}
resp, err := api.NewRaftClient(queryMember.Conn).ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To})
if err != nil {
log.G(ctx).WithError(err).Errorf("could not resolve address of member ID %x", m.To)
return
}
conn, err = n.ConnectToMember(resp.Addr, n.opts.SendTimeout)
if err != nil {
log.G(ctx).WithError(err).Errorf("could connect to member ID %x at %s", m.To, resp.Addr)
return
}
// The temporary connection is only used for this message.
// Eventually, we should catch up and add a long-lived
// connection to the member list.
defer conn.Conn.Close()
}
_, err := api.NewRaftClient(conn.Conn).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
if err != nil {
if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() {
n.removeRaftFunc()
}
if m.Type == raftpb.MsgSnap {
n.raftNode.ReportSnapshot(m.To, raft.SnapshotFailure)
}
if !n.IsMember() {
// node is removed from cluster or stopped
return
}
n.raftNode.ReportUnreachable(m.To)
lastSeenHost := n.cluster.LastSeenHost(m.To)
if lastSeenHost != "" {
// Check if address has changed
officialHost, officialPort, _ := net.SplitHostPort(conn.Addr)
if officialHost != lastSeenHost {
reconnectAddr := net.JoinHostPort(lastSeenHost, officialPort)
log.G(ctx).Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr)
if err := n.handleAddressChange(ctx, conn, reconnectAddr); err != nil {
log.G(ctx).WithError(err).Error("failed to hande address change")
}
return
}
}
// Bounce the connection
newConn, err := n.ConnectToMember(conn.Addr, 0)
if err != nil {
log.G(ctx).WithError(err).Errorf("could connect to member ID %x at %s", m.To, conn.Addr)
return
}
err = n.cluster.ReplaceMemberConnection(m.To, conn, newConn, conn.Addr, false)
if err != nil {
log.G(ctx).WithError(err).Error("failed to replace connection to raft member")
newConn.Conn.Close()
}
} else if m.Type == raftpb.MsgSnap {
n.raftNode.ReportSnapshot(m.To, raft.SnapshotFinish)
}
}
