func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Message) {
defer n.asyncTasks.Done()
if n.cluster.IsIDRemoved(m.To) {
// Should not send to removed members
return
}
ctx, cancel := context.WithTimeout(n.Ctx, n.sendTimeout)
defer cancel()
var (
conn *membership.Member
)
if toMember, ok := members[m.To]; ok {
conn = toMember
} else {
// If we are being asked to send to a member that's not in
// our member list, that could indicate that the current leader
// was added while we were offline. Try to resolve its address.
n.Config.Logger.Warningf("sending message to an unrecognized member ID %x", m.To)
// Choose a random member
var (
queryMember *membership.Member
id uint64
)
for id, queryMember = range members {
if id != n.Config.ID {
break
}
}
if queryMember == nil || queryMember.RaftID == n.Config.ID {
n.Config.Logger.Error("could not find cluster member to query for leader address")
return
}
resp, err := queryMember.ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To})
if err != nil {
n.Config.Logger.Errorf("could not resolve address of member ID %x: %v", m.To, err)
return
}
conn, err = n.ConnectToMember(resp.Addr, n.sendTimeout)
if err != nil {
n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, resp.Addr, err)
return
}
// The temporary connection is only used for this message.
// Eventually, we should catch up and add a long-lived
// connection to the member list.
defer conn.Conn.Close()
}
_, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
if err != nil {
if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() {
n.removeRaftOnce.Do(func() {
close(n.removeRaftCh)
})
}
if m.Type == raftpb.MsgSnap {
n.ReportSnapshot(m.To, raft.SnapshotFailure)
}
if n.IsStopped() {
panic("node is nil")
}
n.ReportUnreachable(m.To)
// Bounce the connection
newConn, err := n.ConnectToMember(conn.Addr, 0)
if err != nil {
n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, conn.Addr, err)
} else {
n.cluster.ReplaceMemberConnection(m.To, newConn)
}
} else if m.Type == raftpb.MsgSnap {
n.ReportSnapshot(m.To, raft.SnapshotFinish)
}
}
func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Message) {
defer n.asyncTasks.Done()
if n.cluster.IsIDRemoved(m.To) {
// Should not send to removed members
return
}
ctx, cancel := context.WithTimeout(n.Ctx, n.sendTimeout)
defer cancel()
var (
conn *membership.Member
)
if toMember, ok := members[m.To]; ok {
conn = toMember
} else {
// If we are being asked to send to a member that's not in
// our member list, that could indicate that the current leader
// was added while we were offline. Try to resolve its address.
n.Config.Logger.Warningf("sending message to an unrecognized member ID %x", m.To)
// Choose a random member
var (
queryMember *membership.Member
id uint64
)
for id, queryMember = range members {
if id != n.Config.ID {
break
}
}
if queryMember == nil || queryMember.RaftID == n.Config.ID {
n.Config.Logger.Error("could not find cluster member to query for leader address")
return
}
resp, err := queryMember.ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To})
if err != nil {
n.Config.Logger.Errorf("could not resolve address of member ID %x: %v", m.To, err)
return
}
conn, err = n.ConnectToMember(resp.Addr, n.sendTimeout)
if err != nil {
n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, resp.Addr, err)
return
}
// The temporary connection is only used for this message.
// Eventually, we should catch up and add a long-lived
// connection to the member list.
defer conn.Conn.Close()
}
_, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
if err != nil {
if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() {
n.removeRaftFunc()
}
if m.Type == raftpb.MsgSnap {
n.ReportSnapshot(m.To, raft.SnapshotFailure)
}
if !n.IsMember() {
// node is removed from cluster or stopped
return
}
n.ReportUnreachable(m.To)
lastSeenHost := n.cluster.LastSeenHost(m.To)
if lastSeenHost != "" {
// Check if address has changed
officialHost, officialPort, _ := net.SplitHostPort(conn.Addr)
if officialHost != lastSeenHost {
reconnectAddr := net.JoinHostPort(lastSeenHost, officialPort)
n.Config.Logger.Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr)
if err := n.handleAddressChange(conn, reconnectAddr); err != nil {
n.Config.Logger.Error(err)
}
return
}
}
// Bounce the connection
newConn, err := n.ConnectToMember(conn.Addr, 0)
if err != nil {
n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, conn.Addr, err)
return
}
err = n.cluster.ReplaceMemberConnection(m.To, conn, newConn, conn.Addr, false)
if err != nil {
n.Config.Logger.Errorf("failed to replace connection to raft member: %v", err)
newConn.Conn.Close()
}
} else if m.Type == raftpb.MsgSnap {
n.ReportSnapshot(m.To, raft.SnapshotFinish)
}
}