コード例 #1
0
ファイル: raft.go プロジェクト: ygf11/docker
func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Message) {
	defer n.asyncTasks.Done()

	if n.cluster.IsIDRemoved(m.To) {
		// Should not send to removed members
		return
	}

	ctx, cancel := context.WithTimeout(n.Ctx, n.sendTimeout)
	defer cancel()

	var (
		conn *membership.Member
	)
	if toMember, ok := members[m.To]; ok {
		conn = toMember
	} else {
		// If we are being asked to send to a member that's not in
		// our member list, that could indicate that the current leader
		// was added while we were offline. Try to resolve its address.
		n.Config.Logger.Warningf("sending message to an unrecognized member ID %x", m.To)

		// Choose a random member
		var (
			queryMember *membership.Member
			id          uint64
		)
		for id, queryMember = range members {
			if id != n.Config.ID {
				break
			}
		}

		if queryMember == nil || queryMember.RaftID == n.Config.ID {
			n.Config.Logger.Error("could not find cluster member to query for leader address")
			return
		}

		resp, err := queryMember.ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To})
		if err != nil {
			n.Config.Logger.Errorf("could not resolve address of member ID %x: %v", m.To, err)
			return
		}
		conn, err = n.ConnectToMember(resp.Addr, n.sendTimeout)
		if err != nil {
			n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, resp.Addr, err)
			return
		}
		// The temporary connection is only used for this message.
		// Eventually, we should catch up and add a long-lived
		// connection to the member list.
		defer conn.Conn.Close()
	}

	_, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
	if err != nil {
		if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() {
			n.removeRaftOnce.Do(func() {
				close(n.removeRaftCh)
			})
		}
		if m.Type == raftpb.MsgSnap {
			n.ReportSnapshot(m.To, raft.SnapshotFailure)
		}
		if n.IsStopped() {
			panic("node is nil")
		}
		n.ReportUnreachable(m.To)

		// Bounce the connection
		newConn, err := n.ConnectToMember(conn.Addr, 0)
		if err != nil {
			n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, conn.Addr, err)
		} else {
			n.cluster.ReplaceMemberConnection(m.To, newConn)
		}
	} else if m.Type == raftpb.MsgSnap {
		n.ReportSnapshot(m.To, raft.SnapshotFinish)
	}
}
コード例 #2
0
ファイル: raft.go プロジェクト: SUSE/docker.mirror
func (n *Node) sendToMember(members map[uint64]*membership.Member, m raftpb.Message) {
	defer n.asyncTasks.Done()

	if n.cluster.IsIDRemoved(m.To) {
		// Should not send to removed members
		return
	}

	ctx, cancel := context.WithTimeout(n.Ctx, n.sendTimeout)
	defer cancel()

	var (
		conn *membership.Member
	)
	if toMember, ok := members[m.To]; ok {
		conn = toMember
	} else {
		// If we are being asked to send to a member that's not in
		// our member list, that could indicate that the current leader
		// was added while we were offline. Try to resolve its address.
		n.Config.Logger.Warningf("sending message to an unrecognized member ID %x", m.To)

		// Choose a random member
		var (
			queryMember *membership.Member
			id          uint64
		)
		for id, queryMember = range members {
			if id != n.Config.ID {
				break
			}
		}

		if queryMember == nil || queryMember.RaftID == n.Config.ID {
			n.Config.Logger.Error("could not find cluster member to query for leader address")
			return
		}

		resp, err := queryMember.ResolveAddress(ctx, &api.ResolveAddressRequest{RaftID: m.To})
		if err != nil {
			n.Config.Logger.Errorf("could not resolve address of member ID %x: %v", m.To, err)
			return
		}
		conn, err = n.ConnectToMember(resp.Addr, n.sendTimeout)
		if err != nil {
			n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, resp.Addr, err)
			return
		}
		// The temporary connection is only used for this message.
		// Eventually, we should catch up and add a long-lived
		// connection to the member list.
		defer conn.Conn.Close()
	}

	_, err := conn.ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
	if err != nil {
		if grpc.ErrorDesc(err) == ErrMemberRemoved.Error() {
			n.removeRaftFunc()
		}
		if m.Type == raftpb.MsgSnap {
			n.ReportSnapshot(m.To, raft.SnapshotFailure)
		}
		if !n.IsMember() {
			// node is removed from cluster or stopped
			return
		}
		n.ReportUnreachable(m.To)

		lastSeenHost := n.cluster.LastSeenHost(m.To)
		if lastSeenHost != "" {
			// Check if address has changed
			officialHost, officialPort, _ := net.SplitHostPort(conn.Addr)
			if officialHost != lastSeenHost {
				reconnectAddr := net.JoinHostPort(lastSeenHost, officialPort)
				n.Config.Logger.Warningf("detected address change for %x (%s -> %s)", m.To, conn.Addr, reconnectAddr)
				if err := n.handleAddressChange(conn, reconnectAddr); err != nil {
					n.Config.Logger.Error(err)
				}
				return
			}
		}

		// Bounce the connection
		newConn, err := n.ConnectToMember(conn.Addr, 0)
		if err != nil {
			n.Config.Logger.Errorf("could connect to member ID %x at %s: %v", m.To, conn.Addr, err)
			return
		}
		err = n.cluster.ReplaceMemberConnection(m.To, conn, newConn, conn.Addr, false)
		if err != nil {
			n.Config.Logger.Errorf("failed to replace connection to raft member: %v", err)
			newConn.Conn.Close()
		}
	} else if m.Type == raftpb.MsgSnap {
		n.ReportSnapshot(m.To, raft.SnapshotFinish)
	}
}