Example #1
0
// Must be called from the processing goroutine.
func processNack(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Nack
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	// If we don't consider ourselves the leader, discard.
	if !amLeader {
		return
	}

	msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
	proposal, leader := store.Proposal()
	if msgProposal == proposal && msgLeader == leader {
		return
	}
	if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
		stopBeingLeader()
		store.SetProposal(msgProposal, msgLeader)
	}
}
Example #2
0
// Must be called from the processing goroutine.
func processAccept(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Accept
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	proposal, leader := store.Proposal()
	msgProposal, msgLeader := *msg.Proposal, node
	if proposal != msgProposal || leader != msgLeader {
		// Send a nack message and return,
		// if this accept relates to an earlier proposal.
		if store.CompareProposals(proposal, leader, msgProposal,
			msgLeader) {

			var nack coproto.Nack
			nack.Proposal = new(uint64)
			nack.Leader = new(uint32)
			*nack.Proposal = proposal
			*nack.Leader = uint32(leader)
			conn.SendProto(6, &nack)

			return
		}

		store.SetProposal(msgProposal, msgLeader)
	}

	addAccept(node, &msg)
}
Example #3
0
func handleLeader(f *followConn, content []byte) {
	var msg fproto.Leader
	if err := proto.Unmarshal(content, &msg); err != nil {
		f.lock.Lock()
		f.Close()
		f.lock.Unlock()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	proposal, leader := store.Proposal()
	msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
	if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
		store.SetProposal(msgProposal, msgLeader)
	}
}
Example #4
0
// Top-level function of the processing goroutine.
func process() {

	connections = make(map[uint16][]*connect.BaseConn)

	// On startup, make an outgoing connection attempt to all other
	// core nodes, before continuing.
	for _, node := range config.CoreNodes() {
		if node == config.Id() {
			continue
		}

		conn, err := connect.Dial(
			connect.CONSENSUS_PROTOCOL, node)
		if err != nil {
			// Can't reach the other node.
			continue
		}

		log.Print("core/consensus: made outgoing connection to ", node)
		connections[node] = append(connections[node], conn)
		go handleConn(node, conn)
	}

	// Retry connections once per config.CHANGE_TIMEOUT_PERIOD.
	// Largely arbitrary.
	reconnectTicker := time.Tick(config.CHANGE_TIMEOUT_PERIOD)

	for {
		select {

		// Connection retry tick.
		// We should try to make an outgoing connection to any node
		// that we do not have at least one connection to.
		// We need to make these asynchronously, because connections
		// are slow.
		case <-reconnectTicker:
			for _, node := range config.CoreNodes() {
				if node == config.Id() {
					continue
				}
				if len(connections[node]) > 0 {
					continue
				}

				go outgoingConn(node)
			}

		// New change request, for us to propose as leader.
		case req := <-newChangeCh:

			store.StartTransaction()

			if !amLeader && receivedPromises != nil {
				waitingRequests = append(waitingRequests, req)
			} else if !amLeader {
				waitingRequests = append(waitingRequests, req)

				// Start attempting to be leader.
				m := make(map[uint16]*coproto.Promise)
				receivedPromises = m

				proposal, _ := store.Proposal()
				proposal++
				store.SetProposal(proposal, config.Id())
				firstUn := store.InstructionFirstUnapplied()

				// Send prepare messages to all other nodes.
				var prepare coproto.Prepare
				prepare.Proposal = new(uint64)
				prepare.FirstUnapplied = new(uint64)
				*prepare.Proposal = proposal
				*prepare.FirstUnapplied = firstUn

				for _, node := range config.CoreNodes() {
					if node == config.Id() {
						continue
					}

					if len(connections[node]) != 0 {
						c := connections[node][0]
						c.SendProto(2, &prepare)
					}
				}

				// Behave as if we got a promise message
				// from ourselves.
				var promise coproto.Promise
				promise.Proposal = prepare.Proposal
				promise.PrevProposal = promise.Proposal
				promise.Leader = new(uint32)
				*promise.Leader = uint32(config.Id())
				promise.PrevLeader = promise.Leader
				addPromise(config.Id(), &promise)
			} else {
				newSlot := nextProposalSlot
				nextProposalSlot++

				addProposalTimeout(newSlot, req)

				proposal, leader := store.Proposal()
				inst := makeInst(newSlot, proposal, leader,
					req)

				sendProposal(inst)
			}

			store.EndTransaction()

		// Leadership attempt timed out.
		case timedOutProposal := <-leaderTimeoutCh:

			store.StartTransaction()

			proposal, leader := store.Proposal()

			// If the proposal has changed since that
			// leadership attempt, ignore it.
			if leader != config.Id() {
				return
			}
			if proposal != timedOutProposal {
				return
			}

			// If we successfully became leader, ignore it.
			if amLeader {
				return
			}

			// Otherwise, stop our attempt to become leader.
			stopBeingLeader()

			store.EndTransaction()

		// Proposal timed out.
		case timeout := <-proposalTimeoutCh:

			store.StartTransaction()

			// If this timeout was not canceled, a proposal failed.
			// We stop being leader.
			if proposalTimeouts[timeout.slot] == timeout {
				stopBeingLeader()
			}

			store.EndTransaction()

		// New received connection.
		case receivedConn := <-receivedConnCh:
			node := receivedConn.node
			conn := receivedConn.conn

			connections[node] = append(connections[node], conn)

		// Received message.
		case recvMsg := <-receivedMsgCh:
			node := recvMsg.node
			conn := recvMsg.conn
			msg := recvMsg.msg

			switch *msg.MsgType {
			case 2:
				processPrepare(node, conn, msg.Content)
			case 3:
				processPromise(node, conn, msg.Content)
			case 4:
				processAccept(node, conn, msg.Content)
			case 5:
				processAccepted(node, conn, msg.Content)
			case 6:
				processNack(node, conn, msg.Content)
			default:
				// Unknown message.
				conn.Close()
			}

		// Terminate received connection.
		case terminatedConn := <-terminatedConnCh:
			node := terminatedConn.node
			conn := terminatedConn.conn

			for i, other := range connections[node] {
				if other != conn {
					continue
				}

				conns := connections[node]
				conns = append(conns[:i], conns[i+1:]...)
				connections[node] = conns
				break
			}
		}
	}
}
Example #5
0
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
	var msg coproto.Prepare
	if err := proto.Unmarshal(content, &msg); err != nil {
		conn.Close()
		return
	}

	store.StartTransaction()
	defer store.EndTransaction()

	newProposal, newLeader := *msg.Proposal, node
	proposal, leader := store.Proposal()
	if store.CompareProposals(newProposal, newLeader, proposal, leader) {

		log.Print("core/consensus: sending promise to ", newLeader)

		// Create a promise message to send back.
		var promise coproto.Promise
		promise.Proposal = new(uint64)
		promise.Leader = new(uint32)
		promise.PrevProposal = new(uint64)
		promise.PrevLeader = new(uint32)
		*promise.Proposal = newProposal
		*promise.Leader = uint32(newLeader)
		*promise.PrevProposal = proposal
		*promise.PrevLeader = uint32(leader)

		// Add all the instructions we've previously accepted or chosen.
		slots := store.InstructionSlots()
		theirFirstUnapplied := *msg.FirstUnapplied
		ourStart := store.InstructionStart()
		relativeSlot := int(theirFirstUnapplied - ourStart)
		if relativeSlot < 0 {
			relativeSlot = 0
		}
		var accepted []*coproto.Instruction
		for ; relativeSlot < len(slots); relativeSlot++ {
			slot := slots[relativeSlot]
			slotNum := ourStart + uint64(relativeSlot)
			for i, _ := range slot {
				if slot[i].IsChosen() {
					appendInst(&accepted, slotNum, slot[i])
					break
				}

				weAccepted := false
				for _, node := range slot[i].Accepted() {
					if node == config.Id() {
						weAccepted = true
						break
					}
				}

				if weAccepted {
					appendInst(&accepted, slotNum, slot[i])
					break
				}
			}
		}

		// Send promise message.
		conn.SendProto(3, &promise)

		// Accept the other node as our new leader.
		store.SetProposal(newProposal, newLeader)
	} else {
		var nack coproto.Nack
		nack.Proposal = new(uint64)
		nack.Leader = new(uint32)
		*nack.Proposal = proposal
		*nack.Leader = uint32(leader)
		conn.SendProto(6, &nack)
	}
}