// Must be called from the processing goroutine.
func processNack(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Nack
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
// If we don't consider ourselves the leader, discard.
if !amLeader {
return
}
msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
proposal, leader := store.Proposal()
if msgProposal == proposal && msgLeader == leader {
return
}
if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
stopBeingLeader()
store.SetProposal(msgProposal, msgLeader)
}
}
// Must be called from the processing goroutine.
func processAccept(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Accept
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
proposal, leader := store.Proposal()
msgProposal, msgLeader := *msg.Proposal, node
if proposal != msgProposal || leader != msgLeader {
// Send a nack message and return,
// if this accept relates to an earlier proposal.
if store.CompareProposals(proposal, leader, msgProposal,
msgLeader) {
var nack coproto.Nack
nack.Proposal = new(uint64)
nack.Leader = new(uint32)
*nack.Proposal = proposal
*nack.Leader = uint32(leader)
conn.SendProto(6, &nack)
return
}
store.SetProposal(msgProposal, msgLeader)
}
addAccept(node, &msg)
}
func handleLeader(f *followConn, content []byte) {
var msg fproto.Leader
if err := proto.Unmarshal(content, &msg); err != nil {
f.lock.Lock()
f.Close()
f.lock.Unlock()
return
}
store.StartTransaction()
defer store.EndTransaction()
proposal, leader := store.Proposal()
msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader)
if store.CompareProposals(msgProposal, msgLeader, proposal, leader) {
store.SetProposal(msgProposal, msgLeader)
}
}
// Must be called from the processing goroutine, inside a transaction.
func addPromise(node uint16, msg *coproto.Promise) {
receivedPromises[node] = msg
// If we have promises from a majority of core nodes,
// become leader.
if len(receivedPromises) > len(config.CoreNodes())/2 {
log.Print("core/consensus: became leader")
stopLeaderTimeout()
amLeader = true
proposal, leader := store.Proposal()
// Find a slot number above all those in promise messages,
// and above our first unapplied.
firstUnapplied := store.InstructionFirstUnapplied()
limit := firstUnapplied
for _, msg := range receivedPromises {
for _, accepted := range msg.Accepted {
if *accepted.Slot >= limit {
limit = *accepted.Slot + 1
}
}
}
// Start our next slot after the limit.
nextProposalSlot = limit
// For all slots between this and our first unapplied,
// submit a previously accepted instruction unless we
// know an instruction was already chosen.
// Fills these slots with proposals.
// This is O(n^2) in the number of instructions between our
// first unapplied and limit.
// TODO: Improve worst-case complexity.
start := store.InstructionStart()
slots := store.InstructionSlots()
for i := firstUnapplied; i < limit; i++ {
// If we already have a chosen instruction, skip.
rel := int(i - start)
if len(slots[rel]) == 1 && slots[rel][0].IsChosen() {
continue
}
// Find the previously accepted instruction
// accepted with the highest proposal number.
var bestInst *coproto.Instruction
var bp uint64 // Best proposal
var bl uint16 // Best leader
for _, msg := range receivedPromises {
for _, accepted := range msg.Accepted {
if *accepted.Slot != i {
continue
}
if bestInst == nil {
bestInst = accepted
bp = *accepted.Proposal
bl = uint16(*accepted.Leader)
continue
}
// TODO: This indent is just absurd.
p := *accepted.Proposal
l := uint16(*accepted.Leader)
if store.CompareProposals(p, l,
bp, bl) {
bestInst = accepted
bp = *accepted.Proposal
bl = uint16(*accepted.Leader)
}
}
}
// If we didn't find an instruction, make an empty one.
if bestInst == nil {
empty := new(coproto.ChangeRequest)
empty.RequestEntity = new(uint64)
empty.RequestNode = new(uint32)
*empty.RequestEntity = uint64(config.Id())
*empty.RequestNode = uint32(config.Id())
bestInst := new(coproto.Instruction)
bestInst.Slot = new(uint64)
*bestInst.Slot = i
bestInst.Request = empty
}
// Add proposal timeout.
req := makeExtChangeRequest(bestInst.Request)
addProposalTimeout(i, req)
// Send proposal.
bestInst.Proposal = new(uint64)
bestInst.Leader = new(uint32)
*bestInst.Proposal = proposal
*bestInst.Leader = uint32(leader)
sendProposal(bestInst)
}
// Discard received promise messages.
receivedPromises = nil
// Make an instruction proposal for each waiting change.
for _, req := range waitingRequests {
slot := nextProposalSlot
nextProposalSlot++
addProposalTimeout(slot, req)
inst := makeInst(slot, proposal, leader, req)
sendProposal(inst)
}
// Clear waiting changes.
waitingRequests = nil
}
}
// Must be called from the processing goroutine.
func processPrepare(node uint16, conn *connect.BaseConn, content []byte) {
var msg coproto.Prepare
if err := proto.Unmarshal(content, &msg); err != nil {
conn.Close()
return
}
store.StartTransaction()
defer store.EndTransaction()
newProposal, newLeader := *msg.Proposal, node
proposal, leader := store.Proposal()
if store.CompareProposals(newProposal, newLeader, proposal, leader) {
log.Print("core/consensus: sending promise to ", newLeader)
// Create a promise message to send back.
var promise coproto.Promise
promise.Proposal = new(uint64)
promise.Leader = new(uint32)
promise.PrevProposal = new(uint64)
promise.PrevLeader = new(uint32)
*promise.Proposal = newProposal
*promise.Leader = uint32(newLeader)
*promise.PrevProposal = proposal
*promise.PrevLeader = uint32(leader)
// Add all the instructions we've previously accepted or chosen.
slots := store.InstructionSlots()
theirFirstUnapplied := *msg.FirstUnapplied
ourStart := store.InstructionStart()
relativeSlot := int(theirFirstUnapplied - ourStart)
if relativeSlot < 0 {
relativeSlot = 0
}
var accepted []*coproto.Instruction
for ; relativeSlot < len(slots); relativeSlot++ {
slot := slots[relativeSlot]
slotNum := ourStart + uint64(relativeSlot)
for i, _ := range slot {
if slot[i].IsChosen() {
appendInst(&accepted, slotNum, slot[i])
break
}
weAccepted := false
for _, node := range slot[i].Accepted() {
if node == config.Id() {
weAccepted = true
break
}
}
if weAccepted {
appendInst(&accepted, slotNum, slot[i])
break
}
}
}
// Send promise message.
conn.SendProto(3, &promise)
// Accept the other node as our new leader.
store.SetProposal(newProposal, newLeader)
} else {
var nack coproto.Nack
nack.Proposal = new(uint64)
nack.Leader = new(uint32)
*nack.Proposal = proposal
*nack.Leader = uint32(leader)
conn.SendProto(6, &nack)
}
}