// Must be called from the processing goroutine. func processNack(node uint16, conn *connect.BaseConn, content []byte) { var msg coproto.Nack if err := proto.Unmarshal(content, &msg); err != nil { conn.Close() return } store.StartTransaction() defer store.EndTransaction() // If we don't consider ourselves the leader, discard. if !amLeader { return } msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader) proposal, leader := store.Proposal() if msgProposal == proposal && msgLeader == leader { return } if store.CompareProposals(msgProposal, msgLeader, proposal, leader) { stopBeingLeader() store.SetProposal(msgProposal, msgLeader) } }
// Must be called from the processing goroutine. func processAccept(node uint16, conn *connect.BaseConn, content []byte) { var msg coproto.Accept if err := proto.Unmarshal(content, &msg); err != nil { conn.Close() return } store.StartTransaction() defer store.EndTransaction() proposal, leader := store.Proposal() msgProposal, msgLeader := *msg.Proposal, node if proposal != msgProposal || leader != msgLeader { // Send a nack message and return, // if this accept relates to an earlier proposal. if store.CompareProposals(proposal, leader, msgProposal, msgLeader) { var nack coproto.Nack nack.Proposal = new(uint64) nack.Leader = new(uint32) *nack.Proposal = proposal *nack.Leader = uint32(leader) conn.SendProto(6, &nack) return } store.SetProposal(msgProposal, msgLeader) } addAccept(node, &msg) }
func handleLeader(f *followConn, content []byte) { var msg fproto.Leader if err := proto.Unmarshal(content, &msg); err != nil { f.lock.Lock() f.Close() f.lock.Unlock() return } store.StartTransaction() defer store.EndTransaction() proposal, leader := store.Proposal() msgProposal, msgLeader := *msg.Proposal, uint16(*msg.Leader) if store.CompareProposals(msgProposal, msgLeader, proposal, leader) { store.SetProposal(msgProposal, msgLeader) } }
// Must be called from the processing goroutine, inside a transaction. func addPromise(node uint16, msg *coproto.Promise) { receivedPromises[node] = msg // If we have promises from a majority of core nodes, // become leader. if len(receivedPromises) > len(config.CoreNodes())/2 { log.Print("core/consensus: became leader") stopLeaderTimeout() amLeader = true proposal, leader := store.Proposal() // Find a slot number above all those in promise messages, // and above our first unapplied. firstUnapplied := store.InstructionFirstUnapplied() limit := firstUnapplied for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot >= limit { limit = *accepted.Slot + 1 } } } // Start our next slot after the limit. nextProposalSlot = limit // For all slots between this and our first unapplied, // submit a previously accepted instruction unless we // know an instruction was already chosen. // Fills these slots with proposals. // This is O(n^2) in the number of instructions between our // first unapplied and limit. // TODO: Improve worst-case complexity. start := store.InstructionStart() slots := store.InstructionSlots() for i := firstUnapplied; i < limit; i++ { // If we already have a chosen instruction, skip. rel := int(i - start) if len(slots[rel]) == 1 && slots[rel][0].IsChosen() { continue } // Find the previously accepted instruction // accepted with the highest proposal number. var bestInst *coproto.Instruction var bp uint64 // Best proposal var bl uint16 // Best leader for _, msg := range receivedPromises { for _, accepted := range msg.Accepted { if *accepted.Slot != i { continue } if bestInst == nil { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) continue } // TODO: This indent is just absurd. p := *accepted.Proposal l := uint16(*accepted.Leader) if store.CompareProposals(p, l, bp, bl) { bestInst = accepted bp = *accepted.Proposal bl = uint16(*accepted.Leader) } } } // If we didn't find an instruction, make an empty one. if bestInst == nil { empty := new(coproto.ChangeRequest) empty.RequestEntity = new(uint64) empty.RequestNode = new(uint32) *empty.RequestEntity = uint64(config.Id()) *empty.RequestNode = uint32(config.Id()) bestInst := new(coproto.Instruction) bestInst.Slot = new(uint64) *bestInst.Slot = i bestInst.Request = empty } // Add proposal timeout. req := makeExtChangeRequest(bestInst.Request) addProposalTimeout(i, req) // Send proposal. bestInst.Proposal = new(uint64) bestInst.Leader = new(uint32) *bestInst.Proposal = proposal *bestInst.Leader = uint32(leader) sendProposal(bestInst) } // Discard received promise messages. receivedPromises = nil // Make an instruction proposal for each waiting change. for _, req := range waitingRequests { slot := nextProposalSlot nextProposalSlot++ addProposalTimeout(slot, req) inst := makeInst(slot, proposal, leader, req) sendProposal(inst) } // Clear waiting changes. waitingRequests = nil } }
// Must be called from the processing goroutine. func processPrepare(node uint16, conn *connect.BaseConn, content []byte) { var msg coproto.Prepare if err := proto.Unmarshal(content, &msg); err != nil { conn.Close() return } store.StartTransaction() defer store.EndTransaction() newProposal, newLeader := *msg.Proposal, node proposal, leader := store.Proposal() if store.CompareProposals(newProposal, newLeader, proposal, leader) { log.Print("core/consensus: sending promise to ", newLeader) // Create a promise message to send back. var promise coproto.Promise promise.Proposal = new(uint64) promise.Leader = new(uint32) promise.PrevProposal = new(uint64) promise.PrevLeader = new(uint32) *promise.Proposal = newProposal *promise.Leader = uint32(newLeader) *promise.PrevProposal = proposal *promise.PrevLeader = uint32(leader) // Add all the instructions we've previously accepted or chosen. slots := store.InstructionSlots() theirFirstUnapplied := *msg.FirstUnapplied ourStart := store.InstructionStart() relativeSlot := int(theirFirstUnapplied - ourStart) if relativeSlot < 0 { relativeSlot = 0 } var accepted []*coproto.Instruction for ; relativeSlot < len(slots); relativeSlot++ { slot := slots[relativeSlot] slotNum := ourStart + uint64(relativeSlot) for i, _ := range slot { if slot[i].IsChosen() { appendInst(&accepted, slotNum, slot[i]) break } weAccepted := false for _, node := range slot[i].Accepted() { if node == config.Id() { weAccepted = true break } } if weAccepted { appendInst(&accepted, slotNum, slot[i]) break } } } // Send promise message. conn.SendProto(3, &promise) // Accept the other node as our new leader. store.SetProposal(newProposal, newLeader) } else { var nack coproto.Nack nack.Proposal = new(uint64) nack.Leader = new(uint32) *nack.Proposal = proposal *nack.Leader = uint32(leader) conn.SendProto(6, &nack) } }