func (lp *libpaxos) ReceiveAccept(args *paxosrpc.ReceiveAcceptArgs, reply *paxosrpc.ReceiveAcceptReply) error {
if lp.interruptFunc != nil {
lp.interruptFunc(lp.myNode.ID, Accept, lp.slotBox.nextUnknownSlotNumber)
}
lp.dataMutex.Lock()
lp.slotBoxMutex.Lock()
slot := lp.slotBox.Get(args.Proposal.CommandSlotNumber)
lp.slotBoxMutex.Unlock()
// Only accept a proposal if it has the highest proposal number so far.
if args.Proposal.Number.LessThan(lp.highestProposalNumberSeen) {
reply.Status = paxosrpc.Reject
} else if lp.highestAcceptedProposal != nil && (args.Proposal.Number.LessThan(&lp.highestAcceptedProposal.Number)) {
reply.Status = paxosrpc.Reject
} else if slot != nil {
// Someone else already filled this slot!
reply.Status = paxosrpc.Reject
} else {
lp.highestAcceptedProposal = &args.Proposal
lp.highestProposalNumberSeen = &args.Proposal.Number
reply.Status = paxosrpc.OK
}
lp.dataMutex.Unlock()
return nil
}
// doPropose will keep attempting to submit a proposal to the Paxos cluster
// with the given value. This may not always succeed because there will be
// competing proposals, but when doPropose returns, it guarantees that the
// value has been decided in a quorum.
func (lp *libpaxos) doPropose(value *paxosrpc.ProposalValue, doneCh chan<- struct{}) {
LOGV.Println("doing propose")
done := false // true if $moves has been Decided, false otherwise.
for !done {
retry := false // true if we should try again, for whatever reason, false otherwise.
// PHASE 1
LOGV.Println("Proposer", lp.myNode.ID, ": PHASE 1")
// Make a new proposal such that my_n > n_h
lp.dataMutex.Lock()
lp.slotBoxMutex.Lock()
myProp := paxosrpc.NewProposal(lp.highestProposalNumberSeen.Number+1, lp.slotBox.GetNextUnknownSlotNumber(), lp.myNode.ID, *value)
lp.highestProposalNumberSeen = &myProp.Number
lp.slotBoxMutex.Unlock()
lp.dataMutex.Unlock()
// Send proposal to everybody
promisedCount := 1 // include myself
var otherProposal *paxosrpc.Proposal
lp.nodesMutex.Lock()
for _, node := range lp.nodes {
if node.Info.ID == lp.myNode.ID {
continue // skip myself
}
client := node.getRPCClient()
if client == nil {
continue
}
args := &paxosrpc.ReceivePrepareArgs{lp.myNode, myProp.Number, myProp.CommandSlotNumber}
var reply paxosrpc.ReceivePrepareReply
reply.Status = paxosrpc.Reject
timedOut, err := rpcCallWithTimeout(client, "PaxosNode.ReceivePrepare", args, &reply)
if err != nil {
node.Client = nil // so it will try to redial in future attempts
continue // skip nodes that are unreachable
} else if timedOut {
LOGE.Println("RPC call PaxosNode.ReceivePrepare to", node.Info.ID, "timed out")
}
lp.dataMutex.Lock()
switch reply.Status {
case paxosrpc.OK:
if reply.HasAcceptedProposal {
if otherProposal == nil || reply.AcceptedProposal.Number.GreaterThan(&otherProposal.Number) {
otherProposal = &reply.AcceptedProposal
}
}
promisedCount++
case paxosrpc.DecidedValueExists:
// Oops, better fill in that value
lp.slotBoxMutex.Lock()
lp.slotBox.Add(NewSlot(reply.DecidedSlotNumber, &reply.DecidedValue))
lp.slotBoxMutex.Unlock()
lp.triggerHandlerCallCh <- struct{}{}
retry = true
break
case paxosrpc.Reject:
// do nothing if REJECTED
}
lp.dataMutex.Unlock()
}
lp.nodesMutex.Unlock()
// Retry?
if retry {
continue
}
// Got majority?
if promisedCount < lp.majorityCount {
LOGV.Println(lp.myNode.ID, " couldn't get a majority. Got", promisedCount, "needed", lp.majorityCount)
// Backoff
num := time.Duration(rand.Int()%100 + 25)
time.Sleep(num * time.Millisecond)
continue // try again
}
// This is the proposal that we will be sending out in PHASE 2
propToAccept := otherProposal
if otherProposal == nil {
propToAccept = myProp
}
LOGV.Println(lp.myNode.ID, "got a majority on", propToAccept.Number.String())
// PHASE 2
LOGV.Println("Proposer", lp.myNode.ID, ": PHASE 2")
// Send <accept, myn, V> to all nodes
acceptedCount := 1
lp.nodesMutex.Lock()
for _, node := range lp.nodes {
if node.Info.ID == lp.myNode.ID {
continue // skip myself
}
client := node.getRPCClient()
if client == nil {
continue
}
args := &paxosrpc.ReceiveAcceptArgs{*propToAccept}
var reply paxosrpc.ReceiveAcceptReply
reply.Status = paxosrpc.Reject
timedOut, err := rpcCallWithTimeout(client, "PaxosNode.ReceiveAccept", args, &reply)
if err != nil {
LOGE.Println(err)
node.Client = nil // so it will try to redial in future attempts
continue // skip nodes that are unreachable
} else if timedOut {
LOGE.Println("RPC call PaxosNode.ReceiveAccept to", node.Info.ID, "timed out")
}
if reply.Status == paxosrpc.OK {
acceptedCount++
} else {
LOGV.Println("Node", node.Info.ID, "rejected my Accept message with status", reply.Status)
} // do nothing if REJECTED
}
lp.nodesMutex.Unlock()
// Got majority?
if acceptedCount < lp.majorityCount {
LOGV.Println("Couldn't get majority for PHASE 2,", acceptedCount, "/", lp.majorityCount)
// Backoff
num := time.Duration(rand.Int()%100 + 25)
time.Sleep(num * time.Millisecond)
continue // try again
}
LOGV.Printf("%d got majority (%d/%d) on ACCEPT for slot %d, seqnum is %s, value is\n%s\n", lp.myNode.ID, acceptedCount, lp.majorityCount, propToAccept.CommandSlotNumber, propToAccept.Number.String(), util.MovesString(propToAccept.Value.Moves))
// Send <decide, va> to all nodes
lp.nodesMutex.Lock()
for _, node := range lp.nodes {
if node.Info.ID == lp.myNode.ID {
continue // skip myself
}
client := node.getRPCClient()
if client == nil {
continue
}
args := &paxosrpc.ReceiveDecideArgs{*propToAccept}
var reply paxosrpc.ReceiveDecideReply
rpcCallWithTimeout(client, "PaxosNode.ReceiveDecide", args, &reply)
// We don't care if that rpc call had an error or timed out
}
LOGV.Printf("%d decided on slot %d, seqnum is %s, value is\n%s\n", lp.myNode.ID, propToAccept.CommandSlotNumber, propToAccept.Number.String(), util.MovesString(propToAccept.Value.Moves))
lp.nodesMutex.Unlock()
lp.dataMutex.Lock()
lp.highestAcceptedProposal = nil
lp.dataMutex.Unlock()
lp.slotBoxMutex.Lock()
lp.slotBox.Add(NewSlot(propToAccept.CommandSlotNumber, &propToAccept.Value))
lp.slotBoxMutex.Unlock()
lp.triggerHandlerCallCh <- struct{}{}
if propToAccept.Number.Equal(&myProp.Number) {
done = true
}
}
LOGV.Println("Done propose")
doneCh <- struct{}{}
}