// Phase1RPC handles ClassicPaxos.Phase1 rpc.
func (this *Paxos) Phase1RPC(header *msgpb.Header,
request *thispb.Phase1Request) (status error) {
if !this.IsAcceptor() {
this.Errorf("this paxos instance is not an acceptor; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header),
"acceptor")
if errLock != nil {
return errLock
}
defer lock.Unlock()
clientID := header.GetMessengerId()
respond := func() error {
response := thispb.Phase1Response{}
response.PromisedBallot = proto.Int64(this.promisedBallot)
if this.votedBallot >= 0 {
response.VotedBallot = proto.Int64(this.votedBallot)
response.VotedValue = this.votedValue
}
message := thispb.PaxosMessage{}
message.Phase1Response = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send phase1 response to %s: %v", clientID,
errSend)
return errSend
}
return nil
}
ballot := request.GetBallotNumber()
if ballot < this.promisedBallot {
this.Warningf("phase1 request from %s is ignored due to stale ballot %d",
clientID, ballot)
return respond()
}
if ballot == this.promisedBallot {
this.Warningf("duplicate phase1 request from client %s with an already "+
"promised ballot number %d", clientID, ballot)
return respond()
}
// Save the promise into the wal.
change := thispb.AcceptorChange{}
change.PromisedBallot = proto.Int64(ballot)
if err := this.doUpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
this.Infof("this acceptor has now promised higher ballot %d from %s", ballot,
clientID)
return respond()
}
// ElectRPC implements the Election.Elect rpc.
func (this *Election) ElectRPC(reqHeader *msgpb.Header,
request *thispb.ElectRequest) (status error) {
if !this.InCommittee() {
return errs.ErrInvalid
}
round := request.GetElectionRound()
current := this.CurrentRound()
if round != current+1 {
this.Errorf("could not begin election round %d because current round "+
"is only at %d", round, current)
return errs.ErrStale
}
uid := this.PaxosUIDFromElectionRound(round)
paxos, errGet := this.GetPaxosInstance(uid)
if errGet != nil {
this.Errorf("could not find paxos instance for uid %s: %v", uid, errGet)
return errGet
}
this.Infof("starting new election for round %d using paxos instance %s",
round, uid)
clientID := reqHeader.GetMessengerId()
proposedValue := []byte(clientID)
chosenValue, errPropose := paxos.Propose(proposedValue,
msg.RequestTimeout(reqHeader))
if errPropose != nil {
this.Errorf("could not propose %s as the leader for election round %d: %v",
proposedValue, round, errPropose)
return errPropose
}
winner := string(chosenValue)
// Send the election result.
response := thispb.ElectResponse{}
response.ElectionRound = proto.Int64(round)
response.ElectionWinner = proto.String(winner)
message := thispb.ElectionMessage{}
message.ElectResponse = &response
if err := msg.SendResponseProto(this.msn, reqHeader, &message); err != nil {
this.Errorf("could not send election response to %s: %v", clientID, err)
return err
}
return nil
}
// Receive waits for a response to a live request.
//
// request: Message header of a live request.
//
// Returns response message header and response data on success.
func (this *Messenger) Receive(request *msgpb.Header) (*msgpb.Header,
[]byte, error) {
timeout := msg.RequestTimeout(request)
lock, errLock := this.ctlr.TimedLock(timeout, "this.requestMap")
if errLock != nil {
return nil, nil, errLock
}
defer lock.Unlock()
requestID := request.GetMessageId()
responseCh, found := this.requestMap[requestID]
if !found {
this.Errorf("no live request [%s] exist", request)
return nil, nil, errs.ErrNotExist
}
// Close the lock early to unblock others.
lock.Unlock()
// Perform a non-blocking receive if timeout is zero.
if timeout == 0 {
select {
case entry := <-responseCh:
return entry.header, entry.data, nil
default:
return nil, nil, errs.ErrRetry
}
}
// Perform a blocking receive with a timeout.
select {
case <-time.After(timeout):
this.Warningf("timedout waiting for response to %s", request)
return nil, nil, errs.ErrTimeout
case entry := <-responseCh:
if entry == nil {
return nil, nil, errs.ErrRetry
}
return entry.header, entry.data, nil
}
}
// LearnRPC handles ClassicPaxos.Learn rpc.
func (this *Paxos) LearnRPC(header *msgpb.Header,
request *thispb.LearnRequest) (status error) {
if !this.IsLearner() {
this.Errorf("this paxos instance is not a learner; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header), "learner")
if errLock != nil {
return errLock
}
defer lock.Unlock()
acceptor := header.GetMessengerId()
if this.chosenValue == nil {
change := thispb.LearnerChange{}
change.VotedBallotList = request.VotedBallotList
change.VotedValueList = request.VotedValueList
change.VotedAcceptor = proto.String(acceptor)
if err := this.doUpdateLearner(&change); err != nil {
this.Errorf("could not update learner state: %v", err)
return err
}
}
response := thispb.LearnResponse{}
if this.chosenValue != nil {
response.KnowsChosenValue = proto.Bool(true)
}
message := thispb.PaxosMessage{}
message.LearnResponse = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not respond to learn request from %s: %v", acceptor,
errSend)
return errSend
}
return nil
}
// Phase2RPC handles ClassicPaxos.Phase2 rpc.
func (this *Paxos) Phase2RPC(header *msgpb.Header,
request *thispb.Phase2Request) (status error) {
if !this.IsAcceptor() {
this.Errorf("this paxos instance is not an acceptor; rejecting %s", header)
return errs.ErrInvalid
}
lock, errLock := this.ctlr.TimedLock(msg.RequestTimeout(header), "acceptor")
if errLock != nil {
return errLock
}
defer lock.Unlock()
clientID := header.GetMessengerId()
respond := func() error {
response := thispb.Phase2Response{}
response.PromisedBallot = proto.Int64(this.promisedBallot)
if this.votedBallot >= 0 {
response.VotedBallot = proto.Int64(this.votedBallot)
response.VotedValue = this.votedValue
}
message := thispb.PaxosMessage{}
message.Phase2Response = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send phase2 response to %s: %v", clientID,
errSend)
return errSend
}
return nil
}
ballot := request.GetBallotNumber()
if ballot < this.promisedBallot {
this.Warningf("phase2 request from %s is ignored due to stale ballot %d",
clientID, ballot)
return respond()
}
if ballot > this.promisedBallot {
this.Errorf("phase2 request from client %s without acquiring a prior "+
"promise", clientID)
return respond()
}
value := request.GetProposedValue()
// Save the phase2 vote into the wal.
change := thispb.AcceptorChange{}
change.VotedBallot = proto.Int64(ballot)
change.VotedValue = value
if err := this.doUpdateAcceptor(&change); err != nil {
this.Errorf("could not update acceptor state: %v", err)
return err
}
this.Infof("this acceptor has voted for %d in ballot %s", ballot, value)
if err := respond(); err != nil {
return err
}
// Schedule a notification to all learners.
_ = this.alarm.ScheduleAt(this.uid, time.Now(), this.NotifyAllLearners)
return nil
}
// ProposeRPC handles ClassicPaxos.Propose rpc.
func (this *Paxos) ProposeRPC(header *msgpb.Header,
request *thispb.ProposeRequest) (status error) {
if !this.IsProposer() {
this.Errorf("this paxos instance is not a proposer; rejecting %s", header)
return errs.ErrInvalid
}
// OPTIMIZATION If this object is also a learner and already knows the
// consensus result, we don't need to perform expensive proposal.
if this.IsLearner() {
lock := this.ctlr.ReadLock("learner")
defer lock.Unlock()
if this.chosenValue != nil {
response := thispb.ProposeResponse{}
response.ChosenValue = this.chosenValue
message := thispb.PaxosMessage{}
message.ProposeResponse = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send known chosen value as the propose "+
"response: %v", errSend)
return errSend
}
return nil
}
lock.Unlock()
}
var chosen []byte
proposal := request.GetProposedValue()
for ii := 0; chosen == nil && msg.RequestTimeout(header) > 0; ii++ {
if ii > 0 {
time.Sleep(this.opts.ProposeRetryInterval)
}
// Get the next proposal ballot number.
ballot, errNext := this.GetNextProposalBallot(msg.RequestTimeout(header))
if errNext != nil {
this.Errorf("could not select higher ballot: %v", errNext)
return errNext
}
this.Infof("using ballot number %d for the proposal", ballot)
lock := this.ctlr.ReadLock("config")
phase1AcceptorList := this.getPhase1AcceptorList(ballot)
lock.Unlock()
// Collect phase1 promises from majority number of acceptors.
votedValue, acceptorList, errPhase1 := this.doPhase1(header, ballot,
phase1AcceptorList)
if errPhase1 != nil {
this.Warningf("could not complete paxos phase1: %v", errPhase1)
continue
}
// If a value was already voted, it may have been chosen, so propose it
// instead.
value := proposal
if votedValue != nil {
value = votedValue
}
// Collect phase2 votes from majority number of acceptors.
errPhase2 := this.doPhase2(header, ballot, value, acceptorList)
if errPhase2 != nil {
this.Warningf("could not complete paxos phase2: %v", errPhase2)
continue
}
// A value is chosen, break out of the loop.
chosen = value
break
}
if chosen == nil {
this.Errorf("could not propose value %s", proposal)
return errs.ErrRetry
}
// If local node is a learner, update him with the consensus result directly.
defer func() {
if this.IsLearner() {
lock, errLock := this.ctlr.Lock("learner")
if errLock != nil {
return
}
defer lock.Unlock()
change := thispb.LearnerChange{}
change.ChosenValue = chosen
if err := this.doUpdateLearner(&change); err != nil {
this.Warningf("could not update local learner with the consensus "+
"result (ignored): %v", err)
}
}
}()
// Send propose response with chosen value.
response := thispb.ProposeResponse{}
response.ChosenValue = chosen
message := thispb.PaxosMessage{}
message.ProposeResponse = &response
errSend := msg.SendResponseProto(this.msn, header, &message)
if errSend != nil {
this.Errorf("could not send propose response: %v", errSend)
return errSend
}
return nil
}