func (r *Replica) handlePropose(propose *genericsmr.Propose) {
//TODO!! Handle client retries
batchSize := len(r.ProposeChan) + 1
if batchSize > MAX_BATCH {
batchSize = MAX_BATCH
}
instNo := r.crtInstance[r.Id]
r.crtInstance[r.Id]++
dlog.Printf("Starting instance %d\n", instNo)
dlog.Printf("Batching %d\n", batchSize)
cmds := make([]state.Command, batchSize)
proposals := make([]*genericsmr.Propose, batchSize)
cmds[0] = propose.Command
proposals[0] = propose
for i := 1; i < batchSize; i++ {
prop := <-r.ProposeChan
cmds[i] = prop.Command
proposals[i] = prop
}
r.startPhase1(r.Id, instNo, 0, proposals, cmds, batchSize)
}
func (r *Replica) handlePropose(propose *genericsmr.Propose) {
if !r.IsLeader {
preply := &genericsmrproto.ProposeReplyTS{FALSE, -1, state.NIL, 0}
r.ReplyProposeTS(preply, propose.Reply)
return
}
for r.instanceSpace[r.crtInstance] != nil {
r.crtInstance++
}
instNo := r.crtInstance
r.crtInstance++
batchSize := len(r.ProposeChan) + 1
if batchSize > MAX_BATCH {
batchSize = MAX_BATCH
}
dlog.Printf("Batched %d\n", batchSize)
cmds := make([]state.Command, batchSize)
proposals := make([]*genericsmr.Propose, batchSize)
cmds[0] = propose.Command
proposals[0] = propose
for i := 1; i < batchSize; i++ {
prop := <-r.ProposeChan
cmds[i] = prop.Command
proposals[i] = prop
}
if r.defaultBallot == -1 {
r.instanceSpace[instNo] = &Instance{
cmds,
r.makeUniqueBallot(0),
PREPARING,
&LeaderBookkeeping{proposals, 0, 0, 0, 0}}
r.bcastPrepare(instNo, r.makeUniqueBallot(0), true)
dlog.Printf("Classic round for instance %d\n", instNo)
} else {
r.instanceSpace[instNo] = &Instance{
cmds,
r.defaultBallot,
PREPARED,
&LeaderBookkeeping{proposals, 0, 0, 0, 0}}
r.recordInstanceMetadata(r.instanceSpace[instNo])
r.recordCommands(cmds)
r.sync()
r.bcastAccept(instNo, r.defaultBallot, cmds)
dlog.Printf("Fast round for instance %d\n", instNo)
}
}
func (r *Replica) updateBlocking(instance int32) {
if instance != r.blockingInstance {
return
}
for r.blockingInstance = r.blockingInstance; true; r.blockingInstance++ {
if r.blockingInstance <= r.skippedTo[int(r.blockingInstance)%r.N] {
continue
}
if r.instanceSpace[r.blockingInstance] == nil {
return
}
inst := r.instanceSpace[r.blockingInstance]
if inst.status == COMMITTED && inst.skipped {
r.skippedTo[int(r.blockingInstance)%r.N] = r.blockingInstance + int32((inst.nbInstSkipped-1)*r.N)
continue
}
if inst.status == ACCEPTED && inst.skipped {
return
}
if r.blockingInstance%int32(r.N) == r.Id || inst.lb != nil {
if inst.status == READY {
//commit my instance
dlog.Printf("Am about to commit instance %d\n", r.blockingInstance)
inst.status = COMMITTED
if inst.lb.clientProposal != nil && !r.Dreply {
// give client the all clear
dlog.Printf("Sending ACK for req. %d\n", inst.lb.clientProposal.CommandId)
r.ReplyProposeTS(&genericsmrproto.ProposeReplyTS{TRUE, inst.lb.clientProposal.CommandId, state.NIL, inst.lb.clientProposal.Timestamp},
inst.lb.clientProposal.Reply)
}
skip := FALSE
if inst.skipped {
skip = TRUE
}
r.recordInstanceMetadata(inst)
r.sync()
r.bcastCommit(r.blockingInstance, skip, int32(inst.nbInstSkipped), *inst.command)
} else if inst.status != COMMITTED && inst.status != EXECUTED {
return
}
if inst.skipped {
r.skippedTo[int(r.blockingInstance)%r.N] = r.blockingInstance + int32((inst.nbInstSkipped-1)*r.N)
}
} else {
if inst.status == PREPARING || (inst.status == ACCEPTED && inst.skipped) {
return
}
}
}
}
func (r *Replica) handleCommitShort(commit *paxosproto.CommitShort) {
inst := r.instanceSpace[commit.Instance]
dlog.Printf("Committing instance %d\n", commit.Instance)
if inst == nil {
r.instanceSpace[commit.Instance] = &Instance{nil,
commit.Ballot,
COMMITTED,
nil}
} else {
r.instanceSpace[commit.Instance].status = COMMITTED
r.instanceSpace[commit.Instance].ballot = commit.Ballot
if inst.lb != nil && inst.lb.clientProposals != nil {
for i := 0; i < len(inst.lb.clientProposals); i++ {
r.ProposeChan <- inst.lb.clientProposals[i]
}
inst.lb.clientProposals = nil
}
}
r.updateCommittedUpTo()
r.recordInstanceMetadata(r.instanceSpace[commit.Instance])
}
func (r *Replica) handlePreAcceptOK(pareply *epaxosproto.PreAcceptOK) {
dlog.Printf("Handling PreAccept reply\n")
inst := r.InstanceSpace[r.Id][pareply.Instance]
if inst.Status != epaxosproto.PREACCEPTED {
// we've moved on, this is a delayed reply
return
}
if !isInitialBallot(inst.ballot) {
return
}
inst.lb.preAcceptOKs++
allCommitted := true
for q := 0; q < r.N; q++ {
if inst.lb.committedDeps[q] < inst.lb.originalDeps[q] {
inst.lb.committedDeps[q] = inst.lb.originalDeps[q]
}
if inst.lb.committedDeps[q] < r.CommittedUpTo[q] {
inst.lb.committedDeps[q] = r.CommittedUpTo[q]
}
if inst.lb.committedDeps[q] < inst.Deps[q] {
allCommitted = false
}
}
//can we commit on the fast path?
if inst.lb.preAcceptOKs >= r.N/2 && inst.lb.allEqual && allCommitted && isInitialBallot(inst.ballot) {
happy++
r.InstanceSpace[r.Id][pareply.Instance].Status = epaxosproto.COMMITTED
r.updateCommitted(r.Id)
if inst.lb.clientProposals != nil && !r.Dreply {
// give clients the all clear
for i := 0; i < len(inst.lb.clientProposals); i++ {
r.ReplyProposeTS(
&genericsmrproto.ProposeReplyTS{
TRUE,
inst.lb.clientProposals[i].CommandId,
state.NIL,
inst.lb.clientProposals[i].Timestamp},
inst.lb.clientProposals[i].Reply)
}
}
r.recordInstanceMetadata(inst)
r.sync() //is this necessary here?
r.bcastCommit(r.Id, pareply.Instance, inst.Cmds, inst.Seq, inst.Deps)
} else if inst.lb.preAcceptOKs >= r.N/2 {
if !allCommitted {
weird++
}
slow++
inst.Status = epaxosproto.ACCEPTED
r.bcastAccept(r.Id, pareply.Instance, inst.ballot, int32(len(inst.Cmds)), inst.Seq, inst.Deps)
}
//TODO: take the slow path if messages are slow to arrive
}
func (r *Replica) handleCommit(commit *lpaxosproto.Commit) {
inst := r.InstanceSpace[commit.Instance]
dlog.Printf("Committing instance %d\n", commit.Instance)
if inst == nil {
r.InstanceSpace[commit.Instance] = &Instance{
commit.LeaseUpdate,
commit.Ballot,
COMMITTED,
nil}
} else {
r.InstanceSpace[commit.Instance].Updates = commit.LeaseUpdate
r.InstanceSpace[commit.Instance].Status = COMMITTED
r.InstanceSpace[commit.Instance].ballot = commit.Ballot
//try to propose in a different instance or just give up?
}
if r.LatestCommitted < commit.Instance {
r.LatestCommitted = commit.Instance
}
/*r.recordInstanceMetadata(r.InstanceSpace[commit.Instance])
r.recordUpdates(commit.Updates)*/
}
func (r *Replica) handleCommit(commit *paxosproto.Commit) {
inst := r.instanceSpace[commit.Instance]
dlog.Printf("Committing instance %d\n", commit.Instance)
if inst == nil {
r.instanceSpace[commit.Instance] = &Instance{
commit.Command,
commit.Ballot,
COMMITTED,
nil,
0, false}
r.addUpdatingKeys(commit.Command)
} else {
r.instanceSpace[commit.Instance].cmds = commit.Command
r.instanceSpace[commit.Instance].status = COMMITTED
r.instanceSpace[commit.Instance].ballot = commit.Ballot
if inst.lb != nil && inst.lb.clientProposals != nil {
for i := 0; i < len(inst.lb.clientProposals); i++ {
r.ProposeChan <- inst.lb.clientProposals[i]
}
inst.lb.clientProposals = nil
}
}
if commit.Instance > r.latestAcceptedInst {
r.latestAcceptedInst = commit.Instance
}
r.updateCommittedUpTo()
r.recordInstanceMetadata(r.instanceSpace[commit.Instance])
r.recordCommands(commit.Command)
}
func (r *Replica) handleAcceptReply(areply *menciusproto.AcceptReply) {
dlog.Printf("AcceptReply for instance %d\n", areply.Instance)
inst := r.instanceSpace[areply.Instance]
if areply.OK == TRUE {
inst.lb.acceptOKs++
if areply.SkippedStartInstance > -1 {
r.instanceSpace[areply.SkippedStartInstance] = &Instance{true,
int(areply.SkippedEndInstance-areply.SkippedStartInstance)/r.N + 1,
nil,
0,
COMMITTED,
nil}
r.updateBlocking(areply.SkippedStartInstance)
}
if inst.status == COMMITTED || inst.status == EXECUTED { //TODO || aargs.Ballot != inst.ballot {
// we've moved on, these are delayed replies, so just ignore
return
}
if inst.lb.acceptOKs+1 > r.N>>1 {
if inst.skipped {
//TODO what if
}
inst.status = READY
if !inst.skipped && areply.Instance > r.latestInstReady {
r.latestInstReady = areply.Instance
}
r.updateBlocking(areply.Instance)
}
} else {
// TODO: there is probably another active leader
inst.lb.nacks++
if areply.Ballot > inst.lb.maxRecvBallot {
inst.lb.maxRecvBallot = areply.Ballot
}
if (areply.Ballot&0x0F)%int32(r.N) == areply.Instance%int32(r.N) {
// the owner of the instance is trying to commit something, I should give up
}
if inst.lb.nacks >= r.N>>1 {
// TODO
if inst.lb.clientProposal != nil {
// I'm the owner of the instance, I'll try again with a higher ballot number
inst.ballot = r.makeBallotLargerThan(inst.lb.maxRecvBallot)
r.bcastPrepare(areply.Instance, inst.ballot)
}
}
}
}
func (r *Replica) handleProposeLease(propose *lpaxosproto.ProposeLease) {
if !r.IsLeader {
//TODO: should notify sender? not necessary, but may speed things up
return
}
for r.InstanceSpace[r.crtInstance] != nil {
r.crtInstance++
}
instNo := r.crtInstance
r.crtInstance++
if r.defaultBallot == -1 {
r.InstanceSpace[instNo] = &Instance{
propose.Updates,
r.makeUniqueBallot(0),
PREPARING,
&LeaderBookkeeping{0, 0, 0, 0}}
r.bcastPrepare(instNo, r.makeUniqueBallot(0), true)
dlog.Printf("Classic round for instance %d\n", instNo)
} else {
r.InstanceSpace[instNo] = &Instance{
propose.Updates,
r.defaultBallot,
PREPARED,
&LeaderBookkeeping{0, 0, 0, 0}}
/*r.recordInstanceMetadata(r.InstanceSpace[instNo])
r.recordCommands(cmds)
r.sync()*/
r.bcastAccept(instNo, r.defaultBallot, propose.Updates)
dlog.Printf("Fast round for instance %d\n", instNo)
}
}
func (r *Replica) handlePrepareReply(preply *menciusproto.PrepareReply) {
dlog.Printf("PrepareReply for instance %d\n", preply.Instance)
inst := r.instanceSpace[preply.Instance]
if inst.status != PREPARING {
// we've moved on -- these are delayed replies, so just ignore
return
}
if preply.OK == TRUE {
inst.lb.prepareOKs++
if preply.Ballot > inst.lb.maxRecvBallot {
inst.command = &preply.Command
inst.skipped = false
if preply.Skip == TRUE {
inst.skipped = true
}
inst.nbInstSkipped = int(preply.NbInstancesToSkip)
inst.lb.maxRecvBallot = preply.Ballot
}
if inst.lb.prepareOKs+1 > r.N>>1 {
inst.status = ACCEPTED
inst.lb.nacks = 0
skip := FALSE
if inst.skipped {
skip = TRUE
}
r.bcastAccept(preply.Instance, inst.ballot, skip, int32(inst.nbInstSkipped), *inst.command)
}
} else {
// TODO: there is probably another active leader
inst.lb.nacks++
if preply.Ballot > inst.lb.maxRecvBallot {
inst.lb.maxRecvBallot = preply.Ballot
}
if inst.lb.nacks >= r.N>>1 && inst.lb != nil {
// TODO: better to wait a while
// some other replica is trying to commit skips for our instance
// increase ballot number and try again
inst.ballot = r.makeBallotLargerThan(inst.lb.maxRecvBallot)
r.bcastPrepare(preply.Instance, inst.ballot)
}
}
}
func (r *Replica) handlePropose(propose *genericsmr.Propose) {
instNo := r.crtInstance
r.crtInstance += int32(r.N)
r.instanceSpace[instNo] = &Instance{false,
0,
&propose.Command,
r.makeBallotLargerThan(0),
ACCEPTED,
&LeaderBookkeeping{propose, 0, 0, 0, 0}}
r.recordInstanceMetadata(r.instanceSpace[instNo])
r.recordCommand(&propose.Command)
r.sync()
r.bcastAccept(instNo, r.instanceSpace[instNo].ballot, FALSE, 0, propose.Command)
dlog.Printf("Choosing req. %d in instance %d\n", propose.CommandId, instNo)
}
func (r *Replica) handleCommit(commit *menciusproto.Commit) {
inst := r.instanceSpace[commit.Instance]
dlog.Printf("Committing instance %d\n", commit.Instance)
if inst == nil {
skip := false
if commit.Skip == TRUE {
skip = true
}
r.instanceSpace[commit.Instance] = &Instance{skip,
int(commit.NbInstancesToSkip),
nil, //&commit.Command,
0,
COMMITTED,
nil}
} else {
//inst.command = &commit.Command
inst.status = COMMITTED
inst.skipped = false
if commit.Skip == TRUE {
inst.skipped = true
}
inst.nbInstSkipped = int(commit.NbInstancesToSkip)
if inst.lb != nil && inst.lb.clientProposal != nil {
// try command in the next available instance
r.ProposeChan <- inst.lb.clientProposal
inst.lb.clientProposal = nil
}
}
r.recordInstanceMetadata(r.instanceSpace[commit.Instance])
if commit.Instance%int32(r.N) == r.Id%int32(r.N) {
if r.crtInstance < commit.Instance+commit.NbInstancesToSkip*int32(r.N) {
r.crtInstance = commit.Instance + commit.NbInstancesToSkip*int32(r.N)
}
}
// Try to commit instances waiting for this one
r.updateBlocking(commit.Instance)
}
func (r *Replica) handle2b(msg *gpaxosproto.M_2b) {
if msg.Balnum != r.crtBalnum {
dlog.Println("2b from a different ballot")
return
}
crtbal := r.ballotArray[r.crtBalnum]
if r.isLeader && crtbal.status != PHASE2 {
log.Println("2b before its time")
return
}
crtbal.lb.cstructs[msg.ReplicaId] = msg.Cstruct
dlog.Printf("Replica %d 2b msg.Cstruct: ", msg.ReplicaId)
dlog.Println(msg.Cstruct)
dlog.Println("my cstruct:", crtbal.cstruct)
crtbal.lb.cstructs[r.Id] = crtbal.cstruct
r.tryToLearn()
}
func (r *Replica) handleCommitShort(commit *lpaxosproto.CommitShort) {
inst := r.InstanceSpace[commit.Instance]
if inst != nil && inst.Updates != nil {
log.Printf("Lease update commit at replica %d\n", r.Id)
}
dlog.Printf("Committing instance %d\n", commit.Instance)
if inst == nil {
r.InstanceSpace[commit.Instance] = &Instance{nil,
commit.Ballot,
COMMITTED,
nil}
} else {
r.InstanceSpace[commit.Instance].Status = COMMITTED
r.InstanceSpace[commit.Instance].ballot = commit.Ballot
if r.LatestCommitted < commit.Instance {
r.LatestCommitted = commit.Instance
}
//try to propose in a different instance or just give up?
}
//r.recordInstanceMetadata(r.InstanceSpace[commit.Instance])
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
}
if r.Id == 0 {
r.IsLeader = true
}
clockChan = make(chan bool, 1)
go r.clock()
clockRang := false
for !r.Shutdown {
select {
case <-clockChan:
clockRang = true
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*paxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*paxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*paxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*paxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
}
if clockRang {
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with op %d\n", propose.Command.Op)
r.handlePropose(propose)
clockRang = false
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*paxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*paxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*paxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*paxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
}
}
}
}
func (r *Replica) executeCommands() {
execedUpTo := int32(-1)
skippedTo := make([]int32, r.N)
skippedToOrig := make([]int32, r.N)
conflicts := make(map[state.Key]int32, 60000)
for q := 0; q < r.N; q++ {
skippedToOrig[q] = -1
}
for !r.Shutdown {
executed := false
jump := false
copy(skippedTo, skippedToOrig)
for i := execedUpTo + 1; i < r.crtInstance; i++ {
if i < skippedTo[i%int32(r.N)] {
continue
}
if r.instanceSpace[i] == nil {
break
}
if r.instanceSpace[i].status == EXECUTED {
continue
}
if r.instanceSpace[i].status != COMMITTED {
if !r.instanceSpace[i].skipped {
confInst, present := conflicts[r.instanceSpace[i].command.K]
if present && r.instanceSpace[confInst].status != EXECUTED {
break
}
conflicts[r.instanceSpace[i].command.K] = i
jump = true
continue
} else {
break
}
}
if r.instanceSpace[i].skipped {
skippedTo[i%int32(r.N)] = i + int32(r.instanceSpace[i].nbInstSkipped*r.N)
if !jump {
skippedToOrig[i%int32(r.N)] = skippedTo[i%int32(r.N)]
}
continue
}
inst := r.instanceSpace[i]
for inst.command == nil {
time.Sleep(1000 * 1000)
}
confInst, present := conflicts[inst.command.K]
if present && confInst < i && r.instanceSpace[confInst].status != EXECUTED && state.Conflict(r.instanceSpace[confInst].command, inst.command) {
break
}
inst.command.Execute(r.State)
if r.Dreply && inst.lb != nil && inst.lb.clientProposal != nil {
dlog.Printf("Sending ACK for req. %d\n", inst.lb.clientProposal.CommandId)
r.ReplyProposeTS(&genericsmrproto.ProposeReplyTS{TRUE, inst.lb.clientProposal.CommandId, state.NIL, inst.lb.clientProposal.Timestamp},
inst.lb.clientProposal.Reply)
}
inst.status = EXECUTED
executed = true
if !jump {
execedUpTo = i
}
}
if !executed {
time.Sleep(1000 * 1000)
}
}
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
}
if r.Id == 0 {
//init quorum read lease
quorum := make([]int32, r.N/2+1)
for i := 0; i <= r.N/2; i++ {
quorum[i] = int32(i)
}
r.UpdatePreferredPeerOrder(quorum)
}
slowClockChan = make(chan bool, 1)
fastClockChan = make(chan bool, 1)
go r.slowClock()
//Enabled when batching for 5ms
if MAX_BATCH > 100 {
go r.fastClock()
}
if r.Beacon {
go r.stopAdapting()
}
for !r.Shutdown {
handleNewProposals := true
// Enabled when batching for 5ms
if MAX_BATCH > 100 {
handleNewProposals = false
}
select {
case <-fastClockChan:
handleNewProposals = true
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*epaxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare for instance %d.%d\n", prepare.Replica, prepare.Instance)
r.handlePrepare(prepare)
break
case preAcceptS := <-r.preAcceptChan:
preAccept := preAcceptS.(*epaxosproto.PreAccept)
//got a PreAccept message
dlog.Printf("Received PreAccept for instance %d.%d\n", preAccept.LeaderId, preAccept.Instance)
r.handlePreAccept(preAccept)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*epaxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept for instance %d.%d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*epaxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*epaxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*epaxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d.%d\n", prepareReply.Replica, prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case preAcceptReplyS := <-r.preAcceptReplyChan:
preAcceptReply := preAcceptReplyS.(*epaxosproto.PreAcceptReply)
//got a PreAccept reply
dlog.Printf("Received PreAcceptReply for instance %d.%d\n", preAcceptReply.Replica, preAcceptReply.Instance)
r.handlePreAcceptReply(preAcceptReply)
break
case preAcceptOKS := <-r.preAcceptOKChan:
preAcceptOK := preAcceptOKS.(*epaxosproto.PreAcceptOK)
//got a PreAccept reply
dlog.Printf("Received PreAcceptOK for instance %d.%d\n", r.Id, preAcceptOK.Instance)
r.handlePreAcceptOK(preAcceptOK)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*epaxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d.%d\n", acceptReply.Replica, acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case tryPreAcceptS := <-r.tryPreAcceptChan:
tryPreAccept := tryPreAcceptS.(*epaxosproto.TryPreAccept)
dlog.Printf("Received TryPreAccept for instance %d.%d\n", tryPreAccept.Replica, tryPreAccept.Instance)
r.handleTryPreAccept(tryPreAccept)
break
case tryPreAcceptReplyS := <-r.tryPreAcceptReplyChan:
tryPreAcceptReply := tryPreAcceptReplyS.(*epaxosproto.TryPreAcceptReply)
dlog.Printf("Received TryPreAcceptReply for instance %d.%d\n", tryPreAcceptReply.Replica, tryPreAcceptReply.Instance)
r.handleTryPreAcceptReply(tryPreAcceptReply)
break
case beacon := <-r.BeaconChan:
dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
r.ReplyBeacon(beacon)
break
case <-slowClockChan:
if r.Beacon {
for q := int32(0); q < int32(r.N); q++ {
if q == r.Id {
continue
}
r.SendBeacon(q)
}
}
break
case <-r.OnClientConnect:
log.Printf("weird %d; conflicted %d; slow %d; happy %d\n", weird, conflicted, slow, happy)
weird, conflicted, slow, happy = 0, 0, 0, 0
case iid := <-r.instancesToRecover:
r.startRecoveryForInstance(iid.replica, iid.instance)
}
if handleNewProposals {
// unfortunately the only way to activate the new proposals channel every X ms
// was to dublicate the body of the select
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with op %d\n", propose.Command.Op)
r.handlePropose(propose)
handleNewProposals = false
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*epaxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare for instance %d.%d\n", prepare.Replica, prepare.Instance)
r.handlePrepare(prepare)
break
case preAcceptS := <-r.preAcceptChan:
preAccept := preAcceptS.(*epaxosproto.PreAccept)
//got a PreAccept message
dlog.Printf("Received PreAccept for instance %d.%d\n", preAccept.LeaderId, preAccept.Instance)
r.handlePreAccept(preAccept)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*epaxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept for instance %d.%d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*epaxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*epaxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*epaxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d.%d\n", prepareReply.Replica, prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case preAcceptReplyS := <-r.preAcceptReplyChan:
preAcceptReply := preAcceptReplyS.(*epaxosproto.PreAcceptReply)
//got a PreAccept reply
dlog.Printf("Received PreAcceptReply for instance %d.%d\n", preAcceptReply.Replica, preAcceptReply.Instance)
r.handlePreAcceptReply(preAcceptReply)
break
case preAcceptOKS := <-r.preAcceptOKChan:
preAcceptOK := preAcceptOKS.(*epaxosproto.PreAcceptOK)
//got a PreAccept reply
dlog.Printf("Received PreAcceptOK for instance %d.%d\n", r.Id, preAcceptOK.Instance)
r.handlePreAcceptOK(preAcceptOK)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*epaxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d.%d\n", acceptReply.Replica, acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case tryPreAcceptS := <-r.tryPreAcceptChan:
tryPreAccept := tryPreAcceptS.(*epaxosproto.TryPreAccept)
dlog.Printf("Received TryPreAccept for instance %d.%d\n", tryPreAccept.Replica, tryPreAccept.Instance)
r.handleTryPreAccept(tryPreAccept)
break
case tryPreAcceptReplyS := <-r.tryPreAcceptReplyChan:
tryPreAcceptReply := tryPreAcceptReplyS.(*epaxosproto.TryPreAcceptReply)
dlog.Printf("Received TryPreAcceptReply for instance %d.%d\n", tryPreAcceptReply.Replica, tryPreAcceptReply.Instance)
r.handleTryPreAcceptReply(tryPreAcceptReply)
break
case beacon := <-r.BeaconChan:
dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
r.ReplyBeacon(beacon)
break
case <-slowClockChan:
if r.Beacon {
for q := int32(0); q < int32(r.N); q++ {
if q == r.Id {
continue
}
r.SendBeacon(q)
}
}
break
case <-r.OnClientConnect:
log.Printf("weird %d; conflicted %d; slow %d; happy %d\n", weird, conflicted, slow, happy)
weird, conflicted, slow, happy = 0, 0, 0, 0
break
case iid := <-r.instancesToRecover:
r.startRecoveryForInstance(iid.replica, iid.instance)
}
}
}
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
}
go r.clock()
for !r.Shutdown {
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with id %d\n", propose.CommandId)
r.handlePropose(propose)
break
case skipS := <-r.skipChan:
skip := skipS.(*menciusproto.Skip)
//got a Skip from another replica
dlog.Printf("Skip for instances %d-%d\n", skip.StartInstance, skip.EndInstance)
r.handleSkip(skip)
case prepareS := <-r.prepareChan:
prepare := prepareS.(*menciusproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*menciusproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*menciusproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*menciusproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*menciusproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case delayedSkip := <-r.delayedSkipChan:
r.handleDelayedSkip(delayedSkip)
break
case <-r.clockChan:
if lastSeenInstance == r.blockingInstance {
r.noCommitFor++
} else {
r.noCommitFor = 0
lastSeenInstance = r.blockingInstance
}
if r.noCommitFor >= 50+int(r.Id) && r.crtInstance >= r.blockingInstance+int32(r.N) {
r.noCommitFor = 0
dlog.Printf("Doing force commit\n")
r.forceCommit()
}
break
}
}
}
func (r *Replica) handlePreAcceptReply(pareply *epaxosproto.PreAcceptReply) {
dlog.Printf("Handling PreAccept reply\n")
inst := r.InstanceSpace[pareply.Replica][pareply.Instance]
if inst.Status != epaxosproto.PREACCEPTED {
// we've moved on, this is a delayed reply
return
}
if inst.ballot != pareply.Ballot {
return
}
if pareply.OK == FALSE {
// TODO: there is probably another active leader
inst.lb.nacks++
if pareply.Ballot > inst.lb.maxRecvBallot {
inst.lb.maxRecvBallot = pareply.Ballot
}
if inst.lb.nacks >= r.N/2 {
// TODO
}
return
}
inst.lb.preAcceptOKs++
var equal bool
inst.Seq, inst.Deps, equal = r.mergeAttributes(inst.Seq, inst.Deps, pareply.Seq, pareply.Deps)
if (r.N <= 3 && !r.Thrifty) || inst.lb.preAcceptOKs > 1 {
inst.lb.allEqual = inst.lb.allEqual && equal
if !equal {
conflicted++
}
}
allCommitted := true
for q := 0; q < r.N; q++ {
if inst.lb.committedDeps[q] < pareply.CommittedDeps[q] {
inst.lb.committedDeps[q] = pareply.CommittedDeps[q]
}
if inst.lb.committedDeps[q] < r.CommittedUpTo[q] {
inst.lb.committedDeps[q] = r.CommittedUpTo[q]
}
if inst.lb.committedDeps[q] < inst.Deps[q] {
allCommitted = false
}
}
//can we commit on the fast path?
if inst.lb.preAcceptOKs >= r.N/2 && inst.lb.allEqual && allCommitted && isInitialBallot(inst.ballot) {
happy++
dlog.Printf("Fast path for instance %d.%d\n", pareply.Replica, pareply.Instance)
r.InstanceSpace[pareply.Replica][pareply.Instance].Status = epaxosproto.COMMITTED
r.updateCommitted(pareply.Replica)
if inst.lb.clientProposals != nil && !r.Dreply {
// give clients the all clear
for i := 0; i < len(inst.lb.clientProposals); i++ {
r.ReplyProposeTS(
&genericsmrproto.ProposeReplyTS{
TRUE,
inst.lb.clientProposals[i].CommandId,
state.NIL,
inst.lb.clientProposals[i].Timestamp},
inst.lb.clientProposals[i].Reply)
}
}
r.recordInstanceMetadata(inst)
r.sync() //is this necessary here?
r.bcastCommit(pareply.Replica, pareply.Instance, inst.Cmds, inst.Seq, inst.Deps)
} else if inst.lb.preAcceptOKs >= r.N/2 {
if !allCommitted {
weird++
}
slow++
inst.Status = epaxosproto.ACCEPTED
r.bcastAccept(pareply.Replica, pareply.Instance, inst.ballot, int32(len(inst.Cmds)), inst.Seq, inst.Deps)
}
//TODO: take the slow path if messages are slow to arrive
}
func (r *Replica) handleAccept(accept *menciusproto.Accept) {
flush := true
inst := r.instanceSpace[accept.Instance]
if inst != nil && inst.ballot > accept.Ballot {
r.replyAccept(accept.LeaderId, &menciusproto.AcceptReply{accept.Instance, FALSE, inst.ballot, -1, -1})
return
}
skipStart := int32(-1)
skipEnd := int32(-1)
if accept.Skip == FALSE && r.crtInstance < accept.Instance {
skipStart = r.crtInstance
skipEnd = accept.Instance/int32(r.N)*int32(r.N) + r.Id
if skipEnd > accept.Instance {
skipEnd -= int32(r.N)
}
if r.skipsWaiting < MAX_SKIPS_WAITING {
//start a timer, waiting for a propose to arrive and fill this hole
go r.timerHelper(&DelayedSkip{skipEnd})
//r.delayedSkipChan <- &DelayedSkip{accept, skipStart}
r.skipsWaiting++
flush = false
}
r.instanceSpace[r.crtInstance] = &Instance{true,
int(skipEnd-r.crtInstance)/r.N + 1,
nil,
-1,
COMMITTED,
nil}
r.recordInstanceMetadata(r.instanceSpace[r.crtInstance])
r.sync()
r.crtInstance = skipEnd + int32(r.N)
}
if inst == nil {
skip := false
if accept.Skip == TRUE {
skip = true
}
r.instanceSpace[accept.Instance] = &Instance{skip,
int(accept.NbInstancesToSkip),
&accept.Command,
accept.Ballot,
ACCEPTED,
nil}
r.recordInstanceMetadata(r.instanceSpace[accept.Instance])
r.recordCommand(&accept.Command)
r.sync()
r.replyAccept(accept.LeaderId, &menciusproto.AcceptReply{accept.Instance, TRUE, -1, skipStart, skipEnd})
} else {
if inst.status == COMMITTED || inst.status == EXECUTED {
if inst.command == nil {
inst.command = &accept.Command
}
dlog.Printf("ATTENTION! Reordered Commit\n")
} else {
inst.command = &accept.Command
inst.ballot = accept.Ballot
inst.status = ACCEPTED
inst.skipped = false
if accept.Skip == TRUE {
inst.skipped = true
}
inst.nbInstSkipped = int(accept.NbInstancesToSkip)
r.recordInstanceMetadata(inst)
r.replyAccept(accept.LeaderId, &menciusproto.AcceptReply{accept.Instance, TRUE, inst.ballot, skipStart, skipEnd})
}
}
if skipStart >= 0 {
dlog.Printf("Skipping!!\n")
r.bcastSkip(skipStart, skipEnd, accept.LeaderId)
r.updateBlocking(skipStart)
if flush {
for _, w := range r.PeerWriters {
if w != nil {
w.Flush()
}
}
}
} else {
r.updateBlocking(accept.Instance)
}
}
func (r *Replica) handlePreAccept(preAccept *epaxosproto.PreAccept) {
inst := r.InstanceSpace[preAccept.LeaderId][preAccept.Instance]
if preAccept.Seq >= r.maxSeq {
r.maxSeq = preAccept.Seq + 1
}
if inst != nil && (inst.Status == epaxosproto.COMMITTED || inst.Status == epaxosproto.ACCEPTED) {
//reordered handling of commit/accept and pre-accept
if inst.Cmds == nil {
r.InstanceSpace[preAccept.LeaderId][preAccept.Instance].Cmds = preAccept.Command
r.updateConflicts(preAccept.Command, preAccept.Replica, preAccept.Instance, preAccept.Seq)
//r.InstanceSpace[preAccept.LeaderId][preAccept.Instance].bfilter = bfFromCommands(preAccept.Command)
}
r.recordCommands(preAccept.Command)
r.sync()
return
}
if preAccept.Instance >= r.crtInstance[preAccept.Replica] {
r.crtInstance[preAccept.Replica] = preAccept.Instance + 1
}
//update attributes for command
seq, deps, changed := r.updateAttributes(preAccept.Command, preAccept.Seq, preAccept.Deps, preAccept.Replica, preAccept.Instance)
uncommittedDeps := false
for q := 0; q < r.N; q++ {
if deps[q] > r.CommittedUpTo[q] {
uncommittedDeps = true
break
}
}
status := epaxosproto.PREACCEPTED_EQ
if changed {
status = epaxosproto.PREACCEPTED
}
if inst != nil {
if preAccept.Ballot < inst.ballot {
r.replyPreAccept(preAccept.LeaderId,
&epaxosproto.PreAcceptReply{
preAccept.Replica,
preAccept.Instance,
FALSE,
inst.ballot,
inst.Seq,
inst.Deps,
r.CommittedUpTo})
return
} else {
inst.Cmds = preAccept.Command
inst.Seq = seq
inst.Deps = deps
inst.ballot = preAccept.Ballot
inst.Status = status
}
} else {
r.InstanceSpace[preAccept.Replica][preAccept.Instance] = &Instance{
preAccept.Command,
preAccept.Ballot,
status,
seq,
deps,
nil, 0, 0,
nil}
}
r.updateConflicts(preAccept.Command, preAccept.Replica, preAccept.Instance, preAccept.Seq)
r.recordInstanceMetadata(r.InstanceSpace[preAccept.Replica][preAccept.Instance])
r.recordCommands(preAccept.Command)
r.sync()
if len(preAccept.Command) == 0 {
//checkpoint
//update latest checkpoint info
r.latestCPReplica = preAccept.Replica
r.latestCPInstance = preAccept.Instance
//discard dependency hashtables
r.clearHashtables()
}
if changed || uncommittedDeps || preAccept.Replica != preAccept.LeaderId || !isInitialBallot(preAccept.Ballot) {
r.replyPreAccept(preAccept.LeaderId,
&epaxosproto.PreAcceptReply{
preAccept.Replica,
preAccept.Instance,
TRUE,
preAccept.Ballot,
seq,
deps,
r.CommittedUpTo})
} else {
pok := &epaxosproto.PreAcceptOK{preAccept.Instance}
r.SendMsg(preAccept.LeaderId, r.preAcceptOKRPC, pok)
}
dlog.Printf("I've replied to the PreAccept\n")
}
func main() {
flag.Parse()
runtime.GOMAXPROCS(*procs)
randObj := rand.New(rand.NewSource(42))
zipf := rand.NewZipf(randObj, *s, *v, uint64(*reqsNb / *rounds + *eps))
if *conflicts > 100 {
log.Fatalf("Conflicts percentage must be between 0 and 100.\n")
}
master, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%d", *masterAddr, *masterPort))
if err != nil {
log.Fatalf("Error connecting to master\n")
}
rlReply := new(masterproto.GetReplicaListReply)
err = master.Call("Master.GetReplicaList", new(masterproto.GetReplicaListArgs), rlReply)
if err != nil {
log.Fatalf("Error making the GetReplicaList RPC")
}
N = len(rlReply.ReplicaList)
servers := make([]net.Conn, N)
readers := make([]*bufio.Reader, N)
writers := make([]*bufio.Writer, N)
rarray = make([]int, *reqsNb / *rounds + *eps)
karray := make([]int64, *reqsNb / *rounds + *eps)
perReplicaCount := make([]int, N)
test := make([]int, *reqsNb / *rounds + *eps)
for i := 0; i < len(rarray); i++ {
r := rand.Intn(N)
rarray[i] = r
if i < *reqsNb / *rounds {
perReplicaCount[r]++
}
if *conflicts >= 0 {
r = rand.Intn(100)
if r < *conflicts {
karray[i] = 42
} else {
karray[i] = int64(43 + i)
}
} else {
karray[i] = int64(zipf.Uint64())
test[karray[i]]++
}
}
if *conflicts >= 0 {
fmt.Println("Uniform distribution")
} else {
fmt.Println("Zipfian distribution:")
//fmt.Println(test[0:100])
}
for i := 0; i < N; i++ {
var err error
servers[i], err = net.Dial("tcp", rlReply.ReplicaList[i])
if err != nil {
log.Printf("Error connecting to replica %d\n", i)
}
readers[i] = bufio.NewReader(servers[i])
writers[i] = bufio.NewWriter(servers[i])
}
successful = make([]int, N)
leader := 0
if *noLeader == false {
reply := new(masterproto.GetLeaderReply)
if err = master.Call("Master.GetLeader", new(masterproto.GetLeaderArgs), reply); err != nil {
log.Fatalf("Error making the GetLeader RPC\n")
}
leader = reply.LeaderId
log.Printf("The leader is replica %d\n", leader)
}
var id int32 = 0
done := make(chan bool, N)
args := genericsmrproto.Propose{id, state.Command{state.PUT, 0, 0}}
before_total := time.Now()
for j := 0; j < *rounds; j++ {
n := *reqsNb / *rounds
if *check {
rsp = make([]bool, n)
for j := 0; j < n; j++ {
rsp[j] = false
}
}
donePrinting := make(chan bool)
readings := make(chan int64, n)
go printer(readings, donePrinting)
if *noLeader {
for i := 0; i < N; i++ {
go waitReplies(readers, i, perReplicaCount[i], done, readings)
}
} else {
go waitReplies(readers, leader, n, done, readings)
}
before := time.Now()
for i := 0; i < n+*eps; i++ {
dlog.Printf("Sending proposal %d\n", id)
args.ClientId = id
args.Command.K = state.Key(karray[i])
args.Command.V = state.Value(time.Now().UnixNano())
if !*fast {
if *noLeader {
leader = rarray[i]
}
writers[leader].WriteByte(genericsmrproto.PROPOSE)
args.Marshal(writers[leader])
} else {
//send to everyone
for rep := 0; rep < N; rep++ {
writers[rep].WriteByte(genericsmrproto.PROPOSE)
args.Marshal(writers[rep])
writers[rep].Flush()
}
}
//fmt.Println("Sent", id)
id++
if i%*batch == 0 {
for i := 0; i < N; i++ {
writers[i].Flush()
}
if *nanosleep > 0 {
time.Sleep(time.Duration(*nanosleep))
}
}
}
for i := 0; i < N; i++ {
writers[i].Flush()
}
err := false
if *noLeader {
for i := 0; i < N; i++ {
e := <-done
err = e || err
}
} else {
err = <-done
}
after := time.Now()
<-donePrinting
fmt.Printf("Round took %v\n", after.Sub(before))
if *check {
for j := 0; j < n; j++ {
if !rsp[j] {
fmt.Println("Didn't receive", j)
}
}
}
if err {
if *noLeader {
N = N - 1
} else {
reply := new(masterproto.GetLeaderReply)
master.Call("Master.GetLeader", new(masterproto.GetLeaderArgs), reply)
leader = reply.LeaderId
log.Printf("New leader is replica %d\n", leader)
}
}
}
after_total := time.Now()
fmt.Printf("Test took %v\n", after_total.Sub(before_total))
s := 0
for _, succ := range successful {
s += succ
}
fmt.Printf("Successful: %d\n", s)
for _, client := range servers {
if client != nil {
client.Close()
}
}
master.Close()
}
func (r *Replica) handlePropose(propose *genericsmr.Propose) {
batches := make(map[int64][]state.Command, 10)
proposals := make(map[int64][]*genericsmr.Propose, 10)
haveWrites := false
totalLen := len(r.ProposeChan) + 1
if totalLen > MAX_BATCH {
totalLen = MAX_BATCH
}
for i := 0; i < totalLen; i++ {
if state.IsRead(&propose.Command) && (r.IsLeader || r.isKeyGranted(propose.Command.K)) {
reads++
//make sure that the channel is not going to be full,
//because this may cause the consumer to block when trying to
//put request back
for len(r.readsChannel) >= READ_CHAN_SIZE-10000 {
time.Sleep(1000)
}
r.readsChannel <- propose
} else if !r.IsLeader {
if state.IsRead(&propose.Command) {
reads++
}
r.fwdId++
r.fwdPropMap[r.fwdId] = propose
r.SendMsg(r.leaderId, r.forwardRPC, &paxosproto.Forward{r.Id, r.fwdId, propose.Command})
} else {
q := quorumToInt64(r.getLeaseQuorumForKey(propose.Command.K, r.Id))
var cmds []state.Command
var present bool
var props []*genericsmr.Propose
props = proposals[q]
if cmds, present = batches[q]; !present {
cmds = make([]state.Command, 0, totalLen/2+1)
props = make([]*genericsmr.Propose, 0, totalLen/2+1)
}
cmds = append(cmds, propose.Command)
batches[q] = cmds
props = append(props, propose)
proposals[q] = props
haveWrites = true
}
if i < totalLen-1 {
propose = <-r.ProposeChan
}
}
if !haveWrites {
return
}
/* if !r.IsLeader {
preply := &genericsmrproto.ProposeReplyTS{FALSE, -1, state.NIL, 0}
r.ReplyProposeTS(preply, propose)
return
}*/
for r.instanceSpace[r.crtInstance] != nil {
r.crtInstance++
}
//dlog.Printf("Batched %d\n", batchSize)
status := PREPARING
ballot := r.makeUniqueBallot(0)
if r.defaultBallot > -1 {
status = PREPARED
ballot = r.defaultBallot
}
for q, cmds := range batches {
if len(cmds) == 0 {
continue
}
//log.Printf("Batching %d\n", len(cmds))
props := proposals[q]
r.instanceSpace[r.crtInstance] = &Instance{
cmds,
ballot,
status,
&LeaderBookkeeping{props, 0, 0, 0, 0, 0},
0, false}
if status == PREPARING {
r.bcastPrepare(r.crtInstance, ballot, true)
dlog.Printf("Classic round for instance %d\n", r.crtInstance)
} else {
r.recordInstanceMetadata(r.instanceSpace[r.crtInstance])
r.recordCommands(cmds)
r.sync()
if r.crtInstance > r.latestAcceptedInst {
r.latestAcceptedInst = r.crtInstance
}
if props[0].FwdReplica >= 0 && props[0].FwdReplica != r.Id {
r.addUpdatingKeys(cmds)
}
//TODO: make sure it supports Forwards
r.instanceSpace[r.crtInstance].lb.acceptOKsToWait, _ = r.bcastAccept(r.crtInstance, ballot, cmds, props[0].FwdReplica, props[0].FwdId)
dlog.Printf("Fast round for instance %d\n", r.crtInstance)
if genericsmr.SendError {
log.Println("BCAST ERROR")
r.delayedInstances <- r.crtInstance
}
}
r.crtInstance++
}
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
go r.reader()
}
if r.Id == 0 {
r.IsLeader = true
r.readStats = NewReadStats(r.N, r.Id)
}
clockChan = make(chan bool, 1)
go r.clock()
r.QLease = qlease.NewLease(r.N)
leaseClockChan = make(chan bool, 1)
leaseClockRestart = make(chan bool)
go r.leaseClock()
// clockRang := false
var tickCounter uint64 = 0
latestBeaconFromReplica := make([]uint64, r.N)
proposedDead := make([]bool, r.N)
for i := 0; i < r.N; i++ {
latestBeaconFromReplica[i] = 0
proposedDead[i] = false
}
stopRenewing := false
for !r.Shutdown {
select {
case <-clockChan:
//clockRang = true
tickCounter++
if tickCounter%20 == 0 {
if r.Beacon {
for q := int32(0); q < int32(r.N); q++ {
if q == r.Id {
continue
}
r.SendBeacon(q)
}
}
if r.IsLeader && r.Beacon {
for rid := int32(0); rid < int32(r.N); rid++ {
if rid == r.Id {
continue
}
if !proposedDead[rid] && tickCounter-latestBeaconFromReplica[rid] >= 200 {
log.Println("Proposing replica dead: ", rid)
//replica might be dead
//propose a lease configuration change
dr := make([]int32, 1)
dr[0] = rid
r.proposeReplicasDead(dr)
proposedDead[rid] = true
}
}
}
rightNow := time.Now().UnixNano()
for rid := int32(0); rid < int32(r.N); rid++ {
if rid == r.Id {
continue
}
if !proposedDead[rid] && r.LastReplyReceivedTimestamp[rid] > 0 && rightNow-r.LastReplyReceivedTimestamp[rid] >= GRACE_PERIOD {
if r.IsLeader {
log.Println("Proposing replica dead: ", rid)
//replica might be dead
//propose a lease configuration change
dr := make([]int32, 1)
dr[0] = rid
r.proposeReplicasDead(dr)
}
proposedDead[rid] = true
stopRenewing = true
}
}
if r.IsLeader {
for i := r.committedUpTo; i >= 0 && i < r.crtInstance; i++ {
r.delayedInstances <- i
}
}
}
break
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with op %d\n", propose.Command.Op)
r.handlePropose(propose)
//clockRang = false
break
case forwardS := <-r.forwardChan:
forward := forwardS.(*paxosproto.Forward)
dlog.Printf("Forward proposal from replica %d\n", forward.ReplicaId)
r.handleForward(forward)
break
case frS := <-r.forwardReplyChan:
fr := frS.(*paxosproto.ForwardReply)
r.handleForwardReply(fr)
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*paxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*paxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*paxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*paxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case instNo := <-r.delayedInstances:
log.Println("Trying to re-start instance")
inst := r.instanceSpace[instNo]
if inst.status < COMMITTED {
inst.status = PREPARED
inst.lb.acceptOKs = 0
inst.ballot = r.makeBallotLargerThan(inst.ballot)
inst.lb.acceptOKsToWait, _ = r.bcastAccept(instNo, inst.ballot, inst.cmds, inst.lb.clientProposals[0].FwdReplica, inst.lb.clientProposals[0].FwdId)
if genericsmr.SendError {
r.delayedInstances <- instNo
}
}
case beacon := <-r.BeaconChan:
dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
//r.ReplyBeacon(beacon)
latestBeaconFromReplica[beacon.Rid] = tickCounter
break
case guardS := <-r.QLGuardChan:
guard := guardS.(*qleaseproto.Guard)
r.HandleQLeaseGuard(r.QLease, guard)
break
case guardReplyS := <-r.QLGuardReplyChan:
guardReply := guardReplyS.(*qleaseproto.GuardReply)
r.HandleQLeaseGuardReply(r.QLease, guardReply, r.latestAcceptedInst)
break
case promiseS := <-r.QLPromiseChan:
promise := promiseS.(*qleaseproto.Promise)
prev := r.QLease.PromisedToMeInst
if !r.HandleQLeasePromise(r.QLease, promise) {
continue
}
if prev != r.QLease.PromisedToMeInst {
r.updateGrantedKeys(prev)
newPromiseCount = 0
}
newPromiseCount++
if newPromiseCount <= r.N/2 {
if r.newestInstanceIDontKnow < promise.LatestAcceptedInst {
r.newestInstanceIDontKnow = promise.LatestAcceptedInst
}
}
break
case preplyS := <-r.QLPromiseReplyChan:
preply := preplyS.(*qleaseproto.PromiseReply)
r.HandleQLeaseReply(r.QLease, preply)
break
case <-leaseClockChan:
if r.QLease.PromisedByMeInst < r.leaseSMR.LatestCommitted {
// wait for previous lease to expire before switching to new config
if r.QLease.CanWriteOutside() {
r.updateKeyQuorumInfo(r.leaseSMR.LatestCommitted)
r.QLease.PromisedByMeInst = r.leaseSMR.LatestCommitted
log.Printf("Replica %d - New lease for instance %d\n", r.Id, r.QLease.PromisedByMeInst)
r.EstablishQLease(r.QLease)
stopRenewing = false
}
} else if r.QLease.PromisedByMeInst >= 0 {
if r.QLease.CanWriteOutside() {
r.EstablishQLease(r.QLease)
stopRenewing = false
} else if !stopRenewing {
r.RenewQLease(r.QLease, r.latestAcceptedInst)
}
}
if r.maintainReadStats {
ticks--
if ticks == 0 {
r.maintainReadStats = false
if r.IsLeader && r.readStats != nil {
go r.proposeLeaseReconf()
}
ticks = TICKS_TO_RECONF_LEASE
//ticks = 60
}
}
// restart the clock
leaseClockRestart <- true
case <-r.OnClientConnect:
log.Printf("reads: %d, local: %d\n", reads, local)
}
}
}
func (r *Replica) run() {
if r.Id == 0 {
r.isLeader = true
}
r.ConnectToPeersNoListeners()
for rid, peerReader := range r.PeerReaders {
if int32(rid) == r.Id {
continue
}
go r.handleReplicaConnection(rid, peerReader)
}
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
/*if r.Exec {
go r.executeCommands()
}*/
clockChan = make(chan bool, 1)
go r.clock()
if r.isLeader {
r.crtBalnum = 0
r.fastRound = true
r.ballotArray[0] = &Ballot{nil, 0, PHASE1, false, &LeaderBookkeeping{cstructs: make([][]int32, 0)}}
r.ballotArray[0].lb.cstructs = append(r.ballotArray[0].lb.cstructs, make([]int32, 0))
r.bcast1a(0, true)
}
for !r.Shutdown {
if r.crtBalnum >= 0 && len(r.ballotArray[r.crtBalnum].cstruct) >= CMDS_PER_BALLOT {
select {
case prepare := <-r.prepareChan:
//got a Prepare message
dlog.Printf("Received Prepare for balnum %d\n", prepare.Balnum)
r.commandsMutex.Lock()
r.handlePrepare(prepare)
r.commandsMutex.Unlock()
break
case msg := <-r.m1aChan:
dlog.Printf("Received 1a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m1bChan:
dlog.Printf("Received 1b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1b(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2aChan:
dlog.Printf("Received 2a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2bChan:
dlog.Printf("Received 2b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2b(msg)
r.commandsMutex.Unlock()
break
case <-clockChan:
//way out of deadlock
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with id %d @ replica %d\n", propose.CommandId, r.Id)
r.commandsMutex.Lock()
r.handlePropose(propose)
r.commandsMutex.Unlock()
break
default:
break
}
}
} else {
select {
case prepare := <-r.prepareChan:
//got a Prepare message
dlog.Printf("Received Prepare for balnum %d\n", prepare.Balnum)
r.commandsMutex.Lock()
r.handlePrepare(prepare)
r.commandsMutex.Unlock()
break
case msg := <-r.m1aChan:
dlog.Printf("Received 1a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m1bChan:
dlog.Printf("Received 1b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1b(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2aChan:
dlog.Printf("Received 2a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2bChan:
dlog.Printf("Received 2b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2b(msg)
r.commandsMutex.Unlock()
break
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with id %d @ replica %d\n", propose.CommandId, r.Id)
r.commandsMutex.Lock()
r.handlePropose(propose)
r.commandsMutex.Unlock()
break
}
}
}
}
func main() {
flag.Parse()
runtime.GOMAXPROCS(*procs)
randObj := rand.New(rand.NewSource(42 + int64(*forceLeader)))
//zipf := rand.NewZipf(randObj, *s, *v, uint64(*reqsNb / *rounds + *eps))
zipf := ycsbzipf.NewZipf(int(*reqsNb / *rounds + *eps), randObj)
if *conflicts > 100 {
log.Fatalf("Conflicts percentage must be between 0 and 100.\n")
}
if *writes > 100 {
log.Fatalf("Write percentage cannot be higher than 100.\n")
}
master, err := rpc.DialHTTP("tcp", fmt.Sprintf("%s:%d", *masterAddr, *masterPort))
if err != nil {
log.Fatalf("Error connecting to master\n")
}
rlReply := new(masterproto.GetReplicaListReply)
err = master.Call("Master.GetReplicaList", new(masterproto.GetReplicaListArgs), rlReply)
if err != nil {
log.Fatalf("Error making the GetReplicaList RPC")
}
N = len(rlReply.ReplicaList)
if *forcedN > N {
log.Fatalf("Cannot connect to more than the total number of replicas. -N parameter too high.\n")
}
if *forcedN > 0 {
N = *forcedN
}
servers := make([]net.Conn, N)
readers := make([]*bufio.Reader, N)
writers := make([]*bufio.Writer, N)
rarray = make([]int, *reqsNb / *rounds + *eps)
karrays := make([][]int64, N)
iarray := make([]int, *reqsNb / *rounds + *eps)
put := make([]bool, *reqsNb / *rounds + *eps)
perReplicaCount := make([]int, N)
test := make([]int, *reqsNb / *rounds + *eps)
for j := 0; j < N; j++ {
karrays[j] = make([]int64, *reqsNb / *rounds + *eps)
for i := 0; i < len(karrays[j]); i++ {
karrays[j][i] = int64(i)
}
robj := rand.New(rand.NewSource(442 + int64(j)))
randperm.Permute(karrays[j], robj)
}
for i := 0; i < len(rarray); i++ {
r := rand.Intn(N)
rarray[i] = r
if i < *reqsNb / *rounds {
perReplicaCount[r]++
}
if *conflicts >= 0 {
r = rand.Intn(100)
if r < *conflicts {
iarray[i] = 0
} else {
iarray[i] = i
}
} else {
iarray[i] = int(zipf.NextInt64())
test[karrays[rarray[i]][iarray[i]]]++
}
r = rand.Intn(100)
if r < *writes {
put[i] = true
} else {
put[i] = false
}
}
if *conflicts >= 0 {
fmt.Println("Uniform distribution")
} else {
fmt.Println("Zipfian distribution:")
//fmt.Println(test[0:100])
}
for i := 0; i < N; i++ {
var err error
servers[i], err = net.Dial("tcp", rlReply.ReplicaList[i])
if err != nil {
log.Printf("Error connecting to replica %d\n", i)
}
readers[i] = bufio.NewReader(servers[i])
writers[i] = bufio.NewWriter(servers[i])
}
successful = make([]int, N)
local = make([]int, N)
leader := 0
if *noLeader == false && *forceLeader < 0 {
reply := new(masterproto.GetLeaderReply)
if err = master.Call("Master.GetLeader", new(masterproto.GetLeaderArgs), reply); err != nil {
log.Fatalf("Error making the GetLeader RPC\n")
}
leader = reply.LeaderId
log.Printf("The leader is replica %d\n", leader)
} else if *forceLeader > 0 {
leader = *forceLeader
log.Printf("My leader is replica %d\n", leader)
}
var id int32 = 0
done := make(chan bool, N)
args := genericsmrproto.Propose{id, state.Command{state.PUT, 0, 0}, 0}
before_total := time.Now()
for j := 0; j < *rounds; j++ {
n := *reqsNb / *rounds
if *check {
rsp = make([]bool, n)
for j := 0; j < n; j++ {
rsp[j] = false
}
}
if *noLeader {
for i := 0; i < N; i++ {
go waitReplies(readers, i, perReplicaCount[i], done)
}
} else {
go waitReplies(readers, leader, n, done)
}
before := time.Now()
for i := 0; i < n+*eps; i++ {
dlog.Printf("Sending proposal %d\n", id)
args.CommandId = id
if put[i] {
args.Command.Op = state.PUT
} else {
args.Command.Op = state.GET
}
if !*fast && *noLeader {
leader = rarray[i]
}
args.Command.K = state.Key(karrays[leader][iarray[i]])
args.Command.V = state.Value(i) + 1
//args.Timestamp = time.Now().UnixNano()
if !*fast {
writers[leader].WriteByte(genericsmrproto.PROPOSE)
args.Marshal(writers[leader])
} else {
//send to everyone
for rep := 0; rep < N; rep++ {
writers[rep].WriteByte(genericsmrproto.PROPOSE)
args.Marshal(writers[rep])
writers[rep].Flush()
}
}
//fmt.Println("Sent", id)
id++
if i%100 == 0 {
for i := 0; i < N; i++ {
writers[i].Flush()
}
}
}
for i := 0; i < N; i++ {
writers[i].Flush()
}
err := false
if *noLeader {
for i := 0; i < N; i++ {
e := <-done
err = e || err
}
} else {
err = <-done
}
after := time.Now()
fmt.Printf("Round took %v\n", after.Sub(before))
if *check {
for j := 0; j < n; j++ {
if !rsp[j] {
fmt.Println("Didn't receive", j)
}
}
}
if err {
if *noLeader {
N = N - 1
} else {
reply := new(masterproto.GetLeaderReply)
master.Call("Master.GetLeader", new(masterproto.GetLeaderArgs), reply)
leader = reply.LeaderId
log.Printf("New leader is replica %d\n", leader)
}
}
}
after_total := time.Now()
fmt.Printf("Test took %v\n", after_total.Sub(before_total))
s := 0
ltot := 0
for _, succ := range successful {
s += succ
}
for _, loc := range local {
ltot += loc
}
fmt.Printf("Successful: %d\n", s)
fmt.Printf("Local Reads: %d\n", ltot)
for _, client := range servers {
if client != nil {
client.Close()
}
}
master.Close()
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
if r.Id == 0 {
r.IsLeader = true
}
for !r.Shutdown {
select {
case proposeS := <-r.ProposeLeaseChan:
propose := proposeS.(*lpaxosproto.ProposeLease)
//got a Propose from a client
dlog.Printf("Lease Update Proposal from replica %d\n", propose.ReplicaId)
r.handleProposeLease(propose)
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*lpaxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*lpaxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*lpaxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*lpaxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received short Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*lpaxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*lpaxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
}
}
}