func (r *Replica) bcastPrepare(replica int32, instance int32, ballot int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Prepare bcast failed:", err)
}
}()
args := &gpaxosproto.Prepare{r.Id, instance, ballot}
n := r.N - 1
//TODO: fix quorum size
if r.Thrifty {
n = r.N >> 1
}
q := r.Id
var w *bufio.Writer
for sent := 0; sent < n; {
q = (q + 1) % int32(r.N)
if q == r.Id {
dlog.Println("Not enough replicas alive!")
break
}
if !r.Alive[q] {
continue
}
sent++
w = r.PeerWriters[q]
w.WriteByte(gpaxosproto.PREPARE)
args.Marshal(w)
w.Flush()
}
}
func (r *Replica) handle1b(msg *gpaxosproto.M_1b) {
if msg.Balnum != r.crtBalnum {
log.Println("1b from a different ballot")
return
}
crtbal := r.ballotArray[r.crtBalnum]
if crtbal.status != PHASE1 {
//delayed 1b
return
}
dlog.Println("msg.Cstruct: ", msg.Cstruct)
crtbal.lb.cstructs = append(crtbal.lb.cstructs, msg.Cstruct)
count := len(crtbal.lb.cstructs)
//is it sufficient to have the same initial quorum size for both fast and slow rounds?
if (r.fastRound && count == r.fastQSize) ||
(!r.fastRound && count == r.N/2+1) {
_, _, crtbal.cstruct = r.learn(true)
dlog.Println("LUB:", crtbal.cstruct)
r.bcast2a(r.crtBalnum, crtbal.cstruct, r.fastRound)
crtbal.lb.cstructs = make([][]int32, r.N)
crtbal.status = PHASE2
if r.fastRound && ALL_TO_ALL {
r.bcast2b(&gpaxosproto.M_2b{r.Id, r.crtBalnum, crtbal.cstruct, crtbal.cstruct})
}
}
}
func (r *Replica) bcastPrepare(replica int32, instance int32, ballot int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Prepare bcast failed:", err)
}
}()
args := &epaxosproto.Prepare{r.Id, replica, instance, ballot}
n := r.N - 1
if r.Thrifty {
n = r.N / 2
}
q := r.Id
for sent := 0; sent < n; {
q = (q + 1) % int32(r.N)
if q == r.Id {
dlog.Println("Not enough replicas alive!")
break
}
if !r.Alive[q] {
continue
}
r.SendMsg(q, r.prepareRPC, args)
sent++
}
}
func (r *Replica) tryToLearn() {
var glb []int32
var conflict bool
crtbal := r.ballotArray[r.crtBalnum]
if conflict, glb, _ = r.learn(false); conflict {
log.Println("Conflict")
if r.isLeader {
r.startHigherBallot()
}
} else if glb != nil {
dlog.Println("Got GLB:", glb)
for _, cid := range glb {
dlog.Println("Committing command ", cid)
r.committed[cid] = true
crtbal.lb.committed++
if prop, present := r.commandReplies[cid]; present {
r.ReplyProposeTS(&genericsmrproto.ProposeReplyTS{TRUE, cid, state.NIL, prop.Timestamp}, prop.Reply)
delete(r.commandReplies, cid)
}
}
if r.isLeader && crtbal.lb.committed >= CMDS_PER_BALLOT {
r.startHigherBallot()
}
}
}
func (r *Replica) bcastCommit(cstruct []int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Commit bcast failed:", err)
}
}()
args := &gpaxosproto.Commit{cstruct}
n := r.N - 1
q := r.Id
var w *bufio.Writer
for sent := 0; sent < n; {
q = (q + 1) % int32(r.N)
if q == r.Id {
break
}
if !r.Alive[q] {
continue
}
sent++
w = r.PeerWriters[q]
w.WriteByte(gpaxosproto.COMMIT)
args.Marshal(w)
w.Flush()
}
}
func (r *Replica) bcast2a(balnum int32, cstruct []int32, fast bool) {
defer func() {
if err := recover(); err != nil {
dlog.Println("1a bcast failed:", err)
}
}()
args := &gpaxosproto.M_2a{r.Id, balnum, cstruct}
n := r.N - 1
if r.Thrifty {
if fast {
n = r.fastQSize - 1
} else {
n = r.N >> 1
}
}
q := r.Id
var w *bufio.Writer
for sent := 0; sent < n; {
q = (q + 1) % int32(r.N)
if q == r.Id {
break
}
if !r.Alive[q] {
continue
}
sent++
w = r.PeerWriters[q]
w.WriteByte(gpaxosproto.M2A)
args.Marshal(w)
w.Flush()
}
}
func (r *Replica) bcastSkip(startInstance int32, endInstance int32, exceptReplica int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Skip bcast failed:", err)
}
}()
sk.LeaderId = r.Id
sk.StartInstance = startInstance
sk.EndInstance = endInstance
args := &sk
//args := &menciusproto.Skip{r.Id, startInstance, endInstance}
n := r.N - 1
q := r.Id
for sent := 0; sent < n; {
q = (q + 1) % int32(r.N)
if q == r.Id {
break
}
if !r.Alive[q] || q == exceptReplica {
continue
}
sent++
r.SendMsgNoFlush(q, r.skipRPC, args)
}
}
func (r *Replica) bcastAccept(replica int32, instance int32, ballot int32, count int32, seq int32, deps [DS]int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Accept bcast failed:", err)
}
}()
ea.LeaderId = r.Id
ea.Replica = replica
ea.Instance = instance
ea.Ballot = ballot
ea.Count = count
ea.Seq = seq
ea.Deps = deps
args := &ea
n := r.N - 1
if r.Thrifty {
n = r.N / 2
}
sent := 0
for q := 0; q < r.N-1; q++ {
if !r.Alive[r.PreferredPeerOrder[q]] {
continue
}
r.SendMsg(r.PreferredPeerOrder[q], r.acceptRPC, args)
sent++
if sent >= n {
break
}
}
}
func (r *Replica) bcastCommit(replica int32, instance int32, cmds []state.Command, seq int32, deps [DS]int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Commit bcast failed:", err)
}
}()
ec.LeaderId = r.Id
ec.Replica = replica
ec.Instance = instance
ec.Command = cmds
ec.Seq = seq
ec.Deps = deps
args := &ec
ecs.LeaderId = r.Id
ecs.Replica = replica
ecs.Instance = instance
ecs.Count = int32(len(cmds))
ecs.Seq = seq
ecs.Deps = deps
argsShort := &ecs
sent := 0
for q := 0; q < r.N-1; q++ {
if !r.Alive[r.PreferredPeerOrder[q]] {
continue
}
if r.Thrifty && sent >= r.N/2 {
r.SendMsg(r.PreferredPeerOrder[q], r.commitRPC, args)
} else {
r.SendMsg(r.PreferredPeerOrder[q], r.commitShortRPC, argsShort)
sent++
}
}
}
func (r *Replica) bcastPreAccept(replica int32, instance int32, ballot int32, cmds []state.Command, seq int32, deps [DS]int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("PreAccept bcast failed:", err)
}
}()
pa.LeaderId = r.Id
pa.Replica = replica
pa.Instance = instance
pa.Ballot = ballot
pa.Command = cmds
pa.Seq = seq
pa.Deps = deps
args := &pa
n := r.N - 1
if r.Thrifty {
n = r.N / 2
}
sent := 0
for q := 0; q < r.N-1; q++ {
if !r.Alive[r.PreferredPeerOrder[q]] {
continue
}
r.SendMsg(r.PreferredPeerOrder[q], r.preAcceptRPC, args)
sent++
if sent >= n {
break
}
}
}
func (r *Replica) bcastCommit(instance int32, skip uint8, nbInstToSkip int32, command state.Command) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Commit bcast failed:", err)
}
}()
mc.LeaderId = r.Id
mc.Instance = instance
mc.Skip = skip
mc.NbInstancesToSkip = nbInstToSkip
//mc.Command = command
//args := &menciusproto.Commit{r.Id, instance, skip, nbInstToSkip, command}
args := &mc
n := r.N - 1
q := r.Id
for sent := 0; sent < n; {
q = (q + 1) % int32(r.N)
if q == r.Id {
break
}
if !r.Alive[q] {
continue
}
sent++
r.SendMsg(q, r.commitRPC, args)
}
}
func (r *Replica) bcastTryPreAccept(replica int32, instance int32, ballot int32, cmds []state.Command, seq int32, deps [DS]int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("PreAccept bcast failed:", err)
}
}()
tpa.LeaderId = r.Id
tpa.Replica = replica
tpa.Instance = instance
tpa.Ballot = ballot
tpa.Command = cmds
tpa.Seq = seq
tpa.Deps = deps
args := &pa
for q := int32(0); q < int32(r.N); q++ {
if q == r.Id {
continue
}
if !r.Alive[q] {
continue
}
r.SendMsg(q, r.tryPreAcceptRPC, args)
}
}
func (r *Replica) bcastPrepare(instance int32, ballot int32) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Prepare bcast failed:", err)
}
}()
args := &menciusproto.Prepare{r.Id, instance, ballot}
n := r.N - 1
if r.Thrifty {
n = r.N >> 1
}
q := r.Id
for sent := 0; sent < n; {
q = (q + 1) % int32(r.N)
if q == r.Id {
break
}
if !r.Alive[q] {
continue
}
sent++
r.SendMsg(q, r.prepareRPC, args)
}
}
func (r *Replica) handle2b(msg *gpaxosproto.M_2b) {
if msg.Balnum != r.crtBalnum {
dlog.Println("2b from a different ballot")
return
}
crtbal := r.ballotArray[r.crtBalnum]
if r.isLeader && crtbal.status != PHASE2 {
log.Println("2b before its time")
return
}
crtbal.lb.cstructs[msg.ReplicaId] = msg.Cstruct
dlog.Printf("Replica %d 2b msg.Cstruct: ", msg.ReplicaId)
dlog.Println(msg.Cstruct)
dlog.Println("my cstruct:", crtbal.cstruct)
crtbal.lb.cstructs[r.Id] = crtbal.cstruct
r.tryToLearn()
}
func (r *Replica) handle2a(msg *gpaxosproto.M_2a) {
if r.isLeader {
log.Println("Received 2a even though I am the leader")
return
}
if r.leaderId != msg.LeaderId {
log.Println("Received 2a from unrecognized leader")
return
}
if r.crtBalnum != msg.Balnum {
log.Println("Received 2a for different ballot: ", msg.Balnum)
return
}
crtbal := r.ballotArray[r.crtBalnum]
crtbal.received2a = true
crtbal.status = PHASE2
dlog.Println("old cstruct", crtbal.cstruct)
cids := make([]int32, 0)
if crtbal.cstruct == nil || len(crtbal.cstruct) == 0 {
crtbal.cstruct = msg.Cstruct
} else {
for _, ocid := range crtbal.cstruct {
present := false
for _, cid := range msg.Cstruct {
if cid == ocid {
present = true
break
}
}
if !present {
msg.Cstruct = append(msg.Cstruct, ocid)
cids = append(cids, ocid)
}
}
crtbal.cstruct = msg.Cstruct
}
if ALL_TO_ALL {
r.bcast2b(&gpaxosproto.M_2b{r.Id, r.crtBalnum, crtbal.cstruct, cids})
r.tryToLearn()
} else {
r.send2b(&gpaxosproto.M_2b{r.Id, r.crtBalnum, crtbal.cstruct, cids}, r.PeerWriters[r.leaderId])
}
}
func (r *Replica) bcast2b(msg *gpaxosproto.M_2b) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Commit bcast failed:", err)
}
}()
for rid, w := range r.PeerWriters {
if int32(rid) == r.Id {
continue
}
w.WriteByte(gpaxosproto.M2B)
msg.Marshal(w)
for _, cid := range msg.Cids {
cmd := r.commands[cid]
cmd.Marshal(w)
}
w.Flush()
}
}
func (r *Replica) handlePrepare(prepare *menciusproto.Prepare) {
inst := r.instanceSpace[prepare.Instance]
if inst == nil {
dlog.Println("Replying OK to null-instance Prepare")
r.replyPrepare(prepare.LeaderId, &menciusproto.PrepareReply{prepare.Instance,
TRUE,
-1,
FALSE,
0,
state.Command{state.NONE, 0, 0}})
r.instanceSpace[prepare.Instance] = &Instance{false,
0,
nil,
prepare.Ballot,
PREPARING,
nil}
} else {
ok := TRUE
if prepare.Ballot < inst.ballot {
ok = FALSE
}
if inst.command == nil {
inst.command = &state.Command{state.NONE, 0, 0}
}
skipped := FALSE
if inst.skipped {
skipped = TRUE
}
r.replyPrepare(prepare.LeaderId, &menciusproto.PrepareReply{prepare.Instance,
ok,
inst.ballot,
skipped,
int32(inst.nbInstSkipped),
*inst.command})
}
}
func (r *Replica) learn(getLub bool) (conflict bool, glb []int32, lub []int32) {
if r.crtBalnum < 0 {
return false, nil, nil
}
crtbal := r.ballotArray[r.crtBalnum]
if len(crtbal.lb.cstructs) == 0 {
return false, nil, nil
}
idToNode := make(map[int32]*node, 2*len(crtbal.cstruct))
// build directed graph
dlog.Println(crtbal.lb.cstructs)
for i := 0; i < len(crtbal.lb.cstructs); i++ {
cs := crtbal.lb.cstructs[i]
for idx, cid := range cs {
var n *node
var present bool
crtCmd := r.commands[cid]
if _, present = r.committed[cid]; present {
continue
}
if n, present = idToNode[cid]; !present {
n = &node{0, make(map[int32]int, 2), WHITE}
idToNode[cid] = n
}
n.count++
for j := 0; j < idx; j++ {
if _, present = r.committed[cs[j]]; present {
continue
}
if crtCmd == nil {
log.Println("crtCmd is nil")
return false, nil, nil
}
if r.commands[cs[j]] == nil {
log.Println("cs[j] is nil")
return false, nil, nil
}
if !state.Conflict(crtCmd, r.commands[cs[j]]) {
continue
}
n.outEdges[cs[j]] = n.outEdges[cs[j]] + 1
}
}
}
// hack
/* conflict = false
glb = make([]int32, 0)
lub = make([]int32, 0)
for cid, n := range idToNode {
if n.count >= r.fastQSize {
glb = append(glb, cid)
}
lub = append(lub, cid)
}
return false, glb, lub*/
// depth-first search
for cid, n := range idToNode {
if n.color == WHITE {
var conf bool
conf, glb, lub = r.dfs(cid, n, glb, lub, idToNode)
conflict = conflict || conf
}
}
/*
if getLub && conflict {
//sort out lub
done := false
for !done {
done = true
for i := 1; i < len(lub) - 1; i++ {
for j := i + 1; j < len(lub); j++ {
u := lub[i]
v := lub[j]
cu := r.commands[u]
cv := r.commands[v]
if !state.Conflict(cu, cv) {
continue
}
nu := idToNode[u]
nv := idToNode[v]
if nv.count - nv.outEdges[u] > nu.count - nu.outEdges[v] {
lub[i] = v
lub[j] = u
done = false
}
}
}
}
}
*/
return conflict, glb, lub
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
}
go r.clock()
for !r.Shutdown {
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with id %d\n", propose.CommandId)
r.handlePropose(propose)
break
case skipS := <-r.skipChan:
skip := skipS.(*menciusproto.Skip)
//got a Skip from another replica
dlog.Printf("Skip for instances %d-%d\n", skip.StartInstance, skip.EndInstance)
r.handleSkip(skip)
case prepareS := <-r.prepareChan:
prepare := prepareS.(*menciusproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*menciusproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*menciusproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*menciusproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*menciusproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case delayedSkip := <-r.delayedSkipChan:
r.handleDelayedSkip(delayedSkip)
break
case <-r.clockChan:
if lastSeenInstance == r.blockingInstance {
r.noCommitFor++
} else {
r.noCommitFor = 0
lastSeenInstance = r.blockingInstance
}
if r.noCommitFor >= 50+int(r.Id) && r.crtInstance >= r.blockingInstance+int32(r.N) {
r.noCommitFor = 0
dlog.Printf("Doing force commit\n")
r.forceCommit()
}
break
}
}
}
func (r *Replica) run() {
if r.Id == 0 {
r.isLeader = true
}
r.ConnectToPeersNoListeners()
for rid, peerReader := range r.PeerReaders {
if int32(rid) == r.Id {
continue
}
go r.handleReplicaConnection(rid, peerReader)
}
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
/*if r.Exec {
go r.executeCommands()
}*/
clockChan = make(chan bool, 1)
go r.clock()
if r.isLeader {
r.crtBalnum = 0
r.fastRound = true
r.ballotArray[0] = &Ballot{nil, 0, PHASE1, false, &LeaderBookkeeping{cstructs: make([][]int32, 0)}}
r.ballotArray[0].lb.cstructs = append(r.ballotArray[0].lb.cstructs, make([]int32, 0))
r.bcast1a(0, true)
}
for !r.Shutdown {
if r.crtBalnum >= 0 && len(r.ballotArray[r.crtBalnum].cstruct) >= CMDS_PER_BALLOT {
select {
case prepare := <-r.prepareChan:
//got a Prepare message
dlog.Printf("Received Prepare for balnum %d\n", prepare.Balnum)
r.commandsMutex.Lock()
r.handlePrepare(prepare)
r.commandsMutex.Unlock()
break
case msg := <-r.m1aChan:
dlog.Printf("Received 1a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m1bChan:
dlog.Printf("Received 1b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1b(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2aChan:
dlog.Printf("Received 2a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2bChan:
dlog.Printf("Received 2b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2b(msg)
r.commandsMutex.Unlock()
break
case <-clockChan:
//way out of deadlock
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with id %d @ replica %d\n", propose.CommandId, r.Id)
r.commandsMutex.Lock()
r.handlePropose(propose)
r.commandsMutex.Unlock()
break
default:
break
}
}
} else {
select {
case prepare := <-r.prepareChan:
//got a Prepare message
dlog.Printf("Received Prepare for balnum %d\n", prepare.Balnum)
r.commandsMutex.Lock()
r.handlePrepare(prepare)
r.commandsMutex.Unlock()
break
case msg := <-r.m1aChan:
dlog.Printf("Received 1a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m1bChan:
dlog.Printf("Received 1b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle1b(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2aChan:
dlog.Printf("Received 2a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2a(msg)
r.commandsMutex.Unlock()
break
case msg := <-r.m2bChan:
dlog.Printf("Received 2b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
r.commandsMutex.Lock()
r.handle2b(msg)
r.commandsMutex.Unlock()
break
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with id %d @ replica %d\n", propose.CommandId, r.Id)
r.commandsMutex.Lock()
r.handlePropose(propose)
r.commandsMutex.Unlock()
break
}
}
}
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
}
if r.Id == 0 {
//init quorum read lease
quorum := make([]int32, r.N/2+1)
for i := 0; i <= r.N/2; i++ {
quorum[i] = int32(i)
}
r.UpdatePreferredPeerOrder(quorum)
}
slowClockChan = make(chan bool, 1)
fastClockChan = make(chan bool, 1)
go r.slowClock()
//Enabled when batching for 5ms
if MAX_BATCH > 100 {
go r.fastClock()
}
if r.Beacon {
go r.stopAdapting()
}
for !r.Shutdown {
handleNewProposals := true
// Enabled when batching for 5ms
if MAX_BATCH > 100 {
handleNewProposals = false
}
select {
case <-fastClockChan:
handleNewProposals = true
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*epaxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare for instance %d.%d\n", prepare.Replica, prepare.Instance)
r.handlePrepare(prepare)
break
case preAcceptS := <-r.preAcceptChan:
preAccept := preAcceptS.(*epaxosproto.PreAccept)
//got a PreAccept message
dlog.Printf("Received PreAccept for instance %d.%d\n", preAccept.LeaderId, preAccept.Instance)
r.handlePreAccept(preAccept)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*epaxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept for instance %d.%d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*epaxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*epaxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*epaxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d.%d\n", prepareReply.Replica, prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case preAcceptReplyS := <-r.preAcceptReplyChan:
preAcceptReply := preAcceptReplyS.(*epaxosproto.PreAcceptReply)
//got a PreAccept reply
dlog.Printf("Received PreAcceptReply for instance %d.%d\n", preAcceptReply.Replica, preAcceptReply.Instance)
r.handlePreAcceptReply(preAcceptReply)
break
case preAcceptOKS := <-r.preAcceptOKChan:
preAcceptOK := preAcceptOKS.(*epaxosproto.PreAcceptOK)
//got a PreAccept reply
dlog.Printf("Received PreAcceptOK for instance %d.%d\n", r.Id, preAcceptOK.Instance)
r.handlePreAcceptOK(preAcceptOK)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*epaxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d.%d\n", acceptReply.Replica, acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case tryPreAcceptS := <-r.tryPreAcceptChan:
tryPreAccept := tryPreAcceptS.(*epaxosproto.TryPreAccept)
dlog.Printf("Received TryPreAccept for instance %d.%d\n", tryPreAccept.Replica, tryPreAccept.Instance)
r.handleTryPreAccept(tryPreAccept)
break
case tryPreAcceptReplyS := <-r.tryPreAcceptReplyChan:
tryPreAcceptReply := tryPreAcceptReplyS.(*epaxosproto.TryPreAcceptReply)
dlog.Printf("Received TryPreAcceptReply for instance %d.%d\n", tryPreAcceptReply.Replica, tryPreAcceptReply.Instance)
r.handleTryPreAcceptReply(tryPreAcceptReply)
break
case beacon := <-r.BeaconChan:
dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
r.ReplyBeacon(beacon)
break
case <-slowClockChan:
if r.Beacon {
for q := int32(0); q < int32(r.N); q++ {
if q == r.Id {
continue
}
r.SendBeacon(q)
}
}
break
case <-r.OnClientConnect:
log.Printf("weird %d; conflicted %d; slow %d; happy %d\n", weird, conflicted, slow, happy)
weird, conflicted, slow, happy = 0, 0, 0, 0
case iid := <-r.instancesToRecover:
r.startRecoveryForInstance(iid.replica, iid.instance)
}
if handleNewProposals {
// unfortunately the only way to activate the new proposals channel every X ms
// was to dublicate the body of the select
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with op %d\n", propose.Command.Op)
r.handlePropose(propose)
handleNewProposals = false
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*epaxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare for instance %d.%d\n", prepare.Replica, prepare.Instance)
r.handlePrepare(prepare)
break
case preAcceptS := <-r.preAcceptChan:
preAccept := preAcceptS.(*epaxosproto.PreAccept)
//got a PreAccept message
dlog.Printf("Received PreAccept for instance %d.%d\n", preAccept.LeaderId, preAccept.Instance)
r.handlePreAccept(preAccept)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*epaxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept for instance %d.%d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*epaxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*epaxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*epaxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d.%d\n", prepareReply.Replica, prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case preAcceptReplyS := <-r.preAcceptReplyChan:
preAcceptReply := preAcceptReplyS.(*epaxosproto.PreAcceptReply)
//got a PreAccept reply
dlog.Printf("Received PreAcceptReply for instance %d.%d\n", preAcceptReply.Replica, preAcceptReply.Instance)
r.handlePreAcceptReply(preAcceptReply)
break
case preAcceptOKS := <-r.preAcceptOKChan:
preAcceptOK := preAcceptOKS.(*epaxosproto.PreAcceptOK)
//got a PreAccept reply
dlog.Printf("Received PreAcceptOK for instance %d.%d\n", r.Id, preAcceptOK.Instance)
r.handlePreAcceptOK(preAcceptOK)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*epaxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d.%d\n", acceptReply.Replica, acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case tryPreAcceptS := <-r.tryPreAcceptChan:
tryPreAccept := tryPreAcceptS.(*epaxosproto.TryPreAccept)
dlog.Printf("Received TryPreAccept for instance %d.%d\n", tryPreAccept.Replica, tryPreAccept.Instance)
r.handleTryPreAccept(tryPreAccept)
break
case tryPreAcceptReplyS := <-r.tryPreAcceptReplyChan:
tryPreAcceptReply := tryPreAcceptReplyS.(*epaxosproto.TryPreAcceptReply)
dlog.Printf("Received TryPreAcceptReply for instance %d.%d\n", tryPreAcceptReply.Replica, tryPreAcceptReply.Instance)
r.handleTryPreAcceptReply(tryPreAcceptReply)
break
case beacon := <-r.BeaconChan:
dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
r.ReplyBeacon(beacon)
break
case <-slowClockChan:
if r.Beacon {
for q := int32(0); q < int32(r.N); q++ {
if q == r.Id {
continue
}
r.SendBeacon(q)
}
}
break
case <-r.OnClientConnect:
log.Printf("weird %d; conflicted %d; slow %d; happy %d\n", weird, conflicted, slow, happy)
weird, conflicted, slow, happy = 0, 0, 0, 0
break
case iid := <-r.instancesToRecover:
r.startRecoveryForInstance(iid.replica, iid.instance)
}
}
}
}
func (r *Replica) handleAccept(accept *paxosproto.Accept) {
inst := r.instanceSpace[accept.Instance]
var areply *paxosproto.AcceptReply
if inst == nil {
if accept.Ballot < r.defaultBallot {
areply = &paxosproto.AcceptReply{accept.Instance, FALSE, r.defaultBallot, -1, accept.OriginReplica, accept.PropId}
} else {
r.instanceSpace[accept.Instance] = &Instance{
accept.Command,
accept.Ballot,
ACCEPTED,
nil,
1, false}
areply = &paxosproto.AcceptReply{accept.Instance, TRUE, accept.Ballot, -1, accept.OriginReplica, accept.PropId}
r.addUpdatingKeys(accept.Command)
}
} else if inst.ballot > accept.Ballot {
areply = &paxosproto.AcceptReply{accept.Instance, FALSE, inst.ballot, -1, accept.OriginReplica, accept.PropId}
} else if inst.ballot < accept.Ballot && inst.status != COMMITTED {
if inst.cmds != nil {
r.removeUpdatingKeys(inst.cmds)
}
r.addUpdatingKeys(accept.Command)
inst.cmds = accept.Command
inst.ballot = accept.Ballot
inst.status = ACCEPTED
inst.directAcks = 1
areply = &paxosproto.AcceptReply{accept.Instance, TRUE, inst.ballot, -1, accept.OriginReplica, accept.PropId}
if inst.lb != nil && inst.lb.clientProposals != nil {
//TODO: is this correct?
// try the proposal in a different instance
for i := 0; i < len(inst.lb.clientProposals); i++ {
r.ProposeChan <- inst.lb.clientProposals[i]
}
inst.lb.clientProposals = nil
}
} else {
// reordered ACCEPT
r.instanceSpace[accept.Instance].cmds = accept.Command
if r.instanceSpace[accept.Instance].status < COMMITTED {
r.instanceSpace[accept.Instance].status = ACCEPTED
}
r.instanceSpace[accept.Instance].ballot = accept.Ballot
areply = &paxosproto.AcceptReply{accept.Instance, TRUE, accept.Ballot, -1, accept.OriginReplica, accept.PropId}
/*} else {
return
}*/
}
if accept.LeaseInstance != r.QLease.PromisedByMeInst && !r.QLease.CanWriteOutside() {
// cannot accept because lease eras do not match
areply.OK = FALSE
areply.LeaseInstance = r.QLease.PromisedByMeInst
log.Println(accept.LeaseInstance, r.leaseSMR.LatestCommitted)
}
if areply.OK == TRUE {
if accept.Instance > r.latestAcceptedInst {
r.latestAcceptedInst = accept.Instance
}
if r.directAcks && r.Id == accept.OriginReplica {
inst = r.instanceSpace[accept.Instance]
inst.directAcks++
if inst.directAcks == int8(r.N/2+1) {
//safe to commit
prop := r.fwdPropMap[accept.PropId]
inst.status = COMMITTED
// give client the all clear
if !r.Dreply && !inst.sentReply {
propreply := &genericsmrproto.ProposeReplyTS{
TRUE,
prop.CommandId,
state.NIL,
prop.Timestamp}
r.ReplyProposeTS(propreply, prop)
inst.sentReply = true
} else if inst.sentReply {
dlog.Println("Trying to send reply twice!")
}
r.updateCommittedUpTo()
}
}
r.recordInstanceMetadata(r.instanceSpace[accept.Instance])
r.recordCommands(accept.Command)
r.sync()
}
r.replyAccept(accept.LeaderId, areply)
if r.directAcks &&
accept.OriginReplica >= 0 &&
accept.OriginReplica != accept.LeaderId &&
accept.OriginReplica != r.Id &&
areply.OK == TRUE {
r.replyAccept(accept.OriginReplica, areply)
}
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
go r.reader()
}
if r.Id == 0 {
r.IsLeader = true
r.readStats = NewReadStats(r.N, r.Id)
}
clockChan = make(chan bool, 1)
go r.clock()
r.QLease = qlease.NewLease(r.N)
leaseClockChan = make(chan bool, 1)
leaseClockRestart = make(chan bool)
go r.leaseClock()
// clockRang := false
var tickCounter uint64 = 0
latestBeaconFromReplica := make([]uint64, r.N)
proposedDead := make([]bool, r.N)
for i := 0; i < r.N; i++ {
latestBeaconFromReplica[i] = 0
proposedDead[i] = false
}
stopRenewing := false
for !r.Shutdown {
select {
case <-clockChan:
//clockRang = true
tickCounter++
if tickCounter%20 == 0 {
if r.Beacon {
for q := int32(0); q < int32(r.N); q++ {
if q == r.Id {
continue
}
r.SendBeacon(q)
}
}
if r.IsLeader && r.Beacon {
for rid := int32(0); rid < int32(r.N); rid++ {
if rid == r.Id {
continue
}
if !proposedDead[rid] && tickCounter-latestBeaconFromReplica[rid] >= 200 {
log.Println("Proposing replica dead: ", rid)
//replica might be dead
//propose a lease configuration change
dr := make([]int32, 1)
dr[0] = rid
r.proposeReplicasDead(dr)
proposedDead[rid] = true
}
}
}
rightNow := time.Now().UnixNano()
for rid := int32(0); rid < int32(r.N); rid++ {
if rid == r.Id {
continue
}
if !proposedDead[rid] && r.LastReplyReceivedTimestamp[rid] > 0 && rightNow-r.LastReplyReceivedTimestamp[rid] >= GRACE_PERIOD {
if r.IsLeader {
log.Println("Proposing replica dead: ", rid)
//replica might be dead
//propose a lease configuration change
dr := make([]int32, 1)
dr[0] = rid
r.proposeReplicasDead(dr)
}
proposedDead[rid] = true
stopRenewing = true
}
}
if r.IsLeader {
for i := r.committedUpTo; i >= 0 && i < r.crtInstance; i++ {
r.delayedInstances <- i
}
}
}
break
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with op %d\n", propose.Command.Op)
r.handlePropose(propose)
//clockRang = false
break
case forwardS := <-r.forwardChan:
forward := forwardS.(*paxosproto.Forward)
dlog.Printf("Forward proposal from replica %d\n", forward.ReplicaId)
r.handleForward(forward)
break
case frS := <-r.forwardReplyChan:
fr := frS.(*paxosproto.ForwardReply)
r.handleForwardReply(fr)
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*paxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*paxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*paxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*paxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
case instNo := <-r.delayedInstances:
log.Println("Trying to re-start instance")
inst := r.instanceSpace[instNo]
if inst.status < COMMITTED {
inst.status = PREPARED
inst.lb.acceptOKs = 0
inst.ballot = r.makeBallotLargerThan(inst.ballot)
inst.lb.acceptOKsToWait, _ = r.bcastAccept(instNo, inst.ballot, inst.cmds, inst.lb.clientProposals[0].FwdReplica, inst.lb.clientProposals[0].FwdId)
if genericsmr.SendError {
r.delayedInstances <- instNo
}
}
case beacon := <-r.BeaconChan:
dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
//r.ReplyBeacon(beacon)
latestBeaconFromReplica[beacon.Rid] = tickCounter
break
case guardS := <-r.QLGuardChan:
guard := guardS.(*qleaseproto.Guard)
r.HandleQLeaseGuard(r.QLease, guard)
break
case guardReplyS := <-r.QLGuardReplyChan:
guardReply := guardReplyS.(*qleaseproto.GuardReply)
r.HandleQLeaseGuardReply(r.QLease, guardReply, r.latestAcceptedInst)
break
case promiseS := <-r.QLPromiseChan:
promise := promiseS.(*qleaseproto.Promise)
prev := r.QLease.PromisedToMeInst
if !r.HandleQLeasePromise(r.QLease, promise) {
continue
}
if prev != r.QLease.PromisedToMeInst {
r.updateGrantedKeys(prev)
newPromiseCount = 0
}
newPromiseCount++
if newPromiseCount <= r.N/2 {
if r.newestInstanceIDontKnow < promise.LatestAcceptedInst {
r.newestInstanceIDontKnow = promise.LatestAcceptedInst
}
}
break
case preplyS := <-r.QLPromiseReplyChan:
preply := preplyS.(*qleaseproto.PromiseReply)
r.HandleQLeaseReply(r.QLease, preply)
break
case <-leaseClockChan:
if r.QLease.PromisedByMeInst < r.leaseSMR.LatestCommitted {
// wait for previous lease to expire before switching to new config
if r.QLease.CanWriteOutside() {
r.updateKeyQuorumInfo(r.leaseSMR.LatestCommitted)
r.QLease.PromisedByMeInst = r.leaseSMR.LatestCommitted
log.Printf("Replica %d - New lease for instance %d\n", r.Id, r.QLease.PromisedByMeInst)
r.EstablishQLease(r.QLease)
stopRenewing = false
}
} else if r.QLease.PromisedByMeInst >= 0 {
if r.QLease.CanWriteOutside() {
r.EstablishQLease(r.QLease)
stopRenewing = false
} else if !stopRenewing {
r.RenewQLease(r.QLease, r.latestAcceptedInst)
}
}
if r.maintainReadStats {
ticks--
if ticks == 0 {
r.maintainReadStats = false
if r.IsLeader && r.readStats != nil {
go r.proposeLeaseReconf()
}
ticks = TICKS_TO_RECONF_LEASE
//ticks = 60
}
}
// restart the clock
leaseClockRestart <- true
case <-r.OnClientConnect:
log.Printf("reads: %d, local: %d\n", reads, local)
}
}
}
func (r *Replica) handleAcceptReply(areply *paxosproto.AcceptReply) {
inst := r.instanceSpace[areply.Instance]
if !r.IsLeader && areply.OK == TRUE && areply.OriginReplica == r.Id {
//direct ACK optimization
prop := r.fwdPropMap[areply.PropId]
if inst == nil {
cmds := make([]state.Command, 1)
cmds[0] = prop.Command
r.instanceSpace[areply.Instance] = &Instance{
cmds,
areply.Ballot,
NONE,
nil,
1, false}
inst = r.instanceSpace[areply.Instance]
r.addUpdatingKeys(cmds)
}
if inst.ballot > areply.Ballot {
return
}
if areply.Ballot > inst.ballot {
inst.ballot = areply.Ballot
inst.directAcks = 2
} else {
inst.directAcks++
}
if (inst.status == COMMITTED && inst.directAcks <= int8(r.N/2+1)) ||
(inst.status != COMMITTED && inst.directAcks == int8(r.N/2+1)) {
//safe to commit
inst.status = COMMITTED
// give the client the all clear
inst.directAcks = int8(r.N/2 + 2)
if !r.Dreply && !inst.sentReply {
propreply := &genericsmrproto.ProposeReplyTS{
TRUE,
prop.CommandId,
state.NIL,
prop.Timestamp}
r.ReplyProposeTS(propreply, prop)
inst.sentReply = true
} else if inst.sentReply {
dlog.Println("Trying to send reply twice!")
}
r.updateCommittedUpTo()
}
return
}
if inst.status >= COMMITTED {
// we've moved on, these are delayed replies, so just ignore
return
}
if inst.ballot != areply.Ballot {
return
}
if areply.OK == TRUE {
inst.lb.acceptOKs++
//if inst.lb.acceptOKs + 1 > r.N >> 1 {
if inst.lb.acceptOKs >= inst.lb.acceptOKsToWait {
inst = r.instanceSpace[areply.Instance]
inst.status = COMMITTED
if inst.lb.clientProposals != nil && !r.Dreply {
// give client the all clear
for i := 0; i < len(inst.cmds); i++ {
propreply := &genericsmrproto.ProposeReplyTS{
TRUE,
inst.lb.clientProposals[i].CommandId,
state.NIL,
inst.lb.clientProposals[i].Timestamp}
r.ReplyProposeTS(propreply, inst.lb.clientProposals[i])
}
}
r.recordInstanceMetadata(r.instanceSpace[areply.Instance])
r.sync() //is this necessary?
if areply.OriginReplica < 0 || areply.OriginReplica == r.Id {
r.addUpdatingKeys(inst.cmds)
}
r.updateCommittedUpTo()
r.bcastCommit(areply.Instance, inst.ballot, inst.cmds)
}
} else {
if areply.LeaseInstance >= 0 {
// acceptor is in different lease era
// re-try later
r.delayedInstances <- areply.Instance
}
// TODO: there is probably another active leader
inst.lb.nacks++
if areply.Ballot > inst.lb.maxRecvBallot {
inst.lb.maxRecvBallot = areply.Ballot
}
if inst.lb.nacks >= r.N>>1 {
// TODO
}
}
}
func (r *Replica) bcastAccept(instance int32, ballot int32, skip uint8, nbInstToSkip int32, command state.Command) {
defer func() {
if err := recover(); err != nil {
dlog.Println("Accept bcast failed:", err)
}
}()
ma.LeaderId = r.Id
ma.Instance = instance
ma.Ballot = ballot
ma.Skip = skip
ma.NbInstancesToSkip = nbInstToSkip
ma.Command = command
args := &ma
//args := &menciusproto.Accept{r.Id, instance, ballot, skip, nbInstToSkip, command}
n := r.N - 1
q := r.Id
sent := 0
for sent < n {
q = (q + 1) % int32(r.N)
if q == r.Id {
break
}
if !r.Alive[q] {
continue
}
if r.Thrifty {
inst := (instance/int32(r.N))*int32(r.N) + q
if inst > instance {
inst -= int32(r.N)
}
if inst < 0 || r.instanceSpace[inst] != nil {
continue
}
}
sent++
r.SendMsg(q, r.acceptRPC, args)
}
for sent < r.N>>1 {
q = (q + 1) % int32(r.N)
if q == r.Id {
break
}
if !r.Alive[q] {
continue
}
if r.Thrifty {
inst := (instance/int32(r.N))*int32(r.N) + q
if inst > instance {
inst -= int32(r.N)
}
if inst >= 0 && r.instanceSpace[inst] == nil {
continue
}
}
sent++
r.SendMsg(q, r.acceptRPC, args)
}
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
go r.WaitForClientConnections()
if r.Exec {
go r.executeCommands()
}
if r.Id == 0 {
r.IsLeader = true
}
clockChan = make(chan bool, 1)
go r.clock()
clockRang := false
for !r.Shutdown {
select {
case <-clockChan:
clockRang = true
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*paxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*paxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*paxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*paxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
}
if clockRang {
select {
case propose := <-r.ProposeChan:
//got a Propose from a client
dlog.Printf("Proposal with op %d\n", propose.Command.Op)
r.handlePropose(propose)
clockRang = false
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*paxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*paxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*paxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*paxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
}
}
}
}
func (r *Replica) run() {
r.ConnectToPeers()
dlog.Println("Waiting for client connections")
if r.Id == 0 {
r.IsLeader = true
}
for !r.Shutdown {
select {
case proposeS := <-r.ProposeLeaseChan:
propose := proposeS.(*lpaxosproto.ProposeLease)
//got a Propose from a client
dlog.Printf("Lease Update Proposal from replica %d\n", propose.ReplicaId)
r.handleProposeLease(propose)
break
case prepareS := <-r.prepareChan:
prepare := prepareS.(*lpaxosproto.Prepare)
//got a Prepare message
dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
r.handlePrepare(prepare)
break
case acceptS := <-r.acceptChan:
accept := acceptS.(*lpaxosproto.Accept)
//got an Accept message
dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
r.handleAccept(accept)
break
case commitS := <-r.commitChan:
commit := commitS.(*lpaxosproto.Commit)
//got a Commit message
dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommit(commit)
break
case commitS := <-r.commitShortChan:
commit := commitS.(*lpaxosproto.CommitShort)
//got a Commit message
dlog.Printf("Received short Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
r.handleCommitShort(commit)
break
case prepareReplyS := <-r.prepareReplyChan:
prepareReply := prepareReplyS.(*lpaxosproto.PrepareReply)
//got a Prepare reply
dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
r.handlePrepareReply(prepareReply)
break
case acceptReplyS := <-r.acceptReplyChan:
acceptReply := acceptReplyS.(*lpaxosproto.AcceptReply)
//got an Accept reply
dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
r.handleAcceptReply(acceptReply)
break
}
}
}