Example #1
0
func (r *Replica) bcastPrepare(replica int32, instance int32, ballot int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Prepare bcast failed:", err)
		}
	}()
	args := &gpaxosproto.Prepare{r.Id, instance, ballot}

	n := r.N - 1
	//TODO: fix quorum size
	if r.Thrifty {
		n = r.N >> 1
	}
	q := r.Id
	var w *bufio.Writer
	for sent := 0; sent < n; {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			dlog.Println("Not enough replicas alive!")
			break
		}
		if !r.Alive[q] {
			continue
		}
		sent++
		w = r.PeerWriters[q]
		w.WriteByte(gpaxosproto.PREPARE)
		args.Marshal(w)
		w.Flush()
	}
}
Example #2
0
func (r *Replica) handle1b(msg *gpaxosproto.M_1b) {
	if msg.Balnum != r.crtBalnum {
		log.Println("1b from a different ballot")
		return
	}

	crtbal := r.ballotArray[r.crtBalnum]

	if crtbal.status != PHASE1 {
		//delayed 1b
		return
	}

	dlog.Println("msg.Cstruct: ", msg.Cstruct)
	crtbal.lb.cstructs = append(crtbal.lb.cstructs, msg.Cstruct)
	count := len(crtbal.lb.cstructs)

	//is it sufficient to have the same initial quorum size for both fast and slow rounds?
	if (r.fastRound && count == r.fastQSize) ||
		(!r.fastRound && count == r.N/2+1) {
		_, _, crtbal.cstruct = r.learn(true)
		dlog.Println("LUB:", crtbal.cstruct)
		r.bcast2a(r.crtBalnum, crtbal.cstruct, r.fastRound)
		crtbal.lb.cstructs = make([][]int32, r.N)
		crtbal.status = PHASE2
		if r.fastRound && ALL_TO_ALL {
			r.bcast2b(&gpaxosproto.M_2b{r.Id, r.crtBalnum, crtbal.cstruct, crtbal.cstruct})
		}
	}
}
Example #3
0
func (r *Replica) bcastPrepare(replica int32, instance int32, ballot int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Prepare bcast failed:", err)
		}
	}()
	args := &epaxosproto.Prepare{r.Id, replica, instance, ballot}

	n := r.N - 1
	if r.Thrifty {
		n = r.N / 2
	}
	q := r.Id
	for sent := 0; sent < n; {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			dlog.Println("Not enough replicas alive!")
			break
		}
		if !r.Alive[q] {
			continue
		}
		r.SendMsg(q, r.prepareRPC, args)
		sent++
	}
}
Example #4
0
func (r *Replica) tryToLearn() {
	var glb []int32
	var conflict bool
	crtbal := r.ballotArray[r.crtBalnum]

	if conflict, glb, _ = r.learn(false); conflict {
		log.Println("Conflict")
		if r.isLeader {
			r.startHigherBallot()
		}
	} else if glb != nil {
		dlog.Println("Got GLB:", glb)
		for _, cid := range glb {
			dlog.Println("Committing command ", cid)
			r.committed[cid] = true
			crtbal.lb.committed++
			if prop, present := r.commandReplies[cid]; present {
				r.ReplyProposeTS(&genericsmrproto.ProposeReplyTS{TRUE, cid, state.NIL, prop.Timestamp}, prop.Reply)
				delete(r.commandReplies, cid)
			}
		}
		if r.isLeader && crtbal.lb.committed >= CMDS_PER_BALLOT {
			r.startHigherBallot()
		}
	}
}
Example #5
0
func (r *Replica) bcastCommit(cstruct []int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Commit bcast failed:", err)
		}
	}()
	args := &gpaxosproto.Commit{cstruct}

	n := r.N - 1
	q := r.Id
	var w *bufio.Writer
	for sent := 0; sent < n; {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			break
		}
		if !r.Alive[q] {
			continue
		}
		sent++
		w = r.PeerWriters[q]
		w.WriteByte(gpaxosproto.COMMIT)
		args.Marshal(w)
		w.Flush()
	}
}
Example #6
0
func (r *Replica) bcast2a(balnum int32, cstruct []int32, fast bool) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("1a bcast failed:", err)
		}
	}()
	args := &gpaxosproto.M_2a{r.Id, balnum, cstruct}

	n := r.N - 1
	if r.Thrifty {
		if fast {
			n = r.fastQSize - 1
		} else {
			n = r.N >> 1
		}
	}
	q := r.Id
	var w *bufio.Writer
	for sent := 0; sent < n; {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			break
		}
		if !r.Alive[q] {
			continue
		}
		sent++
		w = r.PeerWriters[q]
		w.WriteByte(gpaxosproto.M2A)
		args.Marshal(w)
		w.Flush()
	}
}
Example #7
0
func (r *Replica) bcastSkip(startInstance int32, endInstance int32, exceptReplica int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Skip bcast failed:", err)
		}
	}()
	sk.LeaderId = r.Id
	sk.StartInstance = startInstance
	sk.EndInstance = endInstance
	args := &sk
	//args := &menciusproto.Skip{r.Id, startInstance, endInstance}

	n := r.N - 1
	q := r.Id

	for sent := 0; sent < n; {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			break
		}
		if !r.Alive[q] || q == exceptReplica {
			continue
		}
		sent++
		r.SendMsgNoFlush(q, r.skipRPC, args)
	}
}
Example #8
0
func (r *Replica) bcastAccept(replica int32, instance int32, ballot int32, count int32, seq int32, deps [DS]int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Accept bcast failed:", err)
		}
	}()

	ea.LeaderId = r.Id
	ea.Replica = replica
	ea.Instance = instance
	ea.Ballot = ballot
	ea.Count = count
	ea.Seq = seq
	ea.Deps = deps
	args := &ea

	n := r.N - 1
	if r.Thrifty {
		n = r.N / 2
	}

	sent := 0
	for q := 0; q < r.N-1; q++ {
		if !r.Alive[r.PreferredPeerOrder[q]] {
			continue
		}
		r.SendMsg(r.PreferredPeerOrder[q], r.acceptRPC, args)
		sent++
		if sent >= n {
			break
		}
	}
}
Example #9
0
func (r *Replica) bcastCommit(replica int32, instance int32, cmds []state.Command, seq int32, deps [DS]int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Commit bcast failed:", err)
		}
	}()
	ec.LeaderId = r.Id
	ec.Replica = replica
	ec.Instance = instance
	ec.Command = cmds
	ec.Seq = seq
	ec.Deps = deps
	args := &ec
	ecs.LeaderId = r.Id
	ecs.Replica = replica
	ecs.Instance = instance
	ecs.Count = int32(len(cmds))
	ecs.Seq = seq
	ecs.Deps = deps
	argsShort := &ecs

	sent := 0
	for q := 0; q < r.N-1; q++ {
		if !r.Alive[r.PreferredPeerOrder[q]] {
			continue
		}
		if r.Thrifty && sent >= r.N/2 {
			r.SendMsg(r.PreferredPeerOrder[q], r.commitRPC, args)
		} else {
			r.SendMsg(r.PreferredPeerOrder[q], r.commitShortRPC, argsShort)
			sent++
		}
	}
}
Example #10
0
func (r *Replica) bcastPreAccept(replica int32, instance int32, ballot int32, cmds []state.Command, seq int32, deps [DS]int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("PreAccept bcast failed:", err)
		}
	}()
	pa.LeaderId = r.Id
	pa.Replica = replica
	pa.Instance = instance
	pa.Ballot = ballot
	pa.Command = cmds
	pa.Seq = seq
	pa.Deps = deps
	args := &pa

	n := r.N - 1
	if r.Thrifty {
		n = r.N / 2
	}

	sent := 0
	for q := 0; q < r.N-1; q++ {
		if !r.Alive[r.PreferredPeerOrder[q]] {
			continue
		}
		r.SendMsg(r.PreferredPeerOrder[q], r.preAcceptRPC, args)
		sent++
		if sent >= n {
			break
		}
	}
}
Example #11
0
func (r *Replica) bcastCommit(instance int32, skip uint8, nbInstToSkip int32, command state.Command) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Commit bcast failed:", err)
		}
	}()
	mc.LeaderId = r.Id
	mc.Instance = instance
	mc.Skip = skip
	mc.NbInstancesToSkip = nbInstToSkip
	//mc.Command = command
	//args := &menciusproto.Commit{r.Id, instance, skip, nbInstToSkip, command}
	args := &mc

	n := r.N - 1
	q := r.Id

	for sent := 0; sent < n; {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			break
		}
		if !r.Alive[q] {
			continue
		}
		sent++
		r.SendMsg(q, r.commitRPC, args)
	}
}
Example #12
0
func (r *Replica) bcastTryPreAccept(replica int32, instance int32, ballot int32, cmds []state.Command, seq int32, deps [DS]int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("PreAccept bcast failed:", err)
		}
	}()
	tpa.LeaderId = r.Id
	tpa.Replica = replica
	tpa.Instance = instance
	tpa.Ballot = ballot
	tpa.Command = cmds
	tpa.Seq = seq
	tpa.Deps = deps
	args := &pa

	for q := int32(0); q < int32(r.N); q++ {
		if q == r.Id {
			continue
		}
		if !r.Alive[q] {
			continue
		}
		r.SendMsg(q, r.tryPreAcceptRPC, args)
	}
}
Example #13
0
func (r *Replica) bcastPrepare(instance int32, ballot int32) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Prepare bcast failed:", err)
		}
	}()
	args := &menciusproto.Prepare{r.Id, instance, ballot}

	n := r.N - 1
	if r.Thrifty {
		n = r.N >> 1
	}
	q := r.Id

	for sent := 0; sent < n; {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			break
		}
		if !r.Alive[q] {
			continue
		}
		sent++
		r.SendMsg(q, r.prepareRPC, args)
	}
}
Example #14
0
func (r *Replica) handle2b(msg *gpaxosproto.M_2b) {
	if msg.Balnum != r.crtBalnum {
		dlog.Println("2b from a different ballot")
		return
	}

	crtbal := r.ballotArray[r.crtBalnum]

	if r.isLeader && crtbal.status != PHASE2 {
		log.Println("2b before its time")
		return
	}

	crtbal.lb.cstructs[msg.ReplicaId] = msg.Cstruct
	dlog.Printf("Replica %d 2b msg.Cstruct: ", msg.ReplicaId)
	dlog.Println(msg.Cstruct)
	dlog.Println("my cstruct:", crtbal.cstruct)

	crtbal.lb.cstructs[r.Id] = crtbal.cstruct

	r.tryToLearn()
}
Example #15
0
func (r *Replica) handle2a(msg *gpaxosproto.M_2a) {
	if r.isLeader {
		log.Println("Received 2a even though I am the leader")
		return
	}

	if r.leaderId != msg.LeaderId {
		log.Println("Received 2a from unrecognized leader")
		return
	}

	if r.crtBalnum != msg.Balnum {
		log.Println("Received 2a for different ballot: ", msg.Balnum)
		return
	}

	crtbal := r.ballotArray[r.crtBalnum]
	crtbal.received2a = true
	crtbal.status = PHASE2

	dlog.Println("old cstruct", crtbal.cstruct)

	cids := make([]int32, 0)

	if crtbal.cstruct == nil || len(crtbal.cstruct) == 0 {
		crtbal.cstruct = msg.Cstruct
	} else {
		for _, ocid := range crtbal.cstruct {
			present := false
			for _, cid := range msg.Cstruct {
				if cid == ocid {
					present = true
					break
				}
			}
			if !present {
				msg.Cstruct = append(msg.Cstruct, ocid)
				cids = append(cids, ocid)
			}
		}
		crtbal.cstruct = msg.Cstruct
	}

	if ALL_TO_ALL {
		r.bcast2b(&gpaxosproto.M_2b{r.Id, r.crtBalnum, crtbal.cstruct, cids})
		r.tryToLearn()
	} else {
		r.send2b(&gpaxosproto.M_2b{r.Id, r.crtBalnum, crtbal.cstruct, cids}, r.PeerWriters[r.leaderId])
	}
}
Example #16
0
func (r *Replica) bcast2b(msg *gpaxosproto.M_2b) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Commit bcast failed:", err)
		}
	}()

	for rid, w := range r.PeerWriters {
		if int32(rid) == r.Id {
			continue
		}
		w.WriteByte(gpaxosproto.M2B)
		msg.Marshal(w)
		for _, cid := range msg.Cids {
			cmd := r.commands[cid]
			cmd.Marshal(w)
		}
		w.Flush()
	}
}
Example #17
0
func (r *Replica) handlePrepare(prepare *menciusproto.Prepare) {
	inst := r.instanceSpace[prepare.Instance]

	if inst == nil {
		dlog.Println("Replying OK to null-instance Prepare")
		r.replyPrepare(prepare.LeaderId, &menciusproto.PrepareReply{prepare.Instance,
			TRUE,
			-1,
			FALSE,
			0,
			state.Command{state.NONE, 0, 0}})

		r.instanceSpace[prepare.Instance] = &Instance{false,
			0,
			nil,
			prepare.Ballot,
			PREPARING,
			nil}
	} else {
		ok := TRUE
		if prepare.Ballot < inst.ballot {
			ok = FALSE
		}
		if inst.command == nil {
			inst.command = &state.Command{state.NONE, 0, 0}
		}
		skipped := FALSE
		if inst.skipped {
			skipped = TRUE
		}
		r.replyPrepare(prepare.LeaderId, &menciusproto.PrepareReply{prepare.Instance,
			ok,
			inst.ballot,
			skipped,
			int32(inst.nbInstSkipped),
			*inst.command})
	}
}
Example #18
0
func (r *Replica) learn(getLub bool) (conflict bool, glb []int32, lub []int32) {
	if r.crtBalnum < 0 {
		return false, nil, nil
	}

	crtbal := r.ballotArray[r.crtBalnum]

	if len(crtbal.lb.cstructs) == 0 {
		return false, nil, nil
	}

	idToNode := make(map[int32]*node, 2*len(crtbal.cstruct))

	// build directed graph

	dlog.Println(crtbal.lb.cstructs)
	for i := 0; i < len(crtbal.lb.cstructs); i++ {
		cs := crtbal.lb.cstructs[i]
		for idx, cid := range cs {
			var n *node
			var present bool
			crtCmd := r.commands[cid]
			if _, present = r.committed[cid]; present {
				continue
			}
			if n, present = idToNode[cid]; !present {
				n = &node{0, make(map[int32]int, 2), WHITE}
				idToNode[cid] = n
			}
			n.count++
			for j := 0; j < idx; j++ {
				if _, present = r.committed[cs[j]]; present {
					continue
				}
				if crtCmd == nil {
					log.Println("crtCmd is nil")
					return false, nil, nil
				}
				if r.commands[cs[j]] == nil {
					log.Println("cs[j] is nil")
					return false, nil, nil
				}
				if !state.Conflict(crtCmd, r.commands[cs[j]]) {
					continue
				}
				n.outEdges[cs[j]] = n.outEdges[cs[j]] + 1
			}
		}
	}

	// hack

	/*    conflict = false
	      glb = make([]int32, 0)
	      lub = make([]int32, 0)

	      for cid, n := range idToNode {
	          if n.count >= r.fastQSize {
	              glb = append(glb, cid)
	          }
	          lub = append(lub, cid)
	      }

	      return false, glb, lub*/

	// depth-first search

	for cid, n := range idToNode {
		if n.color == WHITE {
			var conf bool
			conf, glb, lub = r.dfs(cid, n, glb, lub, idToNode)
			conflict = conflict || conf
		}
	}
	/*
	   if getLub && conflict {
	       //sort out lub
	       done := false
	       for !done {
	           done = true
	           for i := 1; i < len(lub) - 1; i++ {
	               for j := i + 1; j < len(lub); j++ {
	                   u := lub[i]
	                   v := lub[j]
	                   cu := r.commands[u]
	                   cv := r.commands[v]
	                   if !state.Conflict(cu, cv) {
	                       continue
	                   }
	                   nu := idToNode[u]
	                   nv := idToNode[v]
	                   if nv.count - nv.outEdges[u] > nu.count - nu.outEdges[v] {
	                       lub[i] = v
	                       lub[j] = u
	                       done = false
	                   }
	               }
	           }
	       }
	   }
	*/
	return conflict, glb, lub
}
Example #19
0
func (r *Replica) run() {
	r.ConnectToPeers()

	dlog.Println("Waiting for client connections")

	go r.WaitForClientConnections()

	if r.Exec {
		go r.executeCommands()
	}

	go r.clock()

	for !r.Shutdown {

		select {

		case propose := <-r.ProposeChan:
			//got a Propose from a client
			dlog.Printf("Proposal with id %d\n", propose.CommandId)
			r.handlePropose(propose)
			break

		case skipS := <-r.skipChan:
			skip := skipS.(*menciusproto.Skip)
			//got a Skip from another replica
			dlog.Printf("Skip for instances %d-%d\n", skip.StartInstance, skip.EndInstance)
			r.handleSkip(skip)

		case prepareS := <-r.prepareChan:
			prepare := prepareS.(*menciusproto.Prepare)
			//got a Prepare message
			dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
			r.handlePrepare(prepare)
			break

		case acceptS := <-r.acceptChan:
			accept := acceptS.(*menciusproto.Accept)
			//got an Accept message
			dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
			r.handleAccept(accept)
			break

		case commitS := <-r.commitChan:
			commit := commitS.(*menciusproto.Commit)
			//got a Commit message
			dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
			r.handleCommit(commit)
			break

		case prepareReplyS := <-r.prepareReplyChan:
			prepareReply := prepareReplyS.(*menciusproto.PrepareReply)
			//got a Prepare reply
			dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
			r.handlePrepareReply(prepareReply)
			break

		case acceptReplyS := <-r.acceptReplyChan:
			acceptReply := acceptReplyS.(*menciusproto.AcceptReply)
			//got an Accept reply
			dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
			r.handleAcceptReply(acceptReply)
			break

		case delayedSkip := <-r.delayedSkipChan:
			r.handleDelayedSkip(delayedSkip)
			break

		case <-r.clockChan:
			if lastSeenInstance == r.blockingInstance {
				r.noCommitFor++
			} else {
				r.noCommitFor = 0
				lastSeenInstance = r.blockingInstance
			}
			if r.noCommitFor >= 50+int(r.Id) && r.crtInstance >= r.blockingInstance+int32(r.N) {
				r.noCommitFor = 0
				dlog.Printf("Doing force commit\n")
				r.forceCommit()
			}
			break
		}
	}
}
Example #20
0
func (r *Replica) run() {
	if r.Id == 0 {
		r.isLeader = true
	}

	r.ConnectToPeersNoListeners()

	for rid, peerReader := range r.PeerReaders {
		if int32(rid) == r.Id {
			continue
		}
		go r.handleReplicaConnection(rid, peerReader)
	}

	dlog.Println("Waiting for client connections")

	go r.WaitForClientConnections()

	/*if r.Exec {
	    go r.executeCommands()
	}*/

	clockChan = make(chan bool, 1)
	go r.clock()

	if r.isLeader {
		r.crtBalnum = 0
		r.fastRound = true
		r.ballotArray[0] = &Ballot{nil, 0, PHASE1, false, &LeaderBookkeeping{cstructs: make([][]int32, 0)}}
		r.ballotArray[0].lb.cstructs = append(r.ballotArray[0].lb.cstructs, make([]int32, 0))
		r.bcast1a(0, true)
	}

	for !r.Shutdown {

		if r.crtBalnum >= 0 && len(r.ballotArray[r.crtBalnum].cstruct) >= CMDS_PER_BALLOT {

			select {

			case prepare := <-r.prepareChan:
				//got a Prepare message
				dlog.Printf("Received Prepare for balnum %d\n", prepare.Balnum)
				r.commandsMutex.Lock()
				r.handlePrepare(prepare)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m1aChan:
				dlog.Printf("Received 1a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle1a(msg)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m1bChan:
				dlog.Printf("Received 1b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle1b(msg)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m2aChan:
				dlog.Printf("Received 2a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle2a(msg)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m2bChan:
				dlog.Printf("Received 2b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle2b(msg)
				r.commandsMutex.Unlock()
				break

			case <-clockChan:
				//way out of deadlock

				select {

				case propose := <-r.ProposeChan:
					//got a Propose from a client
					dlog.Printf("Proposal with id %d @ replica %d\n", propose.CommandId, r.Id)
					r.commandsMutex.Lock()
					r.handlePropose(propose)
					r.commandsMutex.Unlock()
					break

				default:
					break
				}
			}

		} else {

			select {

			case prepare := <-r.prepareChan:
				//got a Prepare message
				dlog.Printf("Received Prepare for balnum %d\n", prepare.Balnum)
				r.commandsMutex.Lock()
				r.handlePrepare(prepare)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m1aChan:
				dlog.Printf("Received 1a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle1a(msg)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m1bChan:
				dlog.Printf("Received 1b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle1b(msg)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m2aChan:
				dlog.Printf("Received 2a for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle2a(msg)
				r.commandsMutex.Unlock()
				break

			case msg := <-r.m2bChan:
				dlog.Printf("Received 2b for balnum %d @ replica %d\n", msg.Balnum, r.Id)
				r.commandsMutex.Lock()
				r.handle2b(msg)
				r.commandsMutex.Unlock()
				break

			case propose := <-r.ProposeChan:
				//got a Propose from a client
				dlog.Printf("Proposal with id %d @ replica %d\n", propose.CommandId, r.Id)
				r.commandsMutex.Lock()
				r.handlePropose(propose)
				r.commandsMutex.Unlock()
				break
			}
		}
	}
}
Example #21
0
func (r *Replica) run() {
	r.ConnectToPeers()

	dlog.Println("Waiting for client connections")

	go r.WaitForClientConnections()

	if r.Exec {
		go r.executeCommands()
	}

	if r.Id == 0 {
		//init quorum read lease
		quorum := make([]int32, r.N/2+1)
		for i := 0; i <= r.N/2; i++ {
			quorum[i] = int32(i)
		}
		r.UpdatePreferredPeerOrder(quorum)
	}

	slowClockChan = make(chan bool, 1)
	fastClockChan = make(chan bool, 1)
	go r.slowClock()

	//Enabled when batching for 5ms
	if MAX_BATCH > 100 {
		go r.fastClock()
	}

	if r.Beacon {
		go r.stopAdapting()
	}

	for !r.Shutdown {

		handleNewProposals := true

		// Enabled when batching for 5ms
		if MAX_BATCH > 100 {
			handleNewProposals = false
		}

		select {

		case <-fastClockChan:
			handleNewProposals = true
			break

		case prepareS := <-r.prepareChan:
			prepare := prepareS.(*epaxosproto.Prepare)
			//got a Prepare message
			dlog.Printf("Received Prepare for instance %d.%d\n", prepare.Replica, prepare.Instance)
			r.handlePrepare(prepare)
			break

		case preAcceptS := <-r.preAcceptChan:
			preAccept := preAcceptS.(*epaxosproto.PreAccept)
			//got a PreAccept message
			dlog.Printf("Received PreAccept for instance %d.%d\n", preAccept.LeaderId, preAccept.Instance)
			r.handlePreAccept(preAccept)
			break

		case acceptS := <-r.acceptChan:
			accept := acceptS.(*epaxosproto.Accept)
			//got an Accept message
			dlog.Printf("Received Accept for instance %d.%d\n", accept.LeaderId, accept.Instance)
			r.handleAccept(accept)
			break

		case commitS := <-r.commitChan:
			commit := commitS.(*epaxosproto.Commit)
			//got a Commit message
			dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
			r.handleCommit(commit)
			break

		case commitS := <-r.commitShortChan:
			commit := commitS.(*epaxosproto.CommitShort)
			//got a Commit message
			dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
			r.handleCommitShort(commit)
			break

		case prepareReplyS := <-r.prepareReplyChan:
			prepareReply := prepareReplyS.(*epaxosproto.PrepareReply)
			//got a Prepare reply
			dlog.Printf("Received PrepareReply for instance %d.%d\n", prepareReply.Replica, prepareReply.Instance)
			r.handlePrepareReply(prepareReply)
			break

		case preAcceptReplyS := <-r.preAcceptReplyChan:
			preAcceptReply := preAcceptReplyS.(*epaxosproto.PreAcceptReply)
			//got a PreAccept reply
			dlog.Printf("Received PreAcceptReply for instance %d.%d\n", preAcceptReply.Replica, preAcceptReply.Instance)
			r.handlePreAcceptReply(preAcceptReply)
			break

		case preAcceptOKS := <-r.preAcceptOKChan:
			preAcceptOK := preAcceptOKS.(*epaxosproto.PreAcceptOK)
			//got a PreAccept reply
			dlog.Printf("Received PreAcceptOK for instance %d.%d\n", r.Id, preAcceptOK.Instance)
			r.handlePreAcceptOK(preAcceptOK)
			break

		case acceptReplyS := <-r.acceptReplyChan:
			acceptReply := acceptReplyS.(*epaxosproto.AcceptReply)
			//got an Accept reply
			dlog.Printf("Received AcceptReply for instance %d.%d\n", acceptReply.Replica, acceptReply.Instance)
			r.handleAcceptReply(acceptReply)
			break

		case tryPreAcceptS := <-r.tryPreAcceptChan:
			tryPreAccept := tryPreAcceptS.(*epaxosproto.TryPreAccept)
			dlog.Printf("Received TryPreAccept for instance %d.%d\n", tryPreAccept.Replica, tryPreAccept.Instance)
			r.handleTryPreAccept(tryPreAccept)
			break

		case tryPreAcceptReplyS := <-r.tryPreAcceptReplyChan:
			tryPreAcceptReply := tryPreAcceptReplyS.(*epaxosproto.TryPreAcceptReply)
			dlog.Printf("Received TryPreAcceptReply for instance %d.%d\n", tryPreAcceptReply.Replica, tryPreAcceptReply.Instance)
			r.handleTryPreAcceptReply(tryPreAcceptReply)
			break

		case beacon := <-r.BeaconChan:
			dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
			r.ReplyBeacon(beacon)
			break

		case <-slowClockChan:
			if r.Beacon {
				for q := int32(0); q < int32(r.N); q++ {
					if q == r.Id {
						continue
					}
					r.SendBeacon(q)
				}
			}
			break
		case <-r.OnClientConnect:
			log.Printf("weird %d; conflicted %d; slow %d; happy %d\n", weird, conflicted, slow, happy)
			weird, conflicted, slow, happy = 0, 0, 0, 0

		case iid := <-r.instancesToRecover:
			r.startRecoveryForInstance(iid.replica, iid.instance)
		}

		if handleNewProposals {
			// unfortunately the only way to activate the new proposals channel every X ms
			// was to dublicate the body of the select
			select {

			case propose := <-r.ProposeChan:
				//got a Propose from a client
				dlog.Printf("Proposal with op %d\n", propose.Command.Op)
				r.handlePropose(propose)
				handleNewProposals = false
				break

			case prepareS := <-r.prepareChan:
				prepare := prepareS.(*epaxosproto.Prepare)
				//got a Prepare message
				dlog.Printf("Received Prepare for instance %d.%d\n", prepare.Replica, prepare.Instance)
				r.handlePrepare(prepare)
				break

			case preAcceptS := <-r.preAcceptChan:
				preAccept := preAcceptS.(*epaxosproto.PreAccept)
				//got a PreAccept message
				dlog.Printf("Received PreAccept for instance %d.%d\n", preAccept.LeaderId, preAccept.Instance)
				r.handlePreAccept(preAccept)
				break

			case acceptS := <-r.acceptChan:
				accept := acceptS.(*epaxosproto.Accept)
				//got an Accept message
				dlog.Printf("Received Accept for instance %d.%d\n", accept.LeaderId, accept.Instance)
				r.handleAccept(accept)
				break

			case commitS := <-r.commitChan:
				commit := commitS.(*epaxosproto.Commit)
				//got a Commit message
				dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
				r.handleCommit(commit)
				break

			case commitS := <-r.commitShortChan:
				commit := commitS.(*epaxosproto.CommitShort)
				//got a Commit message
				dlog.Printf("Received Commit for instance %d.%d\n", commit.LeaderId, commit.Instance)
				r.handleCommitShort(commit)
				break

			case prepareReplyS := <-r.prepareReplyChan:
				prepareReply := prepareReplyS.(*epaxosproto.PrepareReply)
				//got a Prepare reply
				dlog.Printf("Received PrepareReply for instance %d.%d\n", prepareReply.Replica, prepareReply.Instance)
				r.handlePrepareReply(prepareReply)
				break

			case preAcceptReplyS := <-r.preAcceptReplyChan:
				preAcceptReply := preAcceptReplyS.(*epaxosproto.PreAcceptReply)
				//got a PreAccept reply
				dlog.Printf("Received PreAcceptReply for instance %d.%d\n", preAcceptReply.Replica, preAcceptReply.Instance)
				r.handlePreAcceptReply(preAcceptReply)
				break

			case preAcceptOKS := <-r.preAcceptOKChan:
				preAcceptOK := preAcceptOKS.(*epaxosproto.PreAcceptOK)
				//got a PreAccept reply
				dlog.Printf("Received PreAcceptOK for instance %d.%d\n", r.Id, preAcceptOK.Instance)
				r.handlePreAcceptOK(preAcceptOK)
				break

			case acceptReplyS := <-r.acceptReplyChan:
				acceptReply := acceptReplyS.(*epaxosproto.AcceptReply)
				//got an Accept reply
				dlog.Printf("Received AcceptReply for instance %d.%d\n", acceptReply.Replica, acceptReply.Instance)
				r.handleAcceptReply(acceptReply)
				break

			case tryPreAcceptS := <-r.tryPreAcceptChan:
				tryPreAccept := tryPreAcceptS.(*epaxosproto.TryPreAccept)
				dlog.Printf("Received TryPreAccept for instance %d.%d\n", tryPreAccept.Replica, tryPreAccept.Instance)
				r.handleTryPreAccept(tryPreAccept)
				break

			case tryPreAcceptReplyS := <-r.tryPreAcceptReplyChan:
				tryPreAcceptReply := tryPreAcceptReplyS.(*epaxosproto.TryPreAcceptReply)
				dlog.Printf("Received TryPreAcceptReply for instance %d.%d\n", tryPreAcceptReply.Replica, tryPreAcceptReply.Instance)
				r.handleTryPreAcceptReply(tryPreAcceptReply)
				break

			case beacon := <-r.BeaconChan:
				dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
				r.ReplyBeacon(beacon)
				break

			case <-slowClockChan:
				if r.Beacon {
					for q := int32(0); q < int32(r.N); q++ {
						if q == r.Id {
							continue
						}
						r.SendBeacon(q)
					}
				}
				break
			case <-r.OnClientConnect:
				log.Printf("weird %d; conflicted %d; slow %d; happy %d\n", weird, conflicted, slow, happy)
				weird, conflicted, slow, happy = 0, 0, 0, 0
				break

			case iid := <-r.instancesToRecover:
				r.startRecoveryForInstance(iid.replica, iid.instance)
			}
		}
	}
}
Example #22
0
func (r *Replica) handleAccept(accept *paxosproto.Accept) {
	inst := r.instanceSpace[accept.Instance]
	var areply *paxosproto.AcceptReply

	if inst == nil {
		if accept.Ballot < r.defaultBallot {
			areply = &paxosproto.AcceptReply{accept.Instance, FALSE, r.defaultBallot, -1, accept.OriginReplica, accept.PropId}
		} else {
			r.instanceSpace[accept.Instance] = &Instance{
				accept.Command,
				accept.Ballot,
				ACCEPTED,
				nil,
				1, false}
			areply = &paxosproto.AcceptReply{accept.Instance, TRUE, accept.Ballot, -1, accept.OriginReplica, accept.PropId}
			r.addUpdatingKeys(accept.Command)
		}
	} else if inst.ballot > accept.Ballot {
		areply = &paxosproto.AcceptReply{accept.Instance, FALSE, inst.ballot, -1, accept.OriginReplica, accept.PropId}
	} else if inst.ballot < accept.Ballot && inst.status != COMMITTED {
		if inst.cmds != nil {
			r.removeUpdatingKeys(inst.cmds)
		}
		r.addUpdatingKeys(accept.Command)
		inst.cmds = accept.Command
		inst.ballot = accept.Ballot
		inst.status = ACCEPTED
		inst.directAcks = 1
		areply = &paxosproto.AcceptReply{accept.Instance, TRUE, inst.ballot, -1, accept.OriginReplica, accept.PropId}
		if inst.lb != nil && inst.lb.clientProposals != nil {
			//TODO: is this correct?
			// try the proposal in a different instance
			for i := 0; i < len(inst.lb.clientProposals); i++ {
				r.ProposeChan <- inst.lb.clientProposals[i]
			}
			inst.lb.clientProposals = nil
		}
	} else {
		// reordered ACCEPT
		r.instanceSpace[accept.Instance].cmds = accept.Command
		if r.instanceSpace[accept.Instance].status < COMMITTED {
			r.instanceSpace[accept.Instance].status = ACCEPTED
		}
		r.instanceSpace[accept.Instance].ballot = accept.Ballot
		areply = &paxosproto.AcceptReply{accept.Instance, TRUE, accept.Ballot, -1, accept.OriginReplica, accept.PropId}
		/*} else {
		    return
		}*/
	}

	if accept.LeaseInstance != r.QLease.PromisedByMeInst && !r.QLease.CanWriteOutside() {
		// cannot accept because lease eras do not match
		areply.OK = FALSE
		areply.LeaseInstance = r.QLease.PromisedByMeInst
		log.Println(accept.LeaseInstance, r.leaseSMR.LatestCommitted)
	}

	if areply.OK == TRUE {
		if accept.Instance > r.latestAcceptedInst {
			r.latestAcceptedInst = accept.Instance
		}

		if r.directAcks && r.Id == accept.OriginReplica {
			inst = r.instanceSpace[accept.Instance]
			inst.directAcks++
			if inst.directAcks == int8(r.N/2+1) {
				//safe to commit
				prop := r.fwdPropMap[accept.PropId]
				inst.status = COMMITTED
				// give client the all clear
				if !r.Dreply && !inst.sentReply {
					propreply := &genericsmrproto.ProposeReplyTS{
						TRUE,
						prop.CommandId,
						state.NIL,
						prop.Timestamp}
					r.ReplyProposeTS(propreply, prop)
					inst.sentReply = true
				} else if inst.sentReply {
					dlog.Println("Trying to send reply twice!")
				}
				r.updateCommittedUpTo()
			}
		}

		r.recordInstanceMetadata(r.instanceSpace[accept.Instance])
		r.recordCommands(accept.Command)
		r.sync()
	}

	r.replyAccept(accept.LeaderId, areply)

	if r.directAcks &&
		accept.OriginReplica >= 0 &&
		accept.OriginReplica != accept.LeaderId &&
		accept.OriginReplica != r.Id &&
		areply.OK == TRUE {
		r.replyAccept(accept.OriginReplica, areply)
	}
}
Example #23
0
func (r *Replica) run() {

	r.ConnectToPeers()

	dlog.Println("Waiting for client connections")

	go r.WaitForClientConnections()

	if r.Exec {
		go r.executeCommands()
		go r.reader()
	}

	if r.Id == 0 {
		r.IsLeader = true
		r.readStats = NewReadStats(r.N, r.Id)
	}

	clockChan = make(chan bool, 1)
	go r.clock()

	r.QLease = qlease.NewLease(r.N)

	leaseClockChan = make(chan bool, 1)
	leaseClockRestart = make(chan bool)
	go r.leaseClock()

	//	clockRang := false

	var tickCounter uint64 = 0
	latestBeaconFromReplica := make([]uint64, r.N)
	proposedDead := make([]bool, r.N)
	for i := 0; i < r.N; i++ {
		latestBeaconFromReplica[i] = 0
		proposedDead[i] = false
	}

	stopRenewing := false

	for !r.Shutdown {

		select {

		case <-clockChan:
			//clockRang = true
			tickCounter++
			if tickCounter%20 == 0 {
				if r.Beacon {
					for q := int32(0); q < int32(r.N); q++ {
						if q == r.Id {
							continue
						}
						r.SendBeacon(q)
					}
				}

				if r.IsLeader && r.Beacon {
					for rid := int32(0); rid < int32(r.N); rid++ {
						if rid == r.Id {
							continue
						}
						if !proposedDead[rid] && tickCounter-latestBeaconFromReplica[rid] >= 200 {
							log.Println("Proposing replica dead: ", rid)
							//replica might be dead
							//propose a lease configuration change
							dr := make([]int32, 1)
							dr[0] = rid
							r.proposeReplicasDead(dr)
							proposedDead[rid] = true
						}
					}
				}

				rightNow := time.Now().UnixNano()
				for rid := int32(0); rid < int32(r.N); rid++ {
					if rid == r.Id {
						continue
					}
					if !proposedDead[rid] && r.LastReplyReceivedTimestamp[rid] > 0 && rightNow-r.LastReplyReceivedTimestamp[rid] >= GRACE_PERIOD {
						if r.IsLeader {
							log.Println("Proposing replica dead: ", rid)
							//replica might be dead
							//propose a lease configuration change
							dr := make([]int32, 1)
							dr[0] = rid
							r.proposeReplicasDead(dr)
						}
						proposedDead[rid] = true
						stopRenewing = true
					}
				}

				if r.IsLeader {
					for i := r.committedUpTo; i >= 0 && i < r.crtInstance; i++ {
						r.delayedInstances <- i
					}
				}
			}
			break

		case propose := <-r.ProposeChan:
			//got a Propose from a client
			dlog.Printf("Proposal with op %d\n", propose.Command.Op)
			r.handlePropose(propose)
			//clockRang = false
			break

		case forwardS := <-r.forwardChan:
			forward := forwardS.(*paxosproto.Forward)
			dlog.Printf("Forward proposal from replica %d\n", forward.ReplicaId)
			r.handleForward(forward)
			break

		case frS := <-r.forwardReplyChan:
			fr := frS.(*paxosproto.ForwardReply)
			r.handleForwardReply(fr)
			break

		case prepareS := <-r.prepareChan:
			prepare := prepareS.(*paxosproto.Prepare)
			//got a Prepare message
			dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
			r.handlePrepare(prepare)
			break

		case acceptS := <-r.acceptChan:
			accept := acceptS.(*paxosproto.Accept)
			//got an Accept message
			dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
			r.handleAccept(accept)
			break

		case commitS := <-r.commitChan:
			commit := commitS.(*paxosproto.Commit)
			//got a Commit message
			dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
			r.handleCommit(commit)
			break

		case commitS := <-r.commitShortChan:
			commit := commitS.(*paxosproto.CommitShort)
			//got a Commit message
			dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
			r.handleCommitShort(commit)
			break

		case prepareReplyS := <-r.prepareReplyChan:
			prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
			//got a Prepare reply
			dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
			r.handlePrepareReply(prepareReply)
			break

		case acceptReplyS := <-r.acceptReplyChan:
			acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
			//got an Accept reply
			dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
			r.handleAcceptReply(acceptReply)
			break

		case instNo := <-r.delayedInstances:
			log.Println("Trying to re-start instance")
			inst := r.instanceSpace[instNo]
			if inst.status < COMMITTED {
				inst.status = PREPARED
				inst.lb.acceptOKs = 0
				inst.ballot = r.makeBallotLargerThan(inst.ballot)
				inst.lb.acceptOKsToWait, _ = r.bcastAccept(instNo, inst.ballot, inst.cmds, inst.lb.clientProposals[0].FwdReplica, inst.lb.clientProposals[0].FwdId)
				if genericsmr.SendError {
					r.delayedInstances <- instNo
				}
			}

		case beacon := <-r.BeaconChan:
			dlog.Printf("Received Beacon from replica %d with timestamp %d\n", beacon.Rid, beacon.Timestamp)
			//r.ReplyBeacon(beacon)
			latestBeaconFromReplica[beacon.Rid] = tickCounter
			break

		case guardS := <-r.QLGuardChan:
			guard := guardS.(*qleaseproto.Guard)
			r.HandleQLeaseGuard(r.QLease, guard)
			break

		case guardReplyS := <-r.QLGuardReplyChan:
			guardReply := guardReplyS.(*qleaseproto.GuardReply)
			r.HandleQLeaseGuardReply(r.QLease, guardReply, r.latestAcceptedInst)
			break

		case promiseS := <-r.QLPromiseChan:
			promise := promiseS.(*qleaseproto.Promise)
			prev := r.QLease.PromisedToMeInst
			if !r.HandleQLeasePromise(r.QLease, promise) {
				continue
			}
			if prev != r.QLease.PromisedToMeInst {
				r.updateGrantedKeys(prev)
				newPromiseCount = 0
			}
			newPromiseCount++
			if newPromiseCount <= r.N/2 {
				if r.newestInstanceIDontKnow < promise.LatestAcceptedInst {
					r.newestInstanceIDontKnow = promise.LatestAcceptedInst
				}
			}
			break

		case preplyS := <-r.QLPromiseReplyChan:
			preply := preplyS.(*qleaseproto.PromiseReply)
			r.HandleQLeaseReply(r.QLease, preply)
			break

		case <-leaseClockChan:
			if r.QLease.PromisedByMeInst < r.leaseSMR.LatestCommitted {
				// wait for previous lease to expire before switching to new config
				if r.QLease.CanWriteOutside() {
					r.updateKeyQuorumInfo(r.leaseSMR.LatestCommitted)
					r.QLease.PromisedByMeInst = r.leaseSMR.LatestCommitted
					log.Printf("Replica %d - New lease for instance %d\n", r.Id, r.QLease.PromisedByMeInst)
					r.EstablishQLease(r.QLease)
					stopRenewing = false
				}
			} else if r.QLease.PromisedByMeInst >= 0 {
				if r.QLease.CanWriteOutside() {
					r.EstablishQLease(r.QLease)
					stopRenewing = false
				} else if !stopRenewing {
					r.RenewQLease(r.QLease, r.latestAcceptedInst)
				}
			}

			if r.maintainReadStats {
				ticks--
				if ticks == 0 {
					r.maintainReadStats = false
					if r.IsLeader && r.readStats != nil {
						go r.proposeLeaseReconf()
					}

					ticks = TICKS_TO_RECONF_LEASE
					//ticks = 60
				}
			}
			// restart the clock
			leaseClockRestart <- true

		case <-r.OnClientConnect:
			log.Printf("reads: %d, local: %d\n", reads, local)
		}
	}
}
Example #24
0
func (r *Replica) handleAcceptReply(areply *paxosproto.AcceptReply) {
	inst := r.instanceSpace[areply.Instance]

	if !r.IsLeader && areply.OK == TRUE && areply.OriginReplica == r.Id {
		//direct ACK optimization
		prop := r.fwdPropMap[areply.PropId]
		if inst == nil {
			cmds := make([]state.Command, 1)
			cmds[0] = prop.Command
			r.instanceSpace[areply.Instance] = &Instance{
				cmds,
				areply.Ballot,
				NONE,
				nil,
				1, false}
			inst = r.instanceSpace[areply.Instance]
			r.addUpdatingKeys(cmds)
		}
		if inst.ballot > areply.Ballot {
			return
		}
		if areply.Ballot > inst.ballot {
			inst.ballot = areply.Ballot
			inst.directAcks = 2
		} else {
			inst.directAcks++
		}
		if (inst.status == COMMITTED && inst.directAcks <= int8(r.N/2+1)) ||
			(inst.status != COMMITTED && inst.directAcks == int8(r.N/2+1)) {
			//safe to commit
			inst.status = COMMITTED
			// give the client the all clear
			inst.directAcks = int8(r.N/2 + 2)
			if !r.Dreply && !inst.sentReply {
				propreply := &genericsmrproto.ProposeReplyTS{
					TRUE,
					prop.CommandId,
					state.NIL,
					prop.Timestamp}
				r.ReplyProposeTS(propreply, prop)
				inst.sentReply = true
			} else if inst.sentReply {
				dlog.Println("Trying to send reply twice!")
			}
			r.updateCommittedUpTo()
		}
		return
	}

	if inst.status >= COMMITTED {
		// we've moved on, these are delayed replies, so just ignore
		return
	}

	if inst.ballot != areply.Ballot {
		return
	}

	if areply.OK == TRUE {
		inst.lb.acceptOKs++
		//if inst.lb.acceptOKs + 1 > r.N >> 1 {
		if inst.lb.acceptOKs >= inst.lb.acceptOKsToWait {
			inst = r.instanceSpace[areply.Instance]
			inst.status = COMMITTED
			if inst.lb.clientProposals != nil && !r.Dreply {
				// give client the all clear
				for i := 0; i < len(inst.cmds); i++ {
					propreply := &genericsmrproto.ProposeReplyTS{
						TRUE,
						inst.lb.clientProposals[i].CommandId,
						state.NIL,
						inst.lb.clientProposals[i].Timestamp}
					r.ReplyProposeTS(propreply, inst.lb.clientProposals[i])
				}
			}

			r.recordInstanceMetadata(r.instanceSpace[areply.Instance])
			r.sync() //is this necessary?

			if areply.OriginReplica < 0 || areply.OriginReplica == r.Id {
				r.addUpdatingKeys(inst.cmds)
			}
			r.updateCommittedUpTo()

			r.bcastCommit(areply.Instance, inst.ballot, inst.cmds)
		}
	} else {
		if areply.LeaseInstance >= 0 {
			// acceptor is in different lease era
			// re-try later
			r.delayedInstances <- areply.Instance
		}
		// TODO: there is probably another active leader
		inst.lb.nacks++
		if areply.Ballot > inst.lb.maxRecvBallot {
			inst.lb.maxRecvBallot = areply.Ballot
		}
		if inst.lb.nacks >= r.N>>1 {
			// TODO
		}
	}
}
Example #25
0
func (r *Replica) bcastAccept(instance int32, ballot int32, skip uint8, nbInstToSkip int32, command state.Command) {
	defer func() {
		if err := recover(); err != nil {
			dlog.Println("Accept bcast failed:", err)
		}
	}()
	ma.LeaderId = r.Id
	ma.Instance = instance
	ma.Ballot = ballot
	ma.Skip = skip
	ma.NbInstancesToSkip = nbInstToSkip
	ma.Command = command
	args := &ma
	//args := &menciusproto.Accept{r.Id, instance, ballot, skip, nbInstToSkip, command}

	n := r.N - 1
	q := r.Id

	sent := 0
	for sent < n {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			break
		}
		if !r.Alive[q] {
			continue
		}
		if r.Thrifty {
			inst := (instance/int32(r.N))*int32(r.N) + q
			if inst > instance {
				inst -= int32(r.N)
			}
			if inst < 0 || r.instanceSpace[inst] != nil {
				continue
			}
		}
		sent++
		r.SendMsg(q, r.acceptRPC, args)
	}

	for sent < r.N>>1 {
		q = (q + 1) % int32(r.N)
		if q == r.Id {
			break
		}
		if !r.Alive[q] {
			continue
		}
		if r.Thrifty {
			inst := (instance/int32(r.N))*int32(r.N) + q
			if inst > instance {
				inst -= int32(r.N)
			}
			if inst >= 0 && r.instanceSpace[inst] == nil {
				continue
			}
		}
		sent++
		r.SendMsg(q, r.acceptRPC, args)
	}
}
Example #26
0
func (r *Replica) run() {

	r.ConnectToPeers()

	dlog.Println("Waiting for client connections")

	go r.WaitForClientConnections()

	if r.Exec {
		go r.executeCommands()
	}

	if r.Id == 0 {
		r.IsLeader = true
	}

	clockChan = make(chan bool, 1)
	go r.clock()

	clockRang := false

	for !r.Shutdown {

		select {

		case <-clockChan:
			clockRang = true
			break

		case prepareS := <-r.prepareChan:
			prepare := prepareS.(*paxosproto.Prepare)
			//got a Prepare message
			dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
			r.handlePrepare(prepare)
			break

		case acceptS := <-r.acceptChan:
			accept := acceptS.(*paxosproto.Accept)
			//got an Accept message
			dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
			r.handleAccept(accept)
			break

		case commitS := <-r.commitChan:
			commit := commitS.(*paxosproto.Commit)
			//got a Commit message
			dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
			r.handleCommit(commit)
			break

		case commitS := <-r.commitShortChan:
			commit := commitS.(*paxosproto.CommitShort)
			//got a Commit message
			dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
			r.handleCommitShort(commit)
			break

		case prepareReplyS := <-r.prepareReplyChan:
			prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
			//got a Prepare reply
			dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
			r.handlePrepareReply(prepareReply)
			break

		case acceptReplyS := <-r.acceptReplyChan:
			acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
			//got an Accept reply
			dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
			r.handleAcceptReply(acceptReply)
			break
		}

		if clockRang {
			select {

			case propose := <-r.ProposeChan:
				//got a Propose from a client
				dlog.Printf("Proposal with op %d\n", propose.Command.Op)
				r.handlePropose(propose)
				clockRang = false
				break

			case prepareS := <-r.prepareChan:
				prepare := prepareS.(*paxosproto.Prepare)
				//got a Prepare message
				dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
				r.handlePrepare(prepare)
				break

			case acceptS := <-r.acceptChan:
				accept := acceptS.(*paxosproto.Accept)
				//got an Accept message
				dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
				r.handleAccept(accept)
				break

			case commitS := <-r.commitChan:
				commit := commitS.(*paxosproto.Commit)
				//got a Commit message
				dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
				r.handleCommit(commit)
				break

			case commitS := <-r.commitShortChan:
				commit := commitS.(*paxosproto.CommitShort)
				//got a Commit message
				dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
				r.handleCommitShort(commit)
				break

			case prepareReplyS := <-r.prepareReplyChan:
				prepareReply := prepareReplyS.(*paxosproto.PrepareReply)
				//got a Prepare reply
				dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
				r.handlePrepareReply(prepareReply)
				break

			case acceptReplyS := <-r.acceptReplyChan:
				acceptReply := acceptReplyS.(*paxosproto.AcceptReply)
				//got an Accept reply
				dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
				r.handleAcceptReply(acceptReply)
				break
			}
		}
	}
}
Example #27
0
func (r *Replica) run() {

	r.ConnectToPeers()

	dlog.Println("Waiting for client connections")

	if r.Id == 0 {
		r.IsLeader = true
	}

	for !r.Shutdown {

		select {

		case proposeS := <-r.ProposeLeaseChan:
			propose := proposeS.(*lpaxosproto.ProposeLease)
			//got a Propose from a client
			dlog.Printf("Lease Update Proposal from replica %d\n", propose.ReplicaId)
			r.handleProposeLease(propose)
			break

		case prepareS := <-r.prepareChan:
			prepare := prepareS.(*lpaxosproto.Prepare)
			//got a Prepare message
			dlog.Printf("Received Prepare from replica %d, for instance %d\n", prepare.LeaderId, prepare.Instance)
			r.handlePrepare(prepare)
			break

		case acceptS := <-r.acceptChan:
			accept := acceptS.(*lpaxosproto.Accept)
			//got an Accept message
			dlog.Printf("Received Accept from replica %d, for instance %d\n", accept.LeaderId, accept.Instance)
			r.handleAccept(accept)
			break

		case commitS := <-r.commitChan:
			commit := commitS.(*lpaxosproto.Commit)
			//got a Commit message
			dlog.Printf("Received Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
			r.handleCommit(commit)
			break

		case commitS := <-r.commitShortChan:
			commit := commitS.(*lpaxosproto.CommitShort)
			//got a Commit message
			dlog.Printf("Received short Commit from replica %d, for instance %d\n", commit.LeaderId, commit.Instance)
			r.handleCommitShort(commit)
			break

		case prepareReplyS := <-r.prepareReplyChan:
			prepareReply := prepareReplyS.(*lpaxosproto.PrepareReply)
			//got a Prepare reply
			dlog.Printf("Received PrepareReply for instance %d\n", prepareReply.Instance)
			r.handlePrepareReply(prepareReply)
			break

		case acceptReplyS := <-r.acceptReplyChan:
			acceptReply := acceptReplyS.(*lpaxosproto.AcceptReply)
			//got an Accept reply
			dlog.Printf("Received AcceptReply for instance %d\n", acceptReply.Instance)
			r.handleAcceptReply(acceptReply)
			break
		}
	}
}