func (kv *KVPaxos) paxosAgreement(op Op) bool {
//initiates a paxos agreement for this operation
glog.Infoln(kv.me, "Initiating Paxos agreement for ", op)
//Start with a sequence number and increment it if it fails
//if agreement reached check if it was really for the value I suggested or not
agreementReached := false
index := kv.px.Max() + 1
to := 100 * time.Millisecond
for !agreementReached {
glog.Infoln(kv.me, "[paxosAgreement]Index to try for paxos agreement", index)
kv.px.Start(index, op)
time.Sleep(to)
agreementReached = kv.waitForPaxosAgreement(index, op)
if agreementReached {
//agreement has been reached interpret value from last time around keeping into account xid
kv.process(index)
break
} else {
if to < 10*time.Second {
to = to + 50
}
glog.Infoln(kv.me, " SPINNING, no decision for seq number ", index, " trying higher number")
index++
}
// if agreement reached then act on it, evaluate state uptill last
//wait till status turns to agreed
}
return false
}
//
// shared by Put and Append.
//
func (ck *Clerk) PutAppend(key string, value string, op string) {
ck.mu.Lock()
defer ck.mu.Unlock()
xid := nrand()
putReq := &PutAppendArgs{}
putRep := &PutAppendReply{}
putReq.Xid = xid
putReq.Value = value
putReq.Op = op
putReq.Key = key
tries := 0
for {
time.Sleep(100 * time.Millisecond)
tries = tries + 1
glog.Infoln(" Loop count", tries)
for index, _ := range ck.servers {
glog.Infoln(ck.uuid, "PutAppend Calling server ", index, " for request ", putReq.Op)
ret := call(ck.servers[index], "KVPaxos.PutAppend", putReq, putRep)
if ret {
//we got a response check if invalid response in which case try again
if putRep.Err != OK {
continue
}
return
} else {
glog.Warningln(ck.uuid, "RPC broke, retrying for a different server ", ck.servers)
continue
}
}
}
}
func (kv *KVPaxos) waitForPaxosAgreement(seq int, op Op) bool {
status, val := kv.px.Status(seq)
if status == paxos.Decided {
//check if value was truly what we sent originally or some morphed value
// check if val was what was trying to be inserted
if val.(Op).Xid == op.Xid {
glog.Infoln(kv.me, "===>> Agreement reached at seq with value", seq, " with value:", val)
return true
} else {
glog.Infoln(kv.me, "++++Agreement reached for seq by other Server#:", val.(Op).Server, " for seq ", seq, " processed till now:", kv.toprocess, " Min/Max", kv.px.Min(), "/", kv.px.Max())
return false
}
}
if status == paxos.Forgotten {
glog.Infoln("WARNING...!!!")
//this has already been applied by the application cannot agree on this
return false
}
//in pending we wait
return false
}
//--------------------------------------------------
func (px *Paxos) accept(seq int, state *State) bool {
//send accept message with state in
acceptReq := &AcceptReq{}
acceptRep := &AcceptReply{}
acceptReq.Proposal = state.proposal
acceptReq.V = state.v
acceptReq.Seq = seq
numAccepts := 0
glog.Infoln(px.me, "[Accept] hadnler with seq", seq)
for indx, server := range px.peers {
ret := false
if indx == px.me {
ret = (px.AcceptHandler(acceptReq, acceptRep) == nil)
} else {
ret = call(server, "Paxos.AcceptHandler", acceptReq, acceptRep)
}
//update highestProposalInReply and value in highestProposaled Reply
if ret {
//RPC or method call successul
if acceptRep.Status == Accept {
numAccepts++
}
}
}
if 2*numAccepts > len(px.peers) {
glog.Infoln(px.me, "[Accept] majority accepted proposal decision reached will send decide soon ", seq)
//do not update internal state yet
return true
} else {
glog.Infoln(px.me, "[Accept]Peers rejected accept proposal having #", state.proposal, " will go to next round for slot#", seq)
}
return false
}
//
// the application on this machine is done with
// all instances <= seq.
//
// see the comments for Min() for more explanation.
//
func (px *Paxos) Done(seq int) {
glog.Infoln(px.me, "[Done] called with ", seq)
px.mu.Lock()
if seq > px.lastDoneSignalled {
px.lastDoneSignalled = seq
glog.Infoln(px.me, "[Done] updating done#", seq, " value of done ", px.lastDoneSignalled)
}
//clear everything below maxCanDisregard
px.mu.Unlock()
}
func (px *Paxos) PromiseHandler(req *PrepareReq, rep *PrepareRep) error {
//handles promise messages following is the pseudo code
//if this is itself the proposer for same instance and incoming seq numner is highr it should back off
//if this proposal is encountered for the first time for instance it should be accepted and acked yes
px.mu.Lock()
rep.LastDoneSignalled = px.lastDoneSignalled
if val, ok := px.stateMap[req.Seq]; ok {
//we have already seen this paxos seq but we might have a different proposal number for it
if val.proposal > req.Proposal {
//reject this proposal
// glog.Infoln(px.me, "[PromisHandler] Proposal #", req.Proposal, " rejected since own proposalNum#", val.proposal, " was higher")
rep.Status = Reject
rep.V = nil
//set proposal number in return msg
rep.Proposal = -1
px.mu.Unlock()
return nil
} else {
//make sure I have not already decided
//proposal will be accepted and value accepted returned if not nil
rep.Status = Accept
rep.ProposalAccepted = val.highestproposalaccepted
rep.V = val.v
//return back locally accepted value if any
//update highest proposal seen
val.proposal = req.Proposal
// glog.Infoln(px.me, "[PromisHandler] Proposal #", req.Proposal, " accepted, since", "incoming propsal is higher, printing v", val, " sending rep", rep)
glog.Infoln(px.me, "[PromiseHandler] printing my state map", px.stateMap)
px.stateMap[req.Seq] = val
px.mu.Unlock()
return nil
}
} else {
//no record exists for this paxos seq
//proposal will be accepted and value accepted returned if not nil
val := State{}
rep.Status = Accept
//return back locally accepted value if any
rep.V = nil
//update highest proposal seen
val.proposal = req.Proposal
val.highestproposalaccepted = -1
rep.Proposal = -1
rep.ProposalAccepted = -1
val.decision = Pending
px.stateMap[req.Seq] = val
glog.Infoln(px.me, "[PromisHandler] Proposal #", req.Proposal, " accepted")
px.mu.Unlock()
return nil
}
}
func (kv *KVPaxos) Get(args *GetArgs, reply *GetReply) error {
glog.Infoln(kv.me, "[GET]Got client request -->", args)
//this server is a client for the paxos library
//initiate a paxos agreement for this operation
op := Op{}
op.Optype = Get
op.Xid = args.Xid
op.Key = args.Key
op.Server = kv.me
kv.paxosAgreement(op)
reply.Value = kv.kvData[args.Key]
reply.Err = OK
glog.Infoln(kv.me, "[Get] returned ", reply.Value)
return nil
}
//
// Min() should return one more than the minimum among z_i,
// where z_i is the highest number ever passed
// to Done() on peer i. A peers z_i is -1 if it has
// never called Done().
//
// Paxos is required to have forgotten all information
// about any instances it knows that are < Min().
// The point is to free up memory in long-running
// Paxos-based servers.
//
// Paxos peers need to exchange their highest Done()
// arguments in order to implement Min(). These
// exchanges can be piggybacked on ordinary Paxos
// agreement protocol messages, so it is OK if one
// peers Min does not reflect another Peers Done()
// until after the next instance is agreed to.
//
// The fact that Min() is defined as a minimum over
// *all* Paxos peers means that Min() cannot increase until
// all peers have been heard from. So if a peer is dead
// or unreachable, other peers Min()s will not increase
// even if all reachable peers call Done. The reason for
// this is that when the unreachable peer comes back to
// life, it will need to catch up on instances that it
// missed -- the other peers therefor cannot forget these
// instances.
//
func (px *Paxos) Min() int {
min := 100000
lastDoneReq := &LastDoneMsg{}
lastDoneRep := &LastDoneReply{}
for indx, server := range px.peers {
if indx == px.me {
px.LastDoneHandler(lastDoneReq, lastDoneRep)
} else {
call(server, "Paxos.LastDoneHandler", lastDoneReq, lastDoneRep)
}
if lastDoneRep.Done < min {
min = lastDoneRep.Done
glog.Infoln(px.me, "Updating min to ", min)
}
//update highestProposalInReply and value in highestProposaled Reply
//we don't care about return code here we just update intrnal state
}
px.mu.Lock()
if min > px.maxCanDisregard {
px.maxCanDisregard = min
}
px.mu.Unlock()
return min + 1
}
func main() {
role := flag.String("role", "master", "Escort role")
flag.Set("alsologtostderr", "true")
flag.Set("v", "5")
flag.Parse()
if role == nil || len(*role) == 0 {
Usage()
return
}
glog.Infoln(*role)
switch *role {
case "master":
// 主服务
server.Work()
case "slave":
// 备份服务
client.Work()
default:
Usage()
}
}
func connectServer() {
//接通
conn, err := net.Dial("tcp", "localhost:8260")
defer func() {
conn.Close()
}()
if err != nil {
glog.Errorln("Escort slave can't connect to master!")
netErrCh <- 1
//return new(NetConnectErr)
} else {
glog.Infoln("Escort slave connected successfully!")
// 定时发送ping
go func() {
timer := time.NewTicker(10 * time.Second)
for {
select {
case <-timer.C:
if err := ping(conn); err != nil {
glog.Errorf("[ERROR] Escort slave send ping err: %s", err.Error())
// ping发生错误，切换
doSwitch()
}
}
}
}()
handler(conn)
}
}
func ping(conn net.Conn) error {
select {
case pingAtCh <- 1:
glog.Infoln("Escort slave send ping!")
default:
// 没有收到上一次ping的pong，超时
glog.Warningln("Escort slave doesn't get pong!")
return new(PingTimeOutErr)
}
pingcmd := new(Packet)
pingcmd.Version = V1
pingcmd.Flags = FLAG_REQUEST
pingcmd.Length = 0
pingcmd.Sequence = 0
data, err := pingcmd.Encode()
_, err = conn.Write(data)
if err != nil {
glog.Errorf("[ERROR] %s", err.Error())
return err
}
return nil
}
func (kv *KVPaxos) PutAppend(args *PutAppendArgs, reply *PutAppendReply) error {
glog.Infoln(kv.me, "[PUTAPPEND]Got client request -->", args)
op := Op{}
if args.Op == "Put" {
op.Optype = Put
} else {
op.Optype = Append
}
op.Xid = args.Xid
op.Key = args.Key
op.Value = args.Value
op.Server = kv.me
kv.paxosAgreement(op)
reply.Err = OK
glog.Infoln(kv.me, "[Put] returned")
return nil
}
func (kv *KVPaxos) process(indexOfAgreement int) {
//processing value in op
// first time we are seeing this xid need to process this
glog.Infoln(kv.me, "In process with index of agreement", indexOfAgreement, "with current kvtprocess", kv.toprocess)
kv.applyBackLogOps(indexOfAgreement) //these are both inclusive
//only signal done till applyBackLogOps
//increment kv.toprocess
//finally call done here signalling we are done processing this
}
func doService(conn net.Conn) {
glog.Infoln("Escort server is connected!")
defer func() {
conn.Close()
}()
for {
buf, err := ReadFromConn(conn)
if err != nil {
glog.Errorf("[ERROR] %s", err.Error())
break
}
p := new(Packet)
p.Decode(buf)
if p.Flags == FLAG_REQUEST && p.Length == 0 { // ping命令
glog.Infoln("Escort server get client ping cmd!")
pong := new(Packet)
pong.Version = V1
pong.Flags = FLAG_RESPONSE
pong.Sequence = 0
pong.Length = 0
var data []byte
var err error
data, err = pong.Encode()
if err != nil {
glog.Errorf("[ERROR] %s", err.Error())
continue
}
_, err = conn.Write(data)
if err == nil {
glog.Infoln("Escort server send pong successfully!")
} else {
glog.Errorf("[ERROR] %s", err.Error())
}
}
}
}
//
// the application wants to know the
// highest instance sequence known to
// this peer.
//
func (px *Paxos) Max() int {
max := 0
px.mu.Lock()
for key, _ := range px.stateMap {
if key > max {
max = key
}
}
px.mu.Unlock()
glog.Infoln("[Max] called with value ", max)
return max
}
func (px *Paxos) AcceptHandler(req *AcceptReq, rep *AcceptReply) error {
//accept request handler will need to toggle global state
px.mu.Lock()
instance := px.stateMap[req.Seq]
if instance.proposal > req.Proposal {
//send reject
rep.Status = Reject
glog.Infoln(px.me, "[ACCEPTHANDLEr]Peer rejected accept message with proposal#", req.Proposal, "since it has seen higher proposal ", instance.proposal)
} else {
//accept this request covers equality case
rep.Status = Accept
instance.highestproposalaccepted = req.Proposal
instance.v = req.V
//this should not be necessary here since accept messages should have same proposal number
instance.proposal = req.Proposal
instance.decision = Pending
px.stateMap[req.Seq] = instance
glog.Infoln(px.me, " [AcceptHandler]Peer accepted accept message with proposal#", req.Proposal, "and value ", req.V, " for seq", req.Seq)
}
px.mu.Unlock()
return nil
}
//
// fetch the current value for a key.
// returns "" if the key does not exist.
// keeps trying forever in the face of all other errors.
//
func (ck *Clerk) Get(key string) string {
ck.mu.Lock()
defer ck.mu.Unlock()
//generate a unique xid
xid := nrand()
retVal := ""
//contact every server for the operation in round robin fashion break if you got a response else move on
getReq := &GetArgs{}
getReq.Xid = xid
getReq.Key = key
getRep := &GetReply{}
tries := 0
for {
tries = 1 + tries
glog.Infoln(ck.uuid, " client trying for ", tries)
time.Sleep(100 * time.Millisecond)
index := 0
for index < len(ck.servers) {
glog.Infoln(ck.uuid, "Get Calling server ", index, " for request ", ck.servers)
ret := call(ck.servers[index], "KVPaxos.Get", getReq, getRep)
if ret {
//we got a response check if invalid response in which case try again
if getRep.Err != OK {
index++
continue
}
retVal = getRep.Value
return retVal
} else {
glog.Infoln(ck.uuid, "RPC broke retrying get request with another server", ck.servers)
index++
}
}
}
return retVal
}
//--------------------------------------------------
func (px *Paxos) decide(seq int, state *State) bool {
//decide handler updates self state and sends decide message to peers
//proposal number is inconsequential here, we can possibly clean state here
decideReq := &DecideReq{}
decideRep := &DecideReply{}
decideReq.V = state.v
decideReq.Seq = seq
glog.Infoln(px.me, "[Decide] handler with seq", seq)
for indx, server := range px.peers {
if indx == px.me {
px.DecideHandler(decideReq, decideRep)
} else {
ret := call(server, "Paxos.DecideHandler", decideReq, decideRep)
if !ret {
glog.Infoln(px.me, "Decide message failed for index", indx)
}
}
//update highestProposalInReply and value in highestProposaled Reply
//we don't care about return code here we just update intrnal state
}
return true
}
func (px *Paxos) DecideHandler(req *DecideReq, rep *DecideReply) error {
//updates internal state here
glog.Infoln(px.me, "[DecideHandler] decision reached with value ", req.V)
px.mu.Lock()
if instance, ok := px.stateMap[req.Seq]; ok {
//instance has to be present
instance.decision = Decided
instance.v = req.V
px.stateMap[req.Seq] = instance
glog.Infoln(px.me, "[DecideHandler]Priting my stateMap", px.stateMap, " for seq", req.Seq)
} else {
instance := State{}
instance.decision = Decided
instance.v = req.V
px.stateMap[req.Seq] = instance
glog.Infoln(px.me, "[DecideHandler] WARNING!! decide message but no state reached, possible crash recovery")
}
px.mu.Unlock()
return nil
}
//
// call() sends an RPC to the rpcname handler on server srv
// with arguments args, waits for the reply, and leaves the
// reply in reply. the reply argument should be a pointer
// to a reply structure.
//
// the return value is true if the server responded, and false
// if call() was not able to contact the server. in particular,
// the reply's contents are only valid if call() returned true.
//
//
//
func call(srv string, rpcname string,
args interface{}, reply interface{}) bool {
c, errx := rpc.Dial("unix", srv)
if errx != nil {
return false
}
defer c.Close()
err := c.Call(rpcname, args, reply)
if err == nil {
return true
}
glog.Infoln(err)
return false
}
//
// call() sends an RPC to the rpcname handler on server srv
// with arguments args, waits for the reply, and leaves the
// reply in reply. the reply argument should be a pointer
// to a reply structure.
//
// the return value is true if the server responded, and false
// if call() was not able to contact the server. in particular,
// tplys contents are only valid if call() returned true.
//
//
func call(srv string, name string, args interface{}, reply interface{}) bool {
c, err := rpc.Dial("unix", srv)
if err != nil {
err1 := err.(*net.OpError)
if err1.Err != syscall.ENOENT && err1.Err != syscall.ECONNREFUSED {
glog.Warningln("paxos Dial() failed: %v\n", err1)
}
return false
}
defer c.Close()
err = c.Call(name, args, reply)
if err == nil {
return true
}
glog.Infoln(err)
return false
}
func startServer() {
listener, err := net.Listen("tcp", "localhost:8260")
if err != nil {
glog.Errorf("[ERROR] %s", err.Error())
os.Exit(1)
}
glog.Infoln("Escort server is up!")
for {
conn, err := listener.Accept()
if err != nil {
glog.Errorf("[ERROR] %s", err.Error())
continue
}
go doService(conn)
}
}
//
// the application wants to know whether this
// peer thinks an instance has been decided,
// and if so what the agreed value is. Status()
// should just inspect the local peer state;
// it should not contact other Paxos peers.
//
func (px *Paxos) Status(seq int) (Fate, interface{}) {
px.mu.Lock()
if val, ok := px.stateMap[seq]; ok {
px.mu.Unlock()
glog.Infoln(px.me, "state printing for seq", seq, " decision ", val.decision, " map", px.stateMap)
return val.decision, val.v
} else {
//might be forgotten or not seen
if seq < px.maxCanDisregard {
px.mu.Unlock()
return Forgotten, nil
}
}
px.mu.Unlock()
return Pending, nil
}
/**
* 处理从对端获取的响应或请求
**/
func handler(conn net.Conn) {
glog.Infoln("Escort slave handler")
for {
data, err := ReadFromConn(conn)
if err != nil {
glog.Errorf("[ERROR] %s", err.Error())
break
} else {
packet := new(Packet)
packet.Decode(data)
if packet.Flags == FLAG_REQUEST {
// master 的请求
} else {
// master 的响应
if packet.Length == 0 {
// pong
pong()
}
}
}
}
}
func pong() {
glog.Infoln("Escort slave get pong!")
<-pingAtCh // 收到pong则清理pingAtCh
}
//--------------------------------------------------
//
// the application wants paxos to start agreement on
// instance seq, with proposed value v.
// Xtart() returns right away; the application will
// call Status() to find out if/when agreement
// is reached.
//
func (px *Paxos) Start(seq int, v interface{}) {
glog.Infoln(px.me, "[Start] called with seq#", seq, " and value#", v)
go px.takeDecision(seq, v)
}
func (px *Paxos) takeDecision(seq int, v interface{}) {
//make sure only one instance of takeDecision is called per instance
//maybe we can use channels to synchronize here btw same application processes
//allocate a new state object
state := State{}
px.mu.Lock()
//check if lastCleared by this peer is > what came in request
if px.maxCanDisregard >= seq {
glog.Infoln(px.me, "WARNING!!! PAXOS election already undertaken and decision reached seq#", seq)
//unlock the mutex
px.mu.Unlock()
return
}
if state, ok := px.stateMap[seq]; ok {
if state.decision == Pending {
//this is a bug calling state again on something which is underway
glog.Infoln(px.me, "WARNING!!! PAXOS election already underway for seq#, please wait before calling ", seq)
//check if I have already accepted a value in which case i use the same value
if state.highestproposalaccepted == -1 {
glog.Infoln(px.me, "Competing proposal ")
state.v = v
}
state.proposal = px.me
px.stateMap[seq] = state
//px.mu.Unlock()
//return
} else {
//could have alrady been decided but not cleared yet since application did not read its value?
glog.Infoln(px.me, "WARNING!!! PAXOS election already done, decision reached:", px.stateMap[seq].decision, " for seq#:", seq)
px.mu.Unlock()
return
}
} else {
//start called for paxos seq which has been already iniated by this application possible error
//initialize the value of state for this paxos instance and put in global map
state.decision = Pending
state.proposal = px.me
state.highestproposalaccepted = -1
state.v = v
px.stateMap[seq] = state
glog.Infoln(px.me, "Initializing state with value", " for seq#", seq)
}
state = px.stateMap[seq]
px.mu.Unlock()
for state.decision != Decided && !px.isdead() {
glog.Infoln(px.me, "[Start] In proposal#", state.proposal, " seq#", seq, " with value", state.v)
//start with a Proposal # and keep on incrementing the proposal untill decision is reached
if !px.promise(seq, &state) {
//promise phase failed will have to restart with updated proposalNum
state.proposal = state.proposal + len(px.peers)
glog.Infoln(px.me, "Incrementing proposal num , new proposal #", state.proposal)
continue
}
glog.Infoln(px.me, " PROMISE STAGE OVER for proposal #", state.proposal)
if !px.accept(seq, &state) {
//accept failed
state.proposal = px.me + len(px.peers)
continue
}
if !px.decide(seq, &state) {
//decide failed
} else {
//PAXOS decision reached / everything is setup in state we store it in the map
px.mu.Lock()
state = px.stateMap[seq]
state.decision = Decided
px.stateMap[seq] = state
px.mu.Unlock()
}
}
glog.Infoln(px.me, "Decision reached with value", state.v)
return
}
func (px *Paxos) promise(seq int, state *State) bool {
glog.Infoln(px.me, "[promise] proposalNum#", state.proposal, " seqNum#", seq, "value ", state.v)
prepareRep := &PrepareRep{}
prepareRep.LastDoneSignalled = -1
prepareReq := &PrepareReq{}
prepareReq.Proposal = state.proposal
prepareReq.Seq = seq
highestProposalInReply := -2
var valueInHighestProposalReply value
numAccepts := 0
maxCanDisregard := 1000000
for indx, server := range px.peers {
ret := false
if indx == px.me {
ret = (px.PromiseHandler(prepareReq, prepareRep) == nil)
} else {
ret = call(server, "Paxos.PromiseHandler", prepareReq, prepareRep)
}
//update highestProposalInReply and value in highestProposaled Reply
if ret {
//RPC or method call successul
if prepareRep.Status == Accept {
glog.Infoln(px.me, "Printing promise reply", prepareRep)
numAccepts++
if prepareRep.ProposalAccepted > highestProposalInReply {
highestProposalInReply = prepareRep.ProposalAccepted
valueInHighestProposalReply = prepareRep.V
glog.Infoln(px.me, " Updating highest proposal in reply", highestProposalInReply, " with value", valueInHighestProposalReply)
}
}
//handle LastDoneSignalled to update maxCanDisregard
if prepareRep.LastDoneSignalled < maxCanDisregard {
maxCanDisregard = prepareRep.LastDoneSignalled
}
}
}
glog.Infoln(px.me, "after promise round MaxCanDisregard set to ", maxCanDisregard)
//maxCanDisregard needs to be updated here
if maxCanDisregard != -1 {
if maxCanDisregard > px.maxCanDisregard {
px.maxCanDisregard = maxCanDisregard
glog.Infoln(px.me, "Updating value of maxCanDisregard to :", px.maxCanDisregard)
//deleting here as much as I can
for key, _ := range px.stateMap {
if key < px.maxCanDisregard {
glog.Infoln(px.me, "Deleting key#", key)
delete(px.stateMap, key)
}
}
}
}
//there are 2 cases: if majority assembled move over to accept phase and update value from reply,
//if reply had nil value proposer uses own value else uses reply from highest valeud proposal
//px.mu.Lock()
glog.Infoln(px.me, "Printing peers for myself", px.peers, " with numAccepts", numAccepts)
if 2*numAccepts > len(px.peers) {
glog.Infoln(px.me, "[promise] Peers accepted proposalNum#", state.proposal, " seqNum#", seq, "value ", state.v)
if highestProposalInReply != -1 {
//we saw a proposal which was accepted
state.v = valueInHighestProposalReply
glog.Infoln(px.me, "[promise] Updating value to be proposed in next round proposalNum#", state.proposal, " seqNum#", seq, "value ", state.v)
} else {
//using own proposal
glog.Infoln(px.me, "[promise] own value to use for accept round ", state.v)
}
return true
} else {
glog.Infoln(px.me, "Peers rejected proposal", state.proposal, " with value ", state.v)
//add some random determinism based on id
rand.Seed(int64(px.me))
tosleep := rand.Intn(50)
glog.Infoln(px.me, "Paxos sleeping for random time", tosleep)
time.Sleep(time.Duration(int(tosleep)) * time.Millisecond)
//if majority not assembled check highestReplied proposal and if > this proposal no chance for this to succeed
//but before replying back immediately wait so as to not start a proposal with higher number immediately
return false
}
}
//apply all operations while keeping track of xid starting from minIndex to maxIndex all included
//assumes lock is held on the map
func (kv *KVPaxos) applyBackLogOps(maxIndx int) {
glog.Infoln(kv.me, "<<<Applying from ", kv.toprocess, " to ", maxIndx, ">>>")
index := kv.toprocess
iter := 1
kv.mu.Lock()
for index <= maxIndx {
//status is guranteed to be decide here or else we woudn't have reached here
status, val := kv.px.Status(index)
//glog.Infoln(kv.me, "reevaluating status at ", index, " status:", status)
if status == paxos.Decided {
paxosStruct := val.(Op)
//apply stuff in paxosStruct and return result
glog.Infoln(kv.me, "SUCCESS:Applying at index ", index, " operation ", paxosStruct.Optype)
switch paxosStruct.Optype {
case Get:
if _, ok := kv.seenMap[paxosStruct.Xid]; !ok {
//old get which can be skipped
glog.Infoln(kv.me, "Applied get fo key", paxosStruct.Key)
kv.seenMap[paxosStruct.Xid] = true
}
case Put:
if _, ok := kv.seenMap[paxosStruct.Xid]; !ok {
kv.kvData[paxosStruct.Key] = paxosStruct.Value
glog.Infoln(kv.me, "Applied put for value", paxosStruct.Value, " key", paxosStruct.Key)
kv.seenMap[paxosStruct.Xid] = true
}
case Append:
if _, ok := kv.seenMap[paxosStruct.Xid]; !ok {
kv.kvData[paxosStruct.Key] = kv.kvData[paxosStruct.Key] + paxosStruct.Value
glog.Infoln(kv.me, "Applied append for value", paxosStruct.Value, " key", paxosStruct.Key)
kv.seenMap[paxosStruct.Xid] = true
}
}
glog.Infoln(kv.me, "State of my server", kv.kvData, "with index:::", index)
kv.px.Done(index)
index++
kv.toprocess = index
} else {
if status == paxos.Pending {
glog.Infoln(kv.me, "Iteration completed", iter, " for seq", index)
iter++
glog.Infoln("STARTING paxos again to converge at index:", index)
//try to converge by applying value last accepted
kv.px.Start(index, dummyOp)
time.Sleep(100 * time.Millisecond)
continue
} else {
glog.Infoln(kv.me, "WARNING!! Paxos forgotten for index", index, " at val", val)
index++
}
}
}
kv.mu.Unlock()
}