//command is the command the app/paxos want to commit/nop
//iter is the current iter round, a new command use this to increase its own N
//index is the index of commitedCommands. When we want to commit a new command, set
//index=-1, and when we want to fill the gap in slot i, sent index=i.
//Now the functon return three values. The first is a boolean varible indicating the
//current Paxos is success or not. The second is a boolean indicating the current
//command is commited or rejected. The third is the current slot's highest Vh.
func (pn *paxosNode) DoReplicate(command *command.Command, iter, index int) (bool, bool, int) {
//Prepare
prepareArgs := paxosrpc.PrepareArgs{}
/*if index == -1 {
LOGV2.Printf("node %d get lock in DoReplicate()\n", pn.nodeID)
pn.cmdMutex.Lock()
prepareArgs.SlotIdx = len(pn.commitedCommands)
LOGV2.Printf("node %d release lock in DoReplicate()\n", pn.nodeID)
pn.cmdMutex.Unlock()
} else {
prepareArgs.SlotIdx = index
}*/
prepareArgs.SlotIdx = index
prepareArgs.N = pn.nodeID + iter*pn.numNodes
prepareArgs.V = *command
prepareReply := paxosrpc.PrepareReply{}
prepareReply.Status = paxosrpc.Reject
pn.DoPrepare(&prepareArgs, &prepareReply)
if prepareReply.Status == paxosrpc.Reject {
if prepareReply.Na > prepareArgs.N {
return false, false, prepareReply.Na
} else {
return false, false, prepareArgs.N
}
}
//Accept
acceptArgs := paxosrpc.AcceptArgs{}
acceptArgs.SlotIdx = prepareArgs.SlotIdx
LOGV.Printf("Before DoAccept:%d %d\n", acceptArgs.SlotIdx, acceptArgs.N)
//TODO:need check, maybe wrong
if prepareReply.Na == prepareArgs.N {
acceptArgs.V = *command
} else {
acceptArgs.V = prepareReply.Va
}
acceptArgs.N = prepareArgs.N
acceptReply := paxosrpc.AcceptReply{}
pn.DoAccept(&acceptArgs, &acceptReply)
if acceptReply.Status == paxosrpc.Reject {
return false, false, acceptArgs.N
}
//Commit
commitArgs := paxosrpc.CommitArgs{}
commitArgs.SlotIdx = prepareArgs.SlotIdx
commitArgs.N = acceptArgs.N
commitArgs.V = acceptArgs.V
pn.DoCommit(&commitArgs)
//TODO:need check about the return number
if prepareReply.Na != prepareArgs.N {
return true, false, prepareArgs.N
} else {
return true, true, prepareArgs.N
}
}
func (pn *paxosNode) DoPrepare(args *paxosrpc.PrepareArgs, reply *paxosrpc.PrepareReply) error {
LOGV.Printf("node %d DoPrepare:%d %s %d\n", pn.nodeID, args.SlotIdx, args.V.ToString(), args.N)
replychan := make(chan *paxosrpc.PrepareReply, len(pn.peers))
for i, n := range pn.peers {
go func(idx int, peernode Node) {
r := paxosrpc.PrepareReply{}
//if localhost, call locally
if peernode.HostPort == pn.addrport {
pn.Prepare(args, &r)
replychan <- &r
} else { //else, call rpc
peer, err := rpcwrapper.DialHTTP("tcp", peernode.HostPort)
if err != nil {
LOGE.Printf("node %d Cannot reach peer %d:%s\n", pn.nodeID, idx, peernode.HostPort)
r.Status = paxosrpc.Reject
replychan <- &r
return
}
prepareCall := peer.Go("PaxosNode.Prepare", args, &r, nil)
select {
case _, _ = <-prepareCall.Done:
replychan <- &r
case _ = <-time.After(time.Second):
//TODO: how to handle timeout correctly?
r.Status = paxosrpc.Reject
replychan <- &r
}
peer.Close()
}
}(i, n)
}
numOK := 0
numRej := 0
reply.Na = -1
for num := 0; num < len(pn.peers); num++ {
r, _ := <-replychan
if r.Status != paxosrpc.Reject {
numOK++
if r.Status == paxosrpc.Existed && r.Na > reply.Na {
reply.Na = r.Na
reply.Va = r.Va
}
} else {
numRej++
}
}
if reply.Na == -1 {
reply.Na = args.N
reply.Va = args.V
}
LOGV.Printf("node %d DoPrepare %d result:%s %d[%dOK %dRej]\n", pn.nodeID, args.SlotIdx, reply.Va.ToString(), reply.Na, numOK, numRej)
if numOK > len(pn.peers)/2 {
reply.Status = paxosrpc.OK
return nil //return nil error, and let caller to do the accept step
} else {
reply.Status = paxosrpc.Reject
return nil
}
}
//Now it is not Multi-Paxos, but just Paxos
//Each command slot has its own na,nh,v, or say, version control
func (pn *paxosNode) Prepare(args *paxosrpc.PrepareArgs, reply *paxosrpc.PrepareReply) error {
LOGV.Printf("node %d OnPrepare:%d %s %d\n", pn.nodeID, args.SlotIdx, args.V.ToString(), args.N)
LOGV2.Printf("node %d get lock in Prepare()\n", pn.nodeID)
pn.cmdMutex.Lock()
v, ok := pn.tempSlots[args.SlotIdx]
if ok {
if args.N <= v.Nh { //n<nh
reply.Status = paxosrpc.Reject
//to speed up, return the Nh the node have seen
//TODO:now it is not used, but we should consider it when we want to optimize the system a little bit
reply.Na = v.Nh
} else {
if v.isAccepted || v.isCommited { //accepted or commited state
v.Nh = args.N
pn.tempSlots[args.SlotIdx] = v
reply.Status = paxosrpc.Existed
reply.Na = v.Na
reply.Va = v.V
} else { //prepare state
ic := IndexCommand{}
ic.Index = args.SlotIdx
ic.Na = -1
ic.Nh = args.N
ic.isAccepted = false
ic.isCommited = false
pn.tempSlots[args.SlotIdx] = ic
reply.Status = paxosrpc.OK
reply.Na = ic.Na
reply.Va = ic.V
}
}
} else { //empty slot
ic := IndexCommand{}
ic.Index = args.SlotIdx
ic.Na = -1
ic.Nh = args.N
ic.isAccepted = false
ic.isCommited = false
pn.tempSlots[args.SlotIdx] = ic
reply.Status = paxosrpc.OK
reply.Na = ic.Na
reply.Va = ic.V
}
LOGV2.Printf("node %d release lock in Prepare()\n", pn.nodeID)
pn.cmdMutex.Unlock()
LOGV.Printf("node %d leaving OnPrepare(%d %s %d)\n\t[%d %d %s]\n", pn.nodeID, args.SlotIdx, args.V.ToString(), args.N, reply.Status, reply.Na, reply.Va.ToString())
return nil
}