// // run election // func (s *Server) runElection() (leader string, err error) { host := GetHostUDPAddr() peers := GetPeerUDPAddr() // Create an election site to start leader election. log.Current.Debugf("Server.runElection(): Local Server %s start election", host) log.Current.Debugf("Server.runElection(): Peer in election") for _, peer := range peers { log.Current.Debugf(" peer : %s", peer) } s.site, err = protocol.CreateElectionSite(host, peers, s.factory, s.handler, false) if err != nil { return "", err } resultCh := s.site.StartElection() if resultCh == nil { return "", common.NewError(common.SERVER_ERROR, "Election Site is in progress or is closed.") } leader, ok := <-resultCh // blocked until leader is elected if !ok { return "", common.NewError(common.SERVER_ERROR, "Election Fails") } return leader, nil }
// // Handle a new incoming request // //func (s *RequestReceiver) NewRequest(message []byte, reply *[]byte) error { func (s *RequestReceiver) NewRequest(req *Request, reply **Reply) error { if s.server.IsDone() { return common.NewError(common.SERVER_ERROR, "Server is terminated. Cannot process new request.") } log.Current.Debugf("RequestReceiver.NewRequest(): Receive request from client") log.Current.Debugf("RequestReceiver.NewRequest(): opCode %s key %s value %s", req.OpCode, req.Key, req.Value) opCode := common.GetOpCode(req.OpCode) if opCode == common.OPCODE_GET { result, err := s.server.GetValue(req.Key) if err != nil { return err } log.Current.Debugf("RequestReceiver.NewRequest(): Receive response from server, len(value) = %d", len(result)) *reply = &Reply{Result: result} return nil } else if opCode == common.OPCODE_ADD || opCode == common.OPCODE_SET || opCode == common.OPCODE_DELETE { if req.Value == nil { req.Value = ([]byte)("") } id, err := common.NewUUID() if err != nil { return err } request := s.server.factory.CreateRequest(id, uint32(common.GetOpCode(req.OpCode)), req.Key, req.Value) handle := newRequestHandle(request) handle.CondVar.L.Lock() defer handle.CondVar.L.Unlock() // push the request to a channel log.Current.Debugf("Handing new request to server. Key %s", req.Key) s.server.state.incomings <- handle // This goroutine will wait until the request has been processed. handle.CondVar.Wait() log.Current.Debugf("Receive Response for request. Key %s", req.Key) *reply = &Reply{Result: nil} return handle.Err } else { return common.NewError(common.CLIENT_ERROR, fmt.Sprintf("Invalid Op code %s", req.OpCode)) } }
// // Cleanup internal state upon exit // func (s *EmbeddedServer) cleanupState() { s.state.mutex.Lock() defer s.state.mutex.Unlock() common.SafeRun("EmbeddedServer.cleanupState()", func() { if s.listener != nil { s.listener.Close() } }) common.SafeRun("EmbeddedServer.cleanupState()", func() { if s.repo != nil { s.repo.Close() } }) for len(s.state.incomings) > 0 { request := <-s.state.incomings request.Err = common.NewError(common.SERVER_ERROR, "Terminate Request due to server termination") common.SafeRun("EmbeddedServer.cleanupState()", func() { request.CondVar.L.Lock() defer request.CondVar.L.Unlock() request.CondVar.Signal() }) } for _, request := range s.state.pendings { request.Err = common.NewError(common.SERVER_ERROR, "Terminate Request due to server termination") common.SafeRun("EmbeddedServer.cleanupState()", func() { request.CondVar.L.Lock() defer request.CondVar.L.Unlock() request.CondVar.Signal() }) } for _, request := range s.state.proposals { request.Err = common.NewError(common.SERVER_ERROR, "Terminate Request due to server termination") common.SafeRun("EmbeddedServer.cleanupState()", func() { request.CondVar.L.Lock() defer request.CondVar.L.Unlock() request.CondVar.Signal() }) } }
func listen(name string, pipe *common.PeerPipe) (common.Packet, error) { reqch := pipe.ReceiveChannel() req, ok := <-reqch if !ok { return nil, common.NewError(common.SERVER_ERROR, "SyncProxy.listen(): channel closed. Terminate") } if req.Name() != name { return nil, common.NewError(common.PROTOCOL_ERROR, "SyncProxy.listen(): Expect message "+name+", Receive message "+req.Name()) } return req, nil }
func (l *LeaderSyncProxy) updateAcceptedEpochAfterQuorum() error { log.Current.Debugf("LeaderSyncProxy.updateAcceptedEpochAfterQuroum()") // Get my follower's vote for the accepted epoch packet, err := listen("FollowerInfo", l.follower) if err != nil { return err } // Get epoch from follower message info := packet.(FollowerInfoMsg) epoch := info.GetAcceptedEpoch() fid := info.GetFid() voting := info.GetVoting() // initialize the follower state l.followerState = &followerState{lastLoggedTxid: 0, currentEpoch: 0, fid: fid, voting: voting} // update my vote and wait for epoch to reach quorum newEpoch, ok := l.state.voteAcceptedEpoch(l.GetFid(), epoch, l.followerState.voting) if !ok { return common.NewError(common.ELECTION_ERROR, "LeaderSyncProxy.updateAcceptedEpochAfterQuorum(): Fail to reach quorum on accepted epoch (FollowerInfo)") } // update the accepted epoch based on the quorum result. This function // will perform update only if the new epoch is larger than existing value. err = l.handler.NotifyNewAcceptedEpoch(newEpoch) if err != nil { return err } return nil }
func (l *LeaderSyncProxy) declareNewLeaderAfterQuorum() error { log.Current.Debugf("LeaderSyncProxy.declareNewLeaderAfterQuorum()") // return the new epoch to the follower epoch, err := l.handler.GetCurrentEpoch() if err != nil { return err } packet := l.factory.CreateNewLeader(epoch) err = send(packet, l.follower) if err != nil { return err } // Get the new leader ack ack, err := listen("NewLeaderAck", l.follower) if err != nil { return err } // TODO : Verify the ack ack = ack // TODO : just to get around compile error // update my vote and wait for quorum of ack from followers ok := l.state.voteNewLeaderAck(l.GetFid(), l.followerState.voting) if !ok { return common.NewError(common.ELECTION_ERROR, "LeaderSyncProxy.declareNewLeaderAfterQuorum(): Fail to reach quorum on NewLeaderAck") } return nil }
// // This is the API for client that is co-located withe gometa server // in the same process. // func NewClientRequest(req *Request, reply **Reply) error { if gHandler == nil { return common.NewError(common.SERVER_ERROR, "Server is not ready to receive new request.") } return gHandler.NewRequest(req, reply) }
// // commit proposal // func (l *Leader) commit(txid common.Txnid) error { // We are skipping proposal. The protocol expects that each follower must // send Accept Msg in the order of Proposal being received. Since the // message is sent out a reliable TCP connection, it is not possible to reach // quorum out of order. Particularly, if a new follower leaves and rejoins, // the leader is responsible for resending all the pending proposals to the // followers. So if we see the txid is out-of-order there, then it is // a fatal condition due to protocol error. // if !common.IsNextInSequence(txid, l.lastCommitted) { log.Current.Debugf("Proposal must committed in sequential order for the same leader term. "+ "Found out-of-order commit. Leader last committed txid %d, commit msg %d", l.lastCommitted, txid) return common.NewError(common.PROTOCOL_ERROR, fmt.Sprintf("Proposal must committed in sequential order for the same leader term. "+ "Found out-of-order commit. Leader last committed txid %d, commit msg %d", l.lastCommitted, txid)) } _, ok := l.proposals[txid] if !ok { return common.NewError(common.SERVER_ERROR, fmt.Sprintf("Cannot find a proposal for the txid %d. Fail to commit the proposal.", txid)) } // marking the proposal as committed. Always do this first before sending to followers. err := l.handler.Commit(txid) if err != nil { return err } // remove the votes delete(l.quorums, txid) delete(l.proposals, txid) // Update lastCommitted l.lastCommitted = txid // Send the commit to followers l.sendCommit(txid) // TODO: do we need to update election site? I don't think so, but need to double check. return nil }
// // Create a new iterator. This is used by LeaderSyncProxy to replicate log // entries to a folower/watcher. txid1 is last logged commited txid from follower/watcher. // txid2 is the txid of the first packet in the observer queue for this follower/watcher. // If the observer queue is empty, then txid2 is 0. // // For any LogEntry returned form the iterator, the following condition must be satisifed: // 1) The txid of any LogEntry returned from the iterator must be greater than or equal to txid1. // 2) The last LogEntry must have txid >= txid2. In other words, the iterator can gaurantee // that it will cover any mutation by txid2, but cannot guarantee that it stops at txid2. // // Since TransientCommitLog does not keep track of history, it only supports the following case: // 1) the beginning txid (txid1) is 0 (sending the full repository) // func (r *TransientCommitLog) NewIterator(txid1, txid2 common.Txnid) (CommitLogIterator, error) { if txid1 != 0 { return nil, common.NewError(common.REPO_ERROR, "TransientLCommitLog.NewIterator: cannot support beginning txid > 0") } // iter can be nil if nothing has been logged retry := true retryLabel: txnid, iter, err := r.repo.AcquireSnapshot(MAIN) if err != nil { return nil, err } // The snapshot txnid must be at least match the ending txid (txid2). If not, it will retry. // This is to ensure that it takes care race condition in which the first msg in an observer's // queue is a commit, and the caller asks for the state of the repository which is at least as // recent as that commit msg (txid2). If retry fails, return an error. if txnid < txid2 { if retry { time.Sleep(common.XACT_COMMIT_WAIT_TIME) retry = false goto retryLabel } return nil, common.NewError(common.REPO_ERROR, fmt.Sprintf("TransientLCommitLog.NewIterator: cannot support ending txid > %d", txnid)) } result := &TransientLogIterator{ iter: iter, txnid: txnid, repo: r.repo, curTxnid: common.Txnid(txid1 + 1), curKey: "", curContent: nil, curError: nil} if result.iter != nil { result.curKey, result.curContent, result.curError = result.iter.Next() } return result, nil }
func send(packet common.Packet, pipe *common.PeerPipe) error { log.Current.Tracef("SyncProxy.send(): sending packet %s to peer (TCP %s)", packet.Name(), pipe.GetAddr()) if !pipe.Send(packet) { return common.NewError(common.SERVER_ERROR, fmt.Sprintf("SyncProxy.listen(): Fail to send packet %s to peer (TCP %s)", packet.Name(), pipe.GetAddr())) } return nil }
// // Create a new FollowerServer. This is a blocking call until // the FollowerServer terminates. Make sure the kilch is a buffered // channel such that if the goroutine running RunFollowerServer goes // away, the sender won't get blocked. // func RunFollowerServer(naddr string, leader string, ss RequestMgr, handler ActionHandler, factory MsgFactory, killch <-chan bool) (err error) { // Catch panic at the main entry point for FollowerServer defer func() { if r := recover(); r != nil { log.Current.Errorf("panic in RunFollowerServer() : %s\n", r) log.Current.Errorf("%s", log.Current.StackTrace()) err = r.(error) } else { log.Current.Debugf("%s", "RunFollowerServer terminates.") log.Current.Tracef(log.Current.StackTrace()) } }() // create connection to leader conn, err := createConnection(leader) if err != nil { return err } pipe := common.NewPeerPipe(conn) log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower %s successfully "+ "created TCP connection to leader %s, local address %s", naddr, leader, conn.LocalAddr()) // close the connection to the leader. If connection is closed, // sync proxy and follower will also terminate by err-ing out. // If sync proxy and follower terminates the pipe upon termination, // it is ok to close it again here. defer common.SafeRun("FollowerServer.runFollowerServer()", func() { pipe.Close() }) // start syncrhorniziing with the leader success := syncWithLeader(naddr, pipe, handler, factory, killch) // run server after synchronization if success { runFollower(pipe, ss, handler, factory, killch) log.Current.Debugf("FollowerServer.RunFollowerServer() : Follower Server %s terminate", naddr) err = nil } else { err = common.NewError(common.SERVER_ERROR, fmt.Sprintf("Follower %s fail to synchronized with leader %s", naddr, leader)) } return err }
// // Retrieve entry from commit log // func (r *TransientCommitLog) Get(txid common.Txnid) (common.OpCode, string, []byte, error) { msg, ok := r.logs[txid] if !ok || msg == nil { err := common.NewError(common.REPO_ERROR, fmt.Sprintf("LogEntry for txid %d does not exist in commit log", txid)) return common.OPCODE_INVALID, "", nil, err } // msg is a protobuf object. Directly pointing slices/strings into a struct // may cause problems for cgo calls key := string([]byte(msg.GetKey())) content := []byte(string(msg.GetContent())) return common.GetOpCodeFromInt(msg.GetOpCode()), key, content, nil }
func (a *ServerAction) persistChange(op common.OpCode, key string, content []byte) error { if op == common.OPCODE_ADD { return a.repo.Set(repo.MAIN, key, content) } if op == common.OPCODE_SET { return a.repo.Set(repo.MAIN, key, content) } if op == common.OPCODE_DELETE { return a.repo.Delete(repo.MAIN, key) } return common.NewError(common.PROTOCOL_ERROR, fmt.Sprintf("ServerAction.persistChange() : Unknown op code %d", op)) }
func (e *Env) initWithArgs() error { if len(os.Args) < 3 { return common.NewError(common.ARG_ERROR, "Missing command line argument") } err := e.resolveHostAddr() if err != nil { return err } if len(os.Args) >= 6 { e.resolvePeerAddr() if err != nil { return err } } return nil }
// // run server (as leader or follower) // func (s *Server) runServer(leader string) (err error) { host := GetHostUDPAddr() // If this host is the leader, then start the leader server. // Otherwise, start the followerServer. if leader == host { log.Current.Debugf("Server.runServer() : Local Server %s is elected as leader. Leading ...", leader) s.state.setStatus(protocol.LEADING) err = protocol.RunLeaderServer(GetHostTCPAddr(), s.listener, s.state, s.handler, s.factory, s.skillch) } else { log.Current.Debugf("Server.runServer() : Remote Server %s is elected as leader. Following ...", leader) s.state.setStatus(protocol.FOLLOWING) leaderAddr := findMatchingPeerTCPAddr(leader) if len(leaderAddr) == 0 { return common.NewError(common.SERVER_ERROR, "Cannot find matching TCP addr for leader "+leader) } err = protocol.RunFollowerServer(GetHostTCPAddr(), leaderAddr, s.state, s.handler, s.factory, s.skillch) } return err }
func (l *LeaderSyncProxy) updateCurrentEpochAfterQuorum() error { log.Current.Debugf("LeaderSyncProxy.updateCurrentEpochAfterQuorum()") // Get my follower's vote for the epoch ack packet, err := listen("EpochAck", l.follower) if err != nil { return err } // Get epoch from follower message // TODO : Validate follower epoch info := packet.(EpochAckMsg) l.followerState.currentEpoch = info.GetCurrentEpoch() l.followerState.lastLoggedTxid = common.Txnid(info.GetLastLoggedTxid()) // update my vote and wait for quorum of ack from followers ok := l.state.voteEpochAck(l.GetFid(), l.followerState.voting) if !ok { return common.NewError(common.ELECTION_ERROR, "LeaderSyncProxy.updateCurrentEpochAfterQuorum(): Fail to reach quorum on current epoch (EpochAck)") } // update the current epock after quorum of followers have ack'ed epoch, err := l.handler.GetAcceptedEpoch() if err != nil { return err } // update the current epoch based on the quorum result. This function // will perform update only if the new epoch is larger than existing value. err = l.handler.NotifyNewCurrentEpoch(epoch) if err != nil { return err } return nil }
// // Goroutine for processing each request one-by-one // func (s *LeaderServer) processRequest(killch <-chan bool, listenerState *ListenerState, reqHandler CustomRequestHandler) (err error) { defer func() { if r := recover(); r != nil { log.Current.Errorf("panic in LeaderServer.processRequest() : %s\n", r) log.Current.Errorf("%s", log.Current.StackTrace()) err = r.(error) } else { log.Current.Debugf("LeaderServer.processRequest() : Terminates.") log.Current.Tracef(log.Current.StackTrace()) } common.SafeRun("LeaderServer.processRequest()", func() { listenerState.killch <- true }) }() // start processing loop after I am being confirmed as a leader (there // is a quorum of followers that have sync'ed with me) if !s.waitTillReady() { return common.NewError(common.ELECTION_ERROR, "LeaderServer.processRequest(): Leader times out waiting for quorum of followers. Terminate") } // At this point, the leader has gotten a majority of followers to follow, so it // can proceed. It is possible that it may loose quorum of followers. But in that // case, the leader will not be able to process any request. log.Current.Debugf("LeaderServer.processRequest(): Leader Server is ready to proces request") // Leader is ready at this time. This implies that there is a quorum of follower has // followed this leader. Get the change channel to keep track of number of followers. // If the leader no longer has quorum, it needs to let go of its leadership. leaderchangech := s.leader.GetEnsembleChangeChannel() ensembleSize := s.handler.GetEnsembleSize() // notify the request processor to start processing new request incomings := s.state.requestMgr.GetRequestChannel() var outgoings <-chan common.Packet = nil if reqHandler != nil { outgoings = reqHandler.GetResponseChannel() } else { outgoings = make(<-chan common.Packet) } for { select { case handle, ok := <-incomings: if ok { // de-queue the request s.state.requestMgr.AddPendingRequest(handle) // forward request to the leader s.leader.QueueRequest(s.leader.GetFollowerId(), handle.Request) } else { // server shutdown. log.Current.Debugf("LeaderServer.processRequest(): channel for receiving client request is closed. Terminate.") return nil } case msg, ok := <-outgoings: if ok { // forward msg to the leader s.leader.QueueResponse(msg) } else { log.Current.Infof("LeaderServer.processRequest(): channel for receiving custom response is closed. Ignore.") } case <-killch: // server shutdown log.Current.Debugf("LeaderServer.processRequest(): receive kill signal. Stop Client request processing.") return nil case <-listenerState.donech: // listener is down. Terminate this request processing loop as well. log.Current.Infof("LeaderServer.processRequest(): follower listener terminates. Stop client request processing.") return nil case <-leaderchangech: // Listen to any change to the leader's active ensemble, and to ensure that the leader maintain majority. // The active ensemble is the set of running followers connected to the leader. numFollowers := s.leader.GetActiveEnsembleSize() if numFollowers <= int(ensembleSize/2) { // leader looses majority of follower. log.Current.Infof("LeaderServer.processRequest(): leader looses majority of follower. Stop client request processing.") return nil } } } return nil }
func (l *FollowerSyncProxy) receiveAndUpdateAcceptedEpoch() error { log.Current.Debugf("FollowerSyncProxy.receiveAndUpdateAcceptedEpoch()") // Get the accepted epoch from the leader. This epoch // is already being voted on by multiple followers (the highest // epoch among the quorum of followers). packet, err := listen("LeaderInfo", l.leader) if err != nil { return err } // Get epoch from leader message info := packet.(LeaderInfoMsg) epoch := info.GetAcceptedEpoch() if err != nil { return err } acceptedEpoch, err := l.handler.GetAcceptedEpoch() if err != nil { return err } if epoch > acceptedEpoch { // Update the accepted epoch based on the quorum result. This function // will perform update only if the new epoch is larger than existing value. // Once the accepted epoch is updated, it will not be reset even if the // sychornization with the leader fails. Therefore, the follower will always // remember the largest accepted epoch known to it, such that it can be used // in the next round of voting. Note that the leader derives this new accepted // epoch only after it has polled from a quorum of followers. So even if sync fails, // it is unlikey that in the next sync, the leader will give a new accepted epoch smaller // than what is being stored now. err = l.handler.NotifyNewAcceptedEpoch(epoch) if err != nil { return err } } else if epoch == acceptedEpoch { // In ZK, if the local epoch (acceptedEpoch) == leader's epoch (epoch), it will replly an EpochAck with epoch = -1. // This is to tell the leader that it should not count this EpockAck when computing quorum of EpochAck. // This is to ensure that this follower does not "double ack" to the leader (e.g. when this follower rejoins a // stable ensemble). In our implementation for ConsentState, it should not be affected by double ack from the same host. } else { return common.NewError(common.PROTOCOL_ERROR, "Accepted Epoch from leader is smaller or equal to my epoch.") } // Notify the leader that I have accepted the epoch. Send // the last logged txid and current epoch to the leader. txid, err := l.handler.GetLastLoggedTxid() if err != nil { return err } currentEpoch, err := l.handler.GetCurrentEpoch() if err != nil { return err } l.state.lastLoggedTxid = common.Txnid(txid) l.state.currentEpoch = currentEpoch packet = l.factory.CreateEpochAck(uint64(txid), currentEpoch) return send(packet, l.leader) }