func (s *testPBServer) Get() (View, error) {
args := new(ConsultArgs)
reply := new(ConsultReply)
err := rpc.Call(s.server, "ViewServer.Get", args, reply)
return reply.View, err
}
func (s *testPBServer) Ping(n uint32) (View, error) {
args := &PingArgs{
Addr: s.addr,
SerialNumber: n,
}
reply := new(PingReply)
err := rpc.Call(s.server, "ViewServer.Ping", args, reply)
return reply.View, err
}
// Get current view from view server.
func (c *testClient) updateView() {
args := &ConsultArgs{}
reply := new(ConsultReply)
//TODO Move retransfer manchism to rpc framework.
for cnt := 0; cnt < RetransferThreshold; cnt++ {
err := rpc.Call(c.server, "ViewServer.Get", args, reply)
if err != nil {
logger.Error(fmt.Sprintf("CLient get view error: %s", err))
} else {
c.view = reply.View
break
}
time.Sleep(RetransferWaitInterval[cnt])
}
}
// Stop the master and all workers.
func (m *Master) shutdown() {
logger.Info("Closing master...")
m.idleWorkers.Close()
rpc.Close(m)
args := new(ShutdownArgs)
reply := new(ShutdownReply)
for _, w := range m.workers {
if err := rpc.Call(w, "Worker.Shutdown", args, reply); err != nil {
logger.Info(fmt.Sprintf("Worker %s shutdown error: %s", w, err))
} else {
logger.Info(fmt.Sprintf("Worker %s runs %d mappers, %d reducers",
w, reply.MapperNum, reply.ReducerNum))
}
}
logger.Info("Close master done")
}
func (w *Worker) run() {
// Register as a rpc server
rpc.Register(w, w.address)
// Register as a worker for master.
args := &RegisterArgs{
Worker: w.address,
}
reply := new(RegisterReply)
err := rpc.Call(masterAddress, "Master.Register", args, reply)
if err != nil {
logger.Error(fmt.Sprintf("Cannot register %s as a worker", err))
rpc.Close(w)
return
}
w.file = reply.File
w.mapper = reply.CMapper
w.reducer = reply.CReducer
w.nMapper = reply.NumMapper
w.nReducer = reply.NumReducer
}
// "op" should be Put or Append.
func (c *testClient) putAppend(key, value string, op string) {
logger.Info(fmt.Sprintf("Client %s send a %s request to %s", c.addr, op,
c.view.Primary))
args := &PutAppendArgs{
OperationArgs: OperationArgs{
Source: Client,
RequsetId: uid(),
},
//Source: Client,
//RequsetId: uid(),
Key: key,
Value: value,
}
reply := new(PutAppendReply)
// Send request until successed.
for {
//TODO Move retransfer mechanism to rpc framework.
for cnt := 0; cnt < RetransferThreshold; cnt++ {
err := rpc.Call(c.view.Primary, "PBServer."+op, args, reply)
if err != nil {
logger.Error(fmt.Sprintf("%s invoke error: %s", op, err))
} else {
break
}
time.Sleep(RetransferWaitInterval[cnt])
}
if reply.Err.Code == OK {
return
} else {
// Unknown, ErrNotPrimary, ErrServerShutdown. Wait a moment and send again.
time.Sleep(PingInterval)
c.updateView()
logger.Error(reply.Err.Description)
}
}
}
func (c *testClient) Get(key string) string {
logger.Info(fmt.Sprintf("Client %s send a Get request to %s", c.addr,
c.view.Primary))
args := &GetArgs{
OperationArgs: OperationArgs{
Source: Client,
RequsetId: uid(),
},
Key: key,
}
reply := new(GetReply)
// Send request until successed.
for {
//TODO Move retransfer mechanism to rpc framework.
for cnt := 0; cnt < RetransferThreshold; cnt++ {
err := rpc.Call(c.view.Primary, "PBServer.Get", args, reply)
if err != nil {
logger.Error(fmt.Sprintf("Get invoke error: %s", err))
} else {
break
}
time.Sleep(RetransferWaitInterval[cnt])
}
if reply.Err.Code == OK {
return reply.Value
} else if reply.Err.Code == ErrKeyNotFound {
logger.Error(reply.Err)
return ""
} else {
// ErrNotPrimary, ErrServerShutdown. wait a moment and send again.
time.Sleep(PingInterval)
c.updateView()
}
}
}
func (s *PBServer) Get(args *GetArgs, reply *GetReply) error {
s.lock()
defer s.unlock()
if !s.alive.Get() {
reply.Err = Error{
Code: ErrServerShutdown,
Description: fmt.Sprintf("Server %s is closed", s.addr),
}
return nil
}
type rValue struct {
err error
reply GetReply
}
// Forward Get request to backup.
forward := func(r *rValue) bool {
format := "Forward Get request from %s to %s"
logger.Info(fmt.Sprintf(format, s.addr, s.view.Backup))
cached := true
args.Source = Primary
//TODO Move retransfer mechanism to rpc framwork.
for cnt := 0; cnt < RetransferThreshold; cnt++ {
err := rpc.Call(s.view.Backup, "PBServer.Get", args,
&(r.reply))
if err != nil {
format := "Forward Get request from %s to" +
" %s error: %s"
logger.Error(fmt.Sprintf(format, s.addr,
s.view.Backup, err))
r.reply.Err.Description = err.Error()
} else {
// Stop transfer.
break
}
time.Sleep(RetransferWaitInterval[cnt])
}
code := r.reply.Err.Code
if code == ErrBackupPromoted {
// backup became new primary, return an error and
// the client will update current view and try
// again.
format := "Server %s is not current primary"
r.reply.Err = Error{
Code: ErrNotPrimary,
Description: fmt.Sprintf(format, s.addr),
}
} else if code != OK &&
code != ErrKeyNotFound {
// r.reply.Err.Code == OK indicated that forward
// success, result is exist, needn't do the query
// in this server any mall.
// r.reply.Err.Code == ErrKeyNotFound indicated
// that key not exist, needn't do query on this
// server any mall.
// May be backup is modified, return an error,
// client will try again.
cached = false
}
return cached
}
// Handle request from client.
clientRequest := func(r *rValue) bool {
format := "Pbserver %s receive Get request from client"
logger.Info(fmt.Sprintf(format, s.addr))
cached := true
if s.addr != s.view.Primary {
format := "Server %s is not current primary"
r.reply.Err = Error{
Code: ErrNotPrimary,
Description: fmt.Sprintf(format, s.addr),
}
// Can't cache this result as this server perhaps don't konw
// it is primary yet.
cached = false
} else {
// Get operation also need forward, used to handle the case as
// follows:
// Primary is overdue but it didn't know, when client send a
// Get request to it, the result on this server may be overdue,
// so it will forward to backup, and backup return an error to
// indicated this case.
if s.view.Backup != "" {
cached = forward(r)
} else {
r.err = s.get(args.Key, &(r.reply))
}
}
return cached
}
// Handle request forward by primary.
forwardRequest := func(r *rValue) bool {
format := "Pbserver %s receive Get request forward by primary"
logger.Info(fmt.Sprintf(format, s.addr))
cached := true
if s.addr != s.view.Backup {
if s.addr == s.view.Primary {
// Backup has beeen promoted as new primary.
format := "Server %s is promoted as new primary"
r.reply.Err = Error{
Code: ErrBackupPromoted,
Description: fmt.Sprintf(format, s.addr),
}
// Can't cache this result as Client will send the same
// request later.
cached = false
} else {
// This server failed or is not backup.
format := "Server %s is not current backup"
r.reply.Err = Error{
Code: ErrNotBackup,
Description: fmt.Sprintf(format, s.addr),
}
// Can't cache this result as this server may become
// backup later.
cached = false
}
} else {
r.err = s.get(args.Key, &(r.reply))
}
return cached
}
executor := func(interface{}) (interface{}, bool) {
r := new(rValue)
cached := true
if args.Source == Client {
// Request from client.
cached = clientRequest(r)
} else if args.Source == Primary {
// Request forward by primary.
cached = forwardRequest(r)
} else {
// Invalid request source.
format := "Server %s can't process a \"Get\" operation from %s"
reply.Err = Error{
Code: ErrInvalidRequestSource,
Description: fmt.Sprintf(format, s.addr, args.Source),
}
}
return r, cached
}
res := s.group.Do(strconv.Itoa(int(args.RequsetId)), nil,
executor).(*rValue)
if res.err != nil {
return res.err
} else {
*reply = res.reply
return nil
}
}
// Send ping signal to view server periodicly.
func (s *PBServer) tick() {
// Only this method can change s.view's value, so s.view is read safe in
// this method.
for s.alive.Get() {
args := &PingArgs{
Addr: s.addr,
SerialNumber: s.view.SerialNumber,
}
reply := new(PingReply)
err := rpc.Call(s.viewServer, "ViewServer.Ping", args, reply)
if err != nil {
//logger.Error(fmt.Sprintf("PBServer(%s) ping error: %s",
// s.addr, err))
continue
}
if reply.View.SerialNumber == s.view.SerialNumber {
// View isn't updated.
time.Sleep(PingInterval)
continue
}
// Update current view.
oldView := s.view
s.lock()
s.view = reply.View
s.unlock()
if s.addr == s.view.Primary {
// This server became new primary.
if oldView.Primary != "" && s.addr != oldView.Primary &&
oldView.Backup != "" && s.addr != oldView.Backup {
format := "New primary(%s) is not the backup(%s) or" +
" primary(%s) before"
logger.Error(fmt.Sprintf(format, s.addr,
oldView.Backup))
}
} else if s.addr == s.view.Backup {
// This server became new backup.
// Get the whole database from primary.
logger.Info(fmt.Sprintf("Pbserver %s send Transfer request to %s",
s.addr, s.view.Primary))
targs := &TransferArgs{
OperationArgs: OperationArgs{
Source: Backup,
RequsetId: uid(),
},
Addr: s.addr,
}
treply := new(TransferReply)
// Send request until successed.
for s.alive.Get() {
//TODO Move the retransfer mechanism to rpc framework.
for cnt := 0; cnt < RetransferThreshold; cnt++ {
err := rpc.Call(s.view.Primary, "PBServer.Transfer", targs,
treply)
if err != nil {
format := "Tansfer data from primary(%s) to backup(%s)" +
" error: %s"
logger.Error(fmt.Sprintf(format, s.view.Primary, s.addr,
err))
// Wait a moment and try again.
time.Sleep(RetransferWaitInterval[cnt])
} else {
break
}
}
if treply.Err.Code == OK {
// Update the whole database.
s.lock()
s.data = treply.Data
s.unlock()
break
} else if treply.Err.Code == ErrNotPrimary {
// Wait a moment and try again.
time.Sleep(PingInterval)
} else if treply.Err.Code == ErrServerShutdown {
// Primary is closed before transfer data, this is bug.
// Just stop sending request.
break
} else {
// Unexpected error.
format := "Can't transfer data from primary(%s) to " +
"backup(%s): %s"
logger.Error(fmt.Sprintf(format, s.view.Primary, s.addr,
treply.Err.Description))
break
}
}
} else {
// Both primary and backup is not empty, this server is not used.
}
time.Sleep(PingInterval)
}
s.done <- true
}
// Executor of Put and Append.
func (s *PBServer) putAppend(args *PutAppendArgs, reply *PutAppendReply,
op string) error {
// Check stopped or not.
if !s.alive.Get() {
reply.Err = Error{
Code: ErrServerShutdown,
Description: fmt.Sprintf("Server %s is closed", s.addr),
}
return nil
}
type rValue struct {
err error
reply PutAppendReply
}
// Forward request to backup.
// The first return value is cached, the second return value is go on
// or not.
forward := func(r *rValue) (bool, bool) {
logger.Info(fmt.Sprintf("Forward %s request from %s to %s",
op, s.addr, s.view.Backup))
cached := true
args.Source = Primary
//TODO Move retransfer mechanism to rpc framework.
for cnt := 0; cnt < RetransferThreshold; cnt++ {
err := rpc.Call(s.view.Backup, "PBServer."+op, args,
&(r.reply))
if err != nil {
format := "Forward %s request from %s to " +
"%s error: %s"
logger.Error(fmt.Sprintf(format, op, s.addr,
s.view.Backup, err))
} else {
break
}
time.Sleep(RetransferWaitInterval[cnt])
}
code := r.reply.Err.Code
if code == OK {
return cached, true
} else if code == ErrBackupPromoted {
// backup became new primary, return an error and
// the client will update current view and try
// again.
format := "Server %s is not current primary"
r.reply.Err = Error{
Code: ErrNotPrimary,
Description: fmt.Sprintf(format, s.addr),
}
return cached, false
} else {
// May be backup is modified, return an error,
// client will try again.
r.err = fmt.Errorf("Forward error: %s",
r.reply.Err.Description)
return false, false
}
}
clientRequest := func(r *rValue) bool {
format := "Pbserver %s receive %s request from client"
logger.Info(fmt.Sprintf(format, s.addr, op))
cached := true
if s.addr != s.view.Primary {
format := "Server %s is not current primary"
r.reply.Err = Error{
Code: ErrNotPrimary,
Description: fmt.Sprintf(format, s.addr),
}
// Can't cache this result as this server perhaps don't konw
// it is primary yet.
cached = false
} else {
if s.view.Backup != "" {
// Forward request to backup
var goOn bool
cached, goOn = forward(r)
if !goOn {
return cached
}
}
if op == "Put" {
s.putValue(args.Key, args.Value)
} else {
s.appendValue(args.Key, args.Value)
}
r.reply.Err.Code = OK
}
return cached
}
forwardRequest := func(r *rValue) bool {
format := "Pbserver %s receive %s request forward by primary"
logger.Info(fmt.Sprintf(format, s.addr, op))
cached := true
if s.addr != s.view.Backup {
if s.addr == s.view.Primary {
// Backup has beeen promoted as new primary.
format := "Server %s is promoted as new primary"
r.reply.Err = Error{
Code: ErrBackupPromoted,
Description: fmt.Sprintf(format, s.addr),
}
// Can't cache this result as Client will send the same
// request later.
cached = false
} else {
// This server failed or is not backup.
format := "Server %s is not current backup"
r.reply.Err = Error{
Code: ErrNotBackup,
Description: fmt.Sprintf(format, s.addr),
}
// Can't cache this result as this server may become
// backup later.
cached = false
}
} else {
if op == "Put" {
s.putValue(args.Key, args.Value)
} else {
s.appendValue(args.Key, args.Value)
}
r.reply.Err.Code = OK
}
return cached
}
f := func(interface{}) (interface{}, bool) {
r := new(rValue)
cached := true
if args.Source == Client {
// Request from client.
cached = clientRequest(r)
} else if args.Source == Primary {
// Request forward by primary.
cached = forwardRequest(r)
} else {
format := "Server %s can't process a \"%s\" operation from %s"
r.reply.Err = Error{
Code: ErrInvalidRequestSource,
Description: fmt.Sprintf(format, s.addr, op, args.Source),
}
}
return r, cached
}
res := s.group.Do(strconv.Itoa(int(args.RequsetId)), nil, f).(*rValue)
if res.err != nil {
return res.err
} else {
*reply = res.reply
return nil
}
}
