func initTribServer(masterServerHostPort string, tribServerPort int) error {
// l, err := net.Listen("tcp", tribServerHostPort)
// if err != nil {
// LOGE.Println("Failed to listen:", err)
// return nil, err
// }
tribServerHostPort := net.JoinHostPort("localhost", strconv.Itoa(tribServerPort))
proxyCounter, err := proxycounter.NewProxyCounter(masterServerHostPort, tribServerHostPort)
if err != nil {
LOGE.Println("Failed to setup test:", err)
return err
}
pc = proxyCounter
rpc.RegisterName("StorageServer", storagerpc.Wrap(pc))
//rpc.HandleHTTP()
//go http.Serve(l, nil)
// Create and start the TribServer.
tribServer, err := tribserver.NewTribServer(masterServerHostPort, tribServerPort)
if err != nil {
LOGE.Println("Failed to create TribServer:", err)
return err
}
ts = tribServer
//rpc.RegisterName("TribServer", tribrpc.Wrap(ts))
return nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
//Create Server Object
server := createStorageServer(numNodes, nodeID)
// Register RPCs
rpc.RegisterName("StorageServer", storagerpc.Wrap(server))
rpc.HandleHTTP()
laddr := fmt.Sprintf(":%d", port)
l, e := net.Listen("tcp", laddr)
if e != nil {
return nil, errors.New("RPC Issue Storage Server\n")
}
go http.Serve(l, nil)
if masterServerHostPort == "" {
//add itself to the empty server ring
hostport := net.JoinHostPort("localhost", strconv.Itoa(port))
server.ring = append(server.ring, storagerpc.Node{hostport, nodeID})
if numNodes > 1 {
<-server.allConnected
}
} else {
master, err := rpc.DialHTTP("tcp", masterServerHostPort)
if err != nil {
return nil, err
}
hostport := net.JoinHostPort("localhost", strconv.Itoa(port))
node := storagerpc.Node{hostport, nodeID}
args := &storagerpc.RegisterArgs{node}
var reply storagerpc.RegisterReply
// Keep trying to register itself
for {
if err := master.Call("StorageServer.RegisterServer", args, &reply); err != nil {
return nil, err
}
if reply.Status != storagerpc.NotReady {
break
} else {
time.Sleep(time.Duration(1) * time.Second)
}
}
server.ring = reply.Servers
}
for _, node := range server.ring {
server.nodeIDs = append(server.nodeIDs, int(node.NodeID))
}
print(len(server.nodeIDs))
sort.Ints(server.nodeIDs)
go incTime(server)
return server, nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
ss := &storageServer{}
ss.nodeID = nodeID
ss.masterServerHostPort = masterServerHostPort
ss.numNodes = numNodes
ss.nodes = make([]storagerpc.Node, numNodes)
ss.port = port
ss.isMaster = (len(masterServerHostPort) == 0)
ss.registerCnt = 0
ss.stringTable = make(map[string]string)
ss.listTable = make(map[string][]string)
ss.leases = make(map[string]LeaseManager)
ss.registerLocker = new(sync.Mutex)
ss.tableLocks = make(map[string]*sync.Mutex)
ss.leasesLock = new(sync.Mutex)
ss.lockForTableLocks = new(sync.Mutex)
if ss.isMaster {
ss.masterServerHostPort = "localhost:" + strconv.Itoa(port)
}
ln, err := net.Listen("tcp", ":"+strconv.Itoa(port))
if err != nil {
fmt.Println("Error on listen: ", err)
return nil, errors.New("Error on listen: " + err.Error())
}
/* Begin listening for incoming connections*/
rpc.RegisterName("StorageServer", storagerpc.Wrap(ss))
rpc.HandleHTTP()
ss.listener = ln
go http.Serve(ln, nil)
/* Wait until all nodes have joined the consistent hashing ring */
for {
client, err := rpc.DialHTTP("tcp", ss.masterServerHostPort)
if err != nil {
time.Sleep(time.Duration(1) * time.Second)
continue
}
args := &storagerpc.RegisterArgs{storagerpc.Node{"localhost:" + strconv.Itoa(port), ss.nodeID}}
var reply storagerpc.RegisterReply
err = client.Call("StorageServer.RegisterServer", args, &reply)
if err != nil {
return nil, err
}
if reply.Status == storagerpc.OK {
ss.nodes = reply.Servers
break
}
/* Sleep for one second before sending another RegisterServer request */
time.Sleep(time.Duration(1) * time.Second)
}
fmt.Printf("StorageServer %d has start up. Listen on port: %d\n", ss.nodeID, port)
return ss, nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
ss := &storageServer{
numNodes: numNodes,
port: port,
nodeID: nodeID,
readyChan: make(chan struct{}),
servers: make(map[uint32]*storagerpc.Node),
store: make(map[string]interface{}),
leases: make(map[string]*list.List),
inRevoking: make(map[string]bool),
rwl: new(sync.RWMutex),
}
ss.cond = sync.NewCond(ss.rwl)
if masterServerHostPort == "" {
ss.isMaster = true
}
ss.servers[nodeID] = &storagerpc.Node{
HostPort: net.JoinHostPort("localhost", strconv.Itoa(port)),
NodeID: ss.nodeID,
}
if numNodes == 1 {
ss.isReady = true
}
// register RPCs
l, err := net.Listen("tcp", ":"+strconv.Itoa(port))
if err != nil {
return nil, err
}
err = rpc.RegisterName("StorageServer", storagerpc.Wrap(ss))
if err != nil {
return nil, err
}
rpc.HandleHTTP()
go http.Serve(l, nil)
if ss.isMaster {
if !ss.isReady {
<-ss.readyChan // wait for slaves
}
} else {
err := ss.joinCluster(masterServerHostPort)
if err != nil {
return nil, err
}
}
return ss, nil
}
// Initialize proxy and libstore
func initLibstore(storage, server, myhostport string, alwaysLease bool) (net.Listener, error) {
l, err := net.Listen("tcp", server)
if err != nil {
LOGE.Println("Failed to listen:", err)
return nil, err
}
// The ProxyServer acts like a "StorageServer" in the system, but also has some
// additional functionalities that allow us to enforce the number of RPCs made
// to the storage server, etc.
proxyCounter, err := proxycounter.NewProxyCounter(storage, server)
if err != nil {
LOGE.Println("Failed to setup test:", err)
return nil, err
}
pc = proxyCounter
// Normally the StorageServer would register itself to receive RPCs,
// but we don't call NewStorageServer here, do we need to do it here instead.
rpc.RegisterName("StorageServer", storagerpc.Wrap(pc))
// Normally the TribServer would call the two methods below when it is first
// created, but these tests mock out the TribServer all together, so we do
// it here instead.
rpc.HandleHTTP()
go http.Serve(l, nil)
var leaseMode libstore.LeaseMode
if alwaysLease {
leaseMode = libstore.Always
} else if myhostport == "" {
leaseMode = libstore.Never
} else {
leaseMode = libstore.Normal
}
// Create and start the Libstore.
libstore, err := libstore.NewLibstore(server, myhostport, leaseMode)
if err != nil {
LOGE.Println("Failed to create Libstore:", err)
return nil, err
}
ls = libstore
return l, nil
}
func initTribServer(masterServerHostPort string, tribServerPort int) error {
tribServerHostPort := net.JoinHostPort("localhost", strconv.Itoa(tribServerPort))
proxyCounter, err := proxycounter.NewProxyCounter(masterServerHostPort, tribServerHostPort)
if err != nil {
LOGE.Println("Failed to setup test:", err)
return err
}
pc = proxyCounter
rpc.RegisterName("StorageServer", storagerpc.Wrap(pc))
// Create and start the TribServer.
tribServer, err := tribserver.NewTribServer(masterServerHostPort, tribServerHostPort)
if err != nil {
LOGE.Println("Failed to create TribServer:", err)
return err
}
ts = tribServer
return nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
server := storageServer{
numNodes: numNodes,
nextNode: 0,
seenNodes: make(map[uint32]bool),
allServersReady: make(chan int),
node: storagerpc.Node{HostPort: net.JoinHostPort("localhost", strconv.Itoa(port)), NodeID: nodeID},
dataStore: make(map[string]interface{}),
dataLock: &sync.Mutex{},
leaseStore: make(map[string]*list.List),
leaseLock: &sync.Mutex{},
ubRange: nodeID,
isTopRing: false,
success: make(chan error),
canGrantLease: make(map[string]bool),
allServersReadyBool: false,
pendingMap: make(map[string]*TrackPending),
connections: make(map[string]*rpc.Client)}
l, err2 := net.Listen("tcp", net.JoinHostPort("localhost", strconv.Itoa(port)))
if err2 != nil {
fmt.Println(err2)
return nil, err2
}
// go http.Serve(l, nil)
err1 := rpc.RegisterName("StorageServer", storagerpc.Wrap(&server))
if err1 != nil {
fmt.Println(err1)
return nil, err1
}
rpc.HandleHTTP()
go http.Serve(l, nil)
if masterServerHostPort == "" { // Server is a master
server.servers = make([]storagerpc.Node, numNodes)
server.servers[server.nextNode] = server.node
server.nextNode++
if numNodes == 1 {
return &server, nil
} else {
for {
select {
case <-server.allServersReady:
server.allServersReadyBool = true
server.lbRange, server.isTopRing = findLowerbound(server.servers, nodeID)
return &server, nil
}
}
}
} else { // Slave server
cli, err1 := rpc.DialHTTP("tcp", masterServerHostPort)
if err1 != nil {
fmt.Println("got in this case")
fmt.Println(err1)
return nil, err1
}
registerArgs := storagerpc.RegisterArgs{ServerInfo: server.node}
var reply storagerpc.RegisterReply
if err2 := cli.Call("StorageServer.RegisterServer", registerArgs, &reply); err2 != nil {
return nil, err2
}
if reply.Status == storagerpc.OK {
server.servers = reply.Servers
return &server, nil
} else {
for {
time.Sleep(time.Second)
cli.Call("StorageServer.RegisterServer", registerArgs, &reply)
if reply.Status == storagerpc.OK {
server.servers = reply.Servers
server.lbRange, server.isTopRing = findLowerbound(server.servers, nodeID)
return &server, nil
}
}
}
}
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
newss := &storageServer{
hostport: fmt.Sprintf("localhost:%d", port),
nodeId: nodeID,
sMutex: &sync.Mutex{},
lMutex: &sync.Mutex{},
mMutex: &sync.Mutex{},
cMutex: &sync.Mutex{},
sMap: make(map[string]*sValue),
lMap: make(map[string]*lValue),
leaseMap: make(map[string](map[string]time.Time)),
keyLockMap: make(map[string]*sync.Mutex),
storageServerReady: false,
libStoreMap: make(map[string]*rpc.Client),
nodeSize: numNodes,
}
if len(masterServerHostPort) == 0 {
// Master Storage Server to be created. First "join" itself.
newss.nodeIdMap = make(map[uint32]storagerpc.Node)
newss.nodesList = make([]storagerpc.Node, 0)
thisNode := storagerpc.Node{newss.hostport, nodeID}
newss.nodeIdMap[nodeID] = thisNode
newss.nodesList = append(newss.nodesList, thisNode)
newss.serverFull = make(chan int, 1)
newss.informedCount = 0
// RPC registration
rpc.RegisterName("StorageServer", storagerpc.Wrap(newss))
rpc.HandleHTTP()
listener, err := net.Listen("tcp", newss.hostport)
if err != nil {
fmt.Println("Error:", err)
return nil, errors.New("Listen error occurs.")
}
// Keep listening to connection requests
go http.Serve(listener, nil)
if numNodes == 1 {
newss.serverFull <- 1
}
<-newss.serverFull
newss.storageServerReady = true
} else {
// Slave Storage Server to be created. First "dial" the master.
client, err2 := rpc.DialHTTP("tcp", masterServerHostPort)
if err2 != nil {
fmt.Println("Error:", err2)
return nil, errors.New("")
}
registerArgs := &storagerpc.RegisterArgs{ServerInfo: storagerpc.Node{HostPort: fmt.Sprintf("localhost:%d", port), NodeID: nodeID}}
var registerReply storagerpc.RegisterReply
client.Call("StorageServer.RegisterServer", registerArgs, ®isterReply)
fmt.Println("Start registration.")
for registerReply.Status != storagerpc.OK {
time.Sleep(time.Second)
client.Call("StorageServer.RegisterServer", registerArgs, ®isterReply)
}
newss.nodesList = registerReply.Servers
rpc.RegisterName("StorageServer", storagerpc.Wrap(newss))
rpc.HandleHTTP()
listener, err3 := net.Listen("tcp", newss.hostport)
if err3 != nil {
fmt.Println("Error:", err3)
return nil, errors.New("Listen error occurs.")
}
go http.Serve(listener, nil)
newss.storageServerReady = true
}
tmp := nodes(newss.nodesList)
sort.Sort(tmp)
newss.nodesList = ([]storagerpc.Node)(tmp)
go newss.CheckLease()
return newss, nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
myHostPort, merr := os.Hostname()
if merr != nil {
fmt.Println("Slave server cannot get the hostname.!")
return nil, merr
}
myHostPort = myHostPort + ":" + strconv.Itoa(port)
storage := make(map[string](*dataUnit))
clientCache := make(map[string](*rpc.Client))
ring := make([]storagerpc.Node, 0, numNodes)
ringSet := make(map[storagerpc.Node](uint32))
masterReady := make(chan int)
server := storageServer{
isMaster: len(masterServerHostPort) == 0,
totalNode: numNodes,
ring: ring,
ringSet: ringSet,
storage: storage,
clientCache: clientCache,
nodeID: nodeID,
masterReady: masterReady}
// server.iCond = sync.NewCond(&server.rMutex)
// Create the server socket that will listen for incoming RPCs.
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
for err != nil {
listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port))
}
// Wrap the storage server before registering it for RPC.
err = rpc.RegisterName("StorageServer", storagerpc.Wrap(&server))
for err != nil {
err = rpc.RegisterName("StorageServer", storagerpc.Wrap(&server))
}
// Setup the HTTP handler that will serve incoming RPCs and
// serve requests in a background goroutine.
rpc.HandleHTTP()
go http.Serve(listener, nil)
// before register for RPC, first register this node to the master node
args := &storagerpc.RegisterArgs{storagerpc.Node{HostPort: myHostPort, NodeID: server.nodeID}}
var reply storagerpc.RegisterReply
if !server.isMaster {
// a slave node
var cli *rpc.Client
var cerr error
for true {
cli, cerr = rpc.DialHTTP("tcp", masterServerHostPort)
if cerr != nil {
fmt.Println("@@Fatal Error: Slave server registration dial failed!!... master not established yet. ", nodeID, cerr)
} else {
break
}
time.Sleep(time.Duration(1000 * time.Millisecond))
}
for true {
if err := cli.Call("StorageServer.RegisterServer", args, &reply); err != nil {
fmt.Println("Slave registration call return error not empty.", err)
}
if reply.Status != storagerpc.OK {
fmt.Println("Retry registration after one second.", server.nodeID)
time.Sleep(time.Second)
} else {
server.ring = reply.Servers
fmt.Println("Slave node successfully registered! -->", nodeID, len(server.ring))
break
}
}
} else {
// fmt.Println("##### Creating a master with nodeID:", nodeID)
// a master node
go server.RegisterServer(args, &reply)
}
/// if this node is the server, then block here until all the slaves are entered
/// we need to use the conditional variable here.
if server.isMaster {
<-server.masterReady
// server.rMutex.Lock()
// if len(server.ring) != server.totalNode {
// server.iCond.Wait()
// }
// // fmt.Println("Master node fully established!.");
// server.rMutex.Unlock()
}
return &server, nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
ss := &storageServer{
nodeID: nodeID,
numNodes: numNodes,
isReady: false,
userList: make(map[string]*list.List),
userValue: make(map[string]string),
userListLease: make(map[string]map[string]*lease),
userValueLease: make(map[string]map[string]*lease),
storageServerReady: make(chan int),
serverList: list.New(),
serverMutex: &sync.Mutex{},
userListMutex: make(map[string]*sync.Mutex),
userValueMutex: make(map[string]*sync.Mutex),
clients: make(map[string]*rpc.Client),
clientsMutex: make(map[string]*sync.Mutex),
}
listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
return nil, err
}
// Wrap the tribServer before registering it for RPC.
err = rpc.RegisterName("StorageServer", storagerpc.Wrap(ss))
if err != nil {
return nil, err
}
// Setup the HTTP handler that will server incoming RPCs and
// serve requests in a background goroutine.
rpc.HandleHTTP()
go http.Serve(listener, nil)
registerArgs := &storagerpc.RegisterArgs{
ServerInfo: storagerpc.Node{
HostPort: ":" + strconv.Itoa(port),
NodeID: nodeID,
},
}
registerReply := &storagerpc.RegisterReply{}
// if a master server
if masterServerHostPort == "" {
ss.isMaster = true
go ss.RegisterServer(registerArgs, registerReply)
<-ss.storageServerReady
} else {
client, err := rpc.DialHTTP("tcp", masterServerHostPort)
if err != nil {
return nil, errors.New("Error DialHTTP")
} else {
err = client.Call("StorageServer.RegisterServer", registerArgs, registerReply)
for err != nil || registerReply.Status != storagerpc.OK {
err = client.Call("StorageServer.RegisterServer", registerArgs, registerReply)
if registerReply.Status != storagerpc.OK {
time.Sleep(1000 * time.Millisecond)
}
}
ss.serverMutex.Lock()
defer ss.serverMutex.Unlock()
ss.serverList.Init()
for i := 0; i < len(registerReply.Servers); i++ {
ss.serverList.PushBack(registerReply.Servers[i])
}
ss.isReady = true
}
}
// if masterServerHostPort == "" {
// fmt.Println("return new master storage server")
// } else {
// fmt.Println("return new slave storage server")
// }
return ss, nil
}