示例#1
0
func initTribServer(masterServerHostPort string, tribServerPort int) error {
	//	l, err := net.Listen("tcp", tribServerHostPort)
	//	if err != nil {
	//		LOGE.Println("Failed to listen:", err)
	//		return nil, err
	//	}

	tribServerHostPort := net.JoinHostPort("localhost", strconv.Itoa(tribServerPort))
	proxyCounter, err := proxycounter.NewProxyCounter(masterServerHostPort, tribServerHostPort)
	if err != nil {
		LOGE.Println("Failed to setup test:", err)
		return err
	}
	pc = proxyCounter

	rpc.RegisterName("StorageServer", storagerpc.Wrap(pc))
	//rpc.HandleHTTP()
	//go http.Serve(l, nil)

	// Create and start the TribServer.
	tribServer, err := tribserver.NewTribServer(masterServerHostPort, tribServerPort)
	if err != nil {
		LOGE.Println("Failed to create TribServer:", err)
		return err
	}
	ts = tribServer
	//rpc.RegisterName("TribServer", tribrpc.Wrap(ts))
	return nil
}
示例#2
0
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {

	//Create Server Object
	server := createStorageServer(numNodes, nodeID)

	// Register RPCs
	rpc.RegisterName("StorageServer", storagerpc.Wrap(server))
	rpc.HandleHTTP()
	laddr := fmt.Sprintf(":%d", port)

	l, e := net.Listen("tcp", laddr)
	if e != nil {
		return nil, errors.New("RPC Issue Storage Server\n")
	}
	go http.Serve(l, nil)

	if masterServerHostPort == "" {
		//add itself to the empty server ring
		hostport := net.JoinHostPort("localhost", strconv.Itoa(port))
		server.ring = append(server.ring, storagerpc.Node{hostport, nodeID})
		if numNodes > 1 {
			<-server.allConnected
		}
	} else {
		master, err := rpc.DialHTTP("tcp", masterServerHostPort)
		if err != nil {
			return nil, err
		}
		hostport := net.JoinHostPort("localhost", strconv.Itoa(port))
		node := storagerpc.Node{hostport, nodeID}
		args := &storagerpc.RegisterArgs{node}
		var reply storagerpc.RegisterReply
		// Keep trying to register itself
		for {

			if err := master.Call("StorageServer.RegisterServer", args, &reply); err != nil {
				return nil, err
			}
			if reply.Status != storagerpc.NotReady {
				break
			} else {
				time.Sleep(time.Duration(1) * time.Second)
			}
		}
		server.ring = reply.Servers
	}

	for _, node := range server.ring {
		server.nodeIDs = append(server.nodeIDs, int(node.NodeID))
	}
	print(len(server.nodeIDs))

	sort.Ints(server.nodeIDs)

	go incTime(server)

	return server, nil

}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
	ss := &storageServer{}
	ss.nodeID = nodeID
	ss.masterServerHostPort = masterServerHostPort
	ss.numNodes = numNodes
	ss.nodes = make([]storagerpc.Node, numNodes)
	ss.port = port
	ss.isMaster = (len(masterServerHostPort) == 0)
	ss.registerCnt = 0
	ss.stringTable = make(map[string]string)
	ss.listTable = make(map[string][]string)
	ss.leases = make(map[string]LeaseManager)
	ss.registerLocker = new(sync.Mutex)
	ss.tableLocks = make(map[string]*sync.Mutex)
	ss.leasesLock = new(sync.Mutex)
	ss.lockForTableLocks = new(sync.Mutex)
	if ss.isMaster {
		ss.masterServerHostPort = "localhost:" + strconv.Itoa(port)
	}

	ln, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		fmt.Println("Error on listen: ", err)
		return nil, errors.New("Error on listen: " + err.Error())
	}
	/* Begin listening for incoming connections*/
	rpc.RegisterName("StorageServer", storagerpc.Wrap(ss))
	rpc.HandleHTTP()

	ss.listener = ln
	go http.Serve(ln, nil)
	/* Wait until all nodes have joined the consistent hashing ring */
	for {
		client, err := rpc.DialHTTP("tcp", ss.masterServerHostPort)
		if err != nil {
			time.Sleep(time.Duration(1) * time.Second)
			continue
		}

		args := &storagerpc.RegisterArgs{storagerpc.Node{"localhost:" + strconv.Itoa(port), ss.nodeID}}
		var reply storagerpc.RegisterReply
		err = client.Call("StorageServer.RegisterServer", args, &reply)
		if err != nil {
			return nil, err
		}
		if reply.Status == storagerpc.OK {
			ss.nodes = reply.Servers
			break
		}
		/* Sleep for one second before sending another RegisterServer request */
		time.Sleep(time.Duration(1) * time.Second)
	}

	fmt.Printf("StorageServer %d has start up. Listen on port: %d\n", ss.nodeID, port)
	return ss, nil
}
示例#4
0
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
	ss := &storageServer{
		numNodes:   numNodes,
		port:       port,
		nodeID:     nodeID,
		readyChan:  make(chan struct{}),
		servers:    make(map[uint32]*storagerpc.Node),
		store:      make(map[string]interface{}),
		leases:     make(map[string]*list.List),
		inRevoking: make(map[string]bool),
		rwl:        new(sync.RWMutex),
	}
	ss.cond = sync.NewCond(ss.rwl)

	if masterServerHostPort == "" {
		ss.isMaster = true
	}
	ss.servers[nodeID] = &storagerpc.Node{
		HostPort: net.JoinHostPort("localhost", strconv.Itoa(port)),
		NodeID:   ss.nodeID,
	}
	if numNodes == 1 {
		ss.isReady = true
	}

	// register RPCs
	l, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		return nil, err
	}
	err = rpc.RegisterName("StorageServer", storagerpc.Wrap(ss))
	if err != nil {
		return nil, err
	}
	rpc.HandleHTTP()
	go http.Serve(l, nil)

	if ss.isMaster {
		if !ss.isReady {
			<-ss.readyChan // wait for slaves
		}

	} else {
		err := ss.joinCluster(masterServerHostPort)
		if err != nil {
			return nil, err
		}
	}

	return ss, nil
}
示例#5
0
// Initialize proxy and libstore
func initLibstore(storage, server, myhostport string, alwaysLease bool) (net.Listener, error) {
	l, err := net.Listen("tcp", server)
	if err != nil {
		LOGE.Println("Failed to listen:", err)
		return nil, err
	}

	// The ProxyServer acts like a "StorageServer" in the system, but also has some
	// additional functionalities that allow us to enforce the number of RPCs made
	// to the storage server, etc.
	proxyCounter, err := proxycounter.NewProxyCounter(storage, server)
	if err != nil {
		LOGE.Println("Failed to setup test:", err)
		return nil, err
	}
	pc = proxyCounter

	// Normally the StorageServer would register itself to receive RPCs,
	// but we don't call NewStorageServer here, do we need to do it here instead.
	rpc.RegisterName("StorageServer", storagerpc.Wrap(pc))

	// Normally the TribServer would call the two methods below when it is first
	// created, but these tests mock out the TribServer all together, so we do
	// it here instead.
	rpc.HandleHTTP()
	go http.Serve(l, nil)

	var leaseMode libstore.LeaseMode
	if alwaysLease {
		leaseMode = libstore.Always
	} else if myhostport == "" {
		leaseMode = libstore.Never
	} else {
		leaseMode = libstore.Normal
	}

	// Create and start the Libstore.
	libstore, err := libstore.NewLibstore(server, myhostport, leaseMode)
	if err != nil {
		LOGE.Println("Failed to create Libstore:", err)
		return nil, err
	}
	ls = libstore
	return l, nil
}
示例#6
0
func initTribServer(masterServerHostPort string, tribServerPort int) error {
	tribServerHostPort := net.JoinHostPort("localhost", strconv.Itoa(tribServerPort))
	proxyCounter, err := proxycounter.NewProxyCounter(masterServerHostPort, tribServerHostPort)
	if err != nil {
		LOGE.Println("Failed to setup test:", err)
		return err
	}
	pc = proxyCounter
	rpc.RegisterName("StorageServer", storagerpc.Wrap(pc))

	// Create and start the TribServer.
	tribServer, err := tribserver.NewTribServer(masterServerHostPort, tribServerHostPort)
	if err != nil {
		LOGE.Println("Failed to create TribServer:", err)
		return err
	}
	ts = tribServer
	return nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
	server := storageServer{
		numNodes:            numNodes,
		nextNode:            0,
		seenNodes:           make(map[uint32]bool),
		allServersReady:     make(chan int),
		node:                storagerpc.Node{HostPort: net.JoinHostPort("localhost", strconv.Itoa(port)), NodeID: nodeID},
		dataStore:           make(map[string]interface{}),
		dataLock:            &sync.Mutex{},
		leaseStore:          make(map[string]*list.List),
		leaseLock:           &sync.Mutex{},
		ubRange:             nodeID,
		isTopRing:           false,
		success:             make(chan error),
		canGrantLease:       make(map[string]bool),
		allServersReadyBool: false,
		pendingMap:          make(map[string]*TrackPending),
		connections:         make(map[string]*rpc.Client)}

	l, err2 := net.Listen("tcp", net.JoinHostPort("localhost", strconv.Itoa(port)))
	if err2 != nil {
		fmt.Println(err2)
		return nil, err2
	}
	// go http.Serve(l, nil)

	err1 := rpc.RegisterName("StorageServer", storagerpc.Wrap(&server))
	if err1 != nil {
		fmt.Println(err1)
		return nil, err1
	}
	rpc.HandleHTTP()
	go http.Serve(l, nil)

	if masterServerHostPort == "" { // Server is a master
		server.servers = make([]storagerpc.Node, numNodes)
		server.servers[server.nextNode] = server.node
		server.nextNode++
		if numNodes == 1 {
			return &server, nil
		} else {
			for {
				select {
				case <-server.allServersReady:
					server.allServersReadyBool = true
					server.lbRange, server.isTopRing = findLowerbound(server.servers, nodeID)
					return &server, nil
				}
			}
		}
	} else { // Slave server
		cli, err1 := rpc.DialHTTP("tcp", masterServerHostPort)
		if err1 != nil {
			fmt.Println("got in this case")
			fmt.Println(err1)
			return nil, err1
		}
		registerArgs := storagerpc.RegisterArgs{ServerInfo: server.node}
		var reply storagerpc.RegisterReply
		if err2 := cli.Call("StorageServer.RegisterServer", registerArgs, &reply); err2 != nil {
			return nil, err2
		}
		if reply.Status == storagerpc.OK {
			server.servers = reply.Servers
			return &server, nil
		} else {
			for {
				time.Sleep(time.Second)
				cli.Call("StorageServer.RegisterServer", registerArgs, &reply)
				if reply.Status == storagerpc.OK {
					server.servers = reply.Servers
					server.lbRange, server.isTopRing = findLowerbound(server.servers, nodeID)
					return &server, nil
				}
			}
		}
	}
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
	newss := &storageServer{
		hostport:           fmt.Sprintf("localhost:%d", port),
		nodeId:             nodeID,
		sMutex:             &sync.Mutex{},
		lMutex:             &sync.Mutex{},
		mMutex:             &sync.Mutex{},
		cMutex:             &sync.Mutex{},
		sMap:               make(map[string]*sValue),
		lMap:               make(map[string]*lValue),
		leaseMap:           make(map[string](map[string]time.Time)),
		keyLockMap:         make(map[string]*sync.Mutex),
		storageServerReady: false,
		libStoreMap:        make(map[string]*rpc.Client),
		nodeSize:           numNodes,
	}

	if len(masterServerHostPort) == 0 {
		// Master Storage Server to be created. First "join" itself.
		newss.nodeIdMap = make(map[uint32]storagerpc.Node)
		newss.nodesList = make([]storagerpc.Node, 0)
		thisNode := storagerpc.Node{newss.hostport, nodeID}
		newss.nodeIdMap[nodeID] = thisNode
		newss.nodesList = append(newss.nodesList, thisNode)
		newss.serverFull = make(chan int, 1)
		newss.informedCount = 0

		// RPC registration
		rpc.RegisterName("StorageServer", storagerpc.Wrap(newss))
		rpc.HandleHTTP()

		listener, err := net.Listen("tcp", newss.hostport)
		if err != nil {
			fmt.Println("Error:", err)
			return nil, errors.New("Listen error occurs.")
		}

		// Keep listening to connection requests
		go http.Serve(listener, nil)
		if numNodes == 1 {
			newss.serverFull <- 1
		}
		<-newss.serverFull

		newss.storageServerReady = true

	} else {
		// Slave Storage Server to be created. First "dial" the master.
		client, err2 := rpc.DialHTTP("tcp", masterServerHostPort)
		if err2 != nil {
			fmt.Println("Error:", err2)
			return nil, errors.New("")
		}

		registerArgs := &storagerpc.RegisterArgs{ServerInfo: storagerpc.Node{HostPort: fmt.Sprintf("localhost:%d", port), NodeID: nodeID}}
		var registerReply storagerpc.RegisterReply

		client.Call("StorageServer.RegisterServer", registerArgs, &registerReply)
		fmt.Println("Start registration.")
		for registerReply.Status != storagerpc.OK {
			time.Sleep(time.Second)
			client.Call("StorageServer.RegisterServer", registerArgs, &registerReply)
		}

		newss.nodesList = registerReply.Servers

		rpc.RegisterName("StorageServer", storagerpc.Wrap(newss))
		rpc.HandleHTTP()

		listener, err3 := net.Listen("tcp", newss.hostport)
		if err3 != nil {
			fmt.Println("Error:", err3)
			return nil, errors.New("Listen error occurs.")
		}

		go http.Serve(listener, nil)
		newss.storageServerReady = true
	}

	tmp := nodes(newss.nodesList)
	sort.Sort(tmp)
	newss.nodesList = ([]storagerpc.Node)(tmp)
	go newss.CheckLease()
	return newss, nil
}
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {

	myHostPort, merr := os.Hostname()
	if merr != nil {
		fmt.Println("Slave server cannot get the hostname.!")
		return nil, merr
	}
	myHostPort = myHostPort + ":" + strconv.Itoa(port)

	storage := make(map[string](*dataUnit))
	clientCache := make(map[string](*rpc.Client))
	ring := make([]storagerpc.Node, 0, numNodes)
	ringSet := make(map[storagerpc.Node](uint32))
	masterReady := make(chan int)
	server := storageServer{
		isMaster:    len(masterServerHostPort) == 0,
		totalNode:   numNodes,
		ring:        ring,
		ringSet:     ringSet,
		storage:     storage,
		clientCache: clientCache,
		nodeID:      nodeID,
		masterReady: masterReady}

	//	server.iCond = sync.NewCond(&server.rMutex)

	// Create the server socket that will listen for incoming RPCs.
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	for err != nil {
		listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port))
	}

	// Wrap the storage server before registering it for RPC.
	err = rpc.RegisterName("StorageServer", storagerpc.Wrap(&server))
	for err != nil {
		err = rpc.RegisterName("StorageServer", storagerpc.Wrap(&server))
	}

	// Setup the HTTP handler that will serve incoming RPCs and
	// serve requests in a background goroutine.
	rpc.HandleHTTP()
	go http.Serve(listener, nil)

	// before register for RPC, first register this node to the master node
	args := &storagerpc.RegisterArgs{storagerpc.Node{HostPort: myHostPort, NodeID: server.nodeID}}
	var reply storagerpc.RegisterReply
	if !server.isMaster {
		// a slave node
		var cli *rpc.Client
		var cerr error
		for true {
			cli, cerr = rpc.DialHTTP("tcp", masterServerHostPort)
			if cerr != nil {
				fmt.Println("@@Fatal Error: Slave server registration dial failed!!... master not established yet. ", nodeID, cerr)
			} else {
				break
			}
			time.Sleep(time.Duration(1000 * time.Millisecond))
		}

		for true {
			if err := cli.Call("StorageServer.RegisterServer", args, &reply); err != nil {
				fmt.Println("Slave registration call return error not empty.", err)
			}

			if reply.Status != storagerpc.OK {
				fmt.Println("Retry registration after one second.", server.nodeID)
				time.Sleep(time.Second)
			} else {
				server.ring = reply.Servers
				fmt.Println("Slave node successfully registered! -->", nodeID, len(server.ring))
				break
			}
		}
	} else {
		//		fmt.Println("##### Creating a master with nodeID:", nodeID)
		// a master node
		go server.RegisterServer(args, &reply)
	}

	/// if this node is the server, then block here until all the slaves are entered
	/// we need to use the conditional variable here.
	if server.isMaster {
		<-server.masterReady
		//		server.rMutex.Lock()
		//		if len(server.ring) != server.totalNode {
		//			server.iCond.Wait()
		//		}
		//		//		fmt.Println("Master node fully established!.");
		//		server.rMutex.Unlock()
	}

	return &server, nil
}
示例#10
0
// NewStorageServer creates and starts a new StorageServer. masterServerHostPort
// is the master storage server's host:port address. If empty, then this server
// is the master; otherwise, this server is a slave. numNodes is the total number of
// servers in the ring. port is the port number that this server should listen on.
// nodeID is a random, unsigned 32-bit ID identifying this server.
//
// This function should return only once all storage servers have joined the ring,
// and should return a non-nil error if the storage server could not be started.
func NewStorageServer(masterServerHostPort string, numNodes, port int, nodeID uint32) (StorageServer, error) {
	ss := &storageServer{
		nodeID:             nodeID,
		numNodes:           numNodes,
		isReady:            false,
		userList:           make(map[string]*list.List),
		userValue:          make(map[string]string),
		userListLease:      make(map[string]map[string]*lease),
		userValueLease:     make(map[string]map[string]*lease),
		storageServerReady: make(chan int),
		serverList:         list.New(),
		serverMutex:        &sync.Mutex{},
		userListMutex:      make(map[string]*sync.Mutex),
		userValueMutex:     make(map[string]*sync.Mutex),
		clients:            make(map[string]*rpc.Client),
		clientsMutex:       make(map[string]*sync.Mutex),
	}
	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
	if err != nil {
		return nil, err
	}
	// Wrap the tribServer before registering it for RPC.
	err = rpc.RegisterName("StorageServer", storagerpc.Wrap(ss))
	if err != nil {
		return nil, err
	}
	// Setup the HTTP handler that will server incoming RPCs and
	// serve requests in a background goroutine.
	rpc.HandleHTTP()
	go http.Serve(listener, nil)
	registerArgs := &storagerpc.RegisterArgs{
		ServerInfo: storagerpc.Node{
			HostPort: ":" + strconv.Itoa(port),
			NodeID:   nodeID,
		},
	}
	registerReply := &storagerpc.RegisterReply{}
	// if a master server
	if masterServerHostPort == "" {
		ss.isMaster = true
		go ss.RegisterServer(registerArgs, registerReply)
		<-ss.storageServerReady
	} else {
		client, err := rpc.DialHTTP("tcp", masterServerHostPort)
		if err != nil {
			return nil, errors.New("Error DialHTTP")
		} else {
			err = client.Call("StorageServer.RegisterServer", registerArgs, registerReply)
			for err != nil || registerReply.Status != storagerpc.OK {
				err = client.Call("StorageServer.RegisterServer", registerArgs, registerReply)
				if registerReply.Status != storagerpc.OK {
					time.Sleep(1000 * time.Millisecond)
				}
			}
			ss.serverMutex.Lock()
			defer ss.serverMutex.Unlock()
			ss.serverList.Init()
			for i := 0; i < len(registerReply.Servers); i++ {
				ss.serverList.PushBack(registerReply.Servers[i])
			}
			ss.isReady = true
		}
	}
	// if masterServerHostPort == "" {
	// 	fmt.Println("return new master storage server")
	// } else {
	// 	fmt.Println("return new slave storage server")
	// }
	return ss, nil
}