Esempio n. 1
0
/**
 * Proxy validates the requests going into a PaxosNode and the responses coming out of it.  * It logs errors that occurs during a test.
 */
func NewProxy(nodePort, myPort int) (Proxy, error) {
	p := new(proxy)
	p.prop = new(proposal)
	p.prop.status = UNSET
	p.prop.num = 0
	p.prop.key = ""
	p.prop.val = 0
	p.err = make([]string, 0)

	// Start server
	l, err := net.Listen("tcp", fmt.Sprintf(":%d", myPort))
	if err != nil {
		LOGE.Println("Failed to listen:", err)
		return nil, err
	}

	// Create RPC connection to paxos node.
	srv, err := rpc.DialHTTP("tcp", fmt.Sprintf("localhost:%d", nodePort))
	if err != nil {
		LOGE.Println("Failed to dial node %d", nodePort)
		return nil, err
	}
	p.srv = srv

	// register RPC
	rpc.RegisterName("PaxosNode", paxosrpc.Wrap(p))
	rpc.HandleHTTP()
	go http.Serve(l, nil)

	// log.Printf("Proxy started")
	return p, nil
}
Esempio n. 2
0
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
	fmt.Println("NewPaxosNode")
	pn := &paxosNode{}
	pn.nodePropNum = srvId
	pn.propNumLock = new(sync.Mutex)
	pn.numNodes = numNodes
	pn.srvID = srvId
	pn.majority_num = numNodes / 2
	pn.nodeHostPort = myHostPort
	pn.hostMap = hostMap

	pn.storage = &Storage{}
	pn.storage.storage = make(map[string]interface{})
	pn.storage.storageLock = new(sync.Mutex)

	pn.propNum = &ProposalNum{}
	pn.propNum.propNum = make(map[string]int)
	pn.propNum.propNumLock = new(sync.Mutex)

	pn.acceptValue = &AcceptValues{}
	pn.acceptValue.acceptValue = make(map[string]interface{})
	pn.acceptValue.acceptNum = make(map[string]int)
	pn.acceptValue.acceptMapLock = new(sync.Mutex)

	pn.connList = &ConnectionList{}
	pn.connList.conn = make(map[int]*rpc.Client)
	pn.connList.connLock = new(sync.Mutex)

	ln, err := net.Listen("tcp", myHostPort)
	if err != nil {
		fmt.Println("Error on listern: ", err)
		return nil, errors.New("Error on listen: " + err.Error())
	}

	// Begin to listen for incoming connections
	rpc.RegisterName("PaxosNode", paxosrpc.Wrap(pn))
	rpc.HandleHTTP()

	pn.listener = ln
	go http.Serve(ln, nil)

	retryCounter := 0
	// Connect with other servers
	pn.connList.connLock.Lock()
	for len(pn.connList.conn) < pn.numNodes {
		for i, host := range pn.hostMap {
			if _, ok := pn.connList.conn[i]; !ok {
				client, err := rpc.DialHTTP("tcp", host)
				if err == nil {
					pn.connList.conn[i] = client
				}
				// If the node is to replace the dead node
				// It needs to call rpc RecvReplaceServer function to other nodes
				if replace && i != srvId {
					replaceArgs := &paxosrpc.ReplaceServerArgs{SrvID: pn.srvID, Hostport: pn.nodeHostPort}
					var replaceReply paxosrpc.ReplaceServerReply
					client.Call("PaxosNode.RecvReplaceServer", replaceArgs, &replaceReply)
				}
			}
		}
		if len(pn.connList.conn) < pn.numNodes {
			time.Sleep(time.Duration(1) * time.Second)
			retryCounter += 1
			if retryCounter > numRetries {
				return nil, errors.New("Cannot connect with other nodes")
			}
		}
	}
	// If the node is to replace the dead node
	// It needs to get all commited information from an arbitrary node
	if replace {
		pn.storage.storageLock.Lock()
		for id, conn := range pn.connList.conn {
			if id == pn.srvID {
				continue
			}
			var replaceCatchupArgs paxosrpc.ReplaceCatchupArgs
			var replaceReply paxosrpc.ReplaceCatchupReply
			conn.Call("PaxosNode.RecvReplaceCatchup", replaceCatchupArgs, &replaceReply)
			err := json.Unmarshal(replaceReply.Data, &pn.storage.storage)
			if err != nil {
				fmt.Println("[ERROR]", err)
			}
			break
		}
		pn.storage.storageLock.Unlock()
	}
	pn.connList.connLock.Unlock()
	fmt.Println("Paxos node started on port " + myHostPort)
	return pn, nil
}
Esempio n. 3
0
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
	fmt.Println("we out here:", myHostPort)

	// register all the other hosts
	otherHosts := make(map[int]*rpc.Client)
	newNode := createNode(srvId, numNodes, replace)
	rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newNode))
	rpc.HandleHTTP()

	port := strings.Split(myHostPort, "localhost")[1]
	l, e := net.Listen("tcp", port)
	if e != nil {
		return nil, errors.New("RPC Issue Storage Server\n")
	}
	go http.Serve(l, nil)

	for id, hostport := range hostMap {
		// keep tryin until you give up, return fail if a node in ring fails to connect
		allGood := false
		for retryCount := 0; retryCount < numRetries; retryCount++ {
			cli, err := rpc.DialHTTP("tcp", hostport)
			if err == nil {
				allGood = true
				otherHosts[id] = cli
				break
			}
			time.Sleep(time.Duration(1) * time.Second)
		}
		if !allGood {
			fmt.Println(hostport)
			return nil, errors.New("Failed to connect to a client")
		}

	}

	newNode.nodes = otherHosts
	// the line below doesnt work but i think we need to do something like this

	//Catchup shit
	if replace {
		var helper *rpc.Client
		for id, cli := range newNode.nodes {
			if id != srvId {
				helper = cli
				args_server := paxosrpc.ReplaceServerArgs{srvId, myHostPort}
				var reply_server paxosrpc.ReplaceServerReply
				if err := cli.Call("PaxosNode.RecvReplaceServer", &args_server, &reply_server); err != nil {
					return nil, err
				}
			}
		}
		args := paxosrpc.ReplaceCatchupArgs{}
		var reply paxosrpc.ReplaceCatchupReply
		if err := helper.Call("PaxosNode.RecvReplaceCatchup", &args, &reply); err != nil {
			return nil, err
		}
		dec := gob.NewDecoder(bytes.NewBuffer(reply.Data))
		var data map[string]interface{}
		err := dec.Decode(&data)
		if err != nil {
			return nil, err
		}

		for key, _ := range data {

			args_num := paxosrpc.ProposalNumberArgs{key}
			var reply_num paxosrpc.ProposalNumberReply
			if err := helper.Call("PaxosNode.GetNextProposalNumber", &args_num, &reply_num); err != nil {
				return nil, err
			}
			newNode.clockMap[key] = (reply_num.N) / numNodes
		}

		newNode.storage = data
	}

	return newNode, nil
}
Esempio n. 4
0
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
	pn := &paxosNode{
		ss:        make(map[int]*rpc.Client),
		numNodes:  numNodes,
		id:        srvId,
		ptimes:    0,
		majorNum:  numNodes/2 + 1,
		instances: make(map[string]*paxosKeyData),
	}
	// Create the server socket that will listen for incoming RPCs.
	_, port, _ := net.SplitHostPort(myHostPort)
	listener, err := net.Listen("tcp", fmt.Sprintf(":%s", port))
	if err != nil {
		return nil, err
	}

	// Wrap the tribServer before registering it for RPC.
	err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(pn))
	if err != nil {
		return nil, err
	}

	// Setup the HTTP handler that will server incoming RPCs and
	// serve requests in a background goroutine.
	rpc.HandleHTTP()
	go http.Serve(listener, nil)
	receivedData := false
	// make connnection to all other nodes
	for servid, hport := range hostMap {
		conn, err := rpc.DialHTTP("tcp", hport)
		ri := 0
		for ; err != nil && ri < numRetries; ri++ {
			time.Sleep(1000 * time.Millisecond)
			conn, err = rpc.DialHTTP("tcp", hport)
		}
		if ri >= numRetries {
			return nil, err
		}
		pn.ss[servid] = conn
		// if replace flag is set
		if replace && servid != srvId {
			var myreply paxosrpc.ReplaceServerReply
			myargs := &paxosrpc.ReplaceServerArgs{
				Hostport: myHostPort,
				SrvID:    srvId,
			}
			//replace the old server entry in other server
			conn.Call("PaxosNode.RecvReplaceServer", myargs, &myreply)
			// flag to check if we have already received data from other nodes
			if !receivedData {
				var serverState paxosrpc.ReplaceCatchupReply
				var catchupArgs paxosrpc.ReplaceCatchupArgs
				// call other server for data
				conn.Call("PaxosNode.RecvReplaceCatchup", catchupArgs, &serverState)
				// the map is decoded from gob
				network := bytes.NewBuffer(serverState.Data)
				dec := gob.NewDecoder(network)
				var instances map[string]*paxosKeyData
				dec.Decode(&instances)

				receivedData = true
				pn.instances = instances
			}
		}
	}

	return pn, nil
}
Esempio n. 5
0
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
	fmt.Println("server id: ", srvId)
	newNode := &paxosNode{
		srvId:           srvId,
		hostMap:         hostMap,
		myHostPort:      myHostPort,
		numNodes:        numNodes,
		replace:         replace,
		numRetries:      numRetries,
		connMap:         make(map[string]*rpc.Client),
		pNumberMap:      make(map[string]int),
		vaMap:           make(map[string]interface{}),
		naMap:           make(map[string]int),
		npMap:           make(map[string]int),
		valueMap:        make(map[string]interface{}),
		prepareDoneChan: make(chan *PrepareData),
		acceptDoneChan:  make(chan bool),
		commitDoneChan:  make(chan bool),
	}

	// register the paxos node for rpc usage
	err := rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newNode))
	if err != nil {
		return nil, errors.New("Errow while register.")
	}

	// running as a server for listening
	rpc.HandleHTTP()
	listener, err := net.Listen("tcp", myHostPort)
	if err != nil {
		fmt.Println("err when listen: ", listener)
		return nil, errors.New(myHostPort)
	}
	go http.Serve(listener, nil)

	// build connection to every other node
	tmpValueMap := make(map[string]interface{})
	count := 0
	succeed := true
	for count < numRetries {
		succeed = true
		for _, v := range hostMap {
			client, err := rpc.DialHTTP("tcp", v)
			if err != nil {
				count += 1
				succeed = false
				break
			}
			newNode.connMap[v] = client
		}

		if succeed {
			// for replacement usage
			if replace {
				for _, client := range newNode.connMap {
					replaceArgs := &paxosrpc.ReplaceServerArgs{srvId, myHostPort}
					replaceReply := &paxosrpc.ReplaceServerReply{}
					client.Call("PaxosNode.RecvReplaceServer", replaceArgs, replaceReply)
					replaceCatchupArgs := &paxosrpc.ReplaceCatchupArgs{}
					replaceCatchupReply := &paxosrpc.ReplaceCatchupReply{}
					client.Call("PaxosNode.RecvReplaceCatchup", replaceCatchupArgs, replaceCatchupReply)
					var nodeValueMap map[string]interface{}
					json.Unmarshal([]byte(replaceCatchupReply.Data), &nodeValueMap)
					for nodeKey, nodeValue := range nodeValueMap {
						tmpValueMap[nodeKey] = nodeValue
					}
				}
			}
			newNode.valueMap = tmpValueMap
			return newNode, nil
		}
		time.Sleep(time.Second * 1)
	}

	return nil, errors.New("cannot connect to all the nodes")
}
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
	fmt.Println(myHostPort)

	var timeFactor int
	for timeFactor = 10; timeFactor < numNodes; timeFactor *= 10 {
	}
	fmt.Printf("number of nodes: %d, factor: %d\n", numNodes, timeFactor)

	myPaxosNode := paxosNode{
		port:        myHostPort,
		idMap:       make(map[int](peerNode)),
		numRetries:  numRetries,
		numNodes:    numNodes,
		id:          srvId,
		logicTime:   0,
		timeFactor:  timeFactor,
		nhMap:       make(map[string](int)),
		naMap:       make(map[string](int)),
		vaMap:       make(map[string](interface{})),
		valueMap:    make(map[string](interface{})),
		countReqMap: make(map[string](chan bool)),
		countResMap: make(map[string](chan int)),
		keyMutex:    make(map[string](*sync.Mutex)),
		nodeMutex:   &sync.Mutex{},
	}

	// create listener for incoming RPCs
	listener, err := net.Listen("tcp", myHostPort)
	for err != nil {
		listener, err = net.Listen("tcp", myHostPort)
	}

	// wrap the paxos node
	err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(&myPaxosNode))
	for err != nil {
		err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(&myPaxosNode))
	}

	// set up http handler for incoming RPCs
	rpc.HandleHTTP()
	go http.Serve(listener, nil)

	for nodeId, nodePort := range hostMap {
		err = myPaxosNode.dialToPeerNode(nodeId, nodePort)
		if err != nil {
			return nil, errors.New(fmt.Sprintf("Not all paxos nodes are ready\nid: %d, port: %d", nodeId, nodePort))
		}
	}

	if replace {
		if err := myPaxosNode.replaceServer(); err != nil {
			return nil, errors.New(fmt.Sprintf("paxos node: %d fail in receiving acks", srvId))
		}

		if err := myPaxosNode.replaceCatchup(); err != nil {
			return nil, errors.New(fmt.Sprintf("paxos node: %d fail to get valueMap", srvId))
		}
	}

	return &myPaxosNode, nil
}
Esempio n. 7
0
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
	newPaxosNode := &paxosNode{
		nodeID:        srvId,
		hostPort:      myHostPort,
		numNodes:      numNodes,
		replace:       replace,
		hostMap:       make(map[int]string),
		keyValue:      make(map[string]*value),
		savedkeyValue: make(map[string]*value),
		keyValueMutex: make(map[string]*sync.Mutex),
		mutex:         &sync.Mutex{},
		nodeMutex:     &sync.Mutex{},
		nodeClientMap: make(map[int]*rpc.Client),
	}
	for nodeID, hostPort := range hostMap {
		newPaxosNode.hostMap[nodeID] = hostPort
	}
	listener, err := net.Listen("tcp", myHostPort)
	if err != nil {
		return nil, err
	}
	err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newPaxosNode))
	if err != nil {
		return nil, err
	}
	newPaxosNode.listener = &listener
	rpc.HandleHTTP()
	go http.Serve(*newPaxosNode.listener, nil)

	// fmt.Println("myHostPort", myHostPort)
	// for nodeID, hostPort := range hostMap {
	// 	fmt.Println(nodeID, hostPort)
	// }

	if replace {
		for nodeID, hostPort := range hostMap {
			if nodeID == srvId {
				continue
			}

			// fmt.Println("my:", newPaxosNode.hostPort, newPaxosNode.nodeID, "dialing:", nodeID, hostPort)
			client, err := rpc.DialHTTP("tcp", hostPort)
			for err != nil {
				// fmt.Println("DialHTTP error", err)
				client, err = rpc.DialHTTP("tcp", hostPort)
			}
			newPaxosNode.nodeClientMap[nodeID] = client

			ReplaceServerReply := paxosrpc.ReplaceServerReply{}
			ReplaceServerArgs := &paxosrpc.ReplaceServerArgs{
				SrvID:    srvId,
				Hostport: myHostPort,
			}
			// fmt.Println(client, nodeID, srvId, hostPort)
			// fmt.Println(ReplaceServerArgs, ReplaceServerReply)
			err = client.Call("PaxosNode.RecvReplaceServer", ReplaceServerArgs, &ReplaceServerReply)
			if err == nil {
				ReplaceCatchupReply := paxosrpc.ReplaceCatchupReply{}
				ReplaceCatchupArgs := &paxosrpc.ReplaceCatchupArgs{}
				err = client.Call("PaxosNode.RecvReplaceCatchup", ReplaceCatchupArgs, &ReplaceCatchupReply)

				var temp map[string]*value
				json.Unmarshal(ReplaceCatchupReply.Data, &temp)

				for key, val := range temp {

					if newPaxosNode.keyValueMutex[key] == nil {
						newPaxosNode.keyValueMutex[key] = &sync.Mutex{}
					}

					oldval, ok := newPaxosNode.keyValue[key]
					val.my_n = 0
					if !ok {
						newPaxosNode.keyValue[key] = val
					} else {
						if val.N_a > oldval.N_a {
							oldval.N_a = val.N_a
							oldval.Value = val.Value
						}
						if val.N_h > oldval.N_h {
							oldval.N_h = val.N_h
						}
						newPaxosNode.keyValue[key] = oldval
					}
				}
			}

		}
	}
	return newPaxosNode, nil
}