/** * Proxy validates the requests going into a PaxosNode and the responses coming out of it. * It logs errors that occurs during a test. */ func NewProxy(nodePort, myPort int) (Proxy, error) { p := new(proxy) p.prop = new(proposal) p.prop.status = UNSET p.prop.num = 0 p.prop.key = "" p.prop.val = 0 p.err = make([]string, 0) // Start server l, err := net.Listen("tcp", fmt.Sprintf(":%d", myPort)) if err != nil { LOGE.Println("Failed to listen:", err) return nil, err } // Create RPC connection to paxos node. srv, err := rpc.DialHTTP("tcp", fmt.Sprintf("localhost:%d", nodePort)) if err != nil { LOGE.Println("Failed to dial node %d", nodePort) return nil, err } p.srv = srv // register RPC rpc.RegisterName("PaxosNode", paxosrpc.Wrap(p)) rpc.HandleHTTP() go http.Serve(l, nil) // log.Printf("Proxy started") return p, nil }
// NewPaxosNode creates a new PaxosNode. This function should return only when // all nodes have joined the ring, and should return a non-nil error if the node // could not be started in spite of dialing the other nodes numRetries times. // // hostMap is a map from node IDs to their hostports, numNodes is the number // of nodes in the ring, replace is a flag which indicates whether this node // is a replacement for a node which failed. func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) { fmt.Println("NewPaxosNode") pn := &paxosNode{} pn.nodePropNum = srvId pn.propNumLock = new(sync.Mutex) pn.numNodes = numNodes pn.srvID = srvId pn.majority_num = numNodes / 2 pn.nodeHostPort = myHostPort pn.hostMap = hostMap pn.storage = &Storage{} pn.storage.storage = make(map[string]interface{}) pn.storage.storageLock = new(sync.Mutex) pn.propNum = &ProposalNum{} pn.propNum.propNum = make(map[string]int) pn.propNum.propNumLock = new(sync.Mutex) pn.acceptValue = &AcceptValues{} pn.acceptValue.acceptValue = make(map[string]interface{}) pn.acceptValue.acceptNum = make(map[string]int) pn.acceptValue.acceptMapLock = new(sync.Mutex) pn.connList = &ConnectionList{} pn.connList.conn = make(map[int]*rpc.Client) pn.connList.connLock = new(sync.Mutex) ln, err := net.Listen("tcp", myHostPort) if err != nil { fmt.Println("Error on listern: ", err) return nil, errors.New("Error on listen: " + err.Error()) } // Begin to listen for incoming connections rpc.RegisterName("PaxosNode", paxosrpc.Wrap(pn)) rpc.HandleHTTP() pn.listener = ln go http.Serve(ln, nil) retryCounter := 0 // Connect with other servers pn.connList.connLock.Lock() for len(pn.connList.conn) < pn.numNodes { for i, host := range pn.hostMap { if _, ok := pn.connList.conn[i]; !ok { client, err := rpc.DialHTTP("tcp", host) if err == nil { pn.connList.conn[i] = client } // If the node is to replace the dead node // It needs to call rpc RecvReplaceServer function to other nodes if replace && i != srvId { replaceArgs := &paxosrpc.ReplaceServerArgs{SrvID: pn.srvID, Hostport: pn.nodeHostPort} var replaceReply paxosrpc.ReplaceServerReply client.Call("PaxosNode.RecvReplaceServer", replaceArgs, &replaceReply) } } } if len(pn.connList.conn) < pn.numNodes { time.Sleep(time.Duration(1) * time.Second) retryCounter += 1 if retryCounter > numRetries { return nil, errors.New("Cannot connect with other nodes") } } } // If the node is to replace the dead node // It needs to get all commited information from an arbitrary node if replace { pn.storage.storageLock.Lock() for id, conn := range pn.connList.conn { if id == pn.srvID { continue } var replaceCatchupArgs paxosrpc.ReplaceCatchupArgs var replaceReply paxosrpc.ReplaceCatchupReply conn.Call("PaxosNode.RecvReplaceCatchup", replaceCatchupArgs, &replaceReply) err := json.Unmarshal(replaceReply.Data, &pn.storage.storage) if err != nil { fmt.Println("[ERROR]", err) } break } pn.storage.storageLock.Unlock() } pn.connList.connLock.Unlock() fmt.Println("Paxos node started on port " + myHostPort) return pn, nil }
// NewPaxosNode creates a new PaxosNode. This function should return only when // all nodes have joined the ring, and should return a non-nil error if the node // could not be started in spite of dialing the other nodes numRetries times. // // hostMap is a map from node IDs to their hostports, numNodes is the number // of nodes in the ring, replace is a flag which indicates whether this node // is a replacement for a node which failed. func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) { fmt.Println("we out here:", myHostPort) // register all the other hosts otherHosts := make(map[int]*rpc.Client) newNode := createNode(srvId, numNodes, replace) rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newNode)) rpc.HandleHTTP() port := strings.Split(myHostPort, "localhost")[1] l, e := net.Listen("tcp", port) if e != nil { return nil, errors.New("RPC Issue Storage Server\n") } go http.Serve(l, nil) for id, hostport := range hostMap { // keep tryin until you give up, return fail if a node in ring fails to connect allGood := false for retryCount := 0; retryCount < numRetries; retryCount++ { cli, err := rpc.DialHTTP("tcp", hostport) if err == nil { allGood = true otherHosts[id] = cli break } time.Sleep(time.Duration(1) * time.Second) } if !allGood { fmt.Println(hostport) return nil, errors.New("Failed to connect to a client") } } newNode.nodes = otherHosts // the line below doesnt work but i think we need to do something like this //Catchup shit if replace { var helper *rpc.Client for id, cli := range newNode.nodes { if id != srvId { helper = cli args_server := paxosrpc.ReplaceServerArgs{srvId, myHostPort} var reply_server paxosrpc.ReplaceServerReply if err := cli.Call("PaxosNode.RecvReplaceServer", &args_server, &reply_server); err != nil { return nil, err } } } args := paxosrpc.ReplaceCatchupArgs{} var reply paxosrpc.ReplaceCatchupReply if err := helper.Call("PaxosNode.RecvReplaceCatchup", &args, &reply); err != nil { return nil, err } dec := gob.NewDecoder(bytes.NewBuffer(reply.Data)) var data map[string]interface{} err := dec.Decode(&data) if err != nil { return nil, err } for key, _ := range data { args_num := paxosrpc.ProposalNumberArgs{key} var reply_num paxosrpc.ProposalNumberReply if err := helper.Call("PaxosNode.GetNextProposalNumber", &args_num, &reply_num); err != nil { return nil, err } newNode.clockMap[key] = (reply_num.N) / numNodes } newNode.storage = data } return newNode, nil }
// NewPaxosNode creates a new PaxosNode. This function should return only when // all nodes have joined the ring, and should return a non-nil error if the node // could not be started in spite of dialing the other nodes numRetries times. // hostMap is a map from node IDs to their hostports, numNodes is the number // of nodes in the ring, replace is a flag which indicates whether this node // is a replacement for a node which failed. func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) { pn := &paxosNode{ ss: make(map[int]*rpc.Client), numNodes: numNodes, id: srvId, ptimes: 0, majorNum: numNodes/2 + 1, instances: make(map[string]*paxosKeyData), } // Create the server socket that will listen for incoming RPCs. _, port, _ := net.SplitHostPort(myHostPort) listener, err := net.Listen("tcp", fmt.Sprintf(":%s", port)) if err != nil { return nil, err } // Wrap the tribServer before registering it for RPC. err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(pn)) if err != nil { return nil, err } // Setup the HTTP handler that will server incoming RPCs and // serve requests in a background goroutine. rpc.HandleHTTP() go http.Serve(listener, nil) receivedData := false // make connnection to all other nodes for servid, hport := range hostMap { conn, err := rpc.DialHTTP("tcp", hport) ri := 0 for ; err != nil && ri < numRetries; ri++ { time.Sleep(1000 * time.Millisecond) conn, err = rpc.DialHTTP("tcp", hport) } if ri >= numRetries { return nil, err } pn.ss[servid] = conn // if replace flag is set if replace && servid != srvId { var myreply paxosrpc.ReplaceServerReply myargs := &paxosrpc.ReplaceServerArgs{ Hostport: myHostPort, SrvID: srvId, } //replace the old server entry in other server conn.Call("PaxosNode.RecvReplaceServer", myargs, &myreply) // flag to check if we have already received data from other nodes if !receivedData { var serverState paxosrpc.ReplaceCatchupReply var catchupArgs paxosrpc.ReplaceCatchupArgs // call other server for data conn.Call("PaxosNode.RecvReplaceCatchup", catchupArgs, &serverState) // the map is decoded from gob network := bytes.NewBuffer(serverState.Data) dec := gob.NewDecoder(network) var instances map[string]*paxosKeyData dec.Decode(&instances) receivedData = true pn.instances = instances } } } return pn, nil }
// NewPaxosNode creates a new PaxosNode. This function should return only when // all nodes have joined the ring, and should return a non-nil error if the node // could not be started in spite of dialing the other nodes numRetries times. // // hostMap is a map from node IDs to their hostports, numNodes is the number // of nodes in the ring, replace is a flag which indicates whether this node // is a replacement for a node which failed. func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) { fmt.Println("server id: ", srvId) newNode := &paxosNode{ srvId: srvId, hostMap: hostMap, myHostPort: myHostPort, numNodes: numNodes, replace: replace, numRetries: numRetries, connMap: make(map[string]*rpc.Client), pNumberMap: make(map[string]int), vaMap: make(map[string]interface{}), naMap: make(map[string]int), npMap: make(map[string]int), valueMap: make(map[string]interface{}), prepareDoneChan: make(chan *PrepareData), acceptDoneChan: make(chan bool), commitDoneChan: make(chan bool), } // register the paxos node for rpc usage err := rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newNode)) if err != nil { return nil, errors.New("Errow while register.") } // running as a server for listening rpc.HandleHTTP() listener, err := net.Listen("tcp", myHostPort) if err != nil { fmt.Println("err when listen: ", listener) return nil, errors.New(myHostPort) } go http.Serve(listener, nil) // build connection to every other node tmpValueMap := make(map[string]interface{}) count := 0 succeed := true for count < numRetries { succeed = true for _, v := range hostMap { client, err := rpc.DialHTTP("tcp", v) if err != nil { count += 1 succeed = false break } newNode.connMap[v] = client } if succeed { // for replacement usage if replace { for _, client := range newNode.connMap { replaceArgs := &paxosrpc.ReplaceServerArgs{srvId, myHostPort} replaceReply := &paxosrpc.ReplaceServerReply{} client.Call("PaxosNode.RecvReplaceServer", replaceArgs, replaceReply) replaceCatchupArgs := &paxosrpc.ReplaceCatchupArgs{} replaceCatchupReply := &paxosrpc.ReplaceCatchupReply{} client.Call("PaxosNode.RecvReplaceCatchup", replaceCatchupArgs, replaceCatchupReply) var nodeValueMap map[string]interface{} json.Unmarshal([]byte(replaceCatchupReply.Data), &nodeValueMap) for nodeKey, nodeValue := range nodeValueMap { tmpValueMap[nodeKey] = nodeValue } } } newNode.valueMap = tmpValueMap return newNode, nil } time.Sleep(time.Second * 1) } return nil, errors.New("cannot connect to all the nodes") }
// NewPaxosNode creates a new PaxosNode. This function should return only when // all nodes have joined the ring, and should return a non-nil error if the node // could not be started in spite of dialing the other nodes numRetries times. // // hostMap is a map from node IDs to their hostports, numNodes is the number // of nodes in the ring, replace is a flag which indicates whether this node // is a replacement for a node which failed. func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) { fmt.Println(myHostPort) var timeFactor int for timeFactor = 10; timeFactor < numNodes; timeFactor *= 10 { } fmt.Printf("number of nodes: %d, factor: %d\n", numNodes, timeFactor) myPaxosNode := paxosNode{ port: myHostPort, idMap: make(map[int](peerNode)), numRetries: numRetries, numNodes: numNodes, id: srvId, logicTime: 0, timeFactor: timeFactor, nhMap: make(map[string](int)), naMap: make(map[string](int)), vaMap: make(map[string](interface{})), valueMap: make(map[string](interface{})), countReqMap: make(map[string](chan bool)), countResMap: make(map[string](chan int)), keyMutex: make(map[string](*sync.Mutex)), nodeMutex: &sync.Mutex{}, } // create listener for incoming RPCs listener, err := net.Listen("tcp", myHostPort) for err != nil { listener, err = net.Listen("tcp", myHostPort) } // wrap the paxos node err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(&myPaxosNode)) for err != nil { err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(&myPaxosNode)) } // set up http handler for incoming RPCs rpc.HandleHTTP() go http.Serve(listener, nil) for nodeId, nodePort := range hostMap { err = myPaxosNode.dialToPeerNode(nodeId, nodePort) if err != nil { return nil, errors.New(fmt.Sprintf("Not all paxos nodes are ready\nid: %d, port: %d", nodeId, nodePort)) } } if replace { if err := myPaxosNode.replaceServer(); err != nil { return nil, errors.New(fmt.Sprintf("paxos node: %d fail in receiving acks", srvId)) } if err := myPaxosNode.replaceCatchup(); err != nil { return nil, errors.New(fmt.Sprintf("paxos node: %d fail to get valueMap", srvId)) } } return &myPaxosNode, nil }
// NewPaxosNode creates a new PaxosNode. This function should return only when // all nodes have joined the ring, and should return a non-nil error if the node // could not be started in spite of dialing the other nodes numRetries times. // // hostMap is a map from node IDs to their hostports, numNodes is the number // of nodes in the ring, replace is a flag which indicates whether this node // is a replacement for a node which failed. func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) { newPaxosNode := &paxosNode{ nodeID: srvId, hostPort: myHostPort, numNodes: numNodes, replace: replace, hostMap: make(map[int]string), keyValue: make(map[string]*value), savedkeyValue: make(map[string]*value), keyValueMutex: make(map[string]*sync.Mutex), mutex: &sync.Mutex{}, nodeMutex: &sync.Mutex{}, nodeClientMap: make(map[int]*rpc.Client), } for nodeID, hostPort := range hostMap { newPaxosNode.hostMap[nodeID] = hostPort } listener, err := net.Listen("tcp", myHostPort) if err != nil { return nil, err } err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newPaxosNode)) if err != nil { return nil, err } newPaxosNode.listener = &listener rpc.HandleHTTP() go http.Serve(*newPaxosNode.listener, nil) // fmt.Println("myHostPort", myHostPort) // for nodeID, hostPort := range hostMap { // fmt.Println(nodeID, hostPort) // } if replace { for nodeID, hostPort := range hostMap { if nodeID == srvId { continue } // fmt.Println("my:", newPaxosNode.hostPort, newPaxosNode.nodeID, "dialing:", nodeID, hostPort) client, err := rpc.DialHTTP("tcp", hostPort) for err != nil { // fmt.Println("DialHTTP error", err) client, err = rpc.DialHTTP("tcp", hostPort) } newPaxosNode.nodeClientMap[nodeID] = client ReplaceServerReply := paxosrpc.ReplaceServerReply{} ReplaceServerArgs := &paxosrpc.ReplaceServerArgs{ SrvID: srvId, Hostport: myHostPort, } // fmt.Println(client, nodeID, srvId, hostPort) // fmt.Println(ReplaceServerArgs, ReplaceServerReply) err = client.Call("PaxosNode.RecvReplaceServer", ReplaceServerArgs, &ReplaceServerReply) if err == nil { ReplaceCatchupReply := paxosrpc.ReplaceCatchupReply{} ReplaceCatchupArgs := &paxosrpc.ReplaceCatchupArgs{} err = client.Call("PaxosNode.RecvReplaceCatchup", ReplaceCatchupArgs, &ReplaceCatchupReply) var temp map[string]*value json.Unmarshal(ReplaceCatchupReply.Data, &temp) for key, val := range temp { if newPaxosNode.keyValueMutex[key] == nil { newPaxosNode.keyValueMutex[key] = &sync.Mutex{} } oldval, ok := newPaxosNode.keyValue[key] val.my_n = 0 if !ok { newPaxosNode.keyValue[key] = val } else { if val.N_a > oldval.N_a { oldval.N_a = val.N_a oldval.Value = val.Value } if val.N_h > oldval.N_h { oldval.N_h = val.N_h } newPaxosNode.keyValue[key] = oldval } } } } } return newPaxosNode, nil }