/**
* Proxy validates the requests going into a PaxosNode and the responses coming out of it. * It logs errors that occurs during a test.
*/
func NewProxy(nodePort, myPort int) (Proxy, error) {
p := new(proxy)
p.prop = new(proposal)
p.prop.status = UNSET
p.prop.num = 0
p.prop.key = ""
p.prop.val = 0
p.err = make([]string, 0)
// Start server
l, err := net.Listen("tcp", fmt.Sprintf(":%d", myPort))
if err != nil {
LOGE.Println("Failed to listen:", err)
return nil, err
}
// Create RPC connection to paxos node.
srv, err := rpc.DialHTTP("tcp", fmt.Sprintf("localhost:%d", nodePort))
if err != nil {
LOGE.Println("Failed to dial node %d", nodePort)
return nil, err
}
p.srv = srv
// register RPC
rpc.RegisterName("PaxosNode", paxosrpc.Wrap(p))
rpc.HandleHTTP()
go http.Serve(l, nil)
// log.Printf("Proxy started")
return p, nil
}
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
fmt.Println("NewPaxosNode")
pn := &paxosNode{}
pn.nodePropNum = srvId
pn.propNumLock = new(sync.Mutex)
pn.numNodes = numNodes
pn.srvID = srvId
pn.majority_num = numNodes / 2
pn.nodeHostPort = myHostPort
pn.hostMap = hostMap
pn.storage = &Storage{}
pn.storage.storage = make(map[string]interface{})
pn.storage.storageLock = new(sync.Mutex)
pn.propNum = &ProposalNum{}
pn.propNum.propNum = make(map[string]int)
pn.propNum.propNumLock = new(sync.Mutex)
pn.acceptValue = &AcceptValues{}
pn.acceptValue.acceptValue = make(map[string]interface{})
pn.acceptValue.acceptNum = make(map[string]int)
pn.acceptValue.acceptMapLock = new(sync.Mutex)
pn.connList = &ConnectionList{}
pn.connList.conn = make(map[int]*rpc.Client)
pn.connList.connLock = new(sync.Mutex)
ln, err := net.Listen("tcp", myHostPort)
if err != nil {
fmt.Println("Error on listern: ", err)
return nil, errors.New("Error on listen: " + err.Error())
}
// Begin to listen for incoming connections
rpc.RegisterName("PaxosNode", paxosrpc.Wrap(pn))
rpc.HandleHTTP()
pn.listener = ln
go http.Serve(ln, nil)
retryCounter := 0
// Connect with other servers
pn.connList.connLock.Lock()
for len(pn.connList.conn) < pn.numNodes {
for i, host := range pn.hostMap {
if _, ok := pn.connList.conn[i]; !ok {
client, err := rpc.DialHTTP("tcp", host)
if err == nil {
pn.connList.conn[i] = client
}
// If the node is to replace the dead node
// It needs to call rpc RecvReplaceServer function to other nodes
if replace && i != srvId {
replaceArgs := &paxosrpc.ReplaceServerArgs{SrvID: pn.srvID, Hostport: pn.nodeHostPort}
var replaceReply paxosrpc.ReplaceServerReply
client.Call("PaxosNode.RecvReplaceServer", replaceArgs, &replaceReply)
}
}
}
if len(pn.connList.conn) < pn.numNodes {
time.Sleep(time.Duration(1) * time.Second)
retryCounter += 1
if retryCounter > numRetries {
return nil, errors.New("Cannot connect with other nodes")
}
}
}
// If the node is to replace the dead node
// It needs to get all commited information from an arbitrary node
if replace {
pn.storage.storageLock.Lock()
for id, conn := range pn.connList.conn {
if id == pn.srvID {
continue
}
var replaceCatchupArgs paxosrpc.ReplaceCatchupArgs
var replaceReply paxosrpc.ReplaceCatchupReply
conn.Call("PaxosNode.RecvReplaceCatchup", replaceCatchupArgs, &replaceReply)
err := json.Unmarshal(replaceReply.Data, &pn.storage.storage)
if err != nil {
fmt.Println("[ERROR]", err)
}
break
}
pn.storage.storageLock.Unlock()
}
pn.connList.connLock.Unlock()
fmt.Println("Paxos node started on port " + myHostPort)
return pn, nil
}
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
fmt.Println("we out here:", myHostPort)
// register all the other hosts
otherHosts := make(map[int]*rpc.Client)
newNode := createNode(srvId, numNodes, replace)
rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newNode))
rpc.HandleHTTP()
port := strings.Split(myHostPort, "localhost")[1]
l, e := net.Listen("tcp", port)
if e != nil {
return nil, errors.New("RPC Issue Storage Server\n")
}
go http.Serve(l, nil)
for id, hostport := range hostMap {
// keep tryin until you give up, return fail if a node in ring fails to connect
allGood := false
for retryCount := 0; retryCount < numRetries; retryCount++ {
cli, err := rpc.DialHTTP("tcp", hostport)
if err == nil {
allGood = true
otherHosts[id] = cli
break
}
time.Sleep(time.Duration(1) * time.Second)
}
if !allGood {
fmt.Println(hostport)
return nil, errors.New("Failed to connect to a client")
}
}
newNode.nodes = otherHosts
// the line below doesnt work but i think we need to do something like this
//Catchup shit
if replace {
var helper *rpc.Client
for id, cli := range newNode.nodes {
if id != srvId {
helper = cli
args_server := paxosrpc.ReplaceServerArgs{srvId, myHostPort}
var reply_server paxosrpc.ReplaceServerReply
if err := cli.Call("PaxosNode.RecvReplaceServer", &args_server, &reply_server); err != nil {
return nil, err
}
}
}
args := paxosrpc.ReplaceCatchupArgs{}
var reply paxosrpc.ReplaceCatchupReply
if err := helper.Call("PaxosNode.RecvReplaceCatchup", &args, &reply); err != nil {
return nil, err
}
dec := gob.NewDecoder(bytes.NewBuffer(reply.Data))
var data map[string]interface{}
err := dec.Decode(&data)
if err != nil {
return nil, err
}
for key, _ := range data {
args_num := paxosrpc.ProposalNumberArgs{key}
var reply_num paxosrpc.ProposalNumberReply
if err := helper.Call("PaxosNode.GetNextProposalNumber", &args_num, &reply_num); err != nil {
return nil, err
}
newNode.clockMap[key] = (reply_num.N) / numNodes
}
newNode.storage = data
}
return newNode, nil
}
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
pn := &paxosNode{
ss: make(map[int]*rpc.Client),
numNodes: numNodes,
id: srvId,
ptimes: 0,
majorNum: numNodes/2 + 1,
instances: make(map[string]*paxosKeyData),
}
// Create the server socket that will listen for incoming RPCs.
_, port, _ := net.SplitHostPort(myHostPort)
listener, err := net.Listen("tcp", fmt.Sprintf(":%s", port))
if err != nil {
return nil, err
}
// Wrap the tribServer before registering it for RPC.
err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(pn))
if err != nil {
return nil, err
}
// Setup the HTTP handler that will server incoming RPCs and
// serve requests in a background goroutine.
rpc.HandleHTTP()
go http.Serve(listener, nil)
receivedData := false
// make connnection to all other nodes
for servid, hport := range hostMap {
conn, err := rpc.DialHTTP("tcp", hport)
ri := 0
for ; err != nil && ri < numRetries; ri++ {
time.Sleep(1000 * time.Millisecond)
conn, err = rpc.DialHTTP("tcp", hport)
}
if ri >= numRetries {
return nil, err
}
pn.ss[servid] = conn
// if replace flag is set
if replace && servid != srvId {
var myreply paxosrpc.ReplaceServerReply
myargs := &paxosrpc.ReplaceServerArgs{
Hostport: myHostPort,
SrvID: srvId,
}
//replace the old server entry in other server
conn.Call("PaxosNode.RecvReplaceServer", myargs, &myreply)
// flag to check if we have already received data from other nodes
if !receivedData {
var serverState paxosrpc.ReplaceCatchupReply
var catchupArgs paxosrpc.ReplaceCatchupArgs
// call other server for data
conn.Call("PaxosNode.RecvReplaceCatchup", catchupArgs, &serverState)
// the map is decoded from gob
network := bytes.NewBuffer(serverState.Data)
dec := gob.NewDecoder(network)
var instances map[string]*paxosKeyData
dec.Decode(&instances)
receivedData = true
pn.instances = instances
}
}
}
return pn, nil
}
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
fmt.Println("server id: ", srvId)
newNode := &paxosNode{
srvId: srvId,
hostMap: hostMap,
myHostPort: myHostPort,
numNodes: numNodes,
replace: replace,
numRetries: numRetries,
connMap: make(map[string]*rpc.Client),
pNumberMap: make(map[string]int),
vaMap: make(map[string]interface{}),
naMap: make(map[string]int),
npMap: make(map[string]int),
valueMap: make(map[string]interface{}),
prepareDoneChan: make(chan *PrepareData),
acceptDoneChan: make(chan bool),
commitDoneChan: make(chan bool),
}
// register the paxos node for rpc usage
err := rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newNode))
if err != nil {
return nil, errors.New("Errow while register.")
}
// running as a server for listening
rpc.HandleHTTP()
listener, err := net.Listen("tcp", myHostPort)
if err != nil {
fmt.Println("err when listen: ", listener)
return nil, errors.New(myHostPort)
}
go http.Serve(listener, nil)
// build connection to every other node
tmpValueMap := make(map[string]interface{})
count := 0
succeed := true
for count < numRetries {
succeed = true
for _, v := range hostMap {
client, err := rpc.DialHTTP("tcp", v)
if err != nil {
count += 1
succeed = false
break
}
newNode.connMap[v] = client
}
if succeed {
// for replacement usage
if replace {
for _, client := range newNode.connMap {
replaceArgs := &paxosrpc.ReplaceServerArgs{srvId, myHostPort}
replaceReply := &paxosrpc.ReplaceServerReply{}
client.Call("PaxosNode.RecvReplaceServer", replaceArgs, replaceReply)
replaceCatchupArgs := &paxosrpc.ReplaceCatchupArgs{}
replaceCatchupReply := &paxosrpc.ReplaceCatchupReply{}
client.Call("PaxosNode.RecvReplaceCatchup", replaceCatchupArgs, replaceCatchupReply)
var nodeValueMap map[string]interface{}
json.Unmarshal([]byte(replaceCatchupReply.Data), &nodeValueMap)
for nodeKey, nodeValue := range nodeValueMap {
tmpValueMap[nodeKey] = nodeValue
}
}
}
newNode.valueMap = tmpValueMap
return newNode, nil
}
time.Sleep(time.Second * 1)
}
return nil, errors.New("cannot connect to all the nodes")
}
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
fmt.Println(myHostPort)
var timeFactor int
for timeFactor = 10; timeFactor < numNodes; timeFactor *= 10 {
}
fmt.Printf("number of nodes: %d, factor: %d\n", numNodes, timeFactor)
myPaxosNode := paxosNode{
port: myHostPort,
idMap: make(map[int](peerNode)),
numRetries: numRetries,
numNodes: numNodes,
id: srvId,
logicTime: 0,
timeFactor: timeFactor,
nhMap: make(map[string](int)),
naMap: make(map[string](int)),
vaMap: make(map[string](interface{})),
valueMap: make(map[string](interface{})),
countReqMap: make(map[string](chan bool)),
countResMap: make(map[string](chan int)),
keyMutex: make(map[string](*sync.Mutex)),
nodeMutex: &sync.Mutex{},
}
// create listener for incoming RPCs
listener, err := net.Listen("tcp", myHostPort)
for err != nil {
listener, err = net.Listen("tcp", myHostPort)
}
// wrap the paxos node
err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(&myPaxosNode))
for err != nil {
err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(&myPaxosNode))
}
// set up http handler for incoming RPCs
rpc.HandleHTTP()
go http.Serve(listener, nil)
for nodeId, nodePort := range hostMap {
err = myPaxosNode.dialToPeerNode(nodeId, nodePort)
if err != nil {
return nil, errors.New(fmt.Sprintf("Not all paxos nodes are ready\nid: %d, port: %d", nodeId, nodePort))
}
}
if replace {
if err := myPaxosNode.replaceServer(); err != nil {
return nil, errors.New(fmt.Sprintf("paxos node: %d fail in receiving acks", srvId))
}
if err := myPaxosNode.replaceCatchup(); err != nil {
return nil, errors.New(fmt.Sprintf("paxos node: %d fail to get valueMap", srvId))
}
}
return &myPaxosNode, nil
}
// NewPaxosNode creates a new PaxosNode. This function should return only when
// all nodes have joined the ring, and should return a non-nil error if the node
// could not be started in spite of dialing the other nodes numRetries times.
//
// hostMap is a map from node IDs to their hostports, numNodes is the number
// of nodes in the ring, replace is a flag which indicates whether this node
// is a replacement for a node which failed.
func NewPaxosNode(myHostPort string, hostMap map[int]string, numNodes, srvId, numRetries int, replace bool) (PaxosNode, error) {
newPaxosNode := &paxosNode{
nodeID: srvId,
hostPort: myHostPort,
numNodes: numNodes,
replace: replace,
hostMap: make(map[int]string),
keyValue: make(map[string]*value),
savedkeyValue: make(map[string]*value),
keyValueMutex: make(map[string]*sync.Mutex),
mutex: &sync.Mutex{},
nodeMutex: &sync.Mutex{},
nodeClientMap: make(map[int]*rpc.Client),
}
for nodeID, hostPort := range hostMap {
newPaxosNode.hostMap[nodeID] = hostPort
}
listener, err := net.Listen("tcp", myHostPort)
if err != nil {
return nil, err
}
err = rpc.RegisterName("PaxosNode", paxosrpc.Wrap(newPaxosNode))
if err != nil {
return nil, err
}
newPaxosNode.listener = &listener
rpc.HandleHTTP()
go http.Serve(*newPaxosNode.listener, nil)
// fmt.Println("myHostPort", myHostPort)
// for nodeID, hostPort := range hostMap {
// fmt.Println(nodeID, hostPort)
// }
if replace {
for nodeID, hostPort := range hostMap {
if nodeID == srvId {
continue
}
// fmt.Println("my:", newPaxosNode.hostPort, newPaxosNode.nodeID, "dialing:", nodeID, hostPort)
client, err := rpc.DialHTTP("tcp", hostPort)
for err != nil {
// fmt.Println("DialHTTP error", err)
client, err = rpc.DialHTTP("tcp", hostPort)
}
newPaxosNode.nodeClientMap[nodeID] = client
ReplaceServerReply := paxosrpc.ReplaceServerReply{}
ReplaceServerArgs := &paxosrpc.ReplaceServerArgs{
SrvID: srvId,
Hostport: myHostPort,
}
// fmt.Println(client, nodeID, srvId, hostPort)
// fmt.Println(ReplaceServerArgs, ReplaceServerReply)
err = client.Call("PaxosNode.RecvReplaceServer", ReplaceServerArgs, &ReplaceServerReply)
if err == nil {
ReplaceCatchupReply := paxosrpc.ReplaceCatchupReply{}
ReplaceCatchupArgs := &paxosrpc.ReplaceCatchupArgs{}
err = client.Call("PaxosNode.RecvReplaceCatchup", ReplaceCatchupArgs, &ReplaceCatchupReply)
var temp map[string]*value
json.Unmarshal(ReplaceCatchupReply.Data, &temp)
for key, val := range temp {
if newPaxosNode.keyValueMutex[key] == nil {
newPaxosNode.keyValueMutex[key] = &sync.Mutex{}
}
oldval, ok := newPaxosNode.keyValue[key]
val.my_n = 0
if !ok {
newPaxosNode.keyValue[key] = val
} else {
if val.N_a > oldval.N_a {
oldval.N_a = val.N_a
oldval.Value = val.Value
}
if val.N_h > oldval.N_h {
oldval.N_h = val.N_h
}
newPaxosNode.keyValue[key] = oldval
}
}
}
}
}
return newPaxosNode, nil
}