func (srv *LspServer) loopServe() { var closeAllReply chan error for { select { case p := <-srv.netReadChan: srv.handleUdpPacket(p) case msg := <-srv.appWriteChan: if conn := srv.getConnById(msg.ConnId); conn != nil { conn.sendChan <- msg } case id := <-srv.closeChan: if conn := srv.getConnById(id); conn != nil { conn.closeChan <- nil } case closeAllReply = <-srv.closeAllChan: srv.stop = true for _, v := range srv.connMap { v.closeChan <- nil } case id := <-srv.removeConnChan: if conn := srv.getConnById(id); conn != nil { delete(srv.connMap, conn.addr.String()) lsplog.Vlogf(2, "[server] remove connection: %v\n", conn.addr.String()) } if srv.stop && len(srv.connMap) == 0 { lsplog.Vlogf(1, "[server] serve stop running\n") if closeAllReply != nil { closeAllReply <- nil } return } } } }
func newLspClient(hostport string, params *LspParams) (*LspClient, error) { if params == nil { params = &LspParams{5, 2000} } addr, err := lspnet.ResolveUDPAddr("udp", hostport) if lsplog.CheckReport(1, err) { return nil, err } udpConn, err := lspnet.DialUDP("udp", nil, addr) if err != nil { lsplog.Vlogf(1, "[client] connect to %v failed: %v\n", addr.String(), err) return nil, err } else { lsplog.Vlogf(1, "[client] connected to %v\n", addr.String()) } removeChan := make(chan uint16) appReadChan := make(chan *LspMsg) conn := newLspConn(params, udpConn, addr, 0, appReadChan, removeChan) cli := &LspClient{ client{ udpConn: udpConn, addr: addr, conn: conn, netReadChan: make(chan *LspMsg), appWriteChan: make(chan *LspMsg), appReadChan: appReadChan, connIdChan: make(chan uint16, 1), removeConnChan: removeChan, closeChan: make(chan error), }, } go cli.loopServe() go cli.loopRead() return cli, nil }
// Wait until server has signaled it is done func (ts *SynchTestSystem) WaitServer() { lsplog.Vlogf(6, "Waiting for server\n") select { case d := <-ts.TChan: ts.Tester.Logf("Test reached time limit of %d waiting for server", d) ts.Tester.FailNow() case <-ts.S2MChan: } lsplog.Vlogf(6, "Got signal from server\n") }
// Enable network & then Wait until it signals that it is done func (ts *SynchTestSystem) SynchNetwork() { lsplog.Vlogf(6, "Enabling Network\n") ts.M2NChan <- true lsplog.Vlogf(6, "Waiting for network\n") select { case d := <-ts.TChan: ts.Tester.Logf("Test reached time limit of %d waiting for network", d) ts.Tester.FailNow() case <-ts.N2MChan: } lsplog.Vlogf(6, "Got signal from network\n") }
// Wait until clients have that they're done func (ts *SynchTestSystem) WaitClients() { lsplog.Vlogf(6, "Waiting for clients\n") for i := 0; i < ts.Nclients; i++ { select { case d := <-ts.TChan: ts.Tester.Logf("Test reached time limit of %d waiting for client", d) ts.Tester.FailNow() case <-ts.C2MChan: } } lsplog.Vlogf(6, "Got signals from all clients\n") }
func (con *UDPConn) Write(b []byte) (int, error) { ncon := con.ncon if dropit(writeDropPercent) { lsplog.Vlogf(5, "UDP: DROPPING written packet of length %v\n", len(b)) // Make it look like write was successful return len(b), nil } else { n, err := ncon.Write(b) lsplog.Vlogf(5, "UDP: Wrote packet of length %v\n", n) return n, err } return 0, nil }
func (con *UDPConn) WriteToUDP(b []byte, addr *UDPAddr) (int, error) { ncon := con.ncon naddr := &net.UDPAddr{IP: addr.IP, Port: addr.Port} if dropit(writeDropPercent) { lsplog.Vlogf(5, "UDP: DROPPING written packet of length %v\n", len(b)) // Make it look like write was successful return len(b), nil } else { n, err := ncon.WriteToUDP(b, naddr) lsplog.Vlogf(5, "UDP: Wrote packet of length %v", n) return n, err } return 0, nil }
// Function to coordinate tests func (ts *SynchTestSystem) Master() { // Wait until server and all clients are ready ts.WaitServer() ts.WaitClients() lsplog.Vlogf(4, "Server + all clients started. Shut off network\n") ts.SynchNetwork() // Network Off // Enable client writes if ts.Mode != doservertoclient { ts.ReadyClients() ts.WaitClients() } // Do fast close of client. The calls will not be able to complete if ts.Mode == doclienttoserver { ts.ReadyClients() } if ts.Mode != doservertoclient { // Turn on network and delay ts.SynchNetwork() if ts.Mode == doclienttoserver { ts.WaitClients() } ts.SynchNetwork() // Network off // Enable server reads ts.ReadyServer() ts.WaitServer() } // Do server writes if ts.Mode != doclienttoserver { ts.ReadyServer() ts.WaitServer() } // Do fast close of server. The calls will not be able to complete if ts.Mode != doroundtrip { ts.ReadyServer() } if ts.Mode != doclienttoserver { // Turn on network ts.SynchNetwork() if ts.Mode != doroundtrip { // If did quick close, should get responses from server ts.WaitServer() } // Network off again ts.SynchNetwork() // Enable client reads ts.ReadyClients() ts.WaitClients() // Enable client closes ts.ReadyClients() ts.WaitClients() } // Final close by server if ts.Mode == doroundtrip { ts.ReadyServer() ts.WaitServer() } // Made it! }
// Have client generate n messages and check that they are echoed. func (ts *TestSystem) runclient(clienti int) { if clienti > ts.NClients { ts.Tester.Logf("Invalid client number %d\n", clienti) ts.Tester.FailNow() } cli := ts.Clients[clienti] for i := 0; i < ts.NMessages && ts.RunFlag; i++ { wt := ts.Rgen.Intn(100) werr := cli.Write(i2b(i + wt)) if werr != nil { ts.Tester.Logf("Client write got error '%s'\n", werr.Error()) ts.RunFlag = false ts.Tester.FailNow() } b := cli.Read() if b == nil { ts.Tester.Logf("Client read got error\n") ts.RunFlag = false ts.Tester.FailNow() } v := b2i(b) if v != wt+i { ts.Tester.Logf("Client got %d. Expected %d\n", v, i+wt) ts.RunFlag = false ts.Tester.FailNow() } } cli.Close() lsplog.Vlogf(0, "Client #%d completed %d messages\n", clienti, ts.NMessages) ts.CChan <- 0 }
func (ts *TestSystem) runtest(timeoutms int) { lspnet.SetWriteDropPercent(ts.DropPercent) if ts.Description != "" { fmt.Printf("Testing: %s\n", ts.Description) } go ts.runserver() for i := 0; i < ts.NClients; i++ { go ts.runclient(i) } go ts.runtimeout(timeoutms) for i := 0; i < ts.NClients; i++ { v := <-ts.CChan if v < 0 { ts.RunFlag = false ts.Tester.Logf("Test timed out after %f secs\n", float64(timeoutms)/1000.0) ts.Tester.FailNow() } } ts.RunFlag = false lsplog.Vlogf(0, "Passed: %d clients, %d messages/client, %.2f maxsleep, %.2f drop rate\n", ts.NClients, ts.NMessages, float64(ts.MaxSleepMilliseconds)/1000.0, float64(ts.DropPercent)/100.0) lsplog.SetVerbose(DefaultVerbosity) lspnet.SetWriteDropPercent(0) }
func (conn *lspConn) udpWrite(msg *LspMsg) { result, err := json.Marshal(msg) if err != nil { lsplog.Vlogf(3, "[conn] Marshal failed: %s\n", err.Error()) return } switch conn.whichSide { case ClientSide: _, err = conn.udpConn.Write(result) case ServerSide: _, err = conn.udpConn.WriteToUDP(result, conn.addr) } if err != nil { lsplog.Vlogf(3, "[conn] udpWrite failed: %s\n", err.Error()) } }
func newLspServer(port int, params *LspParams) (*LspServer, error) { if params == nil { params = &LspParams{5, 2000} } hostport := fmt.Sprintf("localhost:%v", port) addr, err := lspnet.ResolveUDPAddr("udp", hostport) if lsplog.CheckReport(1, err) { return nil, err } udpconn, err := lspnet.ListenUDP("udp", addr) if lsplog.CheckReport(1, err) { return nil, err } else { lsplog.Vlogf(1, "[server] listen on %v\n", addr.String()) } srv := &LspServer{ server{ nextConnId: 1, params: params, connMap: make(map[string]*lspConn), udpConn: udpconn, udpAddr: addr, netReadChan: make(chan *udpPacket), appWriteChan: make(chan *LspMsg), appReadChan: make(chan *LspMsg), closeChan: make(chan uint16), closeAllChan: make(chan chan error), removeConnChan: make(chan uint16), }, } go srv.loopServe() go srv.loopRead() return srv, nil }
func NewLspClient(hostport string, params *LspParams) (*LspClient, error) { cli, err := newLspClient(hostport, params) if err != nil { lsplog.Vlogf(1, "[client] create failure: %v", err) return nil, err } return cli, nil }
func (con *UDPConn) ReadFromUDP(b []byte) (n int, addr *UDPAddr, err error) { var buffer [2000]byte ncon := con.ncon var naddr *net.UDPAddr n, naddr, err = ncon.ReadFromUDP(buffer[0:]) if dropit(readDropPercent) { lsplog.Vlogf(5, "UDP: DROPPING read packet of length %v\n", n) } else { lsplog.Vlogf(6, "UDP: Read packet of length %v\n", n) copy(b, buffer[0:]) } if naddr == nil { addr = nil } else { addr = &UDPAddr{IP: naddr.IP, Port: naddr.Port} } return n, addr, err }
// Turn network off and on func (ts *SynchTestSystem) RunNetwork() { // Network initially on lspnet.SetWriteDropPercent(0) for ts.RunFlag { lsplog.Vlogf(4, "Network running. Waiting for master\n") <-ts.M2NChan lsplog.Vlogf(4, "Turning off network\n") lspnet.SetWriteDropPercent(100) ts.N2MChan <- true lsplog.Vlogf(4, "Network off. Waiting for master\n") <-ts.M2NChan lsplog.Vlogf(4, "Turning network on and delaying\n") lspnet.SetWriteDropPercent(0) ts.N2MChan <- true ts.synchdelay(2.0) } lsplog.Vlogf(4, "Network handler exiting\n") }
func (cli *LspClient) loopRead() { conn := cli.udpConn var buf [2000]byte for { n, _, err := conn.ReadFromUDP(buf[0:]) if err != nil { lsplog.Vlogf(3, "[client] ReadFromUDP error: %s\n", err.Error()) continue } var msg LspMsg err = json.Unmarshal(buf[0:n], &msg) if err != nil { lsplog.Vlogf(3, "[client] Unmarshal error: %s\n", err.Error()) continue } cli.netReadChan <- &msg lsplog.Vlogf(5, "[client] received udp packet\n") } }
func (conn *lspConn) receive(msg *LspMsg) { switch msg.Type { case MsgDATA: if msg.SeqNum == conn.nextRecvSeqNum { conn.recvBuf.Insert(msg) conn.nextRecvSeqNum++ conn.lastAck = genAckMsg(conn.connId, msg.SeqNum) // conn.send(conn.lastAck) } else { lsplog.Vlogf(4, "[conn] ignore data, connId=%v, seqnum=%v, expected=%v\n", conn.connId, msg.SeqNum, conn.nextRecvSeqNum) } conn.udpWrite(conn.lastAck) case MsgACK: if conn.sendBuf.Empty() { lsplog.Vlogf(4, "[conn] ignore ack, nothing to send\n") return } b, _ := conn.sendBuf.Front() expect := b.(*LspMsg) if msg.SeqNum == expect.SeqNum { conn.sendBuf.Remove() if msg.SeqNum == 0 { conn.connId = msg.ConnId lsplog.Vlogf(2, "[conn] connection confirmed, ConnId=%v\n", conn.connId) } if !conn.sendBuf.Empty() { r, _ := conn.sendBuf.Front() m := r.(*LspMsg) conn.udpWrite(m) } } else { conn.udpWrite(expect) lsplog.Vlogf(4, "[conn] ignore ack, ConnId=%v, seqnum=%v, expected=%v\n", conn.connId, msg.SeqNum, expect.SeqNum) } if conn.sendBuf.Empty() && conn.closed { conn.writeDone <- nil } } }
func (srv *LspServer) handleUdpPacket(p *udpPacket) { msg := p.msg addr := p.addr hostport := addr.String() conn := srv.connMap[hostport] switch msg.Type { case MsgCONNECT: if conn != nil { lsplog.Vlogf(5, "[server] Duplicate connect request from: %s\n", hostport) return } conn = newLspConn(srv.params, srv.udpConn, addr, srv.nextConnId, srv.appReadChan, srv.removeConnChan) srv.nextConnId++ srv.connMap[hostport] = conn case MsgDATA: conn.recvChan <- msg case MsgACK: conn.recvChan <- msg default: lsplog.Vlogf(5, "[server] Invalid packet, hostport=%v\n", hostport) } }
func (srv *LspServer) loopRead() { conn := srv.udpConn var buf [2000]byte for { n, addr, err := conn.ReadFromUDP(buf[0:]) if err != nil { lsplog.Vlogf(3, "[server] ReadFromUDP error: %s\n", err.Error()) continue } var msg LspMsg err = json.Unmarshal(buf[0:n], &msg) if err != nil { lsplog.Vlogf(3, "[server] Unmarshal error: %s\n", err.Error()) continue } packet := &udpPacket{ msg: &msg, addr: addr, } srv.netReadChan <- packet lsplog.Vlogf(5, "[server] received udp packet\n") } }
func runserver(srv *lsp.LspServer) { for { // Read from client id, payload, rerr := srv.Read() if rerr != nil { fmt.Printf("Connection %d has died. Error message %s\n", id, rerr.Error()) } else { s := string(payload) lsplog.Vlogf(6, "Connection %d. Received '%s'\n", id, s) payload = []byte(strings.ToUpper(s)) // Echo back to client srv.Write(id, payload) } } }
func (cli *LspClient) loopServe() { var connid uint16 = 0 for { select { case msg := <-cli.netReadChan: cli.conn.recvChan <- msg if connid == 0 { connid = msg.ConnId cli.connIdChan <- connid } case msg := <-cli.appWriteChan: cli.conn.sendChan <- msg case <-cli.removeConnChan: lsplog.Vlogf(1, "[client] exit\n") return case <-cli.closeChan: cli.closeChan <- nil } } }
func (conn *lspConn) serve() { params := conn.params interval := time.Duration(params.EpochMilliseconds) * time.Millisecond timeout := time.After(interval) for { var first *LspMsg var readChan chan<- *LspMsg if !conn.recvBuf.Empty() { b, _ := conn.recvBuf.Front() first = b.(*LspMsg) readChan = conn.readChan } select { case msg := <-conn.sendChan: if !conn.closed { conn.send(msg) } else { lsplog.Vlogf(2, "connection already closed, ignore send msg") } case msg := <-conn.recvChan: conn.lastTime = time.Now() conn.receive(msg) case readChan <- first: conn.recvBuf.Remove() case <-conn.closeChan: conn.closed = true case <-timeout: if conn.epochTrigger() { timeout = time.After(interval) } else { return } case <-conn.writeDone: conn.removeChan <- conn.connId return } } }
// Time Out test func (ts *SynchTestSystem) RunTimeout(epochfraction float64) { lsplog.Vlogf(6, "Setting test to timeout after %.2f epochs\n", epochfraction) ts.synchdelay(epochfraction) ts.RunFlag = false ts.TChan <- int(epochfraction) }
func NewAuxTestSystem(t *testing.T, nclients int, mode int, maxepochs int, params *LspParams) { fmt.Printf("Testing: Mode %s, %d clients\n", ModeName[mode], nclients) ts := new(AuxTestSystem) ts.Mode = mode ts.Params = params ts.ClientChan = make(chan bool) ts.ServerChan = make(chan bool) ts.TimeChan = make(chan bool) ts.Clients = make([]*LspClient, nclients) ts.Nclients = nclients ts.Nmessages = 10 ts.Rgen = *rand.New(rand.NewSource(time.Now().Unix())) ts.Port = randport(ts.Rgen) ts.RunFlag = true ts.Tester = t go ts.BuildServer() for i := 0; i < nclients; i++ { go ts.BuildClient(i) } go ts.runtimeout(float64(maxepochs)) switch ts.Mode { case doclientstop: // Wait for server or timer to complete select { case sok := <-ts.ServerChan: if !sok { ts.Tester.Logf("Server error\n") ts.Tester.FailNow() } case <-ts.TimeChan: ts.Tester.Logf("Test timed out waiting for server\n") ts.Tester.FailNow() } lsplog.Vlogf(0, "Server completed\n") // Wait for the clients for i := 0; i < nclients; i++ { select { case c*k := <-ts.ClientChan: if !c*k { ts.Tester.Logf("Client error\n") ts.Tester.FailNow() } case <-ts.TimeChan: ts.Tester.Logf("Test timed out waiting for client\n") ts.Tester.FailNow() } } lsplog.Vlogf(0, "Clients completed\n") default: // Includes stopserver // Wait for the clients for i := 0; i < nclients; i++ { select { case c*k := <-ts.ClientChan: if !c*k { ts.Tester.Logf("Client error\n") ts.Tester.FailNow() } case <-ts.TimeChan: ts.Tester.Logf("Test timed out waiting for client\n") ts.Tester.FailNow() } } // Wait for server or timer to complete select { case sok := <-ts.ServerChan: if !sok { ts.Tester.Logf("Server error\n") ts.Tester.FailNow() } case <-ts.TimeChan: ts.Tester.Logf("Test timed out waiting for server\n") ts.Tester.FailNow() } } lsplog.SetVerbose(AuxDefaultVerbosity) }
func (srv *LspServer) closeAll() { err := make(chan error) srv.closeAllChan <- err <-err lsplog.Vlogf(1, "[server] all connection closed\n") }
// Create server and and run test func (ts *SynchTestSystem) RunServer() { var err error ts.Server, err = NewLspServer(ts.Port, ts.Params) if err != nil { ts.Tester.Logf("Couldn't create server on port %d\n", ts.Port) ts.Tester.FailNow() } lsplog.Vlogf(1, "Server created on port %d\n", ts.Port) ts.S2MChan <- true // Read if ts.Mode != doservertoclient { lsplog.Vlogf(4, "Server waiting to read\n") <-ts.M2SChan lsplog.Vlogf(3, "Server reading messages\n") // Receive messages from client // Track number received from each client rcvdCount := make([]int, ts.Nclients) n := ts.Nmessages * ts.Nclients for m := 0; m < n; m++ { id, b, err := ts.Server.Read() if err != nil { ts.Tester.Log("Server failed to read\n") ts.Tester.Fail() } v := synchb2i(b) ts.MapLock.Lock() clienti, found := ts.ClientMap[id] ts.MapLock.Unlock() if !found || clienti >= ts.Nclients { ts.Tester.Logf("Server received message from unknown client #%d\n", id) ts.Tester.FailNow() } rc := rcvdCount[clienti] if rc >= ts.Nmessages { ts.Tester.Logf("Server has received too many messages from client #%d\n", id) ts.Tester.FailNow() } ev := ts.Data[clienti][rc] if v != ev { ts.Tester.Logf("Server received element #%v, value %v from connection %v. Expected %v\n", rc, v, id, ev) ts.Tester.Fail() } lsplog.Vlogf(6, "Server received element #%v, value %v from connection %v. Expected %v\n", rc, v, id, ev) rcvdCount[clienti] = rc + 1 } lsplog.Vlogf(3, "Server read all %v messages from clients\n", n) ts.S2MChan <- true } // Write if ts.Mode != doclienttoserver { lsplog.Vlogf(4, "Server waiting to write\n") <-ts.M2SChan lsplog.Vlogf(3, "Server writing messages\n") // Track number sent to each client sentCount := make([]int, ts.Nclients) n := ts.Nmessages * ts.Nclients for nsent := 0; nsent < n; { var clienti int // Choose random client found := false for !found { clienti = ts.Rgen.Intn(ts.Nclients) found = sentCount[clienti] < ts.Nmessages } id := ts.Clients[clienti].ConnId() sc := sentCount[clienti] v := ts.Data[clienti][sc] err := ts.Server.Write(id, synchi2b(v)) if err != nil { ts.Tester.Logf("Server could not send value %v (#%d) to client %d (ID %v)\n", v, sc, clienti, id) ts.Tester.Fail() } sentCount[clienti] = sc + 1 nsent++ } lsplog.Vlogf(3, "Server wrote all %v messages to clients\n", n) ts.S2MChan <- true } lsplog.Vlogf(4, "Server waiting to close\n") <-ts.M2SChan lsplog.Vlogf(4, "Server closing\n") ts.Server.CloseAll() lsplog.Vlogf(4, "Server closed\n") ts.S2MChan <- false }
// Create client and and run test func (ts *SynchTestSystem) RunClient(clienti int) { hostport := fmt.Sprintf("localhost:%d", ts.Port) cli, err := NewLspClient(hostport, ts.Params) if err != nil { ts.Tester.Logf("Couldn't create client %d to server at %s\n", clienti, hostport) ts.Tester.FailNow() } ts.Clients[clienti] = cli id := cli.ConnId() ts.MapLock.Lock() ts.ClientMap[id] = clienti ts.MapLock.Unlock() lsplog.Vlogf(1, "Client %d created with id %d to server at %s\n", clienti, id, hostport) ts.C2MChan <- true // Write if ts.Mode != doservertoclient { lsplog.Vlogf(4, "Client %d (id %d) waiting to write\n", clienti, id) <-ts.M2CChan lsplog.Vlogf(3, "Client %d (id %d) writing messages\n", clienti, id) for nsent := 0; nsent < ts.Nmessages; nsent++ { v := ts.Data[clienti][nsent] cli.Write(synchi2b(v)) if err != nil { ts.Tester.Logf("Client %d (id %d) could not send value %v\n", clienti, id, v) ts.Tester.Fail() } } lsplog.Vlogf(3, "Client %d (id %d) wrote all %v messages to server\n", clienti, id, ts.Nmessages) ts.C2MChan <- true } // Read if ts.Mode != doclienttoserver { lsplog.Vlogf(4, "Client %d (id %d) waiting to read\n", clienti, id) <-ts.M2CChan lsplog.Vlogf(4, "Client %d (id %d) reading messages\n", clienti, id) // Receive messages from server for nrcvd := 0; nrcvd < ts.Nmessages; nrcvd++ { b := cli.Read() if b == nil { ts.Tester.Logf("Client %d (id %d) failed to read value #%d\n", clienti, id, nrcvd) ts.Tester.Fail() } v := synchb2i(b) ev := ts.Data[clienti][nrcvd] if v != ev { ts.Tester.Logf("Client %d (id %d) received element #%v, value %v. Expected %v\n", clienti, id, nrcvd, v, ev) ts.Tester.Fail() } lsplog.Vlogf(6, "Client %d (id %d) received element #%v, value %v. Expected %v\n", clienti, id, nrcvd, v, ev) } lsplog.Vlogf(3, "Client %d (id %d) read all %v messages from servers\n", clienti, id, ts.Nmessages) ts.C2MChan <- true } lsplog.Vlogf(4, "Client %d (id %d) waiting to close\n", clienti, id) <-ts.M2CChan lsplog.Vlogf(4, "Client %d (id %d) closing\n", clienti, id) cli.Close() lsplog.Vlogf(4, "Client %d (id %d) done\n", clienti, id) ts.C2MChan <- false }
// Let server proceed func (ts *SynchTestSystem) ReadyServer() { lsplog.Vlogf(6, "Enabling server\n") ts.M2SChan <- true }
// Let clients proceed func (ts *SynchTestSystem) ReadyClients() { lsplog.Vlogf(6, "Enabling clients\n") for i := 0; i < ts.Nclients; i++ { ts.M2CChan <- true } }
// Time Out test func (ts *AuxTestSystem) runtimeout(epochfraction float64) { lsplog.Vlogf(6, "Setting test to timeout after %.2f epochs\n", epochfraction) ts.auxdelay(epochfraction) ts.RunFlag = false ts.TimeChan <- true }