func (src *Client) Run() error {
src.running = true
defer func() {
src.running = false
}()
slog.Gm.Register(stream.Name(src))
go func(op string, s *Client) { // Update the queue depth on input for each phase
for {
slog.Gm.Update(&op, s.GetInDepth())
time.Sleep(1 * time.Second)
}
}(stream.Name(src), src)
for src.retries < RETRY_MAX {
err := src.connect()
if err == nil {
slog.Logf(logger.Levels.Warn, "Connection failed without error")
return err
} else {
slog.Logf(logger.Levels.Error, "Connection failed with error, retrying: %s", err)
time.Sleep(1 * time.Second)
}
}
slog.Logf(logger.Levels.Error, "Connection failed retries exceeded. Leftover: %d", src.buf.Len())
return nil //>>>>>>>>>>>>>>???????????????????????
}
func (src Server) Run() error {
defer close(src.Out())
ln, err := net.Listen("tcp", src.addr)
if err != nil {
slog.Logf(logger.Levels.Error, "Error listening %v", err)
return err
}
wg_sub := &sync.WaitGroup{}
defer wg_sub.Wait()
//If soft close is enabled, server will exit after last connection exits.
scl := make(chan bool)
wg_scl := &sync.WaitGroup{}
first_connection := true
wg_sub.Add(1)
go func() {
defer wg_sub.Done()
hardCloseListener(src.StopNotifier, scl, ln)
}()
slog.Gm.Register(stream.Name(src))
for {
conn, err := ln.Accept()
if err != nil {
hardClose := false
softClose := false
select {
case _, ok := <-src.StopNotifier:
if !ok {
hardClose = true
}
case _, ok := <-scl:
if !ok {
softClose = true
}
default:
}
if !hardClose && !softClose {
slog.Logf(logger.Levels.Error, "Accept Error %v", err)
}
return nil
}
wg_sub.Add(1)
wg_scl.Add(1)
if first_connection {
first_connection = false
//close scl after all connections exit (need to make sure wg_scl > 1 before launching. Launched once)
if src.EnableSoftClose {
wg_sub.Add(1)
go func() {
defer wg_sub.Done()
softCloserRunner(scl, wg_scl)
}()
}
}
go func() {
defer wg_sub.Done()
defer wg_scl.Done()
defer conn.Close() //handle connection will close conn because of reader and writer. But just as good coding practice
src.handleConnection(conn)
}()
}
}
func (src *Client) connect() error {
defer func() {
src.retries++
}()
conn, err := net.Dial("tcp", src.addr)
if err != nil {
slog.Logf(logger.Levels.Error, "Cannot establish a connection with %s %v", src.addr, err)
return err
}
wg_sub := &sync.WaitGroup{}
defer wg_sub.Wait()
rcvChData := make(chan stream.Object, 10)
receiver := source.NewIOReaderSourceLengthDelim(conn)
receiver.SetOut(rcvChData)
rcvChCloseNotifier := make(chan bool)
wg_sub.Add(1)
go func() {
defer wg_sub.Done()
defer close(rcvChCloseNotifier)
err := receiver.Run()
if err != nil {
slog.Logf(logger.Levels.Error, "Error in client reciever: %v", err)
}
}()
//receiver will be closed by the sender after it is done sending. receiver closed via a hard stop.
writeNotifier := stream.NewNonBlockingProcessedNotifier(2)
sndChData := make(chan stream.Object, src.hwm)
sndChCloseNotifier := make(chan bool)
defer close(sndChData)
sender := sink.NewMultiPartWriterSink(conn)
sender.CompletedNotifier = writeNotifier
sender.SetIn(sndChData)
wg_sub.Add(1)
go func() {
defer receiver.Stop() //close receiver
defer wg_sub.Done()
defer close(sndChCloseNotifier)
err := sender.Run()
if err != nil {
slog.Logf(logger.Levels.Error, "Error in client sender: %v", err)
}
}()
//sender closed by closing the sndChData channel or by a hard stop
if src.buf.Len() > 0 {
leftover := src.buf.Reset()
for i, value := range leftover {
sendData(sndChData, value, i+1)
}
}
timer := src.resetAckTimer()
closing := false
//defer log.Println("Exiting client loop")
opName := stream.Name(src)
writesNotCompleted := uint(0)
for {
upstreamCh := src.In()
if !src.buf.CanAdd() || closing {
//disable upstream listening
upstreamCh = nil
}
if closing && src.buf.Len() == 0 {
sendClose(sndChData, 100)
return nil
}
select {
case msg, ok := <-upstreamCh:
if !ok {
//softClose
//make sure everything was sent
closing = true
} else {
bytes := msg.([]byte)
seq, err := src.buf.Add(bytes)
if err != nil {
slog.Fatalf("Error adding item to buffer %v", err)
return err
}
sendData(sndChData, bytes, seq)
writesNotCompleted += 1
slog.Gm.Event(&opName) // These are batched
//slog.Logf(logger.Levels.Debug, "Sent batch -- length %d seq %d", len(bytes), seq)
}
case cnt := <-writeNotifier.NotificationChannel():
writesNotCompleted -= cnt
if timer == nil {
slog.Logf(logger.Levels.Debug, "Seting timer %v, %v", time.Now(), time.Now().UnixNano())
timer = src.resetAckTimer()
}
case obj, ok := <-rcvChData:
slog.Logf(logger.Levels.Debug, "in Rcv: %v", ok)
if !ok {
return errors.New("Connection to Server was Broken in Recieve Direction")
}
command, seq, _, err := parseMsg(obj.([]byte))
if err != nil {
slog.Fatalf("%v", err)
}
if command == ACK {
if src.processAck(seq) {
timer = src.resetAckTimer()
}
} else {
slog.Fatalf("Unknown Command: %v", command)
}
case <-rcvChCloseNotifier:
//connection threw an eof to the reader?
return errors.New("In Select: Recieve Closed")
case <-sndChCloseNotifier:
return errors.New("Connection to Server was Broken in Send Direction")
case <-timer:
return errors.New(fmt.Sprintf("Time Out Waiting For Ack, %d %v %v", len(rcvChData), time.Now(), time.Now().UnixNano()))
case <-src.StopNotifier:
sender.Stop()
return nil
}
}
}
func (src Server) handleConnection(conn net.Conn) {
wg_sub := &sync.WaitGroup{}
defer wg_sub.Wait()
opName := stream.Name(src)
sndChData := make(chan stream.Object, 100)
sndChCloseNotifier := make(chan bool, 1)
defer close(sndChData)
//side effect: this will close conn on exit
sender := sink.NewMultiPartWriterSink(conn)
sender.SetIn(sndChData)
wg_sub.Add(1)
go func() {
defer wg_sub.Done()
defer close(sndChCloseNotifier)
err := sender.Run()
if err != nil {
slog.Logf(logger.Levels.Error, "Error in server sender %v", err)
}
}()
defer sender.Stop()
//this will actually close conn too
rcvChData := make(chan stream.Object, 100)
receiver := source.NewIOReaderSourceLengthDelim(conn)
receiver.SetOut(rcvChData)
rcvChCloseNotifier := make(chan bool, 1)
wg_sub.Add(1)
go func() {
defer wg_sub.Done()
defer close(rcvChCloseNotifier)
err := receiver.Run()
if err != nil {
slog.Logf(logger.Levels.Error, "Error in server reciever %v", err)
}
}()
defer receiver.Stop()
lastGotAck := 0
lastSentAck := 0
var timer <-chan time.Time
timer = nil
for {
select {
case obj, ok := <-rcvChData:
if !ok {
//send last ack back??
slog.Logf(logger.Levels.Error, "Receive Channel Closed Without Close Message")
return
}
command, seq, payload, err := parseMsg(obj.([]byte))
slog.Gm.Event(&opName)
if err == nil {
if command == DATA {
lastGotAck = seq
if (lastGotAck - lastSentAck) > src.hwm/2 {
sendAck(sndChData, lastGotAck)
lastSentAck = lastGotAck
timer = nil
} else if timer == nil {
slog.Logf(logger.Levels.Debug, "Setting timer %v", time.Now())
timer = time.After(100 * time.Millisecond)
}
src.Out() <- payload
} else if command == CLOSE {
if lastGotAck > lastSentAck {
sendAck(sndChData, lastGotAck)
}
slog.Logf(logger.Levels.Info, "%s", "Server got close")
return
} else {
slog.Fatalf("%v", "Server Got Unknown Command")
}
} else {
slog.Fatalf("Server could not parse packet: %v", err)
}
case <-rcvChCloseNotifier:
if len(rcvChData) > 0 {
continue //drain channel before exiting
}
slog.Logf(logger.Levels.Error, "Client asked for a close on recieve- should not happen, timer is nil = %v, %v", (timer == nil), time.Now())
return
case <-sndChCloseNotifier:
slog.Logf(logger.Levels.Error, "%v", "Server asked for a close on send - should not happen")
return
case <-timer:
sendAck(sndChData, lastGotAck)
lastSentAck = lastGotAck
timer = nil
case <-src.StopNotifier:
return
}
}
}