func (src *Client) Run() error { src.running = true defer func() { src.running = false }() metrics.Gm.Register(stream.Name(src)) go func(op string, s *Client) { // Update the queue depth on input for each phase for { metrics.Gm.Update(&op, s.GetInDepth()) time.Sleep(1 * time.Second) } }(stream.Name(src), src) for src.retries < RETRY_MAX { err := src.connect() if err == nil { slog.Warnf("Connection failed without error") return err } else { slog.Errorf("Connection failed with error, retrying: %s", err) time.Sleep(1 * time.Second) } } slog.Errorf("Connection failed retries exceeded. Leftover: %d", src.buf.Len()) return nil //>>>>>>>>>>>>>>??????????????????????? }
func (src *Client) connect() error { defer func() { src.retries++ }() conn, err := net.Dial("tcp", src.addr) if err != nil { slog.Errorf("Cannot establish a connection with %s %v", src.addr, err) return err } wg_sub := &sync.WaitGroup{} defer wg_sub.Wait() rcvChData := make(chan stream.Object, 10) receiver := source.NewIOReaderSourceLengthDelim(conn) receiver.SetOut(rcvChData) rcvChCloseNotifier := make(chan bool) wg_sub.Add(1) go func() { defer wg_sub.Done() defer close(rcvChCloseNotifier) err := receiver.Run() if err != nil { slog.Errorf("Error in client reciever: %v", err) } }() //receiver will be closed by the sender after it is done sending. receiver closed via a hard stop. writeNotifier := stream.NewNonBlockingProcessedNotifier(2) sndChData := make(chan stream.Object, src.hwm) sndChCloseNotifier := make(chan bool) defer close(sndChData) sender := sink.NewMultiPartWriterSink(conn) sender.CompletedNotifier = writeNotifier sender.SetIn(sndChData) wg_sub.Add(1) go func() { defer receiver.Stop() //close receiver defer wg_sub.Done() defer close(sndChCloseNotifier) err := sender.Run() if err != nil { slog.Errorf("Error in client sender: %v", err) } }() //sender closed by closing the sndChData channel or by a hard stop if src.buf.Len() > 0 { leftover := src.buf.Reset() for i, value := range leftover { sendData(sndChData, value, i+1) } } timer := src.resetAckTimer() closing := false //defer log.Println("Exiting client loop") opName := stream.Name(src) writesNotCompleted := uint(0) for { upstreamCh := src.In() if !src.buf.CanAdd() || closing { //disable upstream listening upstreamCh = nil } if closing && src.buf.Len() == 0 { sendClose(sndChData, 100) return nil } select { case msg, ok := <-upstreamCh: if !ok { //softClose //make sure everything was sent closing = true } else { bytes := msg.([]byte) seq, err := src.buf.Add(bytes) if err != nil { slog.Fatalf("Error adding item to buffer %v", err) return err } sendData(sndChData, bytes, seq) writesNotCompleted += 1 metrics.Gm.Event(&opName) // These are batched //slog.Logf(logger.Levels.Debug, "Sent batch -- length %d seq %d", len(bytes), seq) } case cnt := <-writeNotifier.NotificationChannel(): writesNotCompleted -= cnt if timer == nil { slog.Debugf("Seting timer %v, %v", time.Now(), time.Now().UnixNano()) timer = src.resetAckTimer() } case obj, ok := <-rcvChData: slog.Debugf("in Rcv: %v", ok) if !ok { return errors.New("Connection to Server was Broken in Recieve Direction") } command, seq, _, err := parseMsg(obj.([]byte)) if err != nil { slog.Fatalf("%v", err) } if command == ACK { if src.processAck(seq) { timer = src.resetAckTimer() } } else { slog.Fatalf("Unknown Command: %v", command) } case <-rcvChCloseNotifier: //connection threw an eof to the reader? return errors.New("In Select: Recieve Closed") case <-sndChCloseNotifier: return errors.New("Connection to Server was Broken in Send Direction") case <-timer: return errors.New(fmt.Sprintf("Time Out Waiting For Ack, %d %v %v", len(rcvChData), time.Now(), time.Now().UnixNano())) case <-src.StopNotifier: sender.Stop() return nil } } }