func (s *SyslogSink) Run(inputChan <-chan *logmessage.Message) { s.logger.Infof("Syslog Sink %s: Running.", s.drainUrl) defer s.logger.Errorf("Syslog Sink %s: Stopped. This should never happen", s.drainUrl) backoffStrategy := retrystrategy.NewExponentialRetryStrategy() numberOfTries := 0 buffer := sinks.RunTruncatingBuffer(inputChan, 100, s.logger) timer := time.NewTimer(backoffStrategy(numberOfTries)) defer timer.Stop() for { s.logger.Debugf("Syslog Sink %s: Starting loop. Current backoff: %v", s.drainUrl, backoffStrategy(numberOfTries)) timer.Reset(backoffStrategy(numberOfTries)) select { case <-s.disconnectChannel: return case <-timer.C: } if !s.syslogWriter.IsConnected() { s.logger.Debugf("Syslog Sink %s: Not connected. Trying to connect.", s.drainUrl) err := s.syslogWriter.Connect() if err != nil { errorMsg := fmt.Sprintf("Syslog Sink %s: Error when dialing out. Backing off for %v. Err: %v", s.drainUrl, backoffStrategy(numberOfTries+1), err) numberOfTries++ s.logger.Warnf(errorMsg) logMessage, err := logmessage.GenerateMessage(logmessage.LogMessage_ERR, errorMsg, s.appId, "LGR") if err == nil { s.errorChannel <- logMessage } else { s.logger.Warnf("Error marshalling message: %v", err) } continue } s.logger.Infof("Syslog Sink %s: successfully connected.", s.drainUrl) s.syslogWriter.SetConnected(true) numberOfTries = 0 defer s.syslogWriter.Close() } s.logger.Debugf("Syslog Sink %s: Waiting for activity\n", s.drainUrl) message, ok := <-buffer.GetOutputChannel() if !ok { s.logger.Debugf("Syslog Sink %s: Closed listener channel detected. Closing.\n", s.drainUrl) return } s.logger.Debugf("Syslog Sink %s: Got %d bytes. Sending data\n", s.drainUrl, message.GetRawMessageLength()) var err error switch message.GetLogMessage().GetMessageType() { case logmessage.LogMessage_OUT: _, err = s.syslogWriter.WriteStdout(message.GetLogMessage().GetMessage(), message.GetLogMessage().GetSourceName(), message.GetLogMessage().GetSourceId(), *message.GetLogMessage().Timestamp) case logmessage.LogMessage_ERR: _, err = s.syslogWriter.WriteStderr(message.GetLogMessage().GetMessage(), message.GetLogMessage().GetSourceName(), message.GetLogMessage().GetSourceId(), *message.GetLogMessage().Timestamp) } if err != nil { s.logger.Debugf("Syslog Sink %s: Error when trying to send data to sink. Backing off. Err: %v\n", s.drainUrl, err) numberOfTries++ s.syslogWriter.SetConnected(false) } else { s.logger.Debugf("Syslog Sink %s: Successfully sent data\n", s.drainUrl) numberOfTries = 0 atomic.AddUint64(s.sentMessageCount, 1) atomic.AddUint64(s.sentByteCount, uint64(message.GetRawMessageLength())) } } }
{4, 8000}, {5, 16000}, {11, 1024000}, //1.024s {12, 2048000}, //2.048s {20, 524288000}, //8m and a bit {21, 1048576000}, //17m28.576s {22, 2097152000}, //34m57.152s {23, 4194304000}, //1h9m54.304s {24, 4194304000}, //1h9m54.304s {25, 4194304000}, //1h9m54.304s {26, 4194304000}, //1h9m54.304s } It("should backoff exponentially with different random seeds", func() { rand.Seed(1) strategy := retrystrategy.NewExponentialRetryStrategy() otherStrategy := retrystrategy.NewExponentialRetryStrategy() Expect(strategy(0).String()).To(Equal("0")) Expect(otherStrategy(0).String()).To(Equal("0")) var backoff time.Duration var oldBackoff time.Duration var otherBackoff time.Duration var otherOldBackoff time.Duration for _, bt := range backoffTests { delta := int(bt.expected / 10) for i := 0; i < 10; i++ { backoff = strategy(bt.backoffCount) otherBackoff = otherStrategy(bt.backoffCount)