func (s *SyslogSink) Run(inputChan <-chan *logmessage.Message) {
s.logger.Infof("Syslog Sink %s: Running.", s.drainUrl)
defer s.logger.Errorf("Syslog Sink %s: Stopped. This should never happen", s.drainUrl)
backoffStrategy := retrystrategy.NewExponentialRetryStrategy()
numberOfTries := 0
buffer := sinks.RunTruncatingBuffer(inputChan, 100, s.logger)
timer := time.NewTimer(backoffStrategy(numberOfTries))
defer timer.Stop()
for {
s.logger.Debugf("Syslog Sink %s: Starting loop. Current backoff: %v", s.drainUrl, backoffStrategy(numberOfTries))
timer.Reset(backoffStrategy(numberOfTries))
select {
case <-s.disconnectChannel:
return
case <-timer.C:
}
if !s.syslogWriter.IsConnected() {
s.logger.Debugf("Syslog Sink %s: Not connected. Trying to connect.", s.drainUrl)
err := s.syslogWriter.Connect()
if err != nil {
errorMsg := fmt.Sprintf("Syslog Sink %s: Error when dialing out. Backing off for %v. Err: %v", s.drainUrl, backoffStrategy(numberOfTries+1), err)
numberOfTries++
s.logger.Warnf(errorMsg)
logMessage, err := logmessage.GenerateMessage(logmessage.LogMessage_ERR, errorMsg, s.appId, "LGR")
if err == nil {
s.errorChannel <- logMessage
} else {
s.logger.Warnf("Error marshalling message: %v", err)
}
continue
}
s.logger.Infof("Syslog Sink %s: successfully connected.", s.drainUrl)
s.syslogWriter.SetConnected(true)
numberOfTries = 0
defer s.syslogWriter.Close()
}
s.logger.Debugf("Syslog Sink %s: Waiting for activity\n", s.drainUrl)
message, ok := <-buffer.GetOutputChannel()
if !ok {
s.logger.Debugf("Syslog Sink %s: Closed listener channel detected. Closing.\n", s.drainUrl)
return
}
s.logger.Debugf("Syslog Sink %s: Got %d bytes. Sending data\n", s.drainUrl, message.GetRawMessageLength())
var err error
switch message.GetLogMessage().GetMessageType() {
case logmessage.LogMessage_OUT:
_, err = s.syslogWriter.WriteStdout(message.GetLogMessage().GetMessage(), message.GetLogMessage().GetSourceName(), message.GetLogMessage().GetSourceId(), *message.GetLogMessage().Timestamp)
case logmessage.LogMessage_ERR:
_, err = s.syslogWriter.WriteStderr(message.GetLogMessage().GetMessage(), message.GetLogMessage().GetSourceName(), message.GetLogMessage().GetSourceId(), *message.GetLogMessage().Timestamp)
}
if err != nil {
s.logger.Debugf("Syslog Sink %s: Error when trying to send data to sink. Backing off. Err: %v\n", s.drainUrl, err)
numberOfTries++
s.syslogWriter.SetConnected(false)
} else {
s.logger.Debugf("Syslog Sink %s: Successfully sent data\n", s.drainUrl)
numberOfTries = 0
atomic.AddUint64(s.sentMessageCount, 1)
atomic.AddUint64(s.sentByteCount, uint64(message.GetRawMessageLength()))
}
}
}
{4, 8000},
{5, 16000},
{11, 1024000}, //1.024s
{12, 2048000}, //2.048s
{20, 524288000}, //8m and a bit
{21, 1048576000}, //17m28.576s
{22, 2097152000}, //34m57.152s
{23, 4194304000}, //1h9m54.304s
{24, 4194304000}, //1h9m54.304s
{25, 4194304000}, //1h9m54.304s
{26, 4194304000}, //1h9m54.304s
}
It("should backoff exponentially with different random seeds", func() {
rand.Seed(1)
strategy := retrystrategy.NewExponentialRetryStrategy()
otherStrategy := retrystrategy.NewExponentialRetryStrategy()
Expect(strategy(0).String()).To(Equal("0"))
Expect(otherStrategy(0).String()).To(Equal("0"))
var backoff time.Duration
var oldBackoff time.Duration
var otherBackoff time.Duration
var otherOldBackoff time.Duration
for _, bt := range backoffTests {
delta := int(bt.expected / 10)
for i := 0; i < 10; i++ {
backoff = strategy(bt.backoffCount)
otherBackoff = otherStrategy(bt.backoffCount)