{4, 8000},
{5, 16000},
{11, 1024000}, //1.024s
{12, 2048000}, //2.048s
{20, 524288000}, //8m and a bit
{21, 1048576000}, //17m28.576s
{22, 2097152000}, //34m57.152s
{23, 4194304000}, //1h9m54.304s
{24, 4194304000}, //1h9m54.304s
{25, 4194304000}, //1h9m54.304s
{26, 4194304000}, //1h9m54.304s
}
It("backs off exponentially with different random seeds", func() {
rand.Seed(1)
strategy := retrystrategy.Exponential()
otherStrategy := retrystrategy.Exponential()
Expect(strategy(0).String()).To(Equal("1ms"))
Expect(otherStrategy(0).String()).To(Equal("1ms"))
var backoff time.Duration
var oldBackoff time.Duration
var otherBackoff time.Duration
var otherOldBackoff time.Duration
for _, bt := range backoffTests {
delta := int(bt.expected / 10)
for i := 0; i < 10; i++ {
backoff = strategy(bt.backoffCount)
otherBackoff = otherStrategy(bt.backoffCount)
func (s *SyslogSink) Run(inputChan <-chan *events.Envelope) {
syslogIdentifier := s.Identifier()
s.logger.Infof("Syslog Sink %s: Running.", syslogIdentifier)
defer s.logger.Errorf("Syslog Sink %s: Stopped.", syslogIdentifier)
backoffStrategy := retrystrategy.Exponential()
context := truncatingbuffer.NewLogAllowedContext(s.dropsondeOrigin, syslogIdentifier)
buffer := sinks.RunTruncatingBuffer(inputChan, s.messageDrainBufferSize, context, s.logger, s.disconnectChannel)
timer := time.NewTimer(backoffStrategy(0))
connected := false
defer timer.Stop()
defer s.syslogWriter.Close()
s.logger.Debugf("Syslog Sink %s: Starting loop. Current backoff: %v", syslogIdentifier, backoffStrategy(0))
for {
s.logger.Debugf("Syslog Sink %s: Waiting for activity\n", syslogIdentifier)
select {
case <-s.disconnectChannel:
return
case messageEnvelope, ok := <-buffer.GetOutputChannel():
if !ok {
s.logger.Debugf("Syslog Sink %s: Closed listener channel detected. Closing.\n", syslogIdentifier)
return
}
numberOfTries := 0
for {
for !connected {
s.logger.Debugf("Syslog Sink %s: Not connected. Trying to connect.", syslogIdentifier)
err := s.syslogWriter.Connect()
if err == nil {
s.logger.Infof("Syslog Sink %s: successfully connected.", syslogIdentifier)
connected = true
break
}
sleepDuration := backoffStrategy(numberOfTries)
errorMsg := fmt.Sprintf("Syslog Sink %s: Error when dialing out. Backing off for %v. Err: %v", syslogIdentifier, sleepDuration, err)
s.handleSendError(errorMsg, s.appId)
timer.Reset(sleepDuration)
select {
case <-s.disconnectChannel:
return
case <-timer.C:
}
numberOfTries++
}
err := s.sendLogMessage(messageEnvelope.GetLogMessage())
if err == nil {
connected = true
break
}
s.logger.Debugf("Syslog Sink %s: Error when trying to send data to sink. Backing off. Err: %v\n", syslogIdentifier, err)
connected = false
numberOfTries++
}
}
}
}
func initializeDopplerPool(conf *config.Config, batcher *metricbatcher.MetricBatcher, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
adapter, err := storeAdapterProvider(conf)
if err != nil {
return nil, err
}
backoffStrategy := retrystrategy.Exponential()
err = backoff.Connect(adapter, backoffStrategy, logger, connectionRetries)
if err != nil {
return nil, err
}
var protocols []string
clientPool := make(map[string]clientreader.ClientPool)
writers := make(map[string]eventmarshaller.BatchChainByteWriter)
ip, err := localip.LocalIP()
if err != nil {
return nil, err
}
for protocol := range conf.Protocols {
proto := string(protocol)
protocols = append(protocols, proto)
switch proto {
case "udp":
udpCreator := clientpool.NewUDPClientCreator(logger)
udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
udpPool := clientpool.NewDopplerPool(logger, udpCreator)
udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
clientPool[proto] = udpPool
writers[proto] = udpForwarder
case "tcp":
tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)
tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
dropCounter := batch.NewDroppedCounter(tcpForwarder, batcher, origin, ip, conf)
batchWriter, err := batch.NewWriter(
"tcp",
tcpForwarder,
dropCounter,
conf.TCPBatchSizeBytes,
tcpBatchInterval,
logger,
)
if err != nil {
return nil, err
}
clientPool[proto] = tcpPool
writers[proto] = batchWriter
case "tls":
c := conf.TLSConfig
tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
if err != nil {
return nil, err
}
tlsConfig.ServerName = "doppler"
tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)
tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
dropCounter := batch.NewDroppedCounter(tlsForwarder, batcher, origin, ip, conf)
batchWriter, err := batch.NewWriter(
"tls",
tlsForwarder,
dropCounter,
conf.TCPBatchSizeBytes,
tcpBatchInterval,
logger,
)
if err != nil {
return nil, err
}
clientPool[proto] = tlsPool
writers[proto] = batchWriter
}
}
finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, conf.Protocols.Strings(), conf.Zone, logger)
finder.Start()
marshaller := eventmarshaller.New(batcher, logger)
go func() {
for {
protocol := clientreader.Read(clientPool, conf.Protocols.Strings(), finder.Next())
logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
marshaller.SetWriter(writers[protocol])
}
}()
return marshaller, nil
}
"doppler/sinks/retrystrategy"
"metron/backoff"
"time"
"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("Backoff", func() {
Context("connects successfully", func() {
It("does not return error", func() {
fakeLogger := loggertesthelper.Logger()
mockAdapter := backoff.NewMockAdapter()
retryStrategy := retrystrategy.Exponential()
err := backoff.Connect(mockAdapter, retryStrategy, fakeLogger, 3)
Expect(err).ToNot(HaveOccurred())
})
})
Context("unsuccessfully connects", func() {
It("connects eventually using backoff strategy", func() {
fakeLogger := loggertesthelper.Logger()
mockAdapter := backoff.NewMockAdapter()
mockAdapter.ConnectErr("Etcd connection error")
retryStrategy := retrystrategy.Exponential()