{4, 8000},
			{5, 16000},
			{11, 1024000},    //1.024s
			{12, 2048000},    //2.048s
			{20, 524288000},  //8m and a bit
			{21, 1048576000}, //17m28.576s
			{22, 2097152000}, //34m57.152s
			{23, 4194304000}, //1h9m54.304s
			{24, 4194304000}, //1h9m54.304s
			{25, 4194304000}, //1h9m54.304s
			{26, 4194304000}, //1h9m54.304s
		}

		It("backs off exponentially with different random seeds", func() {
			rand.Seed(1)
			strategy := retrystrategy.Exponential()
			otherStrategy := retrystrategy.Exponential()

			Expect(strategy(0).String()).To(Equal("1ms"))
			Expect(otherStrategy(0).String()).To(Equal("1ms"))

			var backoff time.Duration
			var oldBackoff time.Duration
			var otherBackoff time.Duration
			var otherOldBackoff time.Duration

			for _, bt := range backoffTests {
				delta := int(bt.expected / 10)
				for i := 0; i < 10; i++ {
					backoff = strategy(bt.backoffCount)
					otherBackoff = otherStrategy(bt.backoffCount)
func (s *SyslogSink) Run(inputChan <-chan *events.Envelope) {
	syslogIdentifier := s.Identifier()
	s.logger.Infof("Syslog Sink %s: Running.", syslogIdentifier)
	defer s.logger.Errorf("Syslog Sink %s: Stopped.", syslogIdentifier)

	backoffStrategy := retrystrategy.Exponential()

	context := truncatingbuffer.NewLogAllowedContext(s.dropsondeOrigin, syslogIdentifier)
	buffer := sinks.RunTruncatingBuffer(inputChan, s.messageDrainBufferSize, context, s.logger, s.disconnectChannel)
	timer := time.NewTimer(backoffStrategy(0))
	connected := false
	defer timer.Stop()
	defer s.syslogWriter.Close()

	s.logger.Debugf("Syslog Sink %s: Starting loop. Current backoff: %v", syslogIdentifier, backoffStrategy(0))
	for {
		s.logger.Debugf("Syslog Sink %s: Waiting for activity\n", syslogIdentifier)

		select {
		case <-s.disconnectChannel:
			return
		case messageEnvelope, ok := <-buffer.GetOutputChannel():
			if !ok {
				s.logger.Debugf("Syslog Sink %s: Closed listener channel detected. Closing.\n", syslogIdentifier)
				return
			}

			numberOfTries := 0
			for {
				for !connected {
					s.logger.Debugf("Syslog Sink %s: Not connected. Trying to connect.", syslogIdentifier)
					err := s.syslogWriter.Connect()
					if err == nil {
						s.logger.Infof("Syslog Sink %s: successfully connected.", syslogIdentifier)
						connected = true
						break
					}

					sleepDuration := backoffStrategy(numberOfTries)
					errorMsg := fmt.Sprintf("Syslog Sink %s: Error when dialing out. Backing off for %v. Err: %v", syslogIdentifier, sleepDuration, err)

					s.handleSendError(errorMsg, s.appId)

					timer.Reset(sleepDuration)
					select {
					case <-s.disconnectChannel:
						return
					case <-timer.C:
					}

					numberOfTries++
				}

				err := s.sendLogMessage(messageEnvelope.GetLogMessage())
				if err == nil {
					connected = true
					break
				}

				s.logger.Debugf("Syslog Sink %s: Error when trying to send data to sink. Backing off. Err: %v\n", syslogIdentifier, err)
				connected = false
				numberOfTries++
			}
		}
	}
}
Beispiel #3
0
func initializeDopplerPool(conf *config.Config, batcher *metricbatcher.MetricBatcher, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
	adapter, err := storeAdapterProvider(conf)
	if err != nil {
		return nil, err
	}

	backoffStrategy := retrystrategy.Exponential()
	err = backoff.Connect(adapter, backoffStrategy, logger, connectionRetries)
	if err != nil {
		return nil, err
	}

	var protocols []string
	clientPool := make(map[string]clientreader.ClientPool)
	writers := make(map[string]eventmarshaller.BatchChainByteWriter)

	ip, err := localip.LocalIP()
	if err != nil {
		return nil, err
	}

	for protocol := range conf.Protocols {
		proto := string(protocol)
		protocols = append(protocols, proto)
		switch proto {
		case "udp":
			udpCreator := clientpool.NewUDPClientCreator(logger)
			udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
			udpPool := clientpool.NewDopplerPool(logger, udpCreator)
			udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
			clientPool[proto] = udpPool
			writers[proto] = udpForwarder
		case "tcp":
			tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
			tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
			tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)

			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond

			dropCounter := batch.NewDroppedCounter(tcpForwarder, batcher, origin, ip, conf)
			batchWriter, err := batch.NewWriter(
				"tcp",
				tcpForwarder,
				dropCounter,
				conf.TCPBatchSizeBytes,
				tcpBatchInterval,
				logger,
			)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tcpPool
			writers[proto] = batchWriter
		case "tls":
			c := conf.TLSConfig
			tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
			if err != nil {
				return nil, err
			}
			tlsConfig.ServerName = "doppler"
			tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
			tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
			tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)
			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond

			dropCounter := batch.NewDroppedCounter(tlsForwarder, batcher, origin, ip, conf)
			batchWriter, err := batch.NewWriter(
				"tls",
				tlsForwarder,
				dropCounter,
				conf.TCPBatchSizeBytes,
				tcpBatchInterval,
				logger,
			)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tlsPool
			writers[proto] = batchWriter
		}
	}

	finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, conf.Protocols.Strings(), conf.Zone, logger)
	finder.Start()

	marshaller := eventmarshaller.New(batcher, logger)

	go func() {
		for {
			protocol := clientreader.Read(clientPool, conf.Protocols.Strings(), finder.Next())
			logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
			marshaller.SetWriter(writers[protocol])
		}
	}()

	return marshaller, nil
}
	"doppler/sinks/retrystrategy"
	"metron/backoff"
	"time"

	"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Backoff", func() {
	Context("connects successfully", func() {
		It("does not return error", func() {
			fakeLogger := loggertesthelper.Logger()
			mockAdapter := backoff.NewMockAdapter()

			retryStrategy := retrystrategy.Exponential()

			err := backoff.Connect(mockAdapter, retryStrategy, fakeLogger, 3)
			Expect(err).ToNot(HaveOccurred())

		})
	})

	Context("unsuccessfully connects", func() {
		It("connects eventually using backoff strategy", func() {
			fakeLogger := loggertesthelper.Logger()
			mockAdapter := backoff.NewMockAdapter()

			mockAdapter.ConnectErr("Etcd connection error")
			retryStrategy := retrystrategy.Exponential()