Example #1
0
func initializeDopplerPool(config *config.Config, logger *gosteno.Logger) (*clientpool.DopplerPool, error) {
	adapter, err := storeAdapterProvider(config.EtcdUrls, config.EtcdMaxConcurrentRequests)
	if err != nil {
		return nil, err
	}
	err = adapter.Connect()
	if err != nil {
		logger.Warnd(map[string]interface{}{
			"error": err.Error(),
		}, "Failed to connect to etcd")
	}

	preferInZone := func(relativePath string) bool {
		return strings.HasPrefix(relativePath, "/"+config.Zone+"/")
	}

	var tlsConfig *tls.Config
	if config.PreferredProtocol == "tls" {
		c := config.TLSConfig
		tlsConfig, err = listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
		if err != nil {
			return nil, err
		}
		tlsConfig.ServerName = "doppler"
	}

	clientPool := clientpool.NewDopplerPool(logger, func(logger *gosteno.Logger, url string) (clientpool.Client, error) {
		client, err := clientpool.NewClient(logger, url, tlsConfig)
		if err == nil && client.Scheme() != config.PreferredProtocol {
			logger.Warnd(map[string]interface{}{
				"url": url,
			}, "Doppler advertising UDP only")
		}
		return client, err
	})

	onUpdate := func(all map[string]string, preferred map[string]string) {
		clientPool.Set(all, preferred)
	}

	dopplers, err := dopplerservice.NewFinder(adapter, config.PreferredProtocol, preferInZone, onUpdate, logger)
	if err != nil {
		return nil, err
	}
	dopplers.Start()

	onLegacyUpdate := func(all map[string]string, preferred map[string]string) {
		clientPool.SetLegacy(all, preferred)
	}

	legacyDopplers := dopplerservice.NewLegacyFinder(adapter, config.LoggregatorDropsondePort, preferInZone, onLegacyUpdate, logger)
	legacyDopplers.Start()

	return clientPool, nil
}
Example #2
0
func initializeClientPool(config *config.Config, logger *gosteno.Logger) (*clientpool.DopplerPool, error) {
	adapter := storeAdapterProvider(config.EtcdUrls, config.EtcdMaxConcurrentRequests)
	err := adapter.Connect()
	if err != nil {
		return nil, err
	}

	preferInZone := func(relativePath string) bool {
		return strings.HasPrefix(relativePath, "/"+config.Zone+"/")
	}

	clientPool := clientpool.NewDopplerPool(logger, func(logger *gosteno.Logger, url string) (loggregatorclient.Client, error) {
		client, err := clientpool.NewClient(logger, url)
		if err == nil && client.Scheme() != config.PreferredProtocol {
			logger.Warnd(map[string]interface{}{
				"url": url,
			}, "Doppler advertising UDP only")
		}
		return client, err
	})

	onUpdate := func(all map[string]string, preferred map[string]string) {
		clientPool.Set(all, preferred)
	}

	dopplers, err := dopplerservice.NewFinder(adapter, config.PreferredProtocol, preferInZone, onUpdate, logger)
	if err != nil {
		return nil, err
	}
	dopplers.Start()

	onLegacyUpdate := func(all map[string]string, preferred map[string]string) {
		clientPool.SetLegacy(all, preferred)
	}

	legacyDopplers := dopplerservice.NewLegacyFinder(adapter, config.LoggregatorDropsondePort, preferInZone, onLegacyUpdate, logger)
	legacyDopplers.Start()

	return clientPool, nil
}
			"a": "udp://ahost:aport",
			"b": "udp://bhost:bport",
		}

		preferredLegacy = map[string]string{
			"a": "udp://plahost:plaport",
			"b": "tls://plbhost:plbport",
		}
		allLegacy = map[string]string{
			"a": "udp://lahost:laport",
			"b": "udp://lbhost:lbport",
		}
	})

	JustBeforeEach(func() {
		pool = clientpool.NewDopplerPool(logger, clientFactory)
	})

	Describe("Setters", func() {
		Context("non-Legacy servers", func() {
			Context("with preferred servers", func() {
				It("has only preferred servers", func() {
					pool.Set(allServers, preferredServers)
					Expect(urls(pool.Clients())).To(ConsistOf(values(preferredServers)))
				})
			})

			Context("with no preferred servers", func() {
				It("has only non-preferred servers", func() {
					pool.Set(allServers, nil)
					Expect(urls(pool.Clients())).To(ConsistOf(values(allServers)))
		addresses         []string
	)

	BeforeEach(func() {
		logger = steno.NewLogger("TestLogger")

		addresses = []string{
			"pahost:paport",
			"pbhost:pbport",
		}

		mockClientCreator = newMockClientCreator()
	})

	JustBeforeEach(func() {
		pool = clientpool.NewDopplerPool(logger, mockClientCreator)
	})

	Describe("SetAddresses", func() {

		Context("with successful client creation", func() {

			var clients []clientpool.Client

			BeforeEach(func() {
				clients = []clientpool.Client{
					newMockClient(),
					newMockClient(),
				}
				close(mockClientCreator.CreateClientOutput.err)
				mockClientCreator.CreateClientOutput.client <- clients[0]
Example #5
0
func initializeDopplerPool(conf *config.Config, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
	adapter, err := storeAdapterProvider(conf.EtcdUrls, conf.EtcdMaxConcurrentRequests)
	if err != nil {
		return nil, err
	}
	err = adapter.Connect()
	if err != nil {
		logger.Warnd(map[string]interface{}{
			"error": err.Error(),
		}, "Failed to connect to etcd")
	}

	var protocols []string

	clientPool := make(map[string]clientreader.ClientPool)
	writers := make(map[string]eventmarshaller.ByteWriter)

	for _, protocol := range conf.Protocols {
		proto := string(protocol)
		protocols = append(protocols, proto)
		switch proto {
		case "udp":
			udpCreator := clientpool.NewUDPClientCreator(logger)
			udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
			udpPool := clientpool.NewDopplerPool(logger, udpCreator)
			udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
			clientPool[proto] = udpPool
			writers[proto] = udpForwarder
		case "tcp":
			tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
			tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
			tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)

			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
			batchWriter, err := batch.NewWriter(tcpForwarder, conf.TCPBatchSizeBytes, tcpBatchInterval, logger)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tcpPool
			writers[proto] = batchWriter
		case "tls":
			c := conf.TLSConfig
			tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
			if err != nil {
				return nil, err
			}
			tlsConfig.ServerName = "doppler"
			tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
			tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
			tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)

			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
			batchWriter, err := batch.NewWriter(tlsForwarder, conf.TCPBatchSizeBytes, tcpBatchInterval, logger)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tlsPool
			writers[proto] = batchWriter
		}
	}

	finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, protocols, conf.Zone, logger)
	marshaller := eventmarshaller.New(logger)
	finder.Start()

	go func() {
		for {
			protocol := clientreader.Read(clientPool, protocols, finder.Next())
			logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
			marshaller.SetWriter(writers[protocol])
		}
	}()
	return marshaller, nil
}
Example #6
0
func initializeDopplerPool(conf *config.Config, batcher *metricbatcher.MetricBatcher, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
	adapter, err := storeAdapterProvider(conf)
	if err != nil {
		return nil, err
	}

	backoffStrategy := retrystrategy.Exponential()
	err = backoff.Connect(adapter, backoffStrategy, logger, connectionRetries)
	if err != nil {
		return nil, err
	}

	var protocols []string
	clientPool := make(map[string]clientreader.ClientPool)
	writers := make(map[string]eventmarshaller.BatchChainByteWriter)

	ip, err := localip.LocalIP()
	if err != nil {
		return nil, err
	}

	for protocol := range conf.Protocols {
		proto := string(protocol)
		protocols = append(protocols, proto)
		switch proto {
		case "udp":
			udpCreator := clientpool.NewUDPClientCreator(logger)
			udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
			udpPool := clientpool.NewDopplerPool(logger, udpCreator)
			udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
			clientPool[proto] = udpPool
			writers[proto] = udpForwarder
		case "tcp":
			tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
			tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
			tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)

			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond

			dropCounter := batch.NewDroppedCounter(tcpForwarder, batcher, origin, ip, conf)
			batchWriter, err := batch.NewWriter(
				"tcp",
				tcpForwarder,
				dropCounter,
				conf.TCPBatchSizeBytes,
				tcpBatchInterval,
				logger,
			)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tcpPool
			writers[proto] = batchWriter
		case "tls":
			c := conf.TLSConfig
			tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
			if err != nil {
				return nil, err
			}
			tlsConfig.ServerName = "doppler"
			tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
			tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
			tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)
			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond

			dropCounter := batch.NewDroppedCounter(tlsForwarder, batcher, origin, ip, conf)
			batchWriter, err := batch.NewWriter(
				"tls",
				tlsForwarder,
				dropCounter,
				conf.TCPBatchSizeBytes,
				tcpBatchInterval,
				logger,
			)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tlsPool
			writers[proto] = batchWriter
		}
	}

	finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, conf.Protocols.Strings(), conf.Zone, logger)
	finder.Start()

	marshaller := eventmarshaller.New(batcher, logger)

	go func() {
		for {
			protocol := clientreader.Read(clientPool, conf.Protocols.Strings(), finder.Next())
			logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
			marshaller.SetWriter(writers[protocol])
		}
	}()

	return marshaller, nil
}