示例#1
0
func initializeDopplerPool(config *config.Config, logger *gosteno.Logger) (*clientpool.DopplerPool, error) {
	adapter, err := storeAdapterProvider(config.EtcdUrls, config.EtcdMaxConcurrentRequests)
	if err != nil {
		return nil, err
	}
	err = adapter.Connect()
	if err != nil {
		logger.Warnd(map[string]interface{}{
			"error": err.Error(),
		}, "Failed to connect to etcd")
	}

	preferInZone := func(relativePath string) bool {
		return strings.HasPrefix(relativePath, "/"+config.Zone+"/")
	}

	var tlsConfig *tls.Config
	if config.PreferredProtocol == "tls" {
		c := config.TLSConfig
		tlsConfig, err = listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
		if err != nil {
			return nil, err
		}
		tlsConfig.ServerName = "doppler"
	}

	clientPool := clientpool.NewDopplerPool(logger, func(logger *gosteno.Logger, url string) (clientpool.Client, error) {
		client, err := clientpool.NewClient(logger, url, tlsConfig)
		if err == nil && client.Scheme() != config.PreferredProtocol {
			logger.Warnd(map[string]interface{}{
				"url": url,
			}, "Doppler advertising UDP only")
		}
		return client, err
	})

	onUpdate := func(all map[string]string, preferred map[string]string) {
		clientPool.Set(all, preferred)
	}

	dopplers, err := dopplerservice.NewFinder(adapter, config.PreferredProtocol, preferInZone, onUpdate, logger)
	if err != nil {
		return nil, err
	}
	dopplers.Start()

	onLegacyUpdate := func(all map[string]string, preferred map[string]string) {
		clientPool.SetLegacy(all, preferred)
	}

	legacyDopplers := dopplerservice.NewLegacyFinder(adapter, config.LoggregatorDropsondePort, preferInZone, onLegacyUpdate, logger)
	legacyDopplers.Start()

	return clientPool, nil
}
示例#2
0
func DialTLS(address, cert, key, ca string) (*tls.Conn, error) {
	tlsConfig, err := listeners.NewTLSConfig("../fixtures/client.crt", "../fixtures/client.key", "../fixtures/loggregator-ca.crt")
	Expect(err).NotTo(HaveOccurred())
	tlsConfig.ServerName = "doppler"
	return tls.Dial("tcp", address, tlsConfig)
}
示例#3
0
func initializeDopplerPool(conf *config.Config, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
	adapter, err := storeAdapterProvider(conf.EtcdUrls, conf.EtcdMaxConcurrentRequests)
	if err != nil {
		return nil, err
	}
	err = adapter.Connect()
	if err != nil {
		logger.Warnd(map[string]interface{}{
			"error": err.Error(),
		}, "Failed to connect to etcd")
	}

	var protocols []string

	clientPool := make(map[string]clientreader.ClientPool)
	writers := make(map[string]eventmarshaller.ByteWriter)

	for _, protocol := range conf.Protocols {
		proto := string(protocol)
		protocols = append(protocols, proto)
		switch proto {
		case "udp":
			udpCreator := clientpool.NewUDPClientCreator(logger)
			udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
			udpPool := clientpool.NewDopplerPool(logger, udpCreator)
			udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
			clientPool[proto] = udpPool
			writers[proto] = udpForwarder
		case "tcp":
			tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
			tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
			tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)

			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
			batchWriter, err := batch.NewWriter(tcpForwarder, conf.TCPBatchSizeBytes, tcpBatchInterval, logger)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tcpPool
			writers[proto] = batchWriter
		case "tls":
			c := conf.TLSConfig
			tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
			if err != nil {
				return nil, err
			}
			tlsConfig.ServerName = "doppler"
			tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
			tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
			tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)

			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
			batchWriter, err := batch.NewWriter(tlsForwarder, conf.TCPBatchSizeBytes, tcpBatchInterval, logger)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tlsPool
			writers[proto] = batchWriter
		}
	}

	finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, protocols, conf.Zone, logger)
	marshaller := eventmarshaller.New(logger)
	finder.Start()

	go func() {
		for {
			protocol := clientreader.Read(clientPool, protocols, finder.Next())
			logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
			marshaller.SetWriter(writers[protocol])
		}
	}()
	return marshaller, nil
}
示例#4
0
	"github.com/gogo/protobuf/proto"
)

var _ = Describe("TLSReader", func() {
	var (
		port         int = 3457
		reader       *messagereader.TLSReader
		serverConfig *tls.Config
		clientConfig *tls.Config
	)

	BeforeEach(func() {
		var err error
		serverConfig, err = listeners.NewTLSConfig(
			"fixtures/server.crt",
			"fixtures/server.key",
			"fixtures/loggregator-ca.crt",
		)
		Expect(err).ToNot(HaveOccurred())
		serverConfig.InsecureSkipVerify = true
		clientConfig, err = listeners.NewTLSConfig(
			"fixtures/client.crt",
			"fixtures/client.key",
			"fixtures/loggregator-ca.crt",
		)
		Expect(err).ToNot(HaveOccurred())
		clientConfig.InsecureSkipVerify = true
		reader = messagereader.NewTLS(port, serverConfig)
	})

	AfterEach(func() {
示例#5
0
var _ = Describe("TLSlistener", func() {
	var envelopeChan chan *events.Envelope
	var tlsListener listeners.Listener
	var tlsListenerConfig config.TLSListenerConfig
	var tlsClientConfig *tls.Config

	BeforeEach(func() {
		tlsListenerConfig = config.TLSListenerConfig{
			CertFile: "fixtures/server.crt",
			KeyFile:  "fixtures/server.key",
			CAFile:   "fixtures/loggregator-ca.crt",
		}

		var err error
		tlsClientConfig, err = listeners.NewTLSConfig("fixtures/client.crt", "fixtures/client.key", "fixtures/loggregator-ca.crt")
		Expect(err).NotTo(HaveOccurred())
		tlsClientConfig.ServerName = "doppler"

		envelopeChan = make(chan *events.Envelope)
	})

	JustBeforeEach(func() {
		var err error
		tlsListener, err = listeners.NewTLSListener("aname", "127.0.0.1:0", tlsListenerConfig, envelopeChan, loggertesthelper.Logger())
		Expect(err).NotTo(HaveOccurred())
		go tlsListener.Start()
	})

	AfterEach(func() {
		tlsListener.Stop()
示例#6
0
func initializeDopplerPool(conf *config.Config, batcher *metricbatcher.MetricBatcher, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
	adapter, err := storeAdapterProvider(conf)
	if err != nil {
		return nil, err
	}

	backoffStrategy := retrystrategy.Exponential()
	err = backoff.Connect(adapter, backoffStrategy, logger, connectionRetries)
	if err != nil {
		return nil, err
	}

	var protocols []string
	clientPool := make(map[string]clientreader.ClientPool)
	writers := make(map[string]eventmarshaller.BatchChainByteWriter)

	ip, err := localip.LocalIP()
	if err != nil {
		return nil, err
	}

	for protocol := range conf.Protocols {
		proto := string(protocol)
		protocols = append(protocols, proto)
		switch proto {
		case "udp":
			udpCreator := clientpool.NewUDPClientCreator(logger)
			udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
			udpPool := clientpool.NewDopplerPool(logger, udpCreator)
			udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
			clientPool[proto] = udpPool
			writers[proto] = udpForwarder
		case "tcp":
			tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
			tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
			tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)

			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond

			dropCounter := batch.NewDroppedCounter(tcpForwarder, batcher, origin, ip, conf)
			batchWriter, err := batch.NewWriter(
				"tcp",
				tcpForwarder,
				dropCounter,
				conf.TCPBatchSizeBytes,
				tcpBatchInterval,
				logger,
			)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tcpPool
			writers[proto] = batchWriter
		case "tls":
			c := conf.TLSConfig
			tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
			if err != nil {
				return nil, err
			}
			tlsConfig.ServerName = "doppler"
			tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
			tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
			tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
			tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)
			tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond

			dropCounter := batch.NewDroppedCounter(tlsForwarder, batcher, origin, ip, conf)
			batchWriter, err := batch.NewWriter(
				"tls",
				tlsForwarder,
				dropCounter,
				conf.TCPBatchSizeBytes,
				tcpBatchInterval,
				logger,
			)
			if err != nil {
				return nil, err
			}
			clientPool[proto] = tlsPool
			writers[proto] = batchWriter
		}
	}

	finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, conf.Protocols.Strings(), conf.Zone, logger)
	finder.Start()

	marshaller := eventmarshaller.New(batcher, logger)

	go func() {
		for {
			protocol := clientreader.Read(clientPool, conf.Protocols.Strings(), finder.Next())
			logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
			marshaller.SetWriter(writers[protocol])
		}
	}()

	return marshaller, nil
}
示例#7
0
func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	var (
		interval          time.Duration
		writeRate         int
		stopAfter         time.Duration
		concurrentWriters int
		eventType         events.Envelope_EventType
		protocol          string
		serverCert        string
		serverKey         string
		caCert            string
		burstDelay        time.Duration
	)

	flag.Var(newDurationValue(&interval, time.Second), "interval", "Interval for reported results")
	flag.IntVar(&writeRate, "writeRate", 15000, "Number of writes per second to send to metron")
	flag.Var(newDurationValue(&stopAfter, 5*time.Minute), "stopAfter", "How long to run the experiment for")
	flag.IntVar(&concurrentWriters, "concurrentWriters", 1, "Number of concurrent writers")
	flag.Var(newEventTypeValue(&eventType, events.Envelope_ValueMetric), "eventType", "The event type to test")
	flag.StringVar(&protocol, "protocol", "udp", "The protocol to configure metron to send messages over")
	flag.StringVar(&serverCert, "serverCert", "../../integration_tests/fixtures/server.crt", "The server cert file (for TLS connections)")
	flag.StringVar(&serverKey, "serverKey", "../../integration_tests/fixtures/server.key", "The server key file (for TLS connections)")
	flag.StringVar(&caCert, "caCert", "../../integration_tests/fixtures/loggregator-ca.crt", "The certificate authority cert file (for TLS connections)")
	flag.Var(newDurationValue(&burstDelay, 0), "burstDelay", "The delay between burst sequences.  If this is non-zero then writeRate is used as the number of messages to send each burst.")

	flag.Parse()

	reporter := metricsreporter.New(interval, os.Stdout)
	var generator writestrategies.MessageGenerator
	switch eventType {
	case events.Envelope_ValueMetric:
		generator = messagegenerator.NewValueMetricGenerator()
	case events.Envelope_LogMessage:
		generator = messagegenerator.NewLogMessageGenerator("fake-app-id")
	default:
		log.Fatalf("Unsupported envelope type: %v", eventType)
	}
	var dopplerURLs []string
	var reader eventtypereader.MessageReader
	switch protocol {
	case "udp":
		reader = messagereader.NewUDP(3457)
		dopplerURLs = []string{"udp://127.0.0.1:3457"}
	case "tls":
		tlsConfig, err := listeners.NewTLSConfig(
			serverCert,
			serverKey,
			caCert,
		)
		if err != nil {
			log.Printf("Error: failed to load TLS config: %s", err)
			os.Exit(1)
		}
		tlsConfig.InsecureSkipVerify = true
		reader = messagereader.NewTLS(3458, tlsConfig)
		dopplerURLs = []string{"tls://127.0.0.1:3458"}
	default:
		panic(fmt.Errorf("Unknown protocol %s", protocol))
	}
	valueMetricReader := eventtypereader.New(reporter.ReceivedCounter(), reader, eventType, "test-origin")
	exp := experiment.NewExperiment(valueMetricReader)

	for i := 0; i < concurrentWriters; i++ {
		writer := messagewriter.NewMessageWriter("localhost", 51161, "", reporter.SentCounter())
		writeStrategy := chooseStrategy(generator, writer, writeRate, burstDelay)
		exp.AddWriteStrategy(writeStrategy)
	}

	adapter := announceToEtcd(dopplerURLs...)
	defer func() {
		exp.Stop()
		reporter.Stop()
		err := adapter.Delete(nodeKey)
		if err != nil {
			log.Printf("Warning: Failed to delete etcd key %s: %s", nodeKey, err)
		}
		adapter.Disconnect()
	}()

	exp.Warmup()
	go reporter.Start()
	go exp.Start()

	timer := time.NewTimer(stopAfter)
	<-timer.C
}