func main() {
// Metron is intended to be light-weight so we occupy only one core
runtime.GOMAXPROCS(1)
flag.Parse()
config, logger := parseConfig(*debug, *configFilePath, *logFilePath)
dopplerClientPool := initializeClientPool(config, logger)
dopplerForwarder := dopplerforwarder.New(dopplerClientPool, logger)
byteSigner := signer.New(config.SharedSecret, dopplerForwarder)
marshaller := eventmarshaller.New(byteSigner, logger)
messageTagger := tagger.New(config.Deployment, config.Job, config.Index, marshaller)
aggregator := messageaggregator.New(messageTagger, logger)
initializeMetrics(byteSigner, config, logger)
dropsondeUnmarshaller := eventunmarshaller.New(aggregator, logger)
dropsondeReader := networkreader.New(fmt.Sprintf("localhost:%d", config.DropsondeIncomingMessagesPort), "dropsondeAgentListener", dropsondeUnmarshaller, logger)
// TODO: remove next four lines when legacy support is removed (or extracted to injector)
legacyMarshaller := eventmarshaller.New(byteSigner, logger)
legacyMessageTagger := tagger.New(config.Deployment, config.Job, config.Index, legacyMarshaller)
legacyUnmarshaller := legacyunmarshaller.New(legacyMessageTagger, logger)
legacyReader := networkreader.New(fmt.Sprintf("localhost:%d", config.LegacyIncomingMessagesPort), "legacyAgentListener", legacyUnmarshaller, logger)
go legacyReader.Start()
dropsondeReader.Start()
}
func main() {
// Metron is intended to be light-weight so we occupy only one core
runtime.GOMAXPROCS(1)
flag.Parse()
config, err := config.ParseConfig(*configFilePath)
if err != nil {
panic(err)
}
log := logger.NewLogger(*debug, *logFilePath, "metron", config.Syslog)
log.Info("Startup: Setting up the Metron agent")
dopplerClientPool, err := initializeClientPool(config, log)
if err != nil {
log.Errorf("Error while initializing client pool: %s", err.Error())
os.Exit(-1)
}
dopplerForwarder := dopplerforwarder.New(dopplerClientPool, log)
byteSigner := signer.New(config.SharedSecret, dopplerForwarder)
marshaller := eventmarshaller.New(byteSigner, log)
messageTagger := tagger.New(config.Deployment, config.Job, config.Index, marshaller)
aggregator := messageaggregator.New(messageTagger, log)
initializeMetrics(byteSigner, config, log)
dropsondeUnmarshaller := eventunmarshaller.New(aggregator, log)
dropsondeReader := networkreader.New(fmt.Sprintf("localhost:%d", config.DropsondeIncomingMessagesPort), "dropsondeAgentListener", dropsondeUnmarshaller, log)
log.Info("metron started")
dropsondeReader.Start()
}
func main() {
flag.Parse()
config, logger := parseConfig(*debug, *configFilePath, *logFilePath)
dopplerClientPool := initializeClientPool(config, logger)
dopplerForwarder := dopplerforwarder.New(dopplerClientPool, logger)
byteSigner := signer.New(config.SharedSecret, dopplerForwarder)
marshaller := eventmarshaller.New(byteSigner, logger)
varzShim := varzforwarder.New(config.Job, metricTTL, marshaller, logger)
messageTagger := tagger.New(config.Deployment, config.Job, config.Index, varzShim)
aggregator := messageaggregator.New(messageTagger, logger)
dropsondeUnmarshaller := eventunmarshaller.New(aggregator, logger)
dropsondeReader := networkreader.New(fmt.Sprintf("localhost:%d", config.DropsondeIncomingMessagesPort), "dropsondeAgentListener", dropsondeUnmarshaller, logger)
// TODO: remove next two lines when legacy support is removed (or extracted to injector)
legacyUnmarshaller := legacyunmarshaller.New(aggregator, logger)
legacyReader := networkreader.New(fmt.Sprintf("localhost:%d", config.LegacyIncomingMessagesPort), "legacyAgentListener", legacyUnmarshaller, logger)
instrumentables := []instrumentation.Instrumentable{
legacyReader,
dropsondeReader,
legacyUnmarshaller,
dropsondeUnmarshaller,
aggregator,
varzShim,
marshaller,
}
go startMonitoringEndpoints(config, instrumentables, logger)
go legacyReader.Start()
dropsondeReader.Start()
}
func main() {
// Metron is intended to be light-weight so we occupy only one core
runtime.GOMAXPROCS(1)
flag.Parse()
config, err := config.ParseConfig(*configFilePath)
if err != nil {
panic(err)
}
log := logger.NewLogger(*debug, *logFilePath, "metron", config.Syslog)
log.Info("Startup: Setting up the Metron agent")
dopplerClientPool, err := initializeDopplerPool(config, log)
if err != nil {
log.Errorf("Failed to initialize the doppler pool: %s", err.Error())
os.Exit(-1)
}
dopplerForwarder := dopplerforwarder.New(dopplerClientPool, []byte(config.SharedSecret), uint(config.BufferSize), log)
messageTagger := tagger.New(config.Deployment, config.Job, config.Index, dopplerForwarder)
aggregator := messageaggregator.New(messageTagger, log)
initializeMetrics(messageTagger, config, log)
dropsondeUnmarshaller := eventunmarshaller.New(aggregator, log)
metronAddress := fmt.Sprintf("127.0.0.1:%d", config.DropsondeIncomingMessagesPort)
dropsondeReader, err := networkreader.New(metronAddress, "dropsondeAgentListener", dropsondeUnmarshaller, log)
if err != nil {
log.Errorf("Failed to listen on %s: %s", metronAddress, err)
os.Exit(1)
}
log.Info("metron started")
go dopplerForwarder.Run()
dropsondeReader.Start()
dopplerForwarder.Stop()
}
func main() {
// Put os.Exit in a deferred statement so that other defers get executed prior to
// the os.Exit call.
exitCode := 0
defer func() {
os.Exit(exitCode)
}()
// Metron is intended to be light-weight so we occupy only one core
runtime.GOMAXPROCS(1)
flag.Parse()
config, err := config.ParseConfig(*configFilePath)
if err != nil {
panic(err)
}
log := logger.NewLogger(*debug, *logFilePath, "metron", config.Syslog)
log.Info("Startup: Setting up the Metron agent")
profiler := profiler.New(*cpuprofile, *memprofile, 1*time.Second, log)
profiler.Profile()
defer profiler.Stop()
dopplerClientPool, err := initializeDopplerPool(config, log)
if err != nil {
log.Errorf("Failed to initialize the doppler pool: %s", err.Error())
os.Exit(-1)
}
dopplerForwarder := dopplerforwarder.New(dopplerClientPool, []byte(config.SharedSecret), uint(config.BufferSize), config.EnableBuffer, log)
messageTagger := tagger.New(config.Deployment, config.Job, config.Index, dopplerForwarder)
aggregator := messageaggregator.New(messageTagger, log)
statsStopChan := make(chan struct{})
initializeMetrics(messageTagger, config, statsStopChan, log)
dropsondeUnmarshaller := eventunmarshaller.New(aggregator, log)
metronAddress := fmt.Sprintf("127.0.0.1:%d", config.DropsondeIncomingMessagesPort)
dropsondeReader, err := networkreader.New(metronAddress, "dropsondeAgentListener", dropsondeUnmarshaller, log)
if err != nil {
log.Errorf("Failed to listen on %s: %s", metronAddress, err)
exitCode = 1
return
}
log.Info("metron started")
go dopplerForwarder.Run()
go dropsondeReader.Start()
dumpChan := signalmanager.RegisterGoRoutineDumpSignalChannel()
killChan := signalmanager.RegisterKillSignalChannel()
for {
select {
case <-dumpChan:
signalmanager.DumpGoRoutine()
case <-killChan:
log.Info("Shutting down")
dopplerForwarder.Stop()
close(statsStopChan)
return
}
}
}
func initializeDopplerPool(conf *config.Config, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
adapter, err := storeAdapterProvider(conf.EtcdUrls, conf.EtcdMaxConcurrentRequests)
if err != nil {
return nil, err
}
err = adapter.Connect()
if err != nil {
logger.Warnd(map[string]interface{}{
"error": err.Error(),
}, "Failed to connect to etcd")
}
var protocols []string
clientPool := make(map[string]clientreader.ClientPool)
writers := make(map[string]eventmarshaller.ByteWriter)
for _, protocol := range conf.Protocols {
proto := string(protocol)
protocols = append(protocols, proto)
switch proto {
case "udp":
udpCreator := clientpool.NewUDPClientCreator(logger)
udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
udpPool := clientpool.NewDopplerPool(logger, udpCreator)
udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
clientPool[proto] = udpPool
writers[proto] = udpForwarder
case "tcp":
tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)
tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
batchWriter, err := batch.NewWriter(tcpForwarder, conf.TCPBatchSizeBytes, tcpBatchInterval, logger)
if err != nil {
return nil, err
}
clientPool[proto] = tcpPool
writers[proto] = batchWriter
case "tls":
c := conf.TLSConfig
tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
if err != nil {
return nil, err
}
tlsConfig.ServerName = "doppler"
tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)
tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
batchWriter, err := batch.NewWriter(tlsForwarder, conf.TCPBatchSizeBytes, tcpBatchInterval, logger)
if err != nil {
return nil, err
}
clientPool[proto] = tlsPool
writers[proto] = batchWriter
}
}
finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, protocols, conf.Zone, logger)
marshaller := eventmarshaller.New(logger)
finder.Start()
go func() {
for {
protocol := clientreader.Read(clientPool, protocols, finder.Next())
logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
marshaller.SetWriter(writers[protocol])
}
}()
return marshaller, nil
}
import (
"metron/writers/dopplerforwarder"
"github.com/cloudfoundry/loggregatorlib/cfcomponent/instrumentation"
"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
"github.com/cloudfoundry/loggregatorlib/loggregatorclient"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("DopplerForwarder", func() {
It("sends messages to a random doppler", func() {
clientPool := &mockClientPool{}
logger := loggertesthelper.Logger()
s := dopplerforwarder.New(clientPool, logger)
message := []byte("Some message")
s.Write(message)
Expect(clientPool.randomClient).ToNot(BeNil())
data := clientPool.randomClient.data
Expect(data).To(HaveLen(1))
Expect(data[0]).To(Equal(message))
})
})
type mockClientPool struct {
randomClient *mockClient
}
BeforeEach(func() {
message = []byte("I am a message!")
client = newMockClient()
clientPool = newMockClientPool()
clientPool.RandomClientOutput.Client <- client
close(clientPool.RandomClientOutput.Err)
logger = loggertesthelper.Logger()
loggertesthelper.TestLoggerSink.Clear()
fakeWrapper = newMockNetworkWrapper()
})
JustBeforeEach(func() {
forwarder = dopplerforwarder.New(fakeWrapper, clientPool, logger)
})
Context("client selection", func() {
It("selects a random client", func() {
close(fakeWrapper.WriteOutput.Ret0)
_, err := forwarder.Write(message)
Expect(err).ToNot(HaveOccurred())
Eventually(fakeWrapper.WriteInput.Client).Should(Receive(Equal(client)))
Eventually(fakeWrapper.WriteInput.Message).Should(Receive(Equal(message)))
})
It("passes any chainers to the wrapper", func() {
chainers := []metricbatcher.BatchCounterChainer{
newMockBatchCounterChainer(),
newMockBatchCounterChainer(),
bufferSize uint
)
BeforeEach(func() {
sender = fake.NewFakeMetricSender()
metrics.Initialize(sender, metricbatcher.New(sender, time.Millisecond*10))
client = &fakeclient.FakeClient{}
clientPool = &fakes.FakeClientPool{}
clientPool.RandomClientReturns(client, nil)
logger = loggertesthelper.Logger()
loggertesthelper.TestLoggerSink.Clear()
bufferSize = 10
forwarder = dopplerforwarder.New(clientPool, sharedSecret, bufferSize, logger)
doneChan = make(chan struct{})
go func() {
forwarder.Run()
close(doneChan)
}()
envelope = &events.Envelope{
Origin: proto.String("fake-origin-1"),
EventType: events.Envelope_LogMessage.Enum(),
LogMessage: factories.NewLogMessage(events.LogMessage_OUT, "message", "appid", "sourceType"),
}
})
AfterEach(func() {
forwarder.Stop()
<-doneChan
func initializeDopplerPool(conf *config.Config, batcher *metricbatcher.MetricBatcher, logger *gosteno.Logger) (*eventmarshaller.EventMarshaller, error) {
adapter, err := storeAdapterProvider(conf)
if err != nil {
return nil, err
}
backoffStrategy := retrystrategy.Exponential()
err = backoff.Connect(adapter, backoffStrategy, logger, connectionRetries)
if err != nil {
return nil, err
}
var protocols []string
clientPool := make(map[string]clientreader.ClientPool)
writers := make(map[string]eventmarshaller.BatchChainByteWriter)
ip, err := localip.LocalIP()
if err != nil {
return nil, err
}
for protocol := range conf.Protocols {
proto := string(protocol)
protocols = append(protocols, proto)
switch proto {
case "udp":
udpCreator := clientpool.NewUDPClientCreator(logger)
udpWrapper := dopplerforwarder.NewUDPWrapper([]byte(conf.SharedSecret), logger)
udpPool := clientpool.NewDopplerPool(logger, udpCreator)
udpForwarder := dopplerforwarder.New(udpWrapper, udpPool, logger)
clientPool[proto] = udpPool
writers[proto] = udpForwarder
case "tcp":
tcpCreator := clientpool.NewTCPClientCreator(logger, nil)
tcpWrapper := dopplerforwarder.NewWrapper(logger, proto)
tcpPool := clientpool.NewDopplerPool(logger, tcpCreator)
tcpForwarder := dopplerforwarder.New(tcpWrapper, tcpPool, logger)
tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
dropCounter := batch.NewDroppedCounter(tcpForwarder, batcher, origin, ip, conf)
batchWriter, err := batch.NewWriter(
"tcp",
tcpForwarder,
dropCounter,
conf.TCPBatchSizeBytes,
tcpBatchInterval,
logger,
)
if err != nil {
return nil, err
}
clientPool[proto] = tcpPool
writers[proto] = batchWriter
case "tls":
c := conf.TLSConfig
tlsConfig, err := listeners.NewTLSConfig(c.CertFile, c.KeyFile, c.CAFile)
if err != nil {
return nil, err
}
tlsConfig.ServerName = "doppler"
tlsCreator := clientpool.NewTCPClientCreator(logger, tlsConfig)
tlsWrapper := dopplerforwarder.NewWrapper(logger, proto)
tlsPool := clientpool.NewDopplerPool(logger, tlsCreator)
tlsForwarder := dopplerforwarder.New(tlsWrapper, tlsPool, logger)
tcpBatchInterval := time.Duration(conf.TCPBatchIntervalMilliseconds) * time.Millisecond
dropCounter := batch.NewDroppedCounter(tlsForwarder, batcher, origin, ip, conf)
batchWriter, err := batch.NewWriter(
"tls",
tlsForwarder,
dropCounter,
conf.TCPBatchSizeBytes,
tcpBatchInterval,
logger,
)
if err != nil {
return nil, err
}
clientPool[proto] = tlsPool
writers[proto] = batchWriter
}
}
finder := dopplerservice.NewFinder(adapter, conf.LoggregatorDropsondePort, conf.Protocols.Strings(), conf.Zone, logger)
finder.Start()
marshaller := eventmarshaller.New(batcher, logger)
go func() {
for {
protocol := clientreader.Read(clientPool, conf.Protocols.Strings(), finder.Next())
logger.Infof("Chose protocol %s from last etcd event, updating writer...", protocol)
marshaller.SetWriter(writers[protocol])
}
}()
return marshaller, nil
}
