func New(host string, config *config.Config, logger *gosteno.Logger, storeAdapter storeadapter.StoreAdapter, messageDrainBufferSize uint, dropsondeOrigin string, dialTimeout time.Duration) *Doppler {
cfcomponent.Logger = logger
keepAliveInterval := 30 * time.Second
appStoreCache := cache.NewAppServiceCache()
appStoreWatcher, newAppServiceChan, deletedAppServiceChan := store.NewAppServiceStoreWatcher(storeAdapter, appStoreCache)
var dropsondeUDPListener agentlistener.Listener
var dropsondeTLSListener agentlistener.Listener
var dropsondeBytesChan <-chan []byte
listenerEnvelopeChan := make(chan *events.Envelope)
if config.EnableTLSTransport {
tlsConfig := &tls.Config{
Certificates: []tls.Certificate{config.TLSListenerConfig.Cert},
InsecureSkipVerify: config.TLSListenerConfig.InsecureSkipVerify,
}
dropsondeTLSListener = tlslistener.New(fmt.Sprintf("%s:%d", host, config.TLSListenerConfig.Port), tlsConfig, listenerEnvelopeChan, logger)
}
dropsondeUDPListener, dropsondeBytesChan = agentlistener.NewAgentListener(fmt.Sprintf("%s:%d", host, config.DropsondeIncomingMessagesPort), logger, "dropsondeListener")
signatureVerifier := signature.NewVerifier(logger, config.SharedSecret)
unmarshallerCollection := dropsonde_unmarshaller.NewDropsondeUnmarshallerCollection(logger, config.UnmarshallerCount)
blacklist := blacklist.New(config.BlackListIps)
metricTTL := time.Duration(config.ContainerMetricTTLSeconds) * time.Second
sinkTimeout := time.Duration(config.SinkInactivityTimeoutSeconds) * time.Second
sinkIOTimeout := time.Duration(config.SinkIOTimeoutSeconds) * time.Second
sinkManager := sinkmanager.New(config.MaxRetainedLogMessages, config.SkipCertVerify, blacklist, logger, messageDrainBufferSize, dropsondeOrigin, sinkTimeout, sinkIOTimeout, metricTTL, dialTimeout)
return &Doppler{
Logger: logger,
dropsondeUDPListener: dropsondeUDPListener,
dropsondeTLSListener: dropsondeTLSListener,
sinkManager: sinkManager,
messageRouter: sinkserver.NewMessageRouter(sinkManager, logger),
websocketServer: websocketserver.New(fmt.Sprintf("%s:%d", host, config.OutgoingPort), sinkManager, keepAliveInterval, config.MessageDrainBufferSize, dropsondeOrigin, logger),
newAppServiceChan: newAppServiceChan,
deletedAppServiceChan: deletedAppServiceChan,
appStoreWatcher: appStoreWatcher,
storeAdapter: storeAdapter,
dropsondeBytesChan: dropsondeBytesChan,
dropsondeUnmarshallerCollection: unmarshallerCollection,
envelopeChan: listenerEnvelopeChan,
wrappedEnvelopeChan: make(chan *events.Envelope),
signatureVerifier: signatureVerifier,
dropsondeVerifiedBytesChan: make(chan []byte),
uptimeMonitor: monitor.NewUptimeMonitor(time.Duration(config.MonitorIntervalSeconds) * time.Second),
}
}
func New(host string, config *config.Config, logger *gosteno.Logger, storeAdapter storeadapter.StoreAdapter, dropsondeOrigin string) *Doppler {
cfcomponent.Logger = logger
keepAliveInterval := 30 * time.Second
appStoreCache := cache.NewAppServiceCache()
appStoreWatcher, newAppServiceChan, deletedAppServiceChan := store.NewAppServiceStoreWatcher(storeAdapter, appStoreCache)
dropsondeListener, dropsondeBytesChan := agentlistener.NewAgentListener(fmt.Sprintf("%s:%d", host, config.DropsondeIncomingMessagesPort), logger, "dropsondeListener")
signatureVerifier := signature.NewVerifier(logger, config.SharedSecret)
unmarshallerCollection := dropsonde_unmarshaller.NewDropsondeUnmarshallerCollection(logger, config.UnmarshallerCount)
blacklist := blacklist.New(config.BlackListIps)
metricTTL := time.Duration(config.ContainerMetricTTLSeconds) * time.Second
sinkTimeout := time.Duration(config.SinkInactivityTimeoutSeconds) * time.Second
sinkManager := sinkmanager.New(config.MaxRetainedLogMessages, config.SkipCertVerify, blacklist, logger, dropsondeOrigin, sinkTimeout, metricTTL)
return &Doppler{
Logger: logger,
dropsondeListener: dropsondeListener,
sinkManager: sinkManager,
messageRouter: sinkserver.NewMessageRouter(sinkManager, logger),
websocketServer: websocketserver.New(fmt.Sprintf("%s:%d", host, config.OutgoingPort), sinkManager, keepAliveInterval, config.WSMessageBufferSize, dropsondeOrigin, logger),
newAppServiceChan: newAppServiceChan,
deletedAppServiceChan: deletedAppServiceChan,
appStoreWatcher: appStoreWatcher,
storeAdapter: storeAdapter,
dropsondeBytesChan: dropsondeBytesChan,
dropsondeUnmarshallerCollection: unmarshallerCollection,
envelopeChan: make(chan *events.Envelope),
wrappedEnvelopeChan: make(chan *events.Envelope),
signatureVerifier: signatureVerifier,
dropsondeVerifiedBytesChan: make(chan []byte),
}
}
"time"
"github.com/cloudfoundry/dropsonde/emitter"
"github.com/cloudfoundry/dropsonde/factories"
"github.com/cloudfoundry/loggregatorlib/appservice"
"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
"github.com/cloudfoundry/sonde-go/events"
"github.com/gogo/protobuf/proto"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("SinkManager", func() {
logger := loggertesthelper.Logger()
var blackListManager = blacklist.New([]iprange.IPRange{iprange.IPRange{Start: "10.10.10.10", End: "10.10.10.20"}}, logger)
var sinkManager *sinkmanager.SinkManager
var sinkManagerDone chan struct{}
var newAppServiceChan, deletedAppServiceChan chan appservice.AppService
BeforeEach(func() {
fakeMetricSender.Reset()
sinkManager = sinkmanager.New(1, true, blackListManager, logger, 100, "dropsonde-origin", 1*time.Second, 0, 1*time.Second, 1*time.Second)
newAppServiceChan = make(chan appservice.AppService)
deletedAppServiceChan = make(chan appservice.AppService)
sinkManagerDone = make(chan struct{})
go func() {
defer close(sinkManagerDone)
sinkManager.Start(newAppServiceChan, deletedAppServiceChan)
"github.com/cloudfoundry/dropsonde/factories"
"github.com/cloudfoundry/loggregatorlib/cfcomponent"
"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
"github.com/cloudfoundry/sonde-go/events"
"github.com/gorilla/websocket"
"github.com/cloudfoundry/dropsonde/emitter"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("WebsocketServer", func() {
var server *websocketserver.WebsocketServer
var sinkManager = sinkmanager.New(1024, false, blacklist.New(nil), loggertesthelper.Logger(), 100, "dropsonde-origin", 1*time.Second, 1*time.Second)
var appId = "my-app"
var wsReceivedChan chan []byte
var connectionDropped <-chan struct{}
var apiEndpoint = "127.0.0.1:9091"
BeforeEach(func() {
logger := loggertesthelper.Logger()
cfcomponent.Logger = logger
wsReceivedChan = make(chan []byte)
server = websocketserver.New(apiEndpoint, sinkManager, 100*time.Millisecond, 100, "dropsonde-origin", logger)
go server.Start()
serverUrl := fmt.Sprintf("ws://%s/apps/%s/stream", apiEndpoint, appId)
websocket.DefaultDialer = &websocket.Dialer{HandshakeTimeout: 10 * time.Millisecond}
Eventually(func() error { _, _, err := websocket.DefaultDialer.Dial(serverUrl, http.Header{}); return err }, 1).ShouldNot(HaveOccurred())
services sync.WaitGroup
goRoutineSpawned sync.WaitGroup
serverPort string
)
BeforeEach(func() {
port := 9081 + config.GinkgoConfig.ParallelNode
serverPort = strconv.Itoa(port)
dataReadChannel = make(chan *events.Envelope, 2)
logger := loggertesthelper.Logger()
newAppServiceChan := make(chan appservice.AppService)
deletedAppServiceChan := make(chan appservice.AppService)
emptyBlacklist := blacklist.New(nil)
sinkManager = sinkmanager.New(1024, false, emptyBlacklist, logger, 100, "dropsonde-origin",
2*time.Second, 0, 1*time.Second, 500*time.Millisecond)
services.Add(1)
goRoutineSpawned.Add(1)
go func() {
goRoutineSpawned.Done()
defer services.Done()
sinkManager.Start(newAppServiceChan, deletedAppServiceChan)
}()
TestMessageRouter = sinkserver.NewMessageRouter(sinkManager, logger)
services.Add(1)
goRoutineSpawned.Add(1)
func New(logger *gosteno.Logger,
host string,
config *config.Config,
storeAdapter storeadapter.StoreAdapter,
messageDrainBufferSize uint,
dropsondeOrigin string,
websocketWriteTimeout time.Duration,
dialTimeout time.Duration) (*Doppler, error) {
keepAliveInterval := 30 * time.Second
appStoreCache := cache.NewAppServiceCache()
appStoreWatcher, newAppServiceChan, deletedAppServiceChan := store.NewAppServiceStoreWatcher(storeAdapter, appStoreCache, logger)
var udpListener listeners.Listener
var tlsListener listeners.Listener
var dropsondeBytesChan <-chan []byte
var err error
listenerEnvelopeChan := make(chan *events.Envelope)
if config.EnableTLSTransport {
tlsListener, err = listeners.NewTLSListener("tlsListener", fmt.Sprintf("%s:%d", host, config.TLSListenerConfig.Port), config.TLSListenerConfig, listenerEnvelopeChan, logger)
if err != nil {
return nil, err
}
}
udpListener, dropsondeBytesChan = listeners.NewUDPListener(fmt.Sprintf("%s:%d", host, config.DropsondeIncomingMessagesPort), logger, "dropsondeListener")
signatureVerifier := signature.NewVerifier(logger, config.SharedSecret)
unmarshallerCollection := dropsonde_unmarshaller.NewDropsondeUnmarshallerCollection(logger, config.UnmarshallerCount)
blacklist := blacklist.New(config.BlackListIps)
metricTTL := time.Duration(config.ContainerMetricTTLSeconds) * time.Second
sinkTimeout := time.Duration(config.SinkInactivityTimeoutSeconds) * time.Second
sinkIOTimeout := time.Duration(config.SinkIOTimeoutSeconds) * time.Second
sinkManager := sinkmanager.New(config.MaxRetainedLogMessages, config.SinkSkipCertVerify, blacklist, logger, messageDrainBufferSize, dropsondeOrigin, sinkTimeout, sinkIOTimeout, metricTTL, dialTimeout)
websocketServer, err := websocketserver.New(fmt.Sprintf("%s:%d", host, config.OutgoingPort), sinkManager, websocketWriteTimeout, keepAliveInterval, config.MessageDrainBufferSize, dropsondeOrigin, logger)
if err != nil {
return nil, fmt.Errorf("Failed to create the websocket server: %s", err.Error())
}
return &Doppler{
Logger: logger,
udpListener: udpListener,
tlsListener: tlsListener,
sinkManager: sinkManager,
messageRouter: sinkserver.NewMessageRouter(sinkManager, logger),
websocketServer: websocketServer,
newAppServiceChan: newAppServiceChan,
deletedAppServiceChan: deletedAppServiceChan,
appStoreWatcher: appStoreWatcher,
storeAdapter: storeAdapter,
dropsondeBytesChan: dropsondeBytesChan,
dropsondeUnmarshallerCollection: unmarshallerCollection,
envelopeChan: listenerEnvelopeChan,
signatureVerifier: signatureVerifier,
dropsondeVerifiedBytesChan: make(chan []byte),
uptimeMonitor: monitor.NewUptimeMonitor(time.Duration(config.MonitorIntervalSeconds) * time.Second),
}, nil
}
import (
"doppler/iprange"
"doppler/sinkserver/blacklist"
"net/url"
"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("UrlBlacklistManager", func() {
var urlBlacklistManager *blacklist.URLBlacklistManager
BeforeEach(func() {
urlBlacklistManager = blacklist.New([]iprange.IPRange{iprange.IPRange{Start: "14.15.16.17", End: "14.15.16.20"}}, loggertesthelper.Logger())
})
Describe("CheckUrl", func() {
It("returns the URL and no error if the URL is valid and not blacklisted", func() {
outputURL, err := urlBlacklistManager.CheckUrl("http://10.10.10.10")
Expect(err).NotTo(HaveOccurred())
url, err := url.ParseRequestURI("http://10.10.10.10")
Expect(err).NotTo(HaveOccurred())
Expect(outputURL).To(Equal(url))
})
It("returns the URL and no error if the domain can't be resolved", func() {
outputURL, err := urlBlacklistManager.CheckUrl("http://some.invalid.host")
Expect(err).NotTo(HaveOccurred())
dataReadChannel chan *events.Envelope
services sync.WaitGroup
serverPort string
)
BeforeEach(func() {
port := 9081 + config.GinkgoConfig.ParallelNode
serverPort = strconv.Itoa(port)
dataReadChannel = make(chan *events.Envelope, 2)
logger := loggertesthelper.Logger()
newAppServiceChan := make(chan appservice.AppService)
deletedAppServiceChan := make(chan appservice.AppService)
emptyBlacklist := blacklist.New(nil, logger)
sinkManager = sinkmanager.New(1024, false, emptyBlacklist, logger, 100, "dropsonde-origin",
2*time.Second, 0, 1*time.Second, 500*time.Millisecond)
tempSink := sinkManager
services.Add(1)
go func() {
defer services.Done()
tempSink.Start(newAppServiceChan, deletedAppServiceChan)
}()
TestMessageRouter = sinkserver.NewMessageRouter(sinkManager, logger)
tempMessageRouter := TestMessageRouter
services.Add(1)
go func() {
"github.com/onsi/ginkgo/config"
. "github.com/onsi/gomega"
"github.com/apoydence/eachers/testhelpers"
"github.com/cloudfoundry/dropsonde/emitter"
"github.com/cloudfoundry/dropsonde/factories"
"github.com/cloudfoundry/loggregatorlib/loggertesthelper"
"github.com/cloudfoundry/sonde-go/events"
"github.com/gorilla/websocket"
)
var _ = Describe("WebsocketServer", func() {
var (
logger = loggertesthelper.Logger()
server *websocketserver.WebsocketServer
sinkManager = sinkmanager.New(1024, false, blacklist.New(nil, logger), logger, 100, "dropsonde-origin", 1*time.Second, 0, 1*time.Second, 500*time.Millisecond)
appId = "my-app"
wsReceivedChan chan []byte
apiEndpoint string
mockBatcher *mockBatcher
mockChainer *mockBatchCounterChainer
)
BeforeEach(func() {
mockBatcher = newMockBatcher()
mockChainer = newMockBatchCounterChainer()
testhelpers.AlwaysReturn(mockBatcher.BatchCounterOutput, mockChainer)
testhelpers.AlwaysReturn(mockChainer.SetTagOutput, mockChainer)
wsReceivedChan = make(chan []byte, 100)
package blacklist_test
import (
"doppler/iprange"
"doppler/sinkserver/blacklist"
"net/url"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("UrlBlacklistManager", func() {
var urlBlacklistManager *blacklist.URLBlacklistManager
BeforeEach(func() {
urlBlacklistManager = blacklist.New([]iprange.IPRange{iprange.IPRange{Start: "14.15.16.17", End: "14.15.16.20"}})
})
Describe("CheckUrl", func() {
It("returns the URL and no error if the URL is valid and not blacklisted", func() {
outputURL, err := urlBlacklistManager.CheckUrl("http://10.10.10.10")
url, _ := url.ParseRequestURI("http://10.10.10.10")
Expect(outputURL).To(Equal(url))
Expect(err).To(BeNil())
})
It("returns blacklist error if the URL is blacklisted", func() {
_, err := urlBlacklistManager.CheckUrl("http://14.15.16.18")
Expect(err).ToNot(BeNil())
func New(
logger *gosteno.Logger,
host string,
config *doppler_config.Config,
storeAdapter storeadapter.StoreAdapter,
messageDrainBufferSize uint,
dropsondeOrigin string,
websocketWriteTimeout time.Duration,
dialTimeout time.Duration,
) (*Doppler, error) {
doppler := &Doppler{
Logger: logger,
storeAdapter: storeAdapter,
dropsondeVerifiedBytesChan: make(chan []byte),
}
keepAliveInterval := 30 * time.Second
appStoreCache := cache.NewAppServiceCache()
doppler.appStoreWatcher, doppler.newAppServiceChan, doppler.deletedAppServiceChan = store.NewAppServiceStoreWatcher(storeAdapter, appStoreCache, logger)
doppler.batcher = initializeMetrics(config.MetricBatchIntervalMilliseconds)
doppler.envelopeChan = make(chan *events.Envelope)
doppler.udpListener, doppler.dropsondeBytesChan = listeners.NewUDPListener(
fmt.Sprintf("%s:%d", host, config.IncomingUDPPort),
doppler.batcher,
logger,
"udpListener",
)
var err error
if config.EnableTLSTransport {
tlsConfig := &config.TLSListenerConfig
addr := fmt.Sprintf("%s:%d", host, tlsConfig.Port)
contextName := "tlsListener"
doppler.tlsListener, err = listeners.NewTCPListener(contextName, addr, tlsConfig, doppler.envelopeChan, doppler.batcher, logger)
if err != nil {
return nil, err
}
}
addr := fmt.Sprintf("%s:%d", host, config.IncomingTCPPort)
contextName := "tcpListener"
doppler.tcpListener, err = listeners.NewTCPListener(contextName, addr, nil, doppler.envelopeChan, doppler.batcher, logger)
doppler.signatureVerifier = signature.NewVerifier(logger, config.SharedSecret)
doppler.dropsondeUnmarshallerCollection = dropsonde_unmarshaller.NewDropsondeUnmarshallerCollection(logger, config.UnmarshallerCount)
blacklist := blacklist.New(config.BlackListIps, logger)
metricTTL := time.Duration(config.ContainerMetricTTLSeconds) * time.Second
sinkTimeout := time.Duration(config.SinkInactivityTimeoutSeconds) * time.Second
sinkIOTimeout := time.Duration(config.SinkIOTimeoutSeconds) * time.Second
doppler.sinkManager = sinkmanager.New(
config.MaxRetainedLogMessages,
config.SinkSkipCertVerify,
blacklist,
logger,
messageDrainBufferSize,
dropsondeOrigin,
sinkTimeout,
sinkIOTimeout,
metricTTL,
dialTimeout,
)
doppler.Infof("Listening for GRPC connections on %d", config.GRPCPort)
grpcListener, err := net.Listen("tcp", fmt.Sprintf(":%d", config.GRPCPort))
if err != nil {
return nil, err
}
grpcServer := grpc.NewServer()
plumbing.RegisterDopplerServer(grpcServer, doppler.sinkManager)
go grpcServer.Serve(grpcListener)
doppler.messageRouter = sinkserver.NewMessageRouter(doppler.sinkManager, logger)
doppler.websocketServer, err = websocketserver.New(
fmt.Sprintf(":%d", config.OutgoingPort),
doppler.sinkManager,
websocketWriteTimeout,
keepAliveInterval,
config.MessageDrainBufferSize,
dropsondeOrigin,
doppler.batcher,
logger,
)
if err != nil {
return nil, fmt.Errorf("Failed to create the websocket server: %s", err.Error())
}
monitorInterval := time.Duration(config.MonitorIntervalSeconds) * time.Second
doppler.openFileMonitor = monitor.NewLinuxFD(monitorInterval, logger)
doppler.uptimeMonitor = monitor.NewUptime(monitorInterval)
return doppler, nil
}