// NewListener creates a new listener for carbon datapoints
func NewListener(ctx context.Context, sink dpsink.Sink, conf listenerConfig, listenAddr string,
metricDeconstructor string, metricDeconstructorOptions string) (*Listener, error) {
psocket, err := net.Listen("tcp", listenAddr)
if err != nil {
return nil, err
}
deconstructor, err := metricdeconstructor.Load(metricDeconstructor, metricDeconstructorOptions)
if err != nil {
return nil, err
}
counter := &dpsink.Counter{}
finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter))
receiver := Listener{
sink: finalSink,
psocket: psocket,
conf: conf,
metricDeconstructor: deconstructor,
ctx: ctx,
st: stats.ToKeeperMany(protocol.ListenerDims(conf.name, "carbon"), counter),
}
receiver.wg.Add(1)
go receiver.startListening()
return &receiver, nil
}
// ForwarderLoader1 is a more strictly typed version of ForwarderLoader
func ForwarderLoader1(ctx context.Context, forwardTo *config.ForwardTo) (protocol.Forwarder, *Forwarder, error) {
proxyVersion, ok := ctx.Value("version").(string)
if !ok || proxyVersion == "" {
proxyVersion = "UNKNOWN_VERSION"
}
if forwardTo.FormatVersion == nil {
forwardTo.FormatVersion = workarounds.GolangDoesnotAllowPointerToUintLiteral(3)
}
if *forwardTo.FormatVersion == 1 {
log.WithField("forwardTo", forwardTo).Warn("Old formats not supported in signalfxforwarder. Using newer format. Please update config to use format version 2 or 3")
}
structdefaults.FillDefaultFrom(forwardTo, defaultConfigV2)
log.WithField("forwardTo", forwardTo).Info("Creating signalfx forwarder using final config")
fwd := NewSignalfxJSONForwarder(*forwardTo.URL, *forwardTo.TimeoutDuration,
*forwardTo.DefaultAuthToken, *forwardTo.DrainingThreads,
*forwardTo.DefaultSource, *forwardTo.SourceDimensions, proxyVersion)
fwd.eventURL = *forwardTo.EventURL
counter := &dpsink.Counter{}
dims := protocol.ForwarderDims(*forwardTo.Name, "sfx_protobuf_v2")
buffer := dpbuffered.NewBufferedForwarder(ctx, *(&dpbuffered.Config{}).FromConfig(forwardTo), fwd)
return &protocol.CompositeForwarder{
Sink: dpsink.FromChain(buffer, dpsink.NextWrap(counter)),
Keeper: stats.ToKeeperMany(dims, counter, buffer),
Closer: protocol.CompositeCloser(protocol.OkCloser(buffer.Close)),
}, fwd, nil
}
// SetupHandler is shared between signalfx and here to setup listening for collectd connections.
// Will do shared basic setup like configuring request counters
func SetupHandler(ctx context.Context, name string, sink dpsink.Sink) (*web.Handler, stats.Keeper) {
metricTracking := web.RequestCounter{}
counter := &dpsink.Counter{}
collectdDecoder := JSONDecoder{
SendTo: dpsink.FromChain(sink, dpsink.NextWrap(counter)),
}
h := web.NewHandler(ctx, &collectdDecoder).Add(web.NextHTTP(metricTracking.ServeHTTP))
st := stats.ToKeeperMany(protocol.ListenerDims(name, "collectd"), &metricTracking, counter, &collectdDecoder)
return h, st
}
func setupChain(ctx context.Context, sink dpsink.Sink, name string, chainType string, getReader func(dpsink.Sink) ErrorReader) (*web.Handler, stats.Keeper) {
counter := &dpsink.Counter{}
finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter))
errReader := getReader(finalSink)
errorTracker := ErrorTrackerHandler{
reader: errReader,
}
metricTracking := web.RequestCounter{}
handler := web.NewHandler(ctx, &errorTracker).Add(web.NextHTTP(metricTracking.ServeHTTP))
st := stats.ToKeeperMany(protocol.ListenerDims(name, "sfx_"+chainType), &metricTracking, &errorTracker, counter)
return handler, st
}
// New creates a new forwarder that sends datapoints to multiple recievers
func New(sendTo []dpsink.Sink) *Demultiplexer {
ret := &Demultiplexer{
sendTo: make([]dpsink.Sink, len(sendTo)),
}
for i := range sendTo {
ret.sendTo[i] = dpsink.FromChain(sendTo[i], dpsink.NextWrap(&dpsink.RateLimitErrorLogging{
LogThrottle: time.Second,
Callback: dpsink.LogCallback("Error forwarding points", log.StandardLogger()),
}))
}
return ret
}
// ForwarderLoader loads a carbon forwarder that is buffered
func ForwarderLoader(ctx context.Context, forwardTo *config.ForwardTo) (protocol.Forwarder, error) {
structdefaults.FillDefaultFrom(forwardTo, defaultForwarderConfig)
if forwardTo.Host == nil {
return nil, errRequiredHost
}
fwd, err := NewForwarder(*forwardTo.Host, *forwardTo.Port, *forwardTo.TimeoutDuration, forwardTo.DimensionsOrder, *forwardTo.DrainingThreads)
if err != nil {
return nil, err
}
counter := &dpsink.Counter{}
dims := protocol.ForwarderDims(*forwardTo.Name, "carbon")
buffer := dpbuffered.NewBufferedForwarder(ctx, *(&dpbuffered.Config{}).FromConfig(forwardTo), fwd)
return &protocol.CompositeForwarder{
Sink: dpsink.FromChain(buffer, dpsink.NextWrap(counter)),
Keeper: stats.ToKeeperMany(dims, counter, buffer),
Closer: protocol.CompositeCloser(protocol.OkCloser(buffer.Close), fwd),
}, nil
}
func (proxyCommandLineConfiguration *proxyCommandLineConfigurationT) main() error {
log.WithField("configFile", proxyCommandLineConfiguration.configFileName).Info("Looking for config file")
log.WithField("env", strings.Join(os.Environ(), " -*- ")).Info("Current env at start")
loadedConfig, err := config.Load(proxyCommandLineConfiguration.configFileName)
if err != nil {
log.WithField("err", err).Error("Unable to load config")
return err
}
proxyCommandLineConfiguration.setupLogrus(loadedConfig)
{
pidFilename := proxyCommandLineConfiguration.pidFileName
if loadedConfig.PidFilename != nil {
pidFilename = *loadedConfig.PidFilename
}
writePidFile(pidFilename)
}
log.WithField("config", loadedConfig).Info("Config loaded")
log.WithField("env", strings.Join(os.Environ(), " -*- ")).Info("Current env after loading config")
if loadedConfig.NumProcs != nil {
runtime.GOMAXPROCS(*loadedConfig.NumProcs)
} else {
numProcs := runtime.NumCPU()
runtime.GOMAXPROCS(numProcs)
}
proxyCommandLineConfiguration.allForwarders, err = setupForwarders(proxyCommandLineConfiguration.ctx, loadedConfig)
if err != nil {
return err
}
allKeepers := []stats.Keeper{stats.NewGolangKeeper()}
multiplexerCounter := &dpsink.Counter{}
multiplexer := dpsink.FromChain(demultiplexer.New(recastToSinks(proxyCommandLineConfiguration.allForwarders)), dpsink.NextWrap(multiplexerCounter))
allKeepers = append(allKeepers, stats.ToKeeper(multiplexerCounter, map[string]string{"location": "middle", "name": "proxy", "type": "demultiplexer"}))
proxyCommandLineConfiguration.allListeners, err = setupListeners(proxyCommandLineConfiguration.ctx, loadedConfig, multiplexer)
if err != nil {
return err
}
// I wish I could do this, but golang doesn't allow append() of super/sub types
// allKeepers = append(allKeepers, proxyCommandLineConfiguration.allListeners...)
allKeepers = append(allKeepers, recastListenToKeeper(proxyCommandLineConfiguration.allListeners)...)
for _, forwarder := range proxyCommandLineConfiguration.allForwarders {
allKeepers = append(allKeepers, forwarder)
}
if loadedConfig.StatsDelayDuration != nil && *loadedConfig.StatsDelayDuration != 0 {
proxyCommandLineConfiguration.statDrainThread = stats.NewDrainingThread(*loadedConfig.StatsDelayDuration, recastToSinks(proxyCommandLineConfiguration.allForwarders), allKeepers, proxyCommandLineConfiguration.ctx)
} else {
log.Info("Skipping stat keeping")
}
setupDebugServer(allKeepers, loadedConfig, proxyCommandLineConfiguration.pprofaddr)
log.Infof("Setup done. Blocking!")
if proxyCommandLineConfiguration.closeWhenWaitingToStopChannel != nil {
close(proxyCommandLineConfiguration.closeWhenWaitingToStopChannel)
}
_ = <-proxyCommandLineConfiguration.stopChannel
return nil
}