// NewListener creates a new listener for carbon datapoints func NewListener(ctx context.Context, sink dpsink.Sink, conf listenerConfig, listenAddr string, metricDeconstructor string, metricDeconstructorOptions string) (*Listener, error) { psocket, err := net.Listen("tcp", listenAddr) if err != nil { return nil, err } deconstructor, err := metricdeconstructor.Load(metricDeconstructor, metricDeconstructorOptions) if err != nil { return nil, err } counter := &dpsink.Counter{} finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter)) receiver := Listener{ sink: finalSink, psocket: psocket, conf: conf, metricDeconstructor: deconstructor, ctx: ctx, st: stats.ToKeeperMany(protocol.ListenerDims(conf.name, "carbon"), counter), } receiver.wg.Add(1) go receiver.startListening() return &receiver, nil }
// ForwarderLoader1 is a more strictly typed version of ForwarderLoader func ForwarderLoader1(ctx context.Context, forwardTo *config.ForwardTo) (protocol.Forwarder, *Forwarder, error) { proxyVersion, ok := ctx.Value("version").(string) if !ok || proxyVersion == "" { proxyVersion = "UNKNOWN_VERSION" } if forwardTo.FormatVersion == nil { forwardTo.FormatVersion = workarounds.GolangDoesnotAllowPointerToUintLiteral(3) } if *forwardTo.FormatVersion == 1 { log.WithField("forwardTo", forwardTo).Warn("Old formats not supported in signalfxforwarder. Using newer format. Please update config to use format version 2 or 3") } structdefaults.FillDefaultFrom(forwardTo, defaultConfigV2) log.WithField("forwardTo", forwardTo).Info("Creating signalfx forwarder using final config") fwd := NewSignalfxJSONForwarder(*forwardTo.URL, *forwardTo.TimeoutDuration, *forwardTo.DefaultAuthToken, *forwardTo.DrainingThreads, *forwardTo.DefaultSource, *forwardTo.SourceDimensions, proxyVersion) fwd.eventURL = *forwardTo.EventURL counter := &dpsink.Counter{} dims := protocol.ForwarderDims(*forwardTo.Name, "sfx_protobuf_v2") buffer := dpbuffered.NewBufferedForwarder(ctx, *(&dpbuffered.Config{}).FromConfig(forwardTo), fwd) return &protocol.CompositeForwarder{ Sink: dpsink.FromChain(buffer, dpsink.NextWrap(counter)), Keeper: stats.ToKeeperMany(dims, counter, buffer), Closer: protocol.CompositeCloser(protocol.OkCloser(buffer.Close)), }, fwd, nil }
// SetupHandler is shared between signalfx and here to setup listening for collectd connections. // Will do shared basic setup like configuring request counters func SetupHandler(ctx context.Context, name string, sink dpsink.Sink) (*web.Handler, stats.Keeper) { metricTracking := web.RequestCounter{} counter := &dpsink.Counter{} collectdDecoder := JSONDecoder{ SendTo: dpsink.FromChain(sink, dpsink.NextWrap(counter)), } h := web.NewHandler(ctx, &collectdDecoder).Add(web.NextHTTP(metricTracking.ServeHTTP)) st := stats.ToKeeperMany(protocol.ListenerDims(name, "collectd"), &metricTracking, counter, &collectdDecoder) return h, st }
func setupChain(ctx context.Context, sink dpsink.Sink, name string, chainType string, getReader func(dpsink.Sink) ErrorReader) (*web.Handler, stats.Keeper) { counter := &dpsink.Counter{} finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter)) errReader := getReader(finalSink) errorTracker := ErrorTrackerHandler{ reader: errReader, } metricTracking := web.RequestCounter{} handler := web.NewHandler(ctx, &errorTracker).Add(web.NextHTTP(metricTracking.ServeHTTP)) st := stats.ToKeeperMany(protocol.ListenerDims(name, "sfx_"+chainType), &metricTracking, &errorTracker, counter) return handler, st }
// New creates a new forwarder that sends datapoints to multiple recievers func New(sendTo []dpsink.Sink) *Demultiplexer { ret := &Demultiplexer{ sendTo: make([]dpsink.Sink, len(sendTo)), } for i := range sendTo { ret.sendTo[i] = dpsink.FromChain(sendTo[i], dpsink.NextWrap(&dpsink.RateLimitErrorLogging{ LogThrottle: time.Second, Callback: dpsink.LogCallback("Error forwarding points", log.StandardLogger()), })) } return ret }
// ForwarderLoader loads a carbon forwarder that is buffered func ForwarderLoader(ctx context.Context, forwardTo *config.ForwardTo) (protocol.Forwarder, error) { structdefaults.FillDefaultFrom(forwardTo, defaultForwarderConfig) if forwardTo.Host == nil { return nil, errRequiredHost } fwd, err := NewForwarder(*forwardTo.Host, *forwardTo.Port, *forwardTo.TimeoutDuration, forwardTo.DimensionsOrder, *forwardTo.DrainingThreads) if err != nil { return nil, err } counter := &dpsink.Counter{} dims := protocol.ForwarderDims(*forwardTo.Name, "carbon") buffer := dpbuffered.NewBufferedForwarder(ctx, *(&dpbuffered.Config{}).FromConfig(forwardTo), fwd) return &protocol.CompositeForwarder{ Sink: dpsink.FromChain(buffer, dpsink.NextWrap(counter)), Keeper: stats.ToKeeperMany(dims, counter, buffer), Closer: protocol.CompositeCloser(protocol.OkCloser(buffer.Close), fwd), }, nil }
func (proxyCommandLineConfiguration *proxyCommandLineConfigurationT) main() error { log.WithField("configFile", proxyCommandLineConfiguration.configFileName).Info("Looking for config file") log.WithField("env", strings.Join(os.Environ(), " -*- ")).Info("Current env at start") loadedConfig, err := config.Load(proxyCommandLineConfiguration.configFileName) if err != nil { log.WithField("err", err).Error("Unable to load config") return err } proxyCommandLineConfiguration.setupLogrus(loadedConfig) { pidFilename := proxyCommandLineConfiguration.pidFileName if loadedConfig.PidFilename != nil { pidFilename = *loadedConfig.PidFilename } writePidFile(pidFilename) } log.WithField("config", loadedConfig).Info("Config loaded") log.WithField("env", strings.Join(os.Environ(), " -*- ")).Info("Current env after loading config") if loadedConfig.NumProcs != nil { runtime.GOMAXPROCS(*loadedConfig.NumProcs) } else { numProcs := runtime.NumCPU() runtime.GOMAXPROCS(numProcs) } proxyCommandLineConfiguration.allForwarders, err = setupForwarders(proxyCommandLineConfiguration.ctx, loadedConfig) if err != nil { return err } allKeepers := []stats.Keeper{stats.NewGolangKeeper()} multiplexerCounter := &dpsink.Counter{} multiplexer := dpsink.FromChain(demultiplexer.New(recastToSinks(proxyCommandLineConfiguration.allForwarders)), dpsink.NextWrap(multiplexerCounter)) allKeepers = append(allKeepers, stats.ToKeeper(multiplexerCounter, map[string]string{"location": "middle", "name": "proxy", "type": "demultiplexer"})) proxyCommandLineConfiguration.allListeners, err = setupListeners(proxyCommandLineConfiguration.ctx, loadedConfig, multiplexer) if err != nil { return err } // I wish I could do this, but golang doesn't allow append() of super/sub types // allKeepers = append(allKeepers, proxyCommandLineConfiguration.allListeners...) allKeepers = append(allKeepers, recastListenToKeeper(proxyCommandLineConfiguration.allListeners)...) for _, forwarder := range proxyCommandLineConfiguration.allForwarders { allKeepers = append(allKeepers, forwarder) } if loadedConfig.StatsDelayDuration != nil && *loadedConfig.StatsDelayDuration != 0 { proxyCommandLineConfiguration.statDrainThread = stats.NewDrainingThread(*loadedConfig.StatsDelayDuration, recastToSinks(proxyCommandLineConfiguration.allForwarders), allKeepers, proxyCommandLineConfiguration.ctx) } else { log.Info("Skipping stat keeping") } setupDebugServer(allKeepers, loadedConfig, proxyCommandLineConfiguration.pprofaddr) log.Infof("Setup done. Blocking!") if proxyCommandLineConfiguration.closeWhenWaitingToStopChannel != nil { close(proxyCommandLineConfiguration.closeWhenWaitingToStopChannel) } _ = <-proxyCommandLineConfiguration.stopChannel return nil }