Beispiel #1
0
// NewListener creates a new listener for carbon datapoints
func NewListener(ctx context.Context, sink dpsink.Sink, conf listenerConfig, listenAddr string,
	metricDeconstructor string, metricDeconstructorOptions string) (*Listener, error) {
	psocket, err := net.Listen("tcp", listenAddr)
	if err != nil {
		return nil, err
	}

	deconstructor, err := metricdeconstructor.Load(metricDeconstructor, metricDeconstructorOptions)
	if err != nil {
		return nil, err
	}

	counter := &dpsink.Counter{}
	finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter))

	receiver := Listener{
		sink:                finalSink,
		psocket:             psocket,
		conf:                conf,
		metricDeconstructor: deconstructor,
		ctx:                 ctx,
		st:                  stats.ToKeeperMany(protocol.ListenerDims(conf.name, "carbon"), counter),
	}
	receiver.wg.Add(1)

	go receiver.startListening()
	return &receiver, nil
}
// ForwarderLoader1 is a more strictly typed version of ForwarderLoader
func ForwarderLoader1(ctx context.Context, forwardTo *config.ForwardTo) (protocol.Forwarder, *Forwarder, error) {
	proxyVersion, ok := ctx.Value("version").(string)
	if !ok || proxyVersion == "" {
		proxyVersion = "UNKNOWN_VERSION"
	}
	if forwardTo.FormatVersion == nil {
		forwardTo.FormatVersion = workarounds.GolangDoesnotAllowPointerToUintLiteral(3)
	}
	if *forwardTo.FormatVersion == 1 {
		log.WithField("forwardTo", forwardTo).Warn("Old formats not supported in signalfxforwarder.  Using newer format.  Please update config to use format version 2 or 3")
	}
	structdefaults.FillDefaultFrom(forwardTo, defaultConfigV2)
	log.WithField("forwardTo", forwardTo).Info("Creating signalfx forwarder using final config")
	fwd := NewSignalfxJSONForwarder(*forwardTo.URL, *forwardTo.TimeoutDuration,
		*forwardTo.DefaultAuthToken, *forwardTo.DrainingThreads,
		*forwardTo.DefaultSource, *forwardTo.SourceDimensions, proxyVersion)
	fwd.eventURL = *forwardTo.EventURL
	counter := &dpsink.Counter{}
	dims := protocol.ForwarderDims(*forwardTo.Name, "sfx_protobuf_v2")
	buffer := dpbuffered.NewBufferedForwarder(ctx, *(&dpbuffered.Config{}).FromConfig(forwardTo), fwd)
	return &protocol.CompositeForwarder{
		Sink:   dpsink.FromChain(buffer, dpsink.NextWrap(counter)),
		Keeper: stats.ToKeeperMany(dims, counter, buffer),
		Closer: protocol.CompositeCloser(protocol.OkCloser(buffer.Close)),
	}, fwd, nil
}
// SetupHandler is shared between signalfx and here to setup listening for collectd connections.
// Will do shared basic setup like configuring request counters
func SetupHandler(ctx context.Context, name string, sink dpsink.Sink) (*web.Handler, stats.Keeper) {
	metricTracking := web.RequestCounter{}
	counter := &dpsink.Counter{}
	collectdDecoder := JSONDecoder{
		SendTo: dpsink.FromChain(sink, dpsink.NextWrap(counter)),
	}
	h := web.NewHandler(ctx, &collectdDecoder).Add(web.NextHTTP(metricTracking.ServeHTTP))
	st := stats.ToKeeperMany(protocol.ListenerDims(name, "collectd"), &metricTracking, counter, &collectdDecoder)
	return h, st
}
func setupChain(ctx context.Context, sink dpsink.Sink, name string, chainType string, getReader func(dpsink.Sink) ErrorReader) (*web.Handler, stats.Keeper) {
	counter := &dpsink.Counter{}
	finalSink := dpsink.FromChain(sink, dpsink.NextWrap(counter))
	errReader := getReader(finalSink)
	errorTracker := ErrorTrackerHandler{
		reader: errReader,
	}
	metricTracking := web.RequestCounter{}
	handler := web.NewHandler(ctx, &errorTracker).Add(web.NextHTTP(metricTracking.ServeHTTP))
	st := stats.ToKeeperMany(protocol.ListenerDims(name, "sfx_"+chainType), &metricTracking, &errorTracker, counter)
	return handler, st
}
Beispiel #5
0
// New creates a new forwarder that sends datapoints to multiple recievers
func New(sendTo []dpsink.Sink) *Demultiplexer {
	ret := &Demultiplexer{
		sendTo: make([]dpsink.Sink, len(sendTo)),
	}
	for i := range sendTo {
		ret.sendTo[i] = dpsink.FromChain(sendTo[i], dpsink.NextWrap(&dpsink.RateLimitErrorLogging{
			LogThrottle: time.Second,
			Callback:    dpsink.LogCallback("Error forwarding points", log.StandardLogger()),
		}))
	}
	return ret
}
Beispiel #6
0
// ForwarderLoader loads a carbon forwarder that is buffered
func ForwarderLoader(ctx context.Context, forwardTo *config.ForwardTo) (protocol.Forwarder, error) {
	structdefaults.FillDefaultFrom(forwardTo, defaultForwarderConfig)
	if forwardTo.Host == nil {
		return nil, errRequiredHost
	}
	fwd, err := NewForwarder(*forwardTo.Host, *forwardTo.Port, *forwardTo.TimeoutDuration, forwardTo.DimensionsOrder, *forwardTo.DrainingThreads)
	if err != nil {
		return nil, err
	}
	counter := &dpsink.Counter{}
	dims := protocol.ForwarderDims(*forwardTo.Name, "carbon")
	buffer := dpbuffered.NewBufferedForwarder(ctx, *(&dpbuffered.Config{}).FromConfig(forwardTo), fwd)
	return &protocol.CompositeForwarder{
		Sink:   dpsink.FromChain(buffer, dpsink.NextWrap(counter)),
		Keeper: stats.ToKeeperMany(dims, counter, buffer),
		Closer: protocol.CompositeCloser(protocol.OkCloser(buffer.Close), fwd),
	}, nil
}
Beispiel #7
0
func (proxyCommandLineConfiguration *proxyCommandLineConfigurationT) main() error {
	log.WithField("configFile", proxyCommandLineConfiguration.configFileName).Info("Looking for config file")
	log.WithField("env", strings.Join(os.Environ(), " -*- ")).Info("Current env at start")

	loadedConfig, err := config.Load(proxyCommandLineConfiguration.configFileName)
	if err != nil {
		log.WithField("err", err).Error("Unable to load config")
		return err
	}
	proxyCommandLineConfiguration.setupLogrus(loadedConfig)

	{
		pidFilename := proxyCommandLineConfiguration.pidFileName
		if loadedConfig.PidFilename != nil {
			pidFilename = *loadedConfig.PidFilename
		}
		writePidFile(pidFilename)
	}

	log.WithField("config", loadedConfig).Info("Config loaded")
	log.WithField("env", strings.Join(os.Environ(), " -*- ")).Info("Current env after loading config")
	if loadedConfig.NumProcs != nil {
		runtime.GOMAXPROCS(*loadedConfig.NumProcs)
	} else {
		numProcs := runtime.NumCPU()
		runtime.GOMAXPROCS(numProcs)
	}

	proxyCommandLineConfiguration.allForwarders, err = setupForwarders(proxyCommandLineConfiguration.ctx, loadedConfig)
	if err != nil {
		return err
	}
	allKeepers := []stats.Keeper{stats.NewGolangKeeper()}

	multiplexerCounter := &dpsink.Counter{}
	multiplexer := dpsink.FromChain(demultiplexer.New(recastToSinks(proxyCommandLineConfiguration.allForwarders)), dpsink.NextWrap(multiplexerCounter))

	allKeepers = append(allKeepers, stats.ToKeeper(multiplexerCounter, map[string]string{"location": "middle", "name": "proxy", "type": "demultiplexer"}))

	proxyCommandLineConfiguration.allListeners, err = setupListeners(proxyCommandLineConfiguration.ctx, loadedConfig, multiplexer)
	if err != nil {
		return err
	}
	// I wish I could do this, but golang doesn't allow append() of super/sub types
	// allKeepers = append(allKeepers, proxyCommandLineConfiguration.allListeners...)
	allKeepers = append(allKeepers, recastListenToKeeper(proxyCommandLineConfiguration.allListeners)...)

	for _, forwarder := range proxyCommandLineConfiguration.allForwarders {
		allKeepers = append(allKeepers, forwarder)
	}

	if loadedConfig.StatsDelayDuration != nil && *loadedConfig.StatsDelayDuration != 0 {
		proxyCommandLineConfiguration.statDrainThread = stats.NewDrainingThread(*loadedConfig.StatsDelayDuration, recastToSinks(proxyCommandLineConfiguration.allForwarders), allKeepers, proxyCommandLineConfiguration.ctx)
	} else {
		log.Info("Skipping stat keeping")
	}

	setupDebugServer(allKeepers, loadedConfig, proxyCommandLineConfiguration.pprofaddr)

	log.Infof("Setup done.  Blocking!")
	if proxyCommandLineConfiguration.closeWhenWaitingToStopChannel != nil {
		close(proxyCommandLineConfiguration.closeWhenWaitingToStopChannel)
	}
	_ = <-proxyCommandLineConfiguration.stopChannel
	return nil
}