Beispiel #1
0
func main() {
	flag.Parse()
	dropsonde.Initialize(metronHostPort, origin)

	for {
		fmt.Printf(".")

		switch filter {
		// TODO: Add support for other event types as we add chaining APIs for
		// those event types in NOAA.

		//case "LogMessage":
		//case "HttpStartStop":
		//case "HttpStart":
		//case "HttpStop":
		//case "Error":

		case "CounterEvent":
			sendCounterEvent()
		case "ValueMetric":
			sendValueMetric()
		case "ContainerMetric":
			sendContainerMetric()
		default:
			sendCounterEvent()
			sendValueMetric()
			sendContainerMetric()
		}

		time.Sleep(delay)
	}
}
Beispiel #2
0
func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}
Beispiel #3
0
func main() {
	flag.Parse()

	// ** Config Setup
	config, err := readConfig(*configFile)
	if err != nil {
		panic(err)
	}

	dropsonde.Initialize(config.MetronAddress, "dea_logging_agent")

	log := logger.NewLogger(*logLevel, *logFilePath, "deaagent", config.Syslog)
	log.Info("Startup: Setting up the loggregator dea logging agent")
	// ** END Config Setup

	agent := deaagent.NewAgent(*instancesJsonFilePath, log)

	go agent.Start()

	killChan := signalmanager.RegisterKillSignalChannel()
	dumpChan := signalmanager.RegisterGoRoutineDumpSignalChannel()

	for {
		select {
		case <-dumpChan:
			signalmanager.DumpGoRoutine()
		case <-killChan:
			log.Info("Shutting down")
			os.Exit(0)
			return
		}
	}
}
Beispiel #4
0
func main() {
	flag.Parse()

	config, logger, err := config.ParseConfig(logLevel, configFile, logFilePath)
	if err != nil {
		panic(err)
	}

	profiler := profiler.NewProfiler(*cpuprofile, *memprofile, 1*time.Second, logger)
	profiler.Profile()
	defer profiler.Stop()

	uptimeMonitor := monitor.NewUptimeMonitor(time.Duration(config.MonitorIntervalSeconds) * time.Second)
	go uptimeMonitor.Start()
	defer uptimeMonitor.Stop()

	dropsonde.Initialize("localhost:"+strconv.Itoa(config.MetronPort), "LoggregatorTrafficController")

	adapter := DefaultStoreAdapterProvider(config.EtcdUrls, config.EtcdMaxConcurrentRequests)
	adapter.Connect()

	ipAddress, err := localip.LocalIP()
	if err != nil {
		panic(err)
	}

	dopplerProxy := makeDopplerProxy(adapter, config, logger)
	startOutgoingDopplerProxy(net.JoinHostPort(ipAddress, strconv.FormatUint(uint64(config.OutgoingDropsondePort), 10)), dopplerProxy)

	legacyProxy := makeLegacyProxy(adapter, config, logger)
	startOutgoingProxy(net.JoinHostPort(ipAddress, strconv.FormatUint(uint64(config.OutgoingPort), 10)), legacyProxy)

	rr := routerregistrar.NewRouterRegistrar(config.MbusClient, logger)
	uri := "loggregator." + config.SystemDomain
	err = rr.RegisterWithRouter(ipAddress, config.OutgoingPort, []string{uri})
	if err != nil {
		logger.Fatalf("Startup: Did not get response from router when greeting. Using default keep-alive for now. Err: %v.", err)
	}

	uri = "doppler." + config.SystemDomain
	err = rr.RegisterWithRouter(ipAddress, config.OutgoingDropsondePort, []string{uri})
	if err != nil {
		logger.Fatalf("Startup: Did not get response from router when greeting. Using default keep-alive for now. Err: %v.", err)
	}

	killChan := make(chan os.Signal)
	signal.Notify(killChan, os.Kill, os.Interrupt)

	for {
		select {
		case <-cfcomponent.RegisterGoRoutineDumpSignalChannel():
			cfcomponent.DumpGoRoutine()
		case <-killChan:
			rr.UnregisterFromRouter(ipAddress, config.OutgoingPort, []string{uri})
			break
		}
	}
}
Beispiel #5
0
func main() {
	c := config.DefaultConfig()
	logCounter := vcap.NewLogCounter()

	if configFile != "" {
		c = config.InitConfigFromFile(configFile)
	}

	InitLoggerFromConfig(c, logCounter)
	logger := steno.NewLogger("router.main")

	err := dropsonde.Initialize(c.Logging.MetronAddress, c.Logging.JobName)
	if err != nil {
		logger.Errorf("Dropsonde failed to initialize: %s", err.Error())
		os.Exit(1)
	}

	// setup number of procs
	if c.GoMaxProcs != 0 {
		runtime.GOMAXPROCS(c.GoMaxProcs)
	}

	if c.DebugAddr != "" {
		cf_debug_server.Run(c.DebugAddr)
	}

	logger.Info("Setting up NATs connection")
	natsClient := connectToNatsServer(c, logger)

	registry := rregistry.NewRouteRegistry(c, natsClient)

	logger.Info("Setting up routing_api route fetcher")
	setupRouteFetcher(c, registry)

	varz := rvarz.NewVarz(registry)

	accessLogger, err := access_log.CreateRunningAccessLogger(c)
	if err != nil {
		logger.Fatalf("Error creating access logger: %s\n", err)
	}

	proxy := buildProxy(c, registry, accessLogger, varz)

	router, err := router.NewRouter(c, proxy, natsClient, registry, varz, logCounter)
	if err != nil {
		logger.Errorf("An error occurred: %s", err.Error())
		os.Exit(1)
	}

	errChan := router.Run()

	logger.Info("gorouter.started")

	waitOnErrOrSignal(c, logger, errChan, router)

	os.Exit(0)
}
Beispiel #6
0
func main() {
	flag.Parse()

	config, err := config.ParseConfig(*logLevel, *configFile, *logFilePath)
	if err != nil {
		panic(err)
	}

	log := logger.NewLogger(*logLevel, *logFilePath, "loggregator trafficcontroller", config.Syslog)
	log.Info("Startup: Setting up the loggregator traffic controller")

	dropsonde.Initialize("localhost:"+strconv.Itoa(config.MetronPort), "LoggregatorTrafficController")

	profiler := profiler.NewProfiler(*cpuprofile, *memprofile, 1*time.Second, log)
	profiler.Profile()
	defer profiler.Stop()

	uptimeMonitor := monitor.NewUptimeMonitor(time.Duration(config.MonitorIntervalSeconds) * time.Second)
	go uptimeMonitor.Start()
	defer uptimeMonitor.Stop()

	dopplerAdapter := DefaultStoreAdapterProvider(config.EtcdUrls, config.EtcdMaxConcurrentRequests)
	dopplerAdapter.Connect()

	legacyAdapter := DefaultStoreAdapterProvider(config.EtcdUrls, config.EtcdMaxConcurrentRequests)
	legacyAdapter.Connect()

	ipAddress, err := localip.LocalIP()
	if err != nil {
		panic(err)
	}

	dopplerProxy := makeDopplerProxy(dopplerAdapter, config, log)
	startOutgoingDopplerProxy(net.JoinHostPort(ipAddress, strconv.FormatUint(uint64(config.OutgoingDropsondePort), 10)), dopplerProxy)

	legacyProxy := makeLegacyProxy(legacyAdapter, config, log)
	startOutgoingProxy(net.JoinHostPort(ipAddress, strconv.FormatUint(uint64(config.OutgoingPort), 10)), legacyProxy)

	killChan := make(chan os.Signal)
	signal.Notify(killChan, os.Kill, os.Interrupt)

	dumpChan := registerGoRoutineDumpSignalChannel()

	for {
		select {
		case <-dumpChan:
			logger.DumpGoRoutine()
		case <-killChan:
			break
		}
	}
}
Beispiel #7
0
func main() {
	if dropsondeDestination == "" {
		fmt.Println("dropsondeDestination flag is required")
		os.Exit(1)
	}
	if sourceInstance == "" {
		fmt.Println("sourceInstance flag is required")
		os.Exit(1)
	}

	args := flag.Args()

	if len(args) == 0 {
		fmt.Println("Command not specified!")
		fmt.Println("Usage: tee2metron -dropsondeDestionation=127.0.0.1:3457 -sourceInstance=cell-21 COMMAND")
		os.Exit(3)
	}
	err := dropsonde.Initialize(dropsondeDestination, sourceInstance, args[0])

	if err != nil {
		panic("error initializing dropsonde" + err.Error())
	}

	dropsondeStdoutReader, dropsondeStdoutWriter := io.Pipe()
	dropsondeStderrReader, dropsondeStderrWriter := io.Pipe()

	stdoutTeeWriter := io.MultiWriter(dropsondeStdoutWriter, os.Stdout)
	stderrTeeWriter := io.MultiWriter(dropsondeStderrWriter, os.Stderr)

	defer dropsondeStdoutReader.Close()
	defer dropsondeStderrReader.Close()
	defer dropsondeStdoutWriter.Close()
	defer dropsondeStderrWriter.Close()
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stdout = stdoutTeeWriter
	cmd.Stderr = stderrTeeWriter
	go logs.ScanLogStream(latticeDebugStreamId, args[0], sourceInstance, dropsondeStdoutReader)
	go logs.ScanErrorLogStream(latticeDebugStreamId, args[0], sourceInstance, dropsondeStderrReader)

	err = cmd.Start()
	if err != nil {
		fmt.Println(err)
		os.Exit(3)
	}

	// if the child is killed abnormally we would know
	err = cmd.Wait()
	if err != nil {
		fmt.Println(args[0], ":", err)
		os.Exit(3)
	}
}
Beispiel #8
0
func main() {
	configPath := flag.String("configPath", "", "Path to config file. Optional.")
	stateFilePath := flag.String("stateFilePath", "", "Path to state file. Optional.")
	flag.Parse()

	config = new(observer.Config)
	if *configPath != "" {
		if err := config.Load(*configPath); err != nil {
			errLog.Fatalf("Error loading config file: %s", err.Error())
		}
	}

	state = new(observer.State)
	if *stateFilePath != "" {
		if _, err := os.Stat(*stateFilePath); os.IsNotExist(err) {
			state.Write(*stateFilePath)
		}

		if err := state.Load(*stateFilePath); err != nil {
			errLog.Fatalf("Error loading state file: %s", err.Error())
		}

		setupExitHandler(*stateFilePath)

		defer state.Write(*stateFilePath)
	}

	handleFn := logEvent
	if config.Metron.Endpoint != "" && config.Metron.Origin != "" {
		err := dropsonde.Initialize(
			config.Metron.Endpoint,
			config.Metron.Origin,
		)
		if err != nil {
			errLog.Fatalf("Dropsonde failed to initialize", err)
		}

		handleFn = func(event observer.AddressChange) {
			emitMetric(event)
			logEvent(event)
		}
	}

	addrChanges := make(chan observer.AddressChange)
	done := make(chan struct{})
	go handleObservations(handleFn, addrChanges, done)

	observer.Observe(*config, state, addrChanges)
	<-done
}
Beispiel #9
0
func main() {
	err := dropsonde.Initialize(metronAddr, "METRIC-TEST", "z1", "0")
	if err != nil {
		println(err.Error())
	}

	for i := uint64(0); ; i++ {
		println("emitting metric at counter: ", i)
		metrics.SendContainerMetric(appId, 0, 42.42, 1234, i)
		metrics.SendContainerMetric(appId, 1, 11.41, 1234, i)
		metrics.SendContainerMetric(appId, 2, 11.41, 1234, i)
		metrics.SendContainerMetric("donotseethis", 2, 11.41, 1234, i)
		time.Sleep(1 * time.Second)
	}
}
Beispiel #10
0
func main() {
	redisConn, err := Connect()

	if err != nil {
		fmt.Println(err)
		return
	}

	defer redisConn.Disconnect()

	d := &emitterDetails{
		Destination: "localhost:3457",
		Origin:      "metrics-demo",
		Zone:        "z1",
		Index:       "0",
	}

	err = dropsonde.Initialize(d.Destination, d.Origin, d.Zone, d.Index)
	if err != nil {
		fmt.Println(err)
		return
	}

	fieldName := "uptime_in_seconds"
	uptime, err := redisConn.InfoField(fieldName)
	if err != nil {
		fmt.Println(err)
		return
	}

	uptimeConverted, err := strconv.ParseFloat(uptime, 64)
	if err != nil {
		fmt.Println(err)
	}

	metricData := &metric{
		Name:  fieldName,
		Value: uptimeConverted,
		Unit:  "",
	}

	err = redisConn.EmitMetric(metricData)
	if err != nil {
		fmt.Println(err)
		return
	}
	fmt.Printf("Successfully emitted %+v", metricData)
}
Beispiel #11
0
func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)
	flag.Parse()

	cf_http.Initialize(*communicationTimeout)

	logger, reconfigurableSink := cf_lager.New("ssh-proxy")

	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed-to-initialize-dropsonde", err)
	}

	proxyConfig, err := configureProxy(logger)
	if err != nil {
		logger.Error("configure-failed", err)
		os.Exit(1)
	}

	sshProxy := proxy.New(logger, proxyConfig)
	server := server.NewServer(logger, *address, sshProxy)

	members := grouper.Members{
		{"ssh-proxy", server},
	}

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{{
			"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink),
		}}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)
	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started")

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
	os.Exit(0)
}
func main() {
	err := dropsonde.Initialize("localhost:3457", "METRIC-TEST", "z1", "0")
	if err != nil {
		println(err.Error())
	}

	var i uint64
	i = 0
	for {
		println("emitting metric at counter: ", i)
		metrics.SendContainerMetric(appID, 0, 42.42, 1234, i)
		metrics.SendContainerMetric(appID, 1, 11.41, 1234, i)
		metrics.SendContainerMetric(appID, 2, 11.41, 1234, i)
		metrics.SendContainerMetric("donotseethis", 2, 11.41, 1234, i)
		i++
		time.Sleep(1 * time.Second)
	}
}
Beispiel #13
0
func main() {
	configFilePath := flag.String("configPath", "", "path to the configuration file")

	flag.Parse()

	conf := config.Configuration(*configFilePath)

	if len(conf.Syslog) > 0 {
		logging.SetSysLogger(conf.Syslog)
	}
	logging.SetLevel(conf.LogLevel)

	dropsonde.Initialize("localhost:"+strconv.Itoa(conf.MetronPort), valuemetricsender.ForwarderOrigin)

	go func() {
		err := tcp.Open(conf.IncomingPort, forwarder.StartMessageForwarder(valuemetricsender.NewValueMetricSender()))
		if err != nil {
			logging.Log.Panic("Could not open the TCP port", err)
		}
	}()

	logging.Log.Info("Bosh HM forwarder initialized")

	infoHandler := handlers.NewInfoHandler()
	router := mux.NewRouter()
	router.Handle("/info", infoHandler).Methods("GET")

	if conf.DebugPort > 0 {
		go pprofServer(conf.DebugPort)
	}

	logging.Log.Info(fmt.Sprintf("Starting Info Server on port %d", conf.InfoPort))

	err := http.ListenAndServe(net.JoinHostPort("", fmt.Sprintf("%d", conf.InfoPort)), router)
	if err != nil {
		logging.Log.Panic("Failed to start up alerter: ", err)
	}
}
Beispiel #14
0
func main() {
	flag.StringVar(&configFile, "c", "", "Configuration File")
	cf_lager.AddFlags(flag.CommandLine)
	flag.Parse()

	c := config.DefaultConfig()
	logCounter := schema.NewLogCounter()

	if configFile != "" {
		c = config.InitConfigFromFile(configFile)
	}

	prefix := "gorouter.stdout"
	if c.Logging.Syslog != "" {
		prefix = c.Logging.Syslog
	}
	logger, reconfigurableSink := cf_lager.New(prefix)
	InitLoggerFromConfig(logger, c, logCounter)

	logger.Info("starting")

	err := dropsonde.Initialize(c.Logging.MetronAddress, c.Logging.JobName)
	if err != nil {
		logger.Fatal("dropsonde-initialize-error", err)
	}

	// setup number of procs
	if c.GoMaxProcs != 0 {
		runtime.GOMAXPROCS(c.GoMaxProcs)
	}

	if c.DebugAddr != "" {
		cf_debug_server.Run(c.DebugAddr, reconfigurableSink)
	}

	logger.Info("setting-up-nats-connection")
	natsClient := connectToNatsServer(logger.Session("nats"), c)
	logger.Info("Successfully-connected-to-nats")

	metricsReporter := metrics.NewMetricsReporter()
	registry := rregistry.NewRouteRegistry(logger.Session("registry"), c, metricsReporter)

	varz := rvarz.NewVarz(registry)
	compositeReporter := metrics.NewCompositeReporter(varz, metricsReporter)

	accessLogger, err := access_log.CreateRunningAccessLogger(logger.Session("access-log"), c)
	if err != nil {
		logger.Fatal("error-creating-access-logger", err)
	}

	var crypto secure.Crypto
	var cryptoPrev secure.Crypto
	if c.RouteServiceEnabled {
		crypto = createCrypto(logger, c.RouteServiceSecret)
		if c.RouteServiceSecretPrev != "" {
			cryptoPrev = createCrypto(logger, c.RouteServiceSecretPrev)
		}
	}

	proxy := buildProxy(logger.Session("proxy"), c, registry, accessLogger, compositeReporter, crypto, cryptoPrev)

	router, err := router.NewRouter(logger.Session("router"), c, proxy, natsClient, registry, varz, logCounter, nil)
	if err != nil {
		logger.Fatal("initialize-router-error", err)
	}

	members := grouper.Members{
		{"router", router},
	}
	if c.RoutingApiEnabled() {
		logger.Info("setting-up-routing-api")
		routeFetcher := setupRouteFetcher(logger.Session("route-fetcher"), c, registry)

		// check connectivity to routing api
		err := routeFetcher.FetchRoutes()
		if err != nil {
			logger.Fatal("routing-api-connection-failed", err)
		}
		members = append(members, grouper.Member{"router-fetcher", routeFetcher})
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1))

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("gorouter.exited-with-failure", err)
		os.Exit(1)
	}

	os.Exit(0)
}
Beispiel #15
0
func main() {
	flag.Parse()

	// ** Config Setup
	config, err := readConfig(*configFile)
	if err != nil {
		panic(err)
	}

	dropsonde.Initialize(config.MetronAddress, "dea_logging_agent")

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			f.Close()
		}()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			panic(err)
		}
		go func() {
			defer f.Close()
			ticker := time.NewTicker(time.Second * 1)
			defer ticker.Stop()
			for {
				<-ticker.C
				pprof.WriteHeapProfile(f)
			}
		}()
	}

	log := logger.NewLogger(*logLevel, *logFilePath, "deaagent", config.Syslog)
	log.Info("Startup: Setting up the loggregator dea logging agent")
	// ** END Config Setup

	agent := deaagent.NewAgent(*instancesJsonFilePath, log)

	go agent.Start()

	killChan := make(chan os.Signal)
	signal.Notify(killChan, os.Interrupt)

	dumpChan := registerGoRoutineDumpSignalChannel()

	for {
		select {
		case <-dumpChan:
			logger.DumpGoRoutine()
		case <-killChan:
			log.Info("Shutting down")
			return
		}
	}
}
Beispiel #16
0
func main() {
	seed := time.Now().UnixNano()
	rand.Seed(seed)

	flag.Parse()

	runtime.GOMAXPROCS(runtime.NumCPU())

	localIp, err := localip.LocalIP()
	if err != nil {
		panic(errors.New("Unable to resolve own IP address: " + err.Error()))
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			f.Close()
		}()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			panic(err)
		}
		go func() {
			defer f.Close()
			ticker := time.NewTicker(time.Second * 1)
			defer ticker.Stop()
			for {
				<-ticker.C
				pprof.WriteHeapProfile(f)
			}
		}()
	}

	conf, logger := ParseConfig(logLevel, configFile, logFilePath)

	dropsonde.Initialize(conf.MetronAddress, "DopplerServer")

	if len(conf.NatsHosts) == 0 {
		logger.Warn("Startup: Did not receive a NATS host - not going to regsiter component")
		cfcomponent.DefaultYagnatsClientProvider = func(logger *gosteno.Logger, c *cfcomponent.Config) (yagnats.NATSConn, error) {
			return fakeyagnats.Connect(), nil
		}
	}

	err = conf.Validate(logger)
	if err != nil {
		panic(err)
	}

	storeAdapter := NewStoreAdapter(conf.EtcdUrls, conf.EtcdMaxConcurrentRequests)
	doppler := New(localIp, conf, logger, storeAdapter, conf.MessageDrainBufferSize, "doppler")

	if err != nil {
		panic(err)
	}

	go doppler.Start()
	logger.Info("Startup: doppler server started.")

	killChan := make(chan os.Signal)
	signal.Notify(killChan, os.Kill, os.Interrupt)

	storeAdapter = NewStoreAdapter(conf.EtcdUrls, conf.EtcdMaxConcurrentRequests)
	StartHeartbeats(localIp, config.HeartbeatInterval, conf, storeAdapter, logger)

	for {
		select {
		case <-cfcomponent.RegisterGoRoutineDumpSignalChannel():
			cfcomponent.DumpGoRoutine()
		case <-killChan:
			logger.Info("Shutting down")
			doppler.Stop()
			return
		}
	}
}
Beispiel #17
0
func (cmd *GuardianCommand) initializeDropsonde(log lager.Logger) {
	err := dropsonde.Initialize(cmd.Metrics.DropsondeDestination, cmd.Metrics.DropsondeOrigin)
	if err != nil {
		log.Error("failed to initialize dropsonde", err)
	}
}
Beispiel #18
0
func main() {
	seed := time.Now().UnixNano()
	rand.Seed(seed)

	flag.Parse()

	runtime.GOMAXPROCS(runtime.NumCPU())

	localIp, err := localip.LocalIP()
	if err != nil {
		panic(errors.New("Unable to resolve own IP address: " + err.Error()))
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			f.Close()
		}()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			panic(err)
		}
		go func() {
			defer f.Close()
			ticker := time.NewTicker(time.Second * 1)
			defer ticker.Stop()
			for {
				<-ticker.C
				pprof.WriteHeapProfile(f)
			}
		}()
	}

	conf, err := config.ParseConfig(*configFile)
	if err != nil {
		panic(err)
	}

	log := logger.NewLogger(*logLevel, *logFilePath, "doppler", conf.Syslog)
	log.Info("Startup: Setting up the doppler server")

	dropsonde.Initialize(conf.MetronAddress, DOPPLER_ORIGIN)
	dopplerStoreAdapter := NewStoreAdapter(conf.EtcdUrls, conf.EtcdMaxConcurrentRequests)
	legacyStoreAdapter := NewStoreAdapter(conf.EtcdUrls, conf.EtcdMaxConcurrentRequests)

	doppler := New(localIp, conf, log, dopplerStoreAdapter, conf.MessageDrainBufferSize, DOPPLER_ORIGIN, time.Duration(conf.SinkDialTimeoutSeconds)*time.Second)

	if err != nil {
		panic(err)
	}

	go doppler.Start()
	log.Info("Startup: doppler server started.")

	killChan := make(chan os.Signal)
	signal.Notify(killChan, os.Kill, os.Interrupt)

	dumpChan := registerGoRoutineDumpSignalChannel()

	releaseNodeChan := announcer.Announce(localIp, config.HeartbeatInterval, conf, dopplerStoreAdapter, log)
	legacyReleaseNodeChan := announcer.AnnounceLegacy(localIp, config.HeartbeatInterval, conf, legacyStoreAdapter, log)

	for {
		select {
		case <-dumpChan:
			logger.DumpGoRoutine()
		case <-killChan:
			log.Info("Shutting down")
			doppler.Stop()
			close(releaseNodeChan)
			close(legacyReleaseNodeChan)
			return
		}
	}
}
Beispiel #19
0
func main() {
	flag.StringVar(&configFile, "c", "", "Configuration File")
	cf_lager.AddFlags(flag.CommandLine)
	flag.Parse()

	c := config.DefaultConfig()
	logCounter := vcap.NewLogCounter()

	if configFile != "" {
		c = config.InitConfigFromFile(configFile)
	}

	InitLoggerFromConfig(c, logCounter)
	logger := steno.NewLogger("router.main")

	err := dropsonde.Initialize(c.Logging.MetronAddress, c.Logging.JobName)
	if err != nil {
		logger.Errorf("Dropsonde failed to initialize: %s", err.Error())
		os.Exit(1)
	}

	// setup number of procs
	if c.GoMaxProcs != 0 {
		runtime.GOMAXPROCS(c.GoMaxProcs)
	}

	if c.DebugAddr != "" {
		cf_debug_server.Run(c.DebugAddr)
	}

	logger.Info("Setting up NATs connection")
	natsClient := connectToNatsServer(logger, c)

	metricsReporter := metrics.NewMetricsReporter()
	registry := rregistry.NewRouteRegistry(c, natsClient, metricsReporter)

	varz := rvarz.NewVarz(registry)
	compositeReporter := metrics.NewCompositeReporter(varz, metricsReporter)

	accessLogger, err := access_log.CreateRunningAccessLogger(c)
	if err != nil {
		logger.Fatalf("Error creating access logger: %s\n", err)
	}

	var crypto secure.Crypto
	var cryptoPrev secure.Crypto
	if c.RouteServiceEnabled {
		crypto = createCrypto(logger, c.RouteServiceSecret)
		if c.RouteServiceSecretPrev != "" {
			cryptoPrev = createCrypto(logger, c.RouteServiceSecretPrev)
		}
	}

	proxy := buildProxy(c, registry, accessLogger, compositeReporter, crypto, cryptoPrev)

	router, err := router.NewRouter(c, proxy, natsClient, registry, varz, logCounter, nil)
	if err != nil {
		logger.Errorf("An error occurred: %s", err.Error())
		os.Exit(1)
	}

	members := grouper.Members{
		{"router", router},
	}
	if c.RoutingApiEnabled() {
		logger.Info("Setting up route fetcher")
		routeFetcher := setupRouteFetcher(logger, c, registry)
		members = append(members, grouper.Member{"router-fetcher", routeFetcher})
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1))

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("gorouter.exited-with-failure")
		os.Exit(1)
	}

	os.Exit(0)
}
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("communicating with doppler over TCP", func() {
	It("forwards messages", func() {
		etcdCleanup, etcdClientURL := integration_tests.SetupEtcd()
		defer etcdCleanup()
		metronCleanup, metronPort, metronReady := integration_tests.SetupMetron(etcdClientURL, "tcp")
		defer metronCleanup()
		dopplerCleanup, dopplerOutgoingPort := integration_tests.SetupDoppler(etcdClientURL, metronPort)
		defer dopplerCleanup()
		metronReady()

		err := dropsonde.Initialize(fmt.Sprintf("localhost:%d", metronPort), "test-origin")
		Expect(err).NotTo(HaveOccurred())

		By("sending a message into metron")
		sent := make(chan struct{})
		go func() {
			defer close(sent)
			err = logs.SendAppLog("test-app-id", "An event happened!", "test-app-id", "0")
			Expect(err).NotTo(HaveOccurred())
		}()
		<-sent

		By("reading a message from doppler")
		Eventually(func() ([]byte, error) {
			wsURL := fmt.Sprintf("ws://localhost:%d/apps/test-app-id/recentlogs", dopplerOutgoingPort)
			c, _, err := websocket.DefaultDialer.Dial(wsURL, nil)
Beispiel #21
0
func main() {
	logger := cf_lager.New("routing-api")

	flag.Parse()
	if *configPath == "" {
		logger.Error("failed to start", errors.New("No configuration file provided"))
		os.Exit(1)
	}

	cfg, err := config.NewConfigFromFile(*configPath)
	if err != nil {
		logger.Error("failed to start", err)
		os.Exit(1)
	}

	err = dropsonde.Initialize(cfg.MetronConfig.Address+":"+cfg.MetronConfig.Port, cfg.LogGuid)
	if err != nil {
		logger.Error("failed to initialize Dropsonde", err)
		os.Exit(1)
	}

	logger.Info("database", lager.Data{"etcd-addresses": flag.Args()})
	database := db.NewETCD(flag.Args())
	err = database.Connect()
	if err != nil {
		logger.Error("failed to connect to etcd", err)
		os.Exit(1)
	}
	defer database.Disconnect()

	var token authentication.Token

	if *devMode {
		token = authentication.NullToken{}
	} else {
		token = authentication.NewAccessToken(cfg.UAAPublicKey)
		err = token.CheckPublicToken()
		if err != nil {
			logger.Error("failed to check public token", err)
			os.Exit(1)
		}
	}

	validator := handlers.NewValidator()

	routesHandler := handlers.NewRoutesHandler(token, *maxTTL, validator, database, logger)
	eventStreamHandler := handlers.NewEventStreamHandler(token, database, logger)

	actions := rata.Handlers{
		"Upsert":      route(routesHandler.Upsert),
		"Delete":      route(routesHandler.Delete),
		"List":        route(routesHandler.List),
		"EventStream": route(eventStreamHandler.EventStream),
	}

	handler, err := rata.NewRouter(routing_api.Routes, actions)
	if err != nil {
		logger.Error("failed to create router", err)
		os.Exit(1)
	}

	handler = handlers.LogWrap(handler, logger)

	logger.Info("starting", lager.Data{"port": *port})
	err = http.ListenAndServe(":"+strconv.Itoa(*port), handler)
	if err != nil {
		panic(err)
	}
}
Beispiel #22
0
	. "github.com/onsi/gomega"
)

var _ = Describe("Autowire", func() {

	Describe("Initialize", func() {
		It("resets the HTTP default transport to be instrumented", func() {
			dropsonde.InitializeWithEmitter(&dropsonde.NullEventEmitter{})
			Expect(reflect.TypeOf(http.DefaultTransport).Elem().Name()).To(Equal("instrumentedCancelableRoundTripper"))
		})
	})

	Describe("CreateDefaultEmitter", func() {
		Context("with origin missing", func() {
			It("returns a NullEventEmitter", func() {
				err := dropsonde.Initialize("localhost:2343", "")
				Expect(err).To(HaveOccurred())

				emitter := dropsonde.AutowiredEmitter()
				Expect(emitter).ToNot(BeNil())
				nullEmitter := &dropsonde.NullEventEmitter{}
				Expect(emitter).To(BeAssignableToTypeOf(nullEmitter))
			})
		})
	})
})

type FakeHandler struct{}

func (fh FakeHandler) ServeHTTP(rw http.ResponseWriter, r *http.Request) {}
Beispiel #23
0
func main() {
	flag.Parse()
	cf_lager.AddFlags(flag.CommandLine)
	logger, reconfigurableSink := cf_lager.New("routing-api")

	err := checkFlags()
	if err != nil {
		logger.Error("failed to start", err)
		os.Exit(1)
	}

	cfg, err := config.NewConfigFromFile(*configPath)
	if err != nil {
		logger.Error("failed to start", err)
		os.Exit(1)
	}

	err = dropsonde.Initialize(cfg.MetronConfig.Address+":"+cfg.MetronConfig.Port, cfg.LogGuid)
	if err != nil {
		logger.Error("failed to initialize Dropsonde", err)
		os.Exit(1)
	}

	if cfg.DebugAddress != "" {
		cf_debug_server.Run(cfg.DebugAddress, reconfigurableSink)
	}

	database, err := initializeDatabase(cfg, logger)
	if err != nil {
		logger.Error("failed to initialize database", err)
		os.Exit(1)
	}

	err = database.Connect()
	if err != nil {
		logger.Error("failed to connect to database", err)
		os.Exit(1)
	}
	defer database.Disconnect()

	prefix := "routing_api"
	statsdClient, err := statsd.NewBufferedClient(cfg.StatsdEndpoint, prefix, cfg.StatsdClientFlushInterval, 512)
	if err != nil {
		logger.Error("failed to create a statsd client", err)
		os.Exit(1)
	}
	defer statsdClient.Close()

	stopChan := make(chan struct{})
	apiServer := constructApiServer(cfg, database, statsdClient, stopChan, logger)
	stopper := constructStopper(stopChan)

	routerRegister := constructRouteRegister(cfg.LogGuid, database, logger)

	metricsTicker := time.NewTicker(cfg.MetricsReportingInterval)
	metricsReporter := metrics.NewMetricsReporter(database, statsdClient, metricsTicker)

	members := grouper.Members{
		{"metrics", metricsReporter},
		{"api-server", apiServer},
		{"conn-stopper", stopper},
		{"route-register", routerRegister},
	}

	group := grouper.NewOrdered(os.Interrupt, members)
	process := ifrit.Invoke(sigmon.New(group))

	// This is used by testrunner to signal ready for tests.
	logger.Info("started", lager.Data{"port": *port})

	errChan := process.Wait()
	err = <-errChan
	if err != nil {
		logger.Error("shutdown-error", err)
		os.Exit(1)
	}
	logger.Info("exited")
}
Beispiel #24
0
func main() {
	c := config.DefaultConfig()
	logCounter := vcap.NewLogCounter()

	if configFile != "" {
		c = config.InitConfigFromFile(configFile)
	}

	InitLoggerFromConfig(c, logCounter)
	logger := steno.NewLogger("router.main")

	err := dropsonde.Initialize(c.Logging.MetronAddress, c.Logging.JobName)
	if err != nil {
		logger.Errorf("Dropsonde failed to initialize: %s", err.Error())
		os.Exit(1)
	}

	// setup number of procs
	if c.GoMaxProcs != 0 {
		runtime.GOMAXPROCS(c.GoMaxProcs)
	}

	if c.DebugAddr != "" {
		cf_debug_server.Run(c.DebugAddr)
	}

	logger.Info("Setting up etcd connection")
	etcdEndpoint, err := endpoint.NewEtcdEndpoint(c)
	if err != nil {
		os.Exit(1)
	}
	metricsReporter := metrics.NewMetricsReporter()
	registry := rregistry.NewRouteRegistry(c, etcdEndpoint, metricsReporter)

	logger.Info("Setting up routing_api route fetcher")
	setupRouteFetcher(c, registry)

	varz := rvarz.NewVarz(registry)
	compositeReporter := metrics.NewCompositeReporter(varz, metricsReporter)

	accessLogger, err := access_log.CreateRunningAccessLogger(c)
	if err != nil {
		logger.Fatalf("Error creating access logger: %s\n", err)
	}

	var crypto secure.Crypto
	var cryptoPrev secure.Crypto
	if c.RouteServiceEnabled {
		crypto = createCrypto(c.RouteServiceSecret, logger)
		if c.RouteServiceSecretPrev != "" {
			cryptoPrev = createCrypto(c.RouteServiceSecretPrev, logger)
		}
	}

	proxy := buildProxy(c, registry, accessLogger, compositeReporter, crypto, cryptoPrev)

	router, err := router.NewRouter(c, proxy, etcdEndpoint, registry, varz, logCounter)
	if err != nil {
		logger.Errorf("An error occurred: %s", err.Error())
		os.Exit(1)
	}

	errChan := router.Run()

	logger.Info("gorouter.started")

	waitOnErrOrSignal(c, logger, errChan, router)

	os.Exit(0)
}
	. "github.com/onsi/gomega"
	"net"
	"net/http"
	"strings"
	"sync"
	"time"
)

// these tests need to be invoked individually from an external script,
// since environment variables need to be set/unset before starting the tests
var _ = Describe("Autowire End-to-End", func() {
	Context("with standard initialization", func() {
		origin := []string{"test-origin"}

		BeforeEach(func() {
			dropsonde.Initialize("localhost:3457", origin...)
			metrics.Initialize(metric_sender.NewMetricSender(dropsonde.AutowiredEmitter()))
		})

		It("emits HTTP client/server events and heartbeats", func() {
			udpListener, err := net.ListenPacket("udp4", ":3457")
			Expect(err).ToNot(HaveOccurred())
			defer udpListener.Close()
			udpDataChan := make(chan []byte, 16)

			receivedEvents := make(map[string]bool)
			heartbeatUuidsChan := make(chan string, 1000)

			lock := sync.RWMutex{}
			heartbeatRequest := newHeartbeatRequest()
			marshalledHeartbeatRequest, _ := proto.Marshal(heartbeatRequest)
Beispiel #26
0
func main() {
	flag.Parse()
	config, logger := parseConfig(*debug, *configFile, *logFilePath)

	dropsonde.Initialize(config.MetronAddress, "syslog_drain_binder")

	workPool, err := workpool.NewWorkPool(config.EtcdMaxConcurrentRequests)
	if err != nil {
		panic(err)
	}

	adapter := etcdstoreadapter.NewETCDStoreAdapter(config.EtcdUrls, workPool)

	updateInterval := time.Duration(config.UpdateIntervalSeconds) * time.Second
	politician := elector.NewElector(config.InstanceName, adapter, updateInterval, logger)

	drainTTL := time.Duration(config.DrainUrlTtlSeconds) * time.Second
	store := etcd_syslog_drain_store.NewEtcdSyslogDrainStore(adapter, drainTTL, logger)

	ticker := time.NewTicker(updateInterval)
	for {
		select {
		case <-cfcomponent.RegisterGoRoutineDumpSignalChannel():
			cfcomponent.DumpGoRoutine()
		case <-ticker.C:
			if politician.IsLeader() {
				err = politician.StayAsLeader()
				if err != nil {
					logger.Errorf("Error when staying leader: %s", err.Error())
					politician.Vacate()
					continue
				}
			} else {
				err = politician.RunForElection()

				if err != nil {
					logger.Errorf("Error when running for leader: %s", err.Error())
					politician.Vacate()
					continue
				}
			}

			logger.Debugf("Polling %s for updates", config.CloudControllerAddress)
			drainUrls, err := Poll(config.CloudControllerAddress, config.BulkApiUsername, config.BulkApiPassword, config.PollingBatchSize, config.SkipCertVerify)
			if err != nil {
				logger.Errorf("Error when polling cloud controller: %s", err.Error())
				politician.Vacate()
				continue
			}

			metrics.IncrementCounter("pollCount")

			var totalDrains int
			for _, drainList := range drainUrls {
				totalDrains += len(drainList)
			}

			metrics.SendValue("totalDrains", float64(totalDrains), "drains")

			logger.Debugf("Updating drain URLs for %d application(s)", len(drainUrls))
			err = store.UpdateDrains(drainUrls)
			if err != nil {
				logger.Errorf("Error when updating ETCD: %s", err.Error())
				politician.Vacate()
				continue
			}
		}
	}
}
Beispiel #27
0
func initializeDropsonde(logger lager.Logger) {
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}
	udpConn     net.PacketConn
)

var _ = Describe("LogIntegration", func() {
	Context("with standard initialization", func() {
		origin := []string{"test-origin"}

		BeforeEach(func() {
			var err error
			logMessages = nil
			udpConn, err = net.ListenPacket("udp4", ":0")
			Expect(err).ToNot(HaveOccurred())

			go listenForLogs()
			udpAddr := udpConn.LocalAddr().(*net.UDPAddr)
			dropsonde.Initialize(fmt.Sprintf("localhost:%d", udpAddr.Port), origin...)
			sender := metric_sender.NewMetricSender(dropsonde.AutowiredEmitter())
			batcher := metricbatcher.New(sender, 100*time.Millisecond)
			metrics.Initialize(sender, batcher)
		})

		AfterEach(func() {
			udpConn.Close()
		})

		It("sends dropped error message for messages which are just under 64k and don't fit in UDP packet", func() {
			logSender := log_sender.NewLogSender(dropsonde.AutowiredEmitter(), time.Second, loggertesthelper.Logger())

			const length = 64*1024 - 1
			reader := strings.NewReader(strings.Repeat("s", length) + "\n")
			logSender.ScanErrorLogStream("someId", "app", "0", reader)
Beispiel #29
0
func main() {
	flag.Parse()

	// ** Config Setup
	config := &Config{}
	err := cfcomponent.ReadConfigInto(config, *configFile)
	if err != nil {
		panic(err)
	}

	dropsonde.Initialize(config.MetronAddress, "dea_logging_agent")

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			panic(err)
		}
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			f.Close()
		}()
	}

	if *memprofile != "" {
		f, err := os.Create(*memprofile)
		if err != nil {
			panic(err)
		}
		go func() {
			defer f.Close()
			ticker := time.NewTicker(time.Second * 1)
			defer ticker.Stop()
			for {
				<-ticker.C
				pprof.WriteHeapProfile(f)
			}
		}()
	}

	logger := cfcomponent.NewLogger(*logLevel, *logFilePath, "deaagent", config.Config)
	logger.Info("Startup: Setting up the loggregator dea logging agent")

	if len(config.NatsHosts) == 0 {
		logger.Warn("Startup: Did not receive a NATS host - not going to register component")
		cfcomponent.DefaultYagnatsClientProvider = func(logger *gosteno.Logger, c *cfcomponent.Config) (yagnats.NATSConn, error) {
			return fakeyagnats.Connect(), nil
		}
	}

	err = config.validate(logger)
	if err != nil {
		panic(err)
	}
	// ** END Config Setup

	agent := deaagent.NewAgent(*instancesJsonFilePath, logger)

	go agent.Start()

	killChan := make(chan os.Signal)
	signal.Notify(killChan, os.Interrupt)

	for {
		select {
		case <-cfcomponent.RegisterGoRoutineDumpSignalChannel():
			cfcomponent.DumpGoRoutine()
		case <-killChan:
			logger.Info("Shutting down")
			return
		}
	}
}
Beispiel #30
0
func main() {
	c := config.DefaultConfig()
	logCounter := vcap.NewLogCounter()

	if configFile != "" {
		c = config.InitConfigFromFile(configFile)
	}

	InitLoggerFromConfig(c, logCounter)
	logger := steno.NewLogger("router.main")

	err := dropsonde.Initialize(c.Logging.MetronAddress, c.Logging.JobName)
	if err != nil {
		logger.Errorf("Dropsonde failed to initialize: %s", err.Error())
		os.Exit(1)
	}

	// setup number of procs
	if c.GoMaxProcs != 0 {
		runtime.GOMAXPROCS(c.GoMaxProcs)
	}

	if c.DebugAddr != "" {
		cf_debug_server.Run(c.DebugAddr)
	}

	natsServers := c.NatsServers()
	var natsClient yagnats.NATSConn
	attempts := 3
	for attempts > 0 {
		natsClient, err = yagnats.Connect(natsServers)
		if err == nil {
			break
		} else {
			attempts--
			time.Sleep(100 * time.Millisecond)
		}
	}

	if err != nil {
		logger.Errorf("Error connecting to NATS: %s\n", err)
		os.Exit(1)
	}

	natsClient.AddClosedCB(func(conn *nats.Conn) {
		logger.Errorf("Close on NATS client. nats.Conn: %+v", *conn)
		os.Exit(1)
	})

	registry := rregistry.NewRouteRegistry(c, natsClient)

	if c.RoutingApiEnabled() {
		logger.Info("Setting up routing_api route fetcher")
		tokenFetcher := token_fetcher.NewTokenFetcher(&c.OAuth)
		routingApiUri := fmt.Sprintf("%s:%d", c.RoutingApi.Uri, c.RoutingApi.Port)
		routingApiClient := routing_api.NewClient(routingApiUri)
		routeFetcher := route_fetcher.NewRouteFetcher(steno.NewLogger("router.route_fetcher"), tokenFetcher, registry, c, routingApiClient, 1)
		routeFetcher.StartFetchCycle()
		routeFetcher.StartEventCycle()
	}

	varz := rvarz.NewVarz(registry)

	accessLogger, err := access_log.CreateRunningAccessLogger(c)
	if err != nil {
		logger.Fatalf("Error creating access logger: %s\n", err)
	}

	args := proxy.ProxyArgs{
		EndpointTimeout: c.EndpointTimeout,
		Ip:              c.Ip,
		TraceKey:        c.TraceKey,
		Registry:        registry,
		Reporter:        varz,
		AccessLogger:    accessLogger,
		SecureCookies:   c.SecureCookies,
	}
	p := proxy.NewProxy(args)

	router, err := router.NewRouter(c, p, natsClient, registry, varz, logCounter)
	if err != nil {
		logger.Errorf("An error occurred: %s", err.Error())
		os.Exit(1)
	}

	signals := make(chan os.Signal, 1)
	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT, syscall.SIGUSR1)

	errChan := router.Run()

	logger.Info("gorouter.started")

	select {
	case err := <-errChan:
		if err != nil {
			logger.Errorf("Error occurred: %s", err.Error())
			os.Exit(1)
		}
	case sig := <-signals:
		go func() {
			for sig := range signals {
				logger.Infod(
					map[string]interface{}{
						"signal": sig.String(),
					},
					"gorouter.signal.ignored",
				)
			}
		}()

		if sig == syscall.SIGUSR1 {
			logger.Infod(
				map[string]interface{}{
					"timeout": (c.DrainTimeout).String(),
				},
				"gorouter.draining",
			)

			router.Drain(c.DrainTimeout)
		}

		stoppingAt := time.Now()

		logger.Info("gorouter.stopping")

		router.Stop()

		logger.Infod(
			map[string]interface{}{
				"took": time.Since(stoppingAt).String(),
			},
			"gorouter.stopped",
		)
	}

	os.Exit(0)
}