func main() { if len(os.Args) != 2 { PrintUsageAndExit() } logGuid := os.Args[1] address := os.Getenv("DOPPLER") if address == "" { address = "wss://doppler.ketchup.cf-app.com:4443" } tlsConfig := &tls.Config{ InsecureSkipVerify: true, } logConsumer := noaa.NewConsumer(address, tlsConfig, nil) outputChan := make(chan *events.LogMessage, 0) errorChan := make(chan error, 0) stopChan := make(chan struct{}, 0) go logConsumer.TailingLogs(logGuid, "", outputChan, errorChan, stopChan) say.Println(0, "Fetching logs for log-guid: %s", say.Green(logGuid)) for { select { case message := <-outputChan: t := time.Unix(0, message.GetTimestamp()) say.Println(0, "%s [%s|%s] %s", say.Green(t.Format("02 Jan 15:04")), say.Cyan("%s", message.GetSourceInstance()), say.Cyan("%s", message.GetSourceType()), string(message.GetMessage())) case err := <-errorChan: say.Println(0, say.Red("Error while streaming:\n%s", err.Error())) return } } }
func NewRepositoryLocator(config core_config.ReadWriter, gatewaysByName map[string]net.Gateway) (loc RepositoryLocator) { strategy := strategy.NewEndpointStrategy(config.ApiVersion()) cloudControllerGateway := gatewaysByName["cloud-controller"] routingApiGateway := gatewaysByName["routing-api"] uaaGateway := gatewaysByName["uaa"] loc.authRepo = authentication.NewUAAAuthenticationRepository(uaaGateway, config) // ensure gateway refreshers are set before passing them by value to repositories cloudControllerGateway.SetTokenRefresher(loc.authRepo) uaaGateway.SetTokenRefresher(loc.authRepo) tlsConfig := net.NewTLSConfig([]tls.Certificate{}, config.IsSSLDisabled()) loggregatorConsumer := consumer.New(config.LoggregatorEndpoint(), tlsConfig, http.ProxyFromEnvironment) loggregatorConsumer.SetDebugPrinter(terminal.DebugPrinter{}) noaaLib := noaa.NewConsumer(config.DopplerEndpoint(), tlsConfig, http.ProxyFromEnvironment) noaaLib.SetDebugPrinter(terminal.DebugPrinter{}) logNoaaConsumer := NewNoaaConsumer(noaaLib) loc.appBitsRepo = application_bits.NewCloudControllerApplicationBitsRepository(config, cloudControllerGateway) loc.appEventsRepo = app_events.NewCloudControllerAppEventsRepository(config, cloudControllerGateway, strategy) loc.appFilesRepo = api_app_files.NewCloudControllerAppFilesRepository(config, cloudControllerGateway) loc.appRepo = applications.NewCloudControllerApplicationRepository(config, cloudControllerGateway) loc.appSummaryRepo = NewCloudControllerAppSummaryRepository(config, cloudControllerGateway) loc.appInstancesRepo = app_instances.NewCloudControllerAppInstancesRepository(config, cloudControllerGateway) loc.authTokenRepo = NewCloudControllerServiceAuthTokenRepository(config, cloudControllerGateway) loc.curlRepo = NewCloudControllerCurlRepository(config, cloudControllerGateway) loc.domainRepo = NewCloudControllerDomainRepository(config, cloudControllerGateway, strategy) loc.endpointRepo = NewEndpointRepository(config, cloudControllerGateway) loc.logsNoaaRepo = NewLogsNoaaRepository(config, logNoaaConsumer, loc.authRepo) loc.oldLogsRepo = NewLoggregatorLogsRepository(config, loggregatorConsumer, loc.authRepo) loc.organizationRepo = organizations.NewCloudControllerOrganizationRepository(config, cloudControllerGateway) loc.passwordRepo = password.NewCloudControllerPasswordRepository(config, uaaGateway) loc.quotaRepo = quotas.NewCloudControllerQuotaRepository(config, cloudControllerGateway) loc.routeRepo = NewCloudControllerRouteRepository(config, cloudControllerGateway) loc.routingApiRepo = NewRoutingApiRepository(config, routingApiGateway) loc.stackRepo = stacks.NewCloudControllerStackRepository(config, cloudControllerGateway) loc.serviceRepo = NewCloudControllerServiceRepository(config, cloudControllerGateway) loc.serviceKeyRepo = NewCloudControllerServiceKeyRepository(config, cloudControllerGateway) loc.serviceBindingRepo = NewCloudControllerServiceBindingRepository(config, cloudControllerGateway) loc.serviceBrokerRepo = NewCloudControllerServiceBrokerRepository(config, cloudControllerGateway) loc.servicePlanRepo = NewCloudControllerServicePlanRepository(config, cloudControllerGateway) loc.servicePlanVisibilityRepo = NewCloudControllerServicePlanVisibilityRepository(config, cloudControllerGateway) loc.serviceSummaryRepo = NewCloudControllerServiceSummaryRepository(config, cloudControllerGateway) loc.spaceRepo = spaces.NewCloudControllerSpaceRepository(config, cloudControllerGateway) loc.userProvidedServiceInstanceRepo = NewCCUserProvidedServiceInstanceRepository(config, cloudControllerGateway) loc.userRepo = NewCloudControllerUserRepository(config, uaaGateway, cloudControllerGateway) loc.buildpackRepo = NewCloudControllerBuildpackRepository(config, cloudControllerGateway) loc.buildpackBitsRepo = NewCloudControllerBuildpackBitsRepository(config, cloudControllerGateway, app_files.ApplicationZipper{}) loc.securityGroupRepo = security_groups.NewSecurityGroupRepo(config, cloudControllerGateway) loc.stagingSecurityGroupRepo = staging.NewStagingSecurityGroupsRepo(config, cloudControllerGateway) loc.runningSecurityGroupRepo = running.NewRunningSecurityGroupsRepo(config, cloudControllerGateway) loc.securityGroupSpaceBinder = securitygroupspaces.NewSecurityGroupSpaceBinder(config, cloudControllerGateway) loc.spaceQuotaRepo = space_quotas.NewCloudControllerSpaceQuotaRepository(config, cloudControllerGateway) loc.featureFlagRepo = feature_flags.NewCloudControllerFeatureFlagRepository(config, cloudControllerGateway) loc.environmentVariableGroupRepo = environment_variable_groups.NewCloudControllerEnvironmentVariableGroupsRepository(config, cloudControllerGateway) loc.copyAppSourceRepo = copy_application_source.NewCloudControllerCopyApplicationSourceRepository(config, cloudControllerGateway) return }
func main() { connection := noaa.NewConsumer(dopplerAddress, &tls.Config{InsecureSkipVerify: true}, nil) connection.SetDebugPrinter(ConsoleDebugPrinter{}) messages, err := connection.RecentLogs(appGuid, authToken) if err != nil { fmt.Printf("===== Error getting recent messages: %v\n", err) } else { fmt.Println("===== Recent logs") for _, msg := range messages { fmt.Println(msg) } } fmt.Println("===== Streaming metrics") msgChan := make(chan *events.Envelope) go func() { defer close(msgChan) errorChan := make(chan error) go connection.Stream(appGuid, authToken, msgChan, errorChan) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() for msg := range msgChan { fmt.Printf("%v \n", msg) } }
func main() { cf_debug_server.AddFlags(flag.CommandLine) cf_lager.AddFlags(flag.CommandLine) flag.Parse() logger, reconfigurableSink := cf_lager.New("tps-listener") initializeDropsonde(logger) noaaClient := noaa.NewConsumer(*trafficControllerURL, &tls.Config{InsecureSkipVerify: *skipSSLVerification}, nil) defer noaaClient.Close() apiHandler := initializeHandler(logger, noaaClient, *maxInFlightRequests, initializeBBSClient(logger)) members := grouper.Members{ {"api", http_server.New(*listenAddr, apiHandler)}, } if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" { members = append(grouper.Members{ {"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)}, }, members...) } group := grouper.NewOrdered(os.Interrupt, members) monitor := ifrit.Invoke(sigmon.New(group)) logger.Info("started") err := <-monitor.Wait() if err != nil { logger.Error("exited-with-failure", err) os.Exit(1) } logger.Info("exited") }
func (d *APMFirehoseNozzle) consumeFirehose(authToken string) { d.consumer = noaa.NewConsumer( d.config.TrafficControllerURL, &tls.Config{InsecureSkipVerify: d.config.InsecureSSLSkipVerify}, nil) go d.consumer.Firehose(d.config.FirehoseSubscriptionID, authToken, d.messages, d.errs, nil) }
func (m *MetricsNozzle) consumeFirehose(authToken string) { m.consumer = noaa.NewConsumer( m.config.DopplerAddr, &tls.Config{InsecureSkipVerify: m.config.InsecureSSLSkipVerify}, nil) go m.consumer.Firehose(m.config.FirehoseSubscriptionId, authToken, m.messages, m.errs) }
func (d *DatadogFirehoseNozzle) consumeFirehose(authToken string) { d.consumer = noaa.NewConsumer( d.config.TrafficControllerURL, &tls.Config{InsecureSkipVerify: d.config.InsecureSSLSkipVerify}, nil) d.consumer.SetIdleTimeout(time.Duration(d.config.IdleTimeoutSeconds) * time.Second) go d.consumer.Firehose(d.config.FirehoseSubscriptionID, authToken, d.messages, d.errs) }
func initiateFirehoseConnection() (*noaa.Consumer, chan *events.Envelope) { localIP, _ := localip.LocalIP() firehoseConnection := noaa.NewConsumer("ws://"+localIP+":49629", &tls.Config{InsecureSkipVerify: true}, nil) msgChan := make(chan *events.Envelope, 2000) errorChan := make(chan error) go firehoseConnection.Firehose("uniqueId", "", msgChan, errorChan) return firehoseConnection, msgChan }
func SetUpConsumer() (*noaa.Consumer, *TestDebugPrinter) { tlsConfig := tls.Config{InsecureSkipVerify: config.SkipSSLVerify} printer := &TestDebugPrinter{} connection := noaa.NewConsumer(config.DopplerEndpoint, &tlsConfig, nil) connection.SetDebugPrinter(printer) return connection, printer }
func main() { setupHTTP() uaaURL, err := url.Parse(uaa) if nil != err { panic("Failed to parse uaa url!") } creds, err := uaaclientcredentials.New(uaaURL, true, clientID, clientSecret) if nil != err { panic("Failed to obtain creds!") } consumer := noaa.NewConsumer(dopplerAddress, &tls.Config{InsecureSkipVerify: true}, nil) httpStartStopProcessor := processors.NewHttpStartStopProcessor() sender := statsd.NewStatsdClient(statsdAddress, statsdPrefix) sender.CreateSocket() var processedMetrics []metrics.Metric msgChan := make(chan *events.Envelope) go func() { defer close(msgChan) errorChan := make(chan error) token, err := creds.GetBearerToken() if nil != err { panic(err) } go consumer.Firehose(firehoseSubscriptionID, token, msgChan, errorChan, nil) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() for msg := range msgChan { eventType := msg.GetEventType() switch eventType { case events.Envelope_HttpStartStop: processedMetrics = httpStartStopProcessor.Process(msg) default: atomic.AddUint64(&count, 1) // do nothing } if len(processedMetrics) > 0 { for _, metric := range processedMetrics { metric.Send(sender) } } processedMetrics = nil } }
func main() { var messages map[string][]metricCategory // [origin]{metricCategory To []names}] messages = make(map[string][]metricCategory) connection := noaa.NewConsumer(dopplerAddress, &tls.Config{InsecureSkipVerify: true}, nil) connection.SetDebugPrinter(ConsoleDebugPrinter{}) fmt.Println("===== Streaming Firehose (will only succeed if you have admin credentials)") msgChan := make(chan *events.Envelope) go func() { defer close(msgChan) errorChan := make(chan error) go connection.Firehose(firehoseSubscriptionId, authToken, msgChan, errorChan) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() go startHttp(messages) for msg := range msgChan { vm := msg.GetValueMetric() if vm == nil { continue } origin := msg.GetOrigin() category, subCategory := parseMetric(*vm.Name) index := indexOf(messages[origin], category) if index >= 0 { metricCategoryGroup := messages[origin][index] switch f := metricCategoryGroup.(type) { case *metricCategoryWithSubCategory: if len(subCategory) > 0 { if contains(f.SubCategory, subCategory) == false { f.SubCategory = append(f.SubCategory, subCategory) } } break default: } } else { if len(subCategory) > 0 { messages[origin] = append(messages[origin], &metricCategoryWithSubCategory{ Category: category, SubCategory: []string{subCategory}, }) } else { messages[origin] = append(messages[origin], &metricCategoryOnly{ Category: category, }) } } } }
func main() { consumer := noaa.NewConsumer(DopplerAddress, &tls.Config{InsecureSkipVerify: true}, nil) httpStartStopProcessor := processors.NewHttpStartStopProcessor() valueMetricProcessor := processors.NewValueMetricProcessor() containerMetricProcessor := processors.NewContainerMetricProcessor() heartbeatProcessor := processors.NewHeartbeatProcessor() counterProcessor := processors.NewCounterProcessor() sender := statsd.NewStatsdClient(statsdAddress, statsdPrefix) sender.CreateSocket() var processedMetrics []metrics.Metric msgChan := make(chan *events.Envelope) go func() { defer close(msgChan) errorChan := make(chan error) go consumer.Firehose(firehoseSubscriptionId, authToken, msgChan, errorChan, nil) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() for msg := range msgChan { eventType := msg.GetEventType() // graphite-nozzle can handle CounterEvent, ContainerMetric, Heartbeat, // HttpStartStop and ValueMetric events switch eventType { case events.Envelope_ContainerMetric: processedMetrics = containerMetricProcessor.Process(msg) case events.Envelope_CounterEvent: processedMetrics = counterProcessor.Process(msg) case events.Envelope_Heartbeat: processedMetrics = heartbeatProcessor.Process(msg) case events.Envelope_HttpStartStop: processedMetrics = httpStartStopProcessor.Process(msg) case events.Envelope_ValueMetric: processedMetrics = valueMetricProcessor.Process(msg) default: // do nothing } if len(processedMetrics) > 0 { for _, metric := range processedMetrics { metric.Send(sender) } } processedMetrics = nil } }
func createNoaaClient(dopplerUrl, authToken string) (chan *events.Envelope, chan error) { connection := noaa.NewConsumer(dopplerUrl, &tls.Config{InsecureSkipVerify: true}, nil) msgChan := make(chan *events.Envelope) errorChan := make(chan error) go func() { defer close(msgChan) go connection.Firehose("firehose-a", authToken, msgChan, errorChan) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() return msgChan, errorChan }
func startFirehose(oauthToken string, msgChan chan *events.Envelope) { connection := noaa.NewConsumer(dopplerAddress, &tls.Config{InsecureSkipVerify: true}, nil) // connection.SetDebugPrinter(ConsoleDebugPrinter{}) fmt.Println("===== Streaming Firehose (will only succeed if you have admin credentials)") go func() { defer close(msgChan) errorChan := make(chan error) go connection.Firehose(firehoseSubscriptionId, oauthToken, msgChan, errorChan) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() }
func connectToFirehose(appEnv *cfenv.App, token string) chan *events.Envelope { doppler, err := GetUserProvidedServiceByName("doppler", appEnv) dieIfError("Failed to get doppler service", err) consumer := noaa.NewConsumer(doppler.Credentials["uri"].(string), &tls.Config{InsecureSkipVerify: skipSSLVerify}, nil) msgChan := make(chan *events.Envelope) go func() { defer close(msgChan) errorChan := make(chan error) go consumer.Firehose(subscriptionID, token, msgChan, errorChan) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() return msgChan }
func (c *Client) Start() { dopplerConnection := noaa.NewConsumer(c.dopplerEndpoint, &tls.Config{InsecureSkipVerify: true}, nil) subscriptionID := "firehose-stats" go func() { err := dopplerConnection.FirehoseWithoutReconnect(subscriptionID, c.authToken, c.outputChan) if err != nil { c.ui.Warn(err.Error()) close(c.outputChan) return } }() defer dopplerConnection.Close() c.ui.Say("Starting the nozzle") c.ui.Say("Hit Ctrl+c to exit") }
func main() { connection := noaa.NewConsumer(DopplerAddress, &tls.Config{InsecureSkipVerify: true}, nil) connection.SetDebugPrinter(ConsoleDebugPrinter{}) fmt.Println("===== Streaming ContainerMetrics (will only succeed if you have admin credentials)") for { containerMetrics, err := connection.ContainerMetrics(appId, authToken) for _, cm := range containerMetrics { fmt.Printf("%v \n", cm) } if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } time.Sleep(3 * time.Second) } }
func main() { user := flag.String("user", "example-nozzle", "user who has access to the firehose") password := flag.String("pass", "example-nozzle", "password for the user") trafficControllerURL := flag.String("tcurl", "wss://doppler.bosh-lite.com:443", "loggregator traffic controller URL and port") uaaURL := flag.String("uaaurl", "https://uaa.bosh-lite.com", "UAA URL") flag.Parse() uaaClient, err := uaago.NewClient(*uaaURL) if err != nil { log.Fatalf("Error creating uaa client: %s", err.Error()) } var authToken string authToken, err = uaaClient.GetAuthToken(*user, *password, true) if err != nil { log.Fatalf("Error getting oauth token: %s. Please check your username and password.", err.Error()) } connection := noaa.NewConsumer(*trafficControllerURL, &tls.Config{InsecureSkipVerify: true}, nil) fmt.Println("===== Streaming Firehose (will only succeed if you have admin credentials)") msgChan := make(chan *events.Envelope) go func() { defer close(msgChan) errorChan := make(chan error) const firehoseSubscriptionId = "firehose-a" go connection.Firehose(firehoseSubscriptionId, authToken, msgChan, errorChan) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() for msg := range msgChan { fmt.Printf("%v \n", msg) } }
func CreateFirehoseChan(DopplerEndpoint string, Token string, subId string, skipSSLValidation bool) chan *events.Envelope { connection := noaa.NewConsumer(DopplerEndpoint, &tls.Config{InsecureSkipVerify: skipSSLValidation}, nil) connection.SetDebugPrinter(ConsoleDebugPrinter{}) msgChan := make(chan *events.Envelope) go func() { errorChan := make(chan error) defer close(msgChan) defer func() { if r := recover(); r != nil { log.LogError("Recovered in CreateFirehoseChan Thread!", r) } }() go connection.Firehose(subId, Token, msgChan, errorChan) for err := range errorChan { log.LogError("Firehose Error!", err.Error()) } }() return msgChan }
func main() { kingpin.Parse() fmt.Println(*uaaEndpoint) fmt.Println(*username) fmt.Println(*password) tokenFetcher := &token.UAATokenFetcher{ UaaUrl: *uaaEndpoint, Username: *username, Password: *password, InsecureSSLSkipVerify: *skipSSLValidation, } authToken, err := tokenFetcher.FetchAuthToken() if err != nil { fmt.Println(err) os.Exit(-1) } consumer := noaa.NewConsumer(*dopplerEndpoint, &tls.Config{InsecureSkipVerify: *skipSSLValidation}, nil) httpStartStopProcessor := processors.NewHttpStartStopProcessor() valueMetricProcessor := processors.NewValueMetricProcessor() containerMetricProcessor := processors.NewContainerMetricProcessor() heartbeatProcessor := processors.NewHeartbeatProcessor() counterProcessor := processors.NewCounterProcessor() sender := statsd.NewStatsdClient(*statsdEndpoint, *statsdPrefix) sender.CreateSocket() var processedMetrics []metrics.Metric msgChan := make(chan *events.Envelope) go func() { defer close(msgChan) errorChan := make(chan error) go consumer.Firehose(*subscriptionId, authToken, msgChan, errorChan, nil) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() for msg := range msgChan { eventType := msg.GetEventType() // graphite-nozzle can handle CounterEvent, ContainerMetric, Heartbeat, // HttpStartStop and ValueMetric events switch eventType { case events.Envelope_ContainerMetric: processedMetrics = containerMetricProcessor.Process(msg) case events.Envelope_CounterEvent: processedMetrics = counterProcessor.Process(msg) case events.Envelope_Heartbeat: processedMetrics = heartbeatProcessor.Process(msg) case events.Envelope_HttpStartStop: processedMetrics = httpStartStopProcessor.Process(msg) case events.Envelope_ValueMetric: processedMetrics = valueMetricProcessor.Process(msg) default: // do nothing } if !*debug { if len(processedMetrics) > 0 { for _, metric := range processedMetrics { var prefix string if *prefixJob { prefix = msg.GetJob() + "." + msg.GetIndex() } metric.Send(sender, prefix) } } } else { for _, msg := range processedMetrics { fmt.Println(msg) } } processedMetrics = nil } }
func (cmd *LogalyzerCmd) Run(cliConnection plugin.CliConnection, args []string) { port := 8080 aggregationWindow := 2 * time.Second bufferSize := 100 cmd.ui = terminal.NewUI(os.Stdin, terminal.NewTeePrinter()) cmd.sendChan = make(chan []byte, 256) if len(args) < 2 { cmd.ui.Say("Usage: %s\n", cmd.Usage()) cmd.ui.Failed("No App Name given") } fc := flags.NewFlagContext(setupFlags()) err := fc.Parse(args[2:]...) if err != nil { cmd.ui.Failed(err.Error()) } if fc.IsSet("p") { port = fc.Int("p") } if fc.IsSet("agg-window") { aggregationWindow = time.Duration(fc.Int("agg-window")) * time.Second } if fc.IsSet("buffer-size") { bufferSize = fc.Int("buffer-size") } appName := args[1] dopplerEndpoint, err := cliConnection.DopplerEndpoint() if err != nil { cmd.ui.Failed(err.Error()) } appModel, err := cliConnection.GetApp(appName) if err != nil { cmd.ui.Failed(err.Error()) } authToken, err := cliConnection.AccessToken() if err != nil { cmd.ui.Failed(err.Error()) } outputChan := make(chan *events.LogMessage) errorChan := make(chan error) dopplerConnection := noaa.NewConsumer(dopplerEndpoint, &tls.Config{InsecureSkipVerify: true}, nil) go dopplerConnection.TailingLogs(appModel.Guid, authToken, outputChan, errorChan) go cmd.happyCalcer(aggregationWindow, bufferSize) go cmd.startServer(cmd.getAssetsDir(), port) cmd.ui.Say("Webserver is started at http://localhost:%d", port) for { select { case log := <-outputChan: if log.GetMessageType() == events.LogMessage_ERR { cmd.errCount++ } cmd.logCount++ cmd.sendChan <- log.GetMessage() case err := <-errorChan: cmd.ui.Failed(err.Error()) } } }
return helpers.CurlApp(appName, fmt.Sprintf("/log/sleep/%d", hundredthOfOneSecond)) }, DEFAULT_TIMEOUT).Should(ContainSubstring("Muahaha")) Eventually(func() *Session { appLogsSession := cf.Cf("logs", "--recent", appName) Expect(appLogsSession.Wait(DEFAULT_TIMEOUT)).To(Exit(0)) return appLogsSession }, DEFAULT_TIMEOUT).Should(Say("Muahaha")) }) }) Context("firehose data", func() { It("shows logs and metrics", func() { config := helpers.LoadConfig() noaaConnection := noaa.NewConsumer(getDopplerEndpoint(), &tls.Config{InsecureSkipVerify: config.SkipSSLValidation}, nil) msgChan := make(chan *events.Envelope, 100000) errorChan := make(chan error) stopchan := make(chan struct{}) go noaaConnection.Firehose(generator.RandomName(), getAdminUserAccessToken(), msgChan, errorChan, stopchan) defer close(stopchan) Eventually(func() string { return helpers.CurlApp(appName, fmt.Sprintf("/log/sleep/%d", hundredthOfOneSecond)) }, DEFAULT_TIMEOUT).Should(ContainSubstring("Muahaha")) timeout := time.After(5 * time.Second) messages := make([]*events.Envelope, 0, 100000) for { select {
. "github.com/onsi/gomega" ) var dropsondeEndpoint string const TRAFFIC_CONTROLLER_DROPSONDE_PORT = 4566 var _ = Describe("TrafficController for dropsonde messages", func() { BeforeEach(func() { dropsondeEndpoint = fmt.Sprintf("ws://%s:%d", localIPAddress, TRAFFIC_CONTROLLER_DROPSONDE_PORT) fakeDoppler.ResetMessageChan() }) Context("Streaming", func() { It("passes messages through", func() { client := noaa.NewConsumer(dropsondeEndpoint, &tls.Config{}, nil) messages := make(chan *events.Envelope) go client.StreamWithoutReconnect(APP_ID, AUTH_TOKEN, messages) var request *http.Request Eventually(fakeDoppler.TrafficControllerConnected, 10).Should(Receive(&request)) Expect(request.URL.Path).To(Equal("/apps/1234/stream")) currentTime := time.Now().UnixNano() dropsondeMessage := makeDropsondeMessage("Hello through NOAA", APP_ID, currentTime) fakeDoppler.SendLogMessage(dropsondeMessage) var receivedEnvelope *events.Envelope Eventually(messages).Should(Receive(&receivedEnvelope)) receivedMessage := receivedEnvelope.GetLogMessage()
func cliCommands(ltcConfigRoot string, exitHandler exit_handler.ExitHandler, config *config.Config, logger lager.Logger, targetVerifier target_verifier.TargetVerifier, ui terminal.UI) []cli.Command { receptorClient := receptor.NewClient(config.Receptor()) noaaConsumer := noaa.NewConsumer(LoggregatorUrl(config.Loggregator()), nil, nil) appRunner := app_runner.New(receptorClient, config.Target()) clock := clock.NewClock() logReader := logs.NewLogReader(noaaConsumer) tailedLogsOutputter := console_tailed_logs_outputter.NewConsoleTailedLogsOutputter(ui, logReader) taskExaminer := task_examiner.New(receptorClient) taskExaminerCommandFactory := task_examiner_command_factory.NewTaskExaminerCommandFactory(taskExaminer, ui, exitHandler) taskRunner := task_runner.New(receptorClient, taskExaminer) taskRunnerCommandFactory := task_runner_command_factory.NewTaskRunnerCommandFactory(taskRunner, ui, exitHandler) appExaminer := app_examiner.New(receptorClient, app_examiner.NewNoaaConsumer(noaaConsumer)) graphicalVisualizer := graphical.NewGraphicalVisualizer(appExaminer) appExaminerCommandFactory := app_examiner_command_factory.NewAppExaminerCommandFactory(appExaminer, ui, clock, exitHandler, graphicalVisualizer, taskExaminer, config.Target()) appRunnerCommandFactoryConfig := app_runner_command_factory.AppRunnerCommandFactoryConfig{ AppRunner: appRunner, AppExaminer: appExaminer, UI: ui, Domain: config.Target(), Env: os.Environ(), Clock: clock, Logger: logger, TailedLogsOutputter: tailedLogsOutputter, ExitHandler: exitHandler, } appRunnerCommandFactory := app_runner_command_factory.NewAppRunnerCommandFactory(appRunnerCommandFactoryConfig) dockerRunnerCommandFactoryConfig := docker_runner_command_factory.DockerRunnerCommandFactoryConfig{ AppRunner: appRunner, AppExaminer: appExaminer, UI: ui, Domain: config.Target(), Env: os.Environ(), Clock: clock, Logger: logger, ExitHandler: exitHandler, TailedLogsOutputter: tailedLogsOutputter, DockerMetadataFetcher: docker_metadata_fetcher.New(docker_metadata_fetcher.NewDockerSessionFactory()), } dockerRunnerCommandFactory := docker_runner_command_factory.NewDockerRunnerCommandFactory(dockerRunnerCommandFactoryConfig) logsCommandFactory := logs_command_factory.NewLogsCommandFactory(appExaminer, taskExaminer, ui, tailedLogsOutputter, exitHandler) clusterTestRunner := cluster_test.NewClusterTestRunner(config, ltcConfigRoot) clusterTestCommandFactory := cluster_test_command_factory.NewClusterTestCommandFactory(clusterTestRunner) blobStore := blob_store.New(config) blobStoreVerifier := blob_store.NewVerifier(config) dropletRunner := droplet_runner.New(appRunner, taskRunner, config, blobStore, appExaminer) cfIgnore := cf_ignore.New() zipper := &zipper_package.DropletArtifactZipper{} dropletRunnerCommandFactory := droplet_runner_command_factory.NewDropletRunnerCommandFactory(*appRunnerCommandFactory, blobStoreVerifier, taskExaminer, dropletRunner, cfIgnore, zipper, config) configCommandFactory := config_command_factory.NewConfigCommandFactory(config, ui, targetVerifier, blobStoreVerifier, exitHandler) helpCommand := cli.Command{ Name: "help", Aliases: []string{"h"}, Usage: "Shows a list of commands or help for one command", Description: "ltc help", Action: defaultAction, } return []cli.Command{ appExaminerCommandFactory.MakeCellsCommand(), dockerRunnerCommandFactory.MakeCreateAppCommand(), appRunnerCommandFactory.MakeSubmitLrpCommand(), logsCommandFactory.MakeDebugLogsCommand(), appExaminerCommandFactory.MakeListAppCommand(), logsCommandFactory.MakeLogsCommand(), appRunnerCommandFactory.MakeRemoveAppCommand(), appRunnerCommandFactory.MakeScaleAppCommand(), appExaminerCommandFactory.MakeStatusCommand(), taskRunnerCommandFactory.MakeSubmitTaskCommand(), configCommandFactory.MakeTargetCommand(), taskExaminerCommandFactory.MakeTaskCommand(), taskRunnerCommandFactory.MakeDeleteTaskCommand(), taskRunnerCommandFactory.MakeCancelTaskCommand(), clusterTestCommandFactory.MakeClusterTestCommand(), appRunnerCommandFactory.MakeUpdateRoutesCommand(), appRunnerCommandFactory.MakeUpdateCommand(), appExaminerCommandFactory.MakeVisualizeCommand(), dropletRunnerCommandFactory.MakeBuildDropletCommand(), dropletRunnerCommandFactory.MakeListDropletsCommand(), dropletRunnerCommandFactory.MakeLaunchDropletCommand(), dropletRunnerCommandFactory.MakeRemoveDropletCommand(), dropletRunnerCommandFactory.MakeImportDropletCommand(), dropletRunnerCommandFactory.MakeExportDropletCommand(), helpCommand, } }
var errorChan chan error var finishedChan chan struct{} BeforeEach(func() { errorChan = make(chan error, 10) finishedChan = make(chan struct{}) incomingChan = make(chan *events.Envelope) }) AfterEach(func() { close(messagesToSend) <-finishedChan }) perform := func() { connection = noaa.NewConsumer(endpoint, tlsSettings, consumerProxyFunc) go func() { errorChan <- connection.StreamWithoutReconnect(appGuid, authToken, incomingChan) close(finishedChan) }() } It("connects using valid URL to running consumerProxyFunc server", func() { messagesToSend <- marshalMessage(createMessage("hello", 0)) perform() message := <-incomingChan Expect(message.GetLogMessage().GetMessage()).To(Equal([]byte("hello"))) })
testServer.Close() } }) Describe("SetOnConnectCallback", func() { BeforeEach(func() { testServer = httptest.NewServer(handlers.NewWebsocketHandler(messagesToSend, 100*time.Millisecond, loggertesthelper.Logger())) trafficControllerUrl = "ws://" + testServer.Listener.Addr().String() close(messagesToSend) }) It("sets a callback and calls it when connecting", func() { called := false cb := func() { called = true } connection = noaa.NewConsumer(trafficControllerUrl, tlsSettings, nil) connection.SetOnConnectCallback(cb) logChan := make(chan *events.LogMessage, 100) connection.TailingLogsWithoutReconnect(appGuid, authToken, logChan) Eventually(func() bool { return called }).Should(BeTrue()) }) Context("when the connection fails", func() { It("does not call the callback", func() { trafficControllerUrl = "!!!bad-url" called := false cb := func() { called = true }
func cliCommands(ltcConfigRoot string, exitHandler exit_handler.ExitHandler, config *config.Config, logger lager.Logger, receptorClientCreator receptor_client.Creator, targetVerifier target_verifier.TargetVerifier, ui terminal.UI, latticeVersion string) []cli.Command { receptorClient := receptorClientCreator.CreateReceptorClient(config.Receptor()) noaaConsumer := noaa.NewConsumer(LoggregatorUrl(config.Loggregator()), nil, nil) appRunner := app_runner.New(receptorClient, config.Target(), &keygen_package.KeyGenerator{RandReader: rand.Reader}) clock := clock.NewClock() logReader := logs.NewLogReader(noaaConsumer) tailedLogsOutputter := console_tailed_logs_outputter.NewConsoleTailedLogsOutputter(ui, logReader) taskExaminer := task_examiner.New(receptorClient) taskExaminerCommandFactory := task_examiner_command_factory.NewTaskExaminerCommandFactory(taskExaminer, ui, exitHandler) taskRunner := task_runner.New(receptorClient, taskExaminer, clock) taskRunnerCommandFactory := task_runner_command_factory.NewTaskRunnerCommandFactory(taskRunner, ui, exitHandler) appExaminer := app_examiner.New(receptorClient, app_examiner.NewNoaaConsumer(noaaConsumer)) graphicalVisualizer := graphical.NewGraphicalVisualizer(appExaminer) dockerTerminal := &app_examiner_command_factory.DockerTerminal{} appExaminerCommandFactory := app_examiner_command_factory.NewAppExaminerCommandFactory(appExaminer, ui, dockerTerminal, clock, exitHandler, graphicalVisualizer, taskExaminer, config.Target()) appRunnerCommandFactoryConfig := app_runner_command_factory.AppRunnerCommandFactoryConfig{ AppRunner: appRunner, AppExaminer: appExaminer, UI: ui, Domain: config.Target(), Env: os.Environ(), Clock: clock, Logger: logger, TailedLogsOutputter: tailedLogsOutputter, ExitHandler: exitHandler, } appRunnerCommandFactory := app_runner_command_factory.NewAppRunnerCommandFactory(appRunnerCommandFactoryConfig) dockerRunnerCommandFactoryConfig := docker_runner_command_factory.DockerRunnerCommandFactoryConfig{ AppRunner: appRunner, AppExaminer: appExaminer, UI: ui, Domain: config.Target(), Env: os.Environ(), Clock: clock, Logger: logger, ExitHandler: exitHandler, TailedLogsOutputter: tailedLogsOutputter, DockerMetadataFetcher: docker_metadata_fetcher.New(docker_metadata_fetcher.NewDockerSessionFactory()), } dockerRunnerCommandFactory := docker_runner_command_factory.NewDockerRunnerCommandFactory(dockerRunnerCommandFactoryConfig) logsCommandFactory := logs_command_factory.NewLogsCommandFactory(appExaminer, taskExaminer, ui, tailedLogsOutputter, exitHandler) clusterTestRunner := cluster_test.NewClusterTestRunner(config, ltcConfigRoot) clusterTestCommandFactory := cluster_test_command_factory.NewClusterTestCommandFactory(clusterTestRunner) blobStore := blob_store.New(config) blobStoreVerifier := blob_store.BlobStoreVerifier{ DAVBlobStoreVerifier: dav_blob_store.Verifier{}, S3BlobStoreVerifier: s3_blob_store.Verifier{}, } httpProxyConfReader := &droplet_runner.HTTPProxyConfReader{ URL: fmt.Sprintf("http://%s:8444/proxyconf.json", config.Target()), } dropletRunner := droplet_runner.New(appRunner, taskRunner, config, blobStore, appExaminer, httpProxyConfReader) cfIgnore := cf_ignore.New() zipper := &zipper_package.DropletArtifactZipper{} dropletRunnerCommandFactory := droplet_runner_command_factory.NewDropletRunnerCommandFactory(*appRunnerCommandFactory, blobStoreVerifier, taskExaminer, dropletRunner, cfIgnore, zipper, config) versionManager := version.NewVersionManager(receptorClientCreator, &version.AppFileSwapper{}, defaultLatticeVersion(latticeVersion)) configCommandFactory := config_command_factory.NewConfigCommandFactory(config, ui, targetVerifier, blobStoreVerifier, exitHandler, versionManager) sshCommandFactory := ssh_command_factory.NewSSHCommandFactory(config, ui, exitHandler, appExaminer, ssh.New(exitHandler)) ltcPath, _ := osext.Executable() versionCommandFactory := version_command_factory.NewVersionCommandFactory(config, ui, exitHandler, runtime.GOOS, ltcPath, versionManager) helpCommand := cli.Command{ Name: "help", Aliases: []string{"h"}, Usage: "Shows a list of commands or help for one command", Description: "ltc help", Action: defaultAction, } return []cli.Command{ appExaminerCommandFactory.MakeCellsCommand(), dockerRunnerCommandFactory.MakeCreateAppCommand(), appRunnerCommandFactory.MakeSubmitLrpCommand(), logsCommandFactory.MakeDebugLogsCommand(), appExaminerCommandFactory.MakeListAppCommand(), logsCommandFactory.MakeLogsCommand(), appRunnerCommandFactory.MakeRemoveAppCommand(), appRunnerCommandFactory.MakeScaleAppCommand(), appExaminerCommandFactory.MakeStatusCommand(), taskRunnerCommandFactory.MakeSubmitTaskCommand(), configCommandFactory.MakeTargetCommand(), taskExaminerCommandFactory.MakeTaskCommand(), taskRunnerCommandFactory.MakeDeleteTaskCommand(), taskRunnerCommandFactory.MakeCancelTaskCommand(), clusterTestCommandFactory.MakeClusterTestCommand(), appRunnerCommandFactory.MakeUpdateCommand(), appExaminerCommandFactory.MakeVisualizeCommand(), dropletRunnerCommandFactory.MakeBuildDropletCommand(), dropletRunnerCommandFactory.MakeListDropletsCommand(), dropletRunnerCommandFactory.MakeLaunchDropletCommand(), dropletRunnerCommandFactory.MakeRemoveDropletCommand(), dropletRunnerCommandFactory.MakeImportDropletCommand(), dropletRunnerCommandFactory.MakeExportDropletCommand(), sshCommandFactory.MakeSSHCommand(), versionCommandFactory.MakeSyncCommand(), versionCommandFactory.MakeVersionCommand(), helpCommand, } }
func (fi *FirehoseInspector) Run(cliConnection plugin.CliConnection, args []string) { if args[0] == "firehose-inspector" { confRepo := core_config.NewRepositoryFromFilepath(config_helpers.DefaultFilePath(), fatalIf) dopplerEndpoint := confRepo.DopplerEndpoint() accessToken := confRepo.AccessToken() connection := noaa.NewConsumer(dopplerEndpoint, &tls.Config{InsecureSkipVerify: true}, nil) msgChan := make(chan *events.Envelope) fi.pages = []p.Page{ p.Page{ Title: "Logs", Outputs: []p.Output{ &so.LogsDisplay{ MarginPos: 32, }, }, Foreground: termbox.ColorWhite, Background: termbox.ColorDefault, }, p.Page{ Title: "Page 1", Outputs: []p.Output{ &so.NullDisplay{}, }, Foreground: termbox.ColorWhite, Background: termbox.ColorDefault, }, p.Page{ Title: "Page 2", Outputs: []p.Output{ &so.NullDisplay{}, }, Foreground: termbox.ColorWhite, Background: termbox.ColorDefault, }, } // call init on all Outputs for _, page := range fi.pages { for _, output := range page.Outputs { output.Init() } } fi.currentPage = fi.pages[0] err := termbox.Init() if err != nil { panic(err) } defer termbox.Close() termbox.SetOutputMode(termbox.Output256) fi.currentPage.Draw(fi.pages) go func() { defer close(msgChan) errorChan := make(chan error) go connection.Firehose(firehoseSubscriptionId, accessToken, msgChan, errorChan) for err := range errorChan { fmt.Fprintf(os.Stderr, "%v\n", err.Error()) } }() go func() { for msg := range msgChan { for _, output := range fi.currentPage.Outputs { output.Update(msg) } } }() loop: for { switch ev := termbox.PollEvent(); ev.Type { case termbox.EventKey: for _, output := range fi.currentPage.Outputs { output.KeyEvent(ev.Key) } switch ev.Key { case termbox.KeyCtrlC: break loop case termbox.KeyF1: fi.ChangePage(0) case termbox.KeyF2: fi.ChangePage(1) case termbox.KeyF3: fi.ChangePage(2) // switch_output_mode(1) // draw_all() // case termbox.KeyArrowDown, termbox.KeyArrowLeft: // switch_output_mode(-1) // draw_all() } case termbox.EventResize: fi.currentPage.Draw(fi.pages) // draw_all() } } } }
func main() { app := cli.NewApp() app.Name = "sounder" app.Usage = "acceptance tool for the metric system" app.Commands = []cli.Command{ { Name: "stream", ShortName: "s", Usage: "stream messages", Action: func(c *cli.Context) { consumer := noaa.NewConsumer(c.Args().First(), &tls.Config{InsecureSkipVerify: true}, nil) messages := make(chan *events.Envelope) errs := make(chan error) done := make(chan struct{}) go consumer.Stream(c.Args().Get(1), authToken(), messages, errs, done) displayEnvelopes(messages, errs) }, }, { Name: "recent", ShortName: "r", Usage: "recent log messages", Action: func(c *cli.Context) { consumer := noaa.NewConsumer(c.Args().First(), &tls.Config{InsecureSkipVerify: true}, nil) messages, err := consumer.RecentLogs(c.Args().Get(1), authToken()) if err != nil { panic(err) } for _, message := range messages { displayMessage(message) } }, }, { Name: "tail logs", ShortName: "t", Usage: "tail log messages", Action: func(c *cli.Context) { consumer := noaa.NewConsumer(c.Args().First(), &tls.Config{InsecureSkipVerify: true}, nil) messages := make(chan *events.LogMessage) errs := make(chan error) done := make(chan struct{}) go consumer.TailingLogs(c.Args().Get(1), authToken(), messages, errs, done) for { select { case message, ok := <-messages: if ok { displayMessage(message) } case err, ok := <-errs: if ok { panic(err) } } } }, }, { Name: "firehose", ShortName: "f", Usage: "firehose <client_id>", Action: func(c *cli.Context) { consumer := noaa.NewConsumer(c.Args().First(), &tls.Config{InsecureSkipVerify: true}, nil) messages := make(chan *events.Envelope) errs := make(chan error) done := make(chan struct{}) go consumer.Firehose(c.Args().Get(1), authToken(), messages, errs, done) displayEnvelopes(messages, errs) }, }, } app.Run(os.Args) }
func cliCommands(ltcConfigRoot string, exitHandler exit_handler.ExitHandler, config *config.Config, logger lager.Logger, targetVerifier target_verifier.TargetVerifier, ui terminal.UI) []cli.Command { receptorClient := receptor.NewClient(config.Receptor()) noaaConsumer := noaa.NewConsumer(LoggregatorUrl(config.Loggregator()), nil, nil) appRunner := app_runner.New(receptorClient, config.Target()) clock := clock.NewClock() logReader := logs.NewLogReader(noaaConsumer) tailedLogsOutputter := console_tailed_logs_outputter.NewConsoleTailedLogsOutputter(ui, logReader) taskExaminer := task_examiner.New(receptorClient) taskExaminerCommandFactory := task_examiner_command_factory.NewTaskExaminerCommandFactory(taskExaminer, ui, exitHandler) taskRunner := task_runner.New(receptorClient, taskExaminer) taskRunnerCommandFactory := task_runner_command_factory.NewTaskRunnerCommandFactory(taskRunner, ui, exitHandler) appExaminer := app_examiner.New(receptorClient, app_examiner.NewNoaaConsumer(noaaConsumer)) graphicalVisualizer := graphical.NewGraphicalVisualizer(appExaminer) appExaminerCommandFactory := app_examiner_command_factory.NewAppExaminerCommandFactory(appExaminer, ui, clock, exitHandler, graphicalVisualizer, taskExaminer) appRunnerCommandFactoryConfig := app_runner_command_factory.AppRunnerCommandFactoryConfig{ AppRunner: appRunner, AppExaminer: appExaminer, UI: ui, Domain: config.Target(), Env: os.Environ(), Clock: clock, Logger: logger, TailedLogsOutputter: tailedLogsOutputter, ExitHandler: exitHandler, } appRunnerCommandFactory := app_runner_command_factory.NewAppRunnerCommandFactory(appRunnerCommandFactoryConfig) dockerRunnerCommandFactoryConfig := docker_runner_command_factory.DockerRunnerCommandFactoryConfig{ AppRunner: appRunner, AppExaminer: appExaminer, UI: ui, Domain: config.Target(), Env: os.Environ(), Clock: clock, Logger: logger, ExitHandler: exitHandler, TailedLogsOutputter: tailedLogsOutputter, DockerMetadataFetcher: docker_metadata_fetcher.New(docker_metadata_fetcher.NewDockerSessionFactory()), } dockerRunnerCommandFactory := docker_runner_command_factory.NewDockerRunnerCommandFactory(dockerRunnerCommandFactoryConfig) logsCommandFactory := logs_command_factory.NewLogsCommandFactory(appExaminer, ui, tailedLogsOutputter, exitHandler) configCommandFactory := config_command_factory.NewConfigCommandFactory(config, ui, targetVerifier, exitHandler) testRunner := integration_test.NewIntegrationTestRunner(config, ltcConfigRoot) integrationTestCommandFactory := integration_test_command_factory.NewIntegrationTestCommandFactory(testRunner) s3Auth := aws.Auth{ AccessKey: config.BlobTarget().AccessKey, SecretKey: config.BlobTarget().SecretKey, } s3S3 := s3.New(s3Auth, awsRegion, &http.Client{ Transport: &http.Transport{ Proxy: config.BlobTarget().Proxy(), Dial: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, }).Dial, TLSHandshakeTimeout: 10 * time.Second, }, }) blobStore := blob_store.NewBlobStore(config, s3S3) blobBucket := blobStore.Bucket(config.BlobTarget().BucketName) dropletRunner := droplet_runner.New(appRunner, taskRunner, config, blobStore, blobBucket, targetVerifier, appExaminer) cfIgnore := cf_ignore.New() dropletRunnerCommandFactory := droplet_runner_command_factory.NewDropletRunnerCommandFactory(*appRunnerCommandFactory, taskExaminer, dropletRunner, cfIgnore) helpCommand := cli.Command{ Name: "help", Aliases: []string{"h"}, Usage: "Shows a list of commands or help for one command", Description: "ltc help", Action: defaultAction, } return []cli.Command{ appExaminerCommandFactory.MakeCellsCommand(), dockerRunnerCommandFactory.MakeCreateAppCommand(), appRunnerCommandFactory.MakeSubmitLrpCommand(), logsCommandFactory.MakeDebugLogsCommand(), appExaminerCommandFactory.MakeListAppCommand(), logsCommandFactory.MakeLogsCommand(), appRunnerCommandFactory.MakeRemoveAppCommand(), appRunnerCommandFactory.MakeScaleAppCommand(), appExaminerCommandFactory.MakeStatusCommand(), taskRunnerCommandFactory.MakeSubmitTaskCommand(), configCommandFactory.MakeTargetCommand(), configCommandFactory.MakeTargetBlobCommand(), taskExaminerCommandFactory.MakeTaskCommand(), taskRunnerCommandFactory.MakeDeleteTaskCommand(), taskRunnerCommandFactory.MakeCancelTaskCommand(), integrationTestCommandFactory.MakeIntegrationTestCommand(), appRunnerCommandFactory.MakeUpdateRoutesCommand(), appExaminerCommandFactory.MakeVisualizeCommand(), dropletRunnerCommandFactory.MakeBuildDropletCommand(), dropletRunnerCommandFactory.MakeListDropletsCommand(), dropletRunnerCommandFactory.MakeLaunchDropletCommand(), dropletRunnerCommandFactory.MakeRemoveDropletCommand(), helpCommand, } }