func main() { flag.Parse() //make the out dir logger.Component = "SIMULATOR" if outDir == "" { logger.Fatal("out.dir.unspecified") } err := os.MkdirAll(outDir, 0777) if err != nil { logger.Fatal("out.dir.creation.failed", err) } //set up logging outputFile, err := os.Create(filepath.Join(outDir, "simulator.log")) if err != nil { logger.Fatal("failed.to.create.simulator.log", err) } logger.Writer = io.MultiWriter(os.Stdout, outputFile) cleanup.Register(func() { outputFile.Sync() }) //compile the executor logger.Info("compiling.executor") output, err := exec.Command("go", "install", "github.com/cloudfoundry-incubator/simulator/game_executor").CombinedOutput() if err != nil { logger.Fatal("failed.to.compile.executor", string(output)) } //write info to the output dir writeInfo() //start etcd logger.Info("starting.etcd", etcdNodes) etcd = etcdstorerunner.NewETCDClusterRunner(4001, etcdNodes) etcd.Start() //set up the bbs pool := workerpool.NewWorkerPool(50) etcdAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcd.NodeURLS(), pool) etcdAdapter.Connect() bbs = Bbs.New(etcdAdapter, timeprovider.NewTimeProvider()) //monitor etcd monitorETCD() //start executors startExecutors() cleanup.Register(func() { logger.Info("stopping.etcd", etcdNodes) etcd.Stop() }) //run the simulator runSimulation() cleanup.Exit(0) }
func main() { flag.Parse() runtime.GOMAXPROCS(runtime.NumCPU()) table := map[string]Route{} transport := &http.Transport{ ResponseHeaderTimeout: 10 * time.Second, } handler := &Handler{ table: table, transport: transport, } etcdAdapter := etcdstoreadapter.NewETCDStoreAdapter( strings.Split(*etcdCluster, ","), workerpool.NewWorkerPool(10), ) err := etcdAdapter.Connect() if err != nil { log.Fatalln("can't connect to etcd:", err) } go handler.syncTable(etcdAdapter, *syncInterval) http.Handle("/", handler) http.ListenAndServe(*listenAddr, nil) }
func New(host string, config *Config, logger *gosteno.Logger) *Loggregator { cfcomponent.Logger = logger keepAliveInterval := 30 * time.Second listener, incomingLogChan := agentlistener.NewAgentListener(fmt.Sprintf("%s:%d", host, config.IncomingPort), logger) unmarshaller, messageChan := unmarshaller.NewLogMessageUnmarshaller(config.SharedSecret, incomingLogChan) blacklist := blacklist.New(config.BlackListIps) sinkManager, appStoreInputChan := sinkmanager.NewSinkManager(config.MaxRetainedLogMessages, config.SkipCertVerify, blacklist, logger) workerPool := workerpool.NewWorkerPool(config.EtcdMaxConcurrentRequests) storeAdapter := etcdstoreadapter.NewETCDStoreAdapter(config.EtcdUrls, workerPool) appStoreCache := cache.NewAppServiceCache() appStoreWatcher, newAppServiceChan, deletedAppServiceChan := store.NewAppServiceStoreWatcher(storeAdapter, appStoreCache) appStore := store.NewAppServiceStore(storeAdapter, appStoreWatcher) return &Loggregator{ Logger: logger, listener: listener, unmarshaller: unmarshaller, sinkManager: sinkManager, messageChan: messageChan, appStoreInputChan: appStoreInputChan, appStore: appStore, messageRouter: sinkserver.NewMessageRouter(sinkManager, logger), websocketServer: websocketserver.New(fmt.Sprintf("%s:%d", host, config.OutgoingPort), sinkManager, keepAliveInterval, config.WSMessageBufferSize, logger), newAppServiceChan: newAppServiceChan, deletedAppServiceChan: deletedAppServiceChan, appStoreWatcher: appStoreWatcher, storeAdapter: storeAdapter, } }
func connectToStoreAdapter(l logger.Logger, conf *config.Config) (storeadapter.StoreAdapter, metricsaccountant.UsageTracker) { var adapter storeadapter.StoreAdapter workerPool := workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests) adapter = etcdstoreadapter.NewETCDStoreAdapter(conf.StoreURLs, workerPool) err := adapter.Connect() if err != nil { l.Error("Failed to connect to the store", err) os.Exit(1) } return adapter, workerPool }
func main() { flag.Parse() datadogApiKey := os.Getenv("DATADOG_API_KEY") datadogAppKey := os.Getenv("DATADOG_APP_KEY") var datadogClient *datadog.Client if datadogApiKey != "" { datadogClient = datadog.NewClient(datadogApiKey, datadogAppKey) } store := etcdstoreadapter.NewETCDStoreAdapter( strings.Split(*etcdMachines, ","), workerpool.NewWorkerPool(50), ) err := store.Connect() if err != nil { log.Fatalln("failed to connect to store:", err) } if *emitStates { // logging mode etcd_logger.EmitRunOnceStates(datadogClient, store, strings.Split(*etcdMachines, ",")) } else { // stampede mode runOnce := &models.RunOnce{ Actions: []models.ExecutorAction{ { models.RunAction{ Script: *script, Timeout: *timeout, }, }, }, Stack: *stack, MemoryMB: *memoryMB, DiskMB: *diskMB, } if *logGuid != "" { runOnce.Log = models.LogConfig{ Guid: *logGuid, SourceName: *logSourceName, } } stampede.RunonceStampede(bbs.New(store, timeprovider.NewTimeProvider()), datadogClient, runOnce, *runOnceCount) } }
func connectToStoreAdapter(l logger.Logger, conf *config.Config) (storeadapter.StoreAdapter, metricsaccountant.UsageTracker) { var adapter storeadapter.StoreAdapter workerPool := workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests) if conf.StoreType == "etcd" { adapter = etcdstoreadapter.NewETCDStoreAdapter(conf.StoreURLs, workerPool) } else if conf.StoreType == "ZooKeeper" { adapter = zookeeperstoreadapter.NewZookeeperStoreAdapter(conf.StoreURLs, workerPool, buildTimeProvider(l), time.Second) } else { l.Error(fmt.Sprintf("Unknown store type %s. Choose one of 'etcd' or 'ZooKeeper'", conf.StoreType), fmt.Errorf("Unkown store type")) os.Exit(1) } err := adapter.Connect() if err != nil { l.Error("Failed to connect to the store", err) os.Exit(1) } return adapter, workerPool }
func main() { flag.Parse() cleanup.Register(func() { logger.Info("executor.shuttingdown") close(stop) tasks.Wait() logger.Info("executor.shutdown") }) logger.Component = fmt.Sprintf("EXECUTOR %s", *executorID) lock = &sync.Mutex{} currentMemory = *maxMemory etcdAdapter := etcdstoreadapter.NewETCDStoreAdapter( strings.Split(*etcdCluster, ","), workerpool.NewWorkerPool(10), ) err := etcdAdapter.Connect() if err != nil { logger.Fatal("etcd.connect.fatal", err) } tasks = &sync.WaitGroup{} stop = make(chan bool) bbs := Bbs.New(etcdAdapter, timeprovider.NewTimeProvider()) ready := make(chan bool, 1) err = maintainPresence(bbs, ready) if err != nil { logger.Fatal("executor.initializing-presence.failed", err) } go handleRunOnces(bbs) go convergeRunOnces(bbs) <-ready logger.Info("executor.up") select {} }
store Store storeAdapter storeadapter.StoreAdapter conf *config.Config dea appfixture.DeaFixture app1 appfixture.AppFixture app2 appfixture.AppFixture app3 appfixture.AppFixture app4 appfixture.AppFixture crashCount []models.CrashCount ) conf, _ = config.DefaultConfig() BeforeEach(func() { storeAdapter = etcdstoreadapter.NewETCDStoreAdapter(etcdRunner.NodeURLS(), workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests)) err := storeAdapter.Connect() Ω(err).ShouldNot(HaveOccurred()) store = NewStore(conf, storeAdapter, fakelogger.NewFakeLogger()) dea = appfixture.NewDeaFixture() app1 = dea.GetApp(0) app2 = dea.GetApp(1) app3 = dea.GetApp(2) app4 = dea.GetApp(3) actualState := []models.InstanceHeartbeat{ app1.InstanceAtIndex(0).Heartbeat(), app1.InstanceAtIndex(1).Heartbeat(), app1.InstanceAtIndex(2).Heartbeat(),
func (coordinator *MCATCoordinator) StartETCD() { etcdPort := 5000 + (coordinator.ParallelNode-1)*10 coordinator.StoreRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1) coordinator.StoreRunner.Start() coordinator.StoreAdapter = etcdstoreadapter.NewETCDStoreAdapter(coordinator.StoreRunner.NodeURLS(), workerpool.NewWorkerPool(coordinator.Conf.StoreMaxConcurrentRequests)) err := coordinator.StoreAdapter.Connect() Ω(err).ShouldNot(HaveOccurred()) }
func main() { flag.Parse() runtime.GOMAXPROCS(runtime.NumCPU()) //make the out dir logger.Component = "SIMULATOR" if outDir == "" { logger.Fatal("out.dir.unspecified") } err := os.MkdirAll(outDir, 0777) if err != nil { logger.Fatal("out.dir.creation.failed", err) } //set up logging outputFile, err := os.Create(filepath.Join(outDir, "simulator.log")) if err != nil { logger.Fatal("failed.to.create.simulator.log", err) } logger.Writer = io.MultiWriter(os.Stdout, outputFile) cleanup.Register(func() { outputFile.Sync() }) //start etcd natsClient := yagnats.NewClient() natsMembers := []yagnats.ConnectionProvider{} for _, addr := range strings.Split(*natsAddresses, ",") { natsMembers = append( natsMembers, &yagnats.ConnectionInfo{addr, *natsUsername, *natsPassword}, ) } natsInfo := &yagnats.ConnectionCluster{Members: natsMembers} err = natsClient.Connect(natsInfo) if err != nil { logger.Fatal("could not connect to nats:", err) } logger.Component = "simulator" etcdAdapter := etcdstoreadapter.NewETCDStoreAdapter( strings.Split(*etcdCluster, ","), workerpool.NewWorkerPool(10), ) err = etcdAdapter.Connect() if err != nil { logger.Fatal("etcd.connect-failed", map[string]interface{}{ "error": err.Error(), }) } //write info to the output dir writeInfo() //monitor etcd monitorETCD(etcdAdapter) //run the simulator runSimulation(natsClient) cleanup.Exit(0) }
func main() { var err error runtime.GOMAXPROCS(runtime.NumCPU()) rand.Seed(time.Now().UnixNano()) flag.Parse() executorUUID, err := uuid.NewV4() if err != nil { log.Fatalln("could not generate guid:", err) } executorID = executorUUID.String() cleanup.Register(func() { once.Do(func() { logger.Info("shutting-down", map[string]interface{}{}) close(stop) tasks.Wait() logger.Info("shutdown", map[string]interface{}{}) }) }) natsClient := yagnats.NewClient() natsMembers := []yagnats.ConnectionProvider{} for _, addr := range strings.Split(*natsAddresses, ",") { natsMembers = append( natsMembers, &yagnats.ConnectionInfo{addr, *natsUsername, *natsPassword}, ) } natsInfo := &yagnats.ConnectionCluster{Members: natsMembers} err = logger.Connect(natsInfo) if err != nil { log.Fatalln("could not connect logger:", err) } err = natsClient.Connect(natsInfo) if err != nil { log.Fatalln("could not connect to nats:", err) } logger.Component = fmt.Sprintf("executor.%s", executorID) etcdAdapter := etcdstoreadapter.NewETCDStoreAdapter( strings.Split(*etcdCluster, ","), workerpool.NewWorkerPool(10), ) err = etcdAdapter.Connect() if err != nil { logger.Fatal("etcd.connect-failed", map[string]interface{}{ "error": err.Error(), }) } bbs := bbs.New(bbs.NewHurlerKicker(*hurlerAddress), etcdAdapter, timeprovider.NewTimeProvider()) ready := make(chan bool, 1) err = maintainPresence(bbs, ready) if err != nil { logger.Fatal("initializing-presence", map[string]interface{}{ "error": err.Error(), }) } err = registerHandler(etcdAdapter, *listenAddr, ready) if err != nil { logger.Fatal("initializing-route", map[string]interface{}{ "error": err.Error(), }) } go handleTasks(bbs, *listenAddr) go convergeTasks(bbs) <-ready <-ready logger.Info("up", map[string]interface{}{ "executor": executorID, }) select {} }
func (coordinator *MCATCoordinator) StartZooKeeper() { coordinator.CurrentStoreType = "ZooKeeper" zookeeperPort := 2181 + (coordinator.ParallelNode-1)*10 coordinator.StoreRunner = zookeeperstorerunner.NewZookeeperClusterRunner(zookeeperPort, 1) coordinator.StoreRunner.Start() coordinator.StoreAdapter = zookeeperstoreadapter.NewZookeeperStoreAdapter(coordinator.StoreRunner.NodeURLS(), workerpool.NewWorkerPool(coordinator.Conf.StoreMaxConcurrentRequests), &timeprovider.RealTimeProvider{}, time.Second) err := coordinator.StoreAdapter.Connect() Ω(err).ShouldNot(HaveOccurred()) }