Beispiel #1
0
func ServeAPI(l logger.Logger, conf *config.Config) {
	store := connectToStore(l, conf)

	apiHandler, err := handlers.New(l, store, buildTimeProvider(l))
	if err != nil {
		l.Error("initialize-handler.failed", err)
		panic(err)
	}
	handler := handlers.BasicAuthWrap(apiHandler, conf.APIServerUsername, conf.APIServerPassword)

	listenAddr := fmt.Sprintf("%s:%d", conf.APIServerAddress, conf.APIServerPort)

	members := grouper.Members{
		{"api", http_server.New(listenAddr, handler)},
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	l.Info("started")
	l.Info(listenAddr)

	err = <-monitor.Wait()
	if err != nil {
		l.Error("exited", err)
		os.Exit(1)
	}

	l.Info("exited")
	os.Exit(0)
}
Beispiel #2
0
func connectToMessageBus(l logger.Logger, conf *config.Config) yagnats.NATSClient {
	members := []yagnats.ConnectionProvider{}

	for _, natsConf := range conf.NATS {
		members = append(members, &yagnats.ConnectionInfo{
			Addr: fmt.Sprintf("%s:%d", natsConf.Host, natsConf.Port),

			Username: natsConf.User,
			Password: natsConf.Password,
		})
	}

	connectionInfo := &yagnats.ConnectionCluster{
		Members: members,
	}

	natsClient := yagnats.NewClient()

	err := natsClient.Connect(connectionInfo)

	if err != nil {
		l.Error("Failed to connect to the message bus", err)
		os.Exit(1)
	}

	return natsClient
}
Beispiel #3
0
func ServeMetrics(steno *gosteno.Logger, l logger.Logger, conf *config.Config) {
	store := connectToStore(l, conf)
	messageBus := connectToMessageBus(l, conf)

	acquireLock(l, conf, "metrics-server")

	collectorRegistrar := collectorregistrar.NewCollectorRegistrar(messageBus, steno)

	metricsServer := metricsserver.New(
		collectorRegistrar,
		steno,
		metricsaccountant.New(store),
		l,
		store,
		buildTimeProvider(l),
		conf,
	)

	err := metricsServer.Start()
	if err != nil {
		l.Error("Failed to serve metrics", err)
	}
	l.Info("Serving Metrics")
	select {}
}
func connectToStore(l logger.Logger, conf *config.Config) (store.Store, metricsaccountant.UsageTracker) {
	if conf.StoreType == "etcd" || conf.StoreType == "ZooKeeper" {
		adapter, workerPool := connectToStoreAdapter(l, conf)
		return store.NewStore(conf, adapter, l), workerPool
	} else {
		l.Error(fmt.Sprintf("Unknown store type %s.  Choose one of 'etcd' or 'ZooKeeper'", conf.StoreType), fmt.Errorf("Unkown store type"))
		os.Exit(1)
	}

	return nil, nil
}
Beispiel #5
0
func buildClock(l logger.Logger) clock.Clock {
	if os.Getenv("HM9000_FAKE_TIME") == "" {
		return clock.NewClock()
	} else {
		timestamp, err := strconv.Atoi(os.Getenv("HM9000_FAKE_TIME"))
		if err != nil {
			l.Error("Failed to load timestamp", err)
			os.Exit(1)
		}
		return NewFixedClock(time.Unix(int64(timestamp), 0))
	}
}
Beispiel #6
0
func connectToStoreAdapter(l logger.Logger, conf *config.Config) (storeadapter.StoreAdapter, metricsaccountant.UsageTracker) {
	var adapter storeadapter.StoreAdapter
	workerPool := workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests)
	adapter = etcdstoreadapter.NewETCDStoreAdapter(conf.StoreURLs, workerPool)
	err := adapter.Connect()
	if err != nil {
		l.Error("Failed to connect to the store", err)
		os.Exit(1)
	}

	return adapter, workerPool
}
func StartEvacuator(l logger.Logger, conf *config.Config) {
	messageBus := connectToMessageBus(l, conf)
	store, _ := connectToStore(l, conf)

	acquireLock(l, conf, "evacuator")

	evacuator := evacuatorpackage.New(messageBus, store, buildTimeProvider(l), conf, l)

	evacuator.Listen()
	l.Info("Listening for DEA Evacuations")
	select {}
}
Beispiel #8
0
func acquireLock(l logger.Logger, conf *config.Config, lockName string) {
	adapter, _ := connectToStoreAdapter(l, conf)
	l.Info("Acquiring lock for " + lockName)

	lock := storeadapter.StoreNode{
		Key: "/hm/locks/" + lockName,
		TTL: 10,
	}

	status, _, err := adapter.MaintainNode(lock)
	if err != nil {
		l.Error("Failed to talk to lock store", err)
		os.Exit(1)
	}

	lockAcquired := make(chan bool)

	go func() {
		for {
			if <-status {
				if lockAcquired != nil {
					close(lockAcquired)
					lockAcquired = nil
				}
			} else {
				l.Error("Lost the lock", errors.New("Lost the lock"))
				os.Exit(197)
			}
		}
	}()

	<-lockAcquired
	l.Info("Acquired lock for " + lockName)
}
Beispiel #9
0
func buildTimeProvider(l logger.Logger) timeprovider.TimeProvider {
	if os.Getenv("HM9000_FAKE_TIME") == "" {
		return timeprovider.NewTimeProvider()
	} else {
		timestamp, err := strconv.Atoi(os.Getenv("HM9000_FAKE_TIME"))
		if err != nil {
			l.Error("Failed to load timestamp", err)
			os.Exit(1)
		}
		return &faketimeprovider.FakeTimeProvider{
			TimeToProvide: time.Unix(int64(timestamp), 0),
		}
	}
}
func StartListeningForActual(l logger.Logger, c *cli.Context) {
	conf := loadConfig(l, c)
	messageBus := connectToMessageBus(l, conf)
	store := connectToStore(l, conf)

	listener := actualstatelistener.New(conf,
		messageBus,
		store,
		timeprovider.NewTimeProvider(),
		l)

	listener.Start()
	l.Info("Listening for Actual State", nil)
	select {}
}
Beispiel #11
0
func connectToStoreAdapter(l logger.Logger, conf *config.Config, usage *usageTracker) storeadapter.StoreAdapter {
	var adapter storeadapter.StoreAdapter
	var around workpool.AroundWork = workpool.DefaultAround
	if usage != nil {
		around = usage
	}
	workPool := workpool.New(conf.StoreMaxConcurrentRequests, 0, around)
	adapter = etcdstoreadapter.NewETCDStoreAdapter(conf.StoreURLs, workPool)
	err := adapter.Connect()
	if err != nil {
		l.Error("Failed to connect to the store", err)
		os.Exit(1)
	}

	return adapter
}
func connectToStoreAdapter(l logger.Logger, conf *config.Config) (storeadapter.StoreAdapter, metricsaccountant.UsageTracker) {
	var adapter storeadapter.StoreAdapter
	workerPool := workerpool.NewWorkerPool(conf.StoreMaxConcurrentRequests)
	if conf.StoreType == "etcd" {
		adapter = etcdstoreadapter.NewETCDStoreAdapter(conf.StoreURLs, workerPool)
	} else if conf.StoreType == "ZooKeeper" {
		adapter = zookeeperstoreadapter.NewZookeeperStoreAdapter(conf.StoreURLs, workerPool, buildTimeProvider(l), time.Second)
	} else {
		l.Error(fmt.Sprintf("Unknown store type %s.  Choose one of 'etcd' or 'ZooKeeper'", conf.StoreType), fmt.Errorf("Unkown store type"))
		os.Exit(1)
	}
	err := adapter.Connect()
	if err != nil {
		l.Error("Failed to connect to the store", err)
		os.Exit(1)
	}

	return adapter, workerPool
}
func StartListeningForActual(l logger.Logger, conf *config.Config) {
	messageBus := connectToMessageBus(l, conf)
	store, usageTracker := connectToStoreAndTrack(l, conf)

	acquireLock(l, conf, "listener")

	listener := actualstatelistener.New(conf,
		messageBus,
		store,
		usageTracker,
		metricsaccountant.New(store),
		buildClock(l),
		l,
	)

	listener.Start()
	l.Info("Listening for Actual State")
	select {}
}
Beispiel #14
0
func ServeAPI(l logger.Logger, conf *config.Config) {
	store, _ := connectToStore(l, conf)
	messageBus := connectToMessageBus(l, conf)

	//no locking necessary for the api server.  it's ok to have multiples of these running.
	//NATS will distribute the requests and ensure that only one api-server handles a given request
	//because we use a NATS queue.

	apiServer := apiserver.New(
		messageBus,
		store,
		buildTimeProvider(l),
		l,
	)

	apiServer.Listen()
	l.Info(fmt.Sprintf("Serving API over NATS (subject: app.state)"))
	select {}
}
Beispiel #15
0
func Daemonize(
	component string,
	callback func() error,
	period time.Duration,
	timeout time.Duration,
	logger logger.Logger,
	adapter storeadapter.StoreAdapter,
) error {
	logger.Info("Acquiring lock for " + component)

	lostLockChannel, releaseLockChannel, err := adapter.GetAndMaintainLock(component, 10)
	if err != nil {
		logger.Info(fmt.Sprintf("Failed to acquire lock: %s", err))
		return err
	}

	go func() {
		<-lostLockChannel
		logger.Error("Lost the lock", errors.New("Lock the lock"))
		os.Exit(197)
	}()

	logger.Info("Acquired lock for " + component)

	logger.Info(fmt.Sprintf("Running Daemon every %d seconds with a timeout of %d", int(period.Seconds()), int(timeout.Seconds())))

	for {
		afterChan := time.After(period)
		timeoutChan := time.After(timeout)
		errorChan := make(chan error, 1)

		t := time.Now()

		go func() {
			errorChan <- callback()
		}()

		select {
		case err := <-errorChan:
			logger.Info("Daemonize Time", map[string]string{
				"Component": component,
				"Duration":  fmt.Sprintf("%.4f", time.Since(t).Seconds()),
			})
			if err != nil {
				logger.Error("Daemon returned an error. Continuining...", err)
			}
		case <-timeoutChan:
			releaseLockChannel <- true
			return errors.New("Daemon timed out. Aborting!")
		}

		<-afterChan
	}

	return nil
}
func FetchDesiredState(l logger.Logger, conf *config.Config, poll bool) {
	store, _ := connectToStore(l, conf)

	if poll {
		l.Info("Starting Desired State Daemon...")

		adapter, _ := connectToStoreAdapter(l, conf)

		err := Daemonize("Fetcher", func() error {
			return fetchDesiredState(l, conf, store)
		}, conf.FetcherPollingInterval(), conf.FetcherTimeout(), l, adapter)
		if err != nil {
			l.Error("Desired State Daemon Errored", err)
		}
		l.Info("Desired State Daemon is Down")
		os.Exit(1)
	} else {
		err := fetchDesiredState(l, conf, store)
		if err != nil {
			os.Exit(1)
		} else {
			os.Exit(0)
		}
	}
}
Beispiel #17
0
func Analyze(l logger.Logger, conf *config.Config, poll bool) {
	store := connectToStore(l, conf)

	if poll {
		l.Info("Starting Analyze Daemon...")

		adapter := connectToStoreAdapter(l, conf, nil)
		err := Daemonize("Analyzer", func() error {
			return analyze(l, conf, store)
		}, conf.AnalyzerPollingInterval(), conf.AnalyzerTimeout(), l, adapter)

		if err != nil {
			l.Error("Analyze Daemon Errored", err)
		}
		l.Info("Analyze Daemon is Down")
		os.Exit(1)
	} else {
		err := analyze(l, conf, store)
		if err != nil {
			os.Exit(1)
		} else {
			os.Exit(0)
		}
	}
}
Beispiel #18
0
func connectToMessageBus(l logger.Logger, conf *config.Config) yagnats.NATSConn {
	members := make([]string, len(conf.NATS))

	for _, natsConf := range conf.NATS {
		uri := url.URL{
			Scheme: "nats",
			User:   url.UserPassword(natsConf.User, natsConf.Password),
			Host:   fmt.Sprintf("%s:%d", natsConf.Host, natsConf.Port),
		}
		members = append(members, uri.String())
	}

	natsClient, err := yagnats.Connect(members)
	if err != nil {
		l.Error("Failed to connect to the message bus", err)
		os.Exit(1)
	}

	natsClient.AddReconnectedCB(func(conn *nats.Conn) {
		l.Info(fmt.Sprintf("NATS Client Reconnected. Server URL: %s", conn.Opts.Url))
	})

	natsClient.AddClosedCB(func(conn *nats.Conn) {
		err := errors.New(fmt.Sprintf("NATS Client Closed. nats.Conn: %+v", conn))
		l.Error("NATS Closed", err)
		os.Exit(1)
	})

	return natsClient
}
Beispiel #19
0
func connectToStoreAdapter(l logger.Logger, conf *config.Config) storeadapter.StoreAdapter {
	var adapter storeadapter.StoreAdapter
	workPool, err := workpool.NewWorkPool(conf.StoreMaxConcurrentRequests)
	if err != nil {
		l.Error("Failed to create workpool", err)
		os.Exit(1)
	}

	options := &etcdstoreadapter.ETCDOptions{
		ClusterUrls: conf.StoreURLs,
	}
	adapter, err = etcdstoreadapter.New(options, workPool)
	if err != nil {
		l.Error("Failed to create the store adapter", err)
		os.Exit(1)
	}

	err = adapter.Connect()
	if err != nil {
		l.Error("Failed to connect to the store", err)
		os.Exit(1)
	}

	return adapter
}
Beispiel #20
0
func Send(l logger.Logger, conf *config.Config, poll bool) {
	messageBus := connectToMessageBus(l, conf)
	store, _ := connectToStore(l, conf)

	if poll {
		l.Info("Starting Sender Daemon...")

		adapter, _ := connectToStoreAdapter(l, conf)

		err := Daemonize("Sender", func() error {
			return send(l, conf, messageBus, store)
		}, conf.SenderPollingInterval(), conf.SenderTimeout(), l, adapter)
		if err != nil {
			l.Error("Sender Daemon Errored", err)
		}
		l.Info("Sender Daemon is Down")
		os.Exit(1)
	} else {
		err := send(l, conf, messageBus, store)
		if err != nil {
			os.Exit(1)
		} else {
			os.Exit(0)
		}
	}
}
Beispiel #21
0
func FetchDesiredState(l logger.Logger, c *cli.Context) {
	conf := loadConfig(l, c)
	messageBus := connectToMessageBus(l, conf)
	store := connectToStore(l, conf)

	fetcher := desiredstatefetcher.New(conf,
		messageBus,
		store,
		httpclient.NewHttpClient(),
		timeprovider.NewTimeProvider(),
	)

	resultChan := make(chan desiredstatefetcher.DesiredStateFetcherResult, 1)
	fetcher.Fetch(resultChan)

	select {
	case result := <-resultChan:
		if result.Success {
			l.Info("Success", map[string]string{"Number of Desired Apps Fetched": strconv.Itoa(result.NumResults)})
			os.Exit(0)
		} else {
			l.Info(result.Message, map[string]string{"Error": result.Error.Error(), "Message": result.Message})
			os.Exit(1)
		}
	case <-time.After(600 * time.Second):
		l.Info("Timed out when fetching desired state", nil)
		os.Exit(1)
	}
}
func acquireLock(l logger.Logger, conf *config.Config, lockName string) {
	adapter, _ := connectToStoreAdapter(l, conf)
	l.Info("Acquiring lock for " + lockName)
	lostLockChannel, _, err := adapter.GetAndMaintainLock(lockName, 10)
	if err != nil {
		l.Error("Failed to talk to lock store", err)
		os.Exit(1)
	}

	go func() {
		<-lostLockChannel
		l.Error("Lost the lock", errors.New("Lock the lock"))
		os.Exit(197)
	}()

	l.Info("Acquired lock for " + lockName)
}
Beispiel #23
0
func analyze(l logger.Logger, conf *config.Config, store store.Store) error {
	l.Info("Analyzing...")

	analyzer := analyzer.New(store, buildTimeProvider(l), l, conf)
	err := analyzer.Analyze()

	if err != nil {
		l.Error("Analyzer failed with error", err)
		return err
	} else {
		l.Info("Analyzer completed succesfully")
		return nil
	}
}
Beispiel #24
0
func send(l logger.Logger, conf *config.Config, messageBus yagnats.NATSClient, store store.Store) error {
	l.Info("Sending...")

	sender := sender.New(store, metricsaccountant.New(store), conf, messageBus, buildTimeProvider(l), l)
	err := sender.Send()

	if err != nil {
		l.Error("Sender failed with error", err)
		return err
	} else {
		l.Info("Sender completed succesfully")
		return nil
	}
}
Beispiel #25
0
func acquireLock(l logger.Logger, conf *config.Config, lockName string) {
	adapter, _ := connectToStoreAdapter(l, conf)
	l.Info("Acquiring lock for " + lockName)

	lock := storeadapter.StoreNode{
		Key: "/hm/locks/" + lockName,
		TTL: 10,
	}

	lostLockChannel, _, err := adapter.MaintainNode(lock)
	if err != nil {
		l.Error("Failed to talk to lock store", err)
		os.Exit(1)
	}

	go func() {
		<-lostLockChannel
		l.Error("Lost the lock", errors.New("Lock the lock"))
		os.Exit(197)
	}()

	l.Info("Acquired lock for " + lockName)
}
func fetchDesiredState(l logger.Logger, conf *config.Config, store store.Store) error {
	l.Info("Fetching Desired State")
	fetcher := desiredstatefetcher.New(conf,
		store,
		metricsaccountant.New(store),
		httpclient.NewHttpClient(conf.SkipSSLVerification, conf.FetcherNetworkTimeout()),
		buildTimeProvider(l),
		l,
	)

	resultChan := make(chan desiredstatefetcher.DesiredStateFetcherResult, 1)
	fetcher.Fetch(resultChan)

	result := <-resultChan

	if result.Success {
		l.Info("Success", map[string]string{"Number of Desired Apps Fetched": strconv.Itoa(result.NumResults)})
		return nil
	} else {
		l.Error(result.Message, result.Error)
		return result.Error
	}
	return nil
}
Beispiel #27
0
func shred(l logger.Logger, store store.Store) error {
	l.Info("Shredding Store")
	theShredder := shredder.New(store)
	return theShredder.Shred()
}
Beispiel #28
0
func Daemonize(
	component string,
	callback func() error,
	period time.Duration,
	timeout time.Duration,
	logger logger.Logger,
	adapter storeadapter.StoreAdapter,
) error {
	logger.Info("Acquiring lock for " + component)

	lock := storeadapter.StoreNode{
		Key: "/hm/locks/" + component,
		TTL: 10,
	}

	status, releaseLockChannel, err := adapter.MaintainNode(lock)
	if err != nil {
		logger.Info(fmt.Sprintf("Failed to acquire lock: %s", err))
		return err
	}

	lockAcquired := make(chan bool)

	go func() {
		for {
			if <-status {
				if lockAcquired != nil {
					close(lockAcquired)
					lockAcquired = nil
				}
			} else {
				logger.Error("Lost the lock", errors.New("Lock the lock"))
				os.Exit(197)
			}
		}
	}()

	<-lockAcquired

	logger.Info("Acquired lock for " + component)

	logger.Info(fmt.Sprintf("Running Daemon every %d seconds with a timeout of %d", int(period.Seconds()), int(timeout.Seconds())))

	for {
		afterChan := time.After(period)
		timeoutChan := time.After(timeout)
		errorChan := make(chan error, 1)

		t := time.Now()

		go func() {
			errorChan <- callback()
		}()

		select {
		case err := <-errorChan:
			logger.Info("Daemonize Time", map[string]string{
				"Component": component,
				"Duration":  fmt.Sprintf("%.4f", time.Since(t).Seconds()),
			})
			if err != nil {
				logger.Error("Daemon returned an error. Continuining...", err)
			}
		case <-timeoutChan:
			released := make(chan bool)
			releaseLockChannel <- released
			<-released

			return errors.New("Daemon timed out. Aborting!")
		}

		<-afterChan
	}

	return nil
}