var (
logger *lagertest.TestLogger
fakeTaskDB *dbfakes.FakeTaskDB
fakeAuctioneerClient *auctioneerfakes.FakeClient
fakeTaskCompletionClient *taskworkpoolfakes.FakeTaskCompletionClient
controller *controllers.TaskController
)
BeforeEach(func() {
fakeTaskDB = new(dbfakes.FakeTaskDB)
fakeAuctioneerClient = new(auctioneerfakes.FakeClient)
fakeTaskCompletionClient = new(taskworkpoolfakes.FakeTaskCompletionClient)
logger = lagertest.NewTestLogger("test")
controller = controllers.NewTaskController(fakeTaskDB, fakeTaskCompletionClient, fakeAuctioneerClient, fakeServiceClient, fakeRepClientFactory)
})
Describe("Tasks", func() {
var (
domain, cellId string
task1 models.Task
task2 models.Task
actualTasks []*models.Task
err error
)
BeforeEach(func() {
task1 = models.Task{Domain: "domain-1"}
task2 = models.Task{CellId: "cell-id"}
domain = ""
func New(
logger, accessLogger lager.Logger,
updateWorkers int,
convergenceWorkersSize int,
db db.DB,
desiredHub, actualHub events.Hub,
taskCompletionClient taskworkpool.TaskCompletionClient,
serviceClient bbs.ServiceClient,
auctioneerClient auctioneer.Client,
repClientFactory rep.ClientFactory,
migrationsDone <-chan struct{},
exitChan chan struct{},
) http.Handler {
retirer := controllers.NewActualLRPRetirer(db, actualHub, repClientFactory, serviceClient)
pingHandler := NewPingHandler()
domainHandler := NewDomainHandler(db, exitChan)
actualLRPHandler := NewActualLRPHandler(db, exitChan)
actualLRPLifecycleHandler := NewActualLRPLifecycleHandler(db, db, actualHub, auctioneerClient, retirer, exitChan)
evacuationHandler := NewEvacuationHandler(db, db, db, actualHub, auctioneerClient, exitChan)
desiredLRPHandler := NewDesiredLRPHandler(updateWorkers, db, db, desiredHub, actualHub, auctioneerClient, repClientFactory, serviceClient, exitChan)
taskController := controllers.NewTaskController(db, taskCompletionClient, auctioneerClient, serviceClient, repClientFactory)
taskHandler := NewTaskHandler(taskController, exitChan)
eventsHandler := NewEventHandler(desiredHub, actualHub)
cellsHandler := NewCellHandler(serviceClient, exitChan)
emitter := middleware.NewLatencyEmitter(logger)
actions := rata.Handlers{
// Ping
bbs.PingRoute: emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, pingHandler.Ping)),
// Domains
bbs.DomainsRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, domainHandler.Domains))),
bbs.UpsertDomainRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, domainHandler.Upsert))),
// Actual LRPs
bbs.ActualLRPGroupsRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPHandler.ActualLRPGroups))),
bbs.ActualLRPGroupsByProcessGuidRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPHandler.ActualLRPGroupsByProcessGuid))),
bbs.ActualLRPGroupByProcessGuidAndIndexRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPHandler.ActualLRPGroupByProcessGuidAndIndex))),
// Actual LRP Lifecycle
bbs.ClaimActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPLifecycleHandler.ClaimActualLRP))),
bbs.StartActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPLifecycleHandler.StartActualLRP))),
bbs.CrashActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPLifecycleHandler.CrashActualLRP))),
bbs.RetireActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPLifecycleHandler.RetireActualLRP))),
bbs.FailActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPLifecycleHandler.FailActualLRP))),
bbs.RemoveActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, actualLRPLifecycleHandler.RemoveActualLRP))),
// Evacuation
bbs.RemoveEvacuatingActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, evacuationHandler.RemoveEvacuatingActualLRP))),
bbs.EvacuateClaimedActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, evacuationHandler.EvacuateClaimedActualLRP))),
bbs.EvacuateCrashedActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, evacuationHandler.EvacuateCrashedActualLRP))),
bbs.EvacuateStoppedActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, evacuationHandler.EvacuateStoppedActualLRP))),
bbs.EvacuateRunningActualLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, evacuationHandler.EvacuateRunningActualLRP))),
// Desired LRPs
bbs.DesiredLRPsRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesiredLRPs))),
bbs.DesiredLRPByProcessGuidRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesiredLRPByProcessGuid))),
bbs.DesiredLRPSchedulingInfosRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesiredLRPSchedulingInfos))),
bbs.DesireDesiredLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesireDesiredLRP))),
bbs.UpdateDesiredLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.UpdateDesiredLRP))),
bbs.RemoveDesiredLRPRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.RemoveDesiredLRP))),
bbs.DesiredLRPsRoute_r0: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesiredLRPs_r0))),
bbs.DesiredLRPsRoute_r1: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesiredLRPs_r1))),
bbs.DesiredLRPByProcessGuidRoute_r0: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesiredLRPByProcessGuid_r0))),
bbs.DesiredLRPByProcessGuidRoute_r1: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesiredLRPByProcessGuid_r1))),
bbs.DesireDesiredLRPRoute_r0: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesireDesiredLRP_r0))),
bbs.DesireDesiredLRPRoute_r1: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, desiredLRPHandler.DesireDesiredLRP_r1))),
// Tasks
bbs.TasksRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.Tasks))),
bbs.TaskByGuidRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.TaskByGuid))),
bbs.DesireTaskRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.DesireTask))),
bbs.StartTaskRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.StartTask))),
bbs.CancelTaskRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.CancelTask))),
bbs.FailTaskRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.FailTask))),
bbs.CompleteTaskRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.CompleteTask))),
bbs.ResolvingTaskRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.ResolvingTask))),
bbs.DeleteTaskRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.DeleteTask))),
bbs.TasksRoute_r1: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.Tasks_r1))),
bbs.TasksRoute_r0: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.Tasks_r0))),
bbs.TaskByGuidRoute_r1: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.TaskByGuid_r1))),
bbs.TaskByGuidRoute_r0: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.TaskByGuid_r0))),
bbs.DesireTaskRoute_r1: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.DesireTask_r1))),
bbs.DesireTaskRoute_r0: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, taskHandler.DesireTask_r0))),
// Events
bbs.EventStreamRoute_r0: route(middleware.LogWrap(logger, accessLogger, eventsHandler.Subscribe_r0)),
// Cells
bbs.CellsRoute: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, cellsHandler.Cells))),
bbs.CellsRoute_r1: route(emitter.EmitLatency(middleware.LogWrap(logger, accessLogger, cellsHandler.Cells))),
}
handler, err := rata.NewRouter(bbs.Routes, actions)
if err != nil {
panic("unable to create router: " + err.Error())
}
return middleware.RequestCountWrap(
UnavailableWrap(handler,
migrationsDone,
),
)
}
func main() {
debugserver.AddFlags(flag.CommandLine)
lagerflags.AddFlags(flag.CommandLine)
etcdFlags := AddETCDFlags(flag.CommandLine)
encryptionFlags := encryption.AddEncryptionFlags(flag.CommandLine)
flag.Parse()
cfhttp.Initialize(*communicationTimeout)
logger, reconfigurableSink := lagerflags.New("bbs")
logger.Info("starting")
initializeDropsonde(logger)
clock := clock.NewClock()
consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
if err != nil {
logger.Fatal("new-consul-client-failed", err)
}
serviceClient := bbs.NewServiceClient(consulClient, clock)
maintainer := initializeLockMaintainer(logger, serviceClient)
_, portString, err := net.SplitHostPort(*listenAddress)
if err != nil {
logger.Fatal("failed-invalid-listen-address", err)
}
portNum, err := net.LookupPort("tcp", portString)
if err != nil {
logger.Fatal("failed-invalid-listen-port", err)
}
_, portString, err = net.SplitHostPort(*healthAddress)
if err != nil {
logger.Fatal("failed-invalid-health-address", err)
}
_, err = net.LookupPort("tcp", portString)
if err != nil {
logger.Fatal("failed-invalid-health-port", err)
}
registrationRunner := initializeRegistrationRunner(logger, consulClient, portNum, clock)
var activeDB db.DB
var sqlDB *sqldb.SQLDB
var sqlConn *sql.DB
var storeClient etcddb.StoreClient
var etcdDB *etcddb.ETCDDB
key, keys, err := encryptionFlags.Parse()
if err != nil {
logger.Fatal("cannot-setup-encryption", err)
}
keyManager, err := encryption.NewKeyManager(key, keys)
if err != nil {
logger.Fatal("cannot-setup-encryption", err)
}
cryptor := encryption.NewCryptor(keyManager, rand.Reader)
etcdOptions, err := etcdFlags.Validate()
if err != nil {
logger.Fatal("etcd-validation-failed", err)
}
if etcdOptions.IsConfigured {
storeClient = initializeEtcdStoreClient(logger, etcdOptions)
etcdDB = initializeEtcdDB(logger, cryptor, storeClient, serviceClient, *desiredLRPCreationTimeout)
activeDB = etcdDB
}
// If SQL database info is passed in, use SQL instead of ETCD
if *databaseDriver != "" && *databaseConnectionString != "" {
var err error
connectionString := appendSSLConnectionStringParam(logger, *databaseDriver, *databaseConnectionString, *sqlCACertFile)
sqlConn, err = sql.Open(*databaseDriver, connectionString)
if err != nil {
logger.Fatal("failed-to-open-sql", err)
}
defer sqlConn.Close()
sqlConn.SetMaxOpenConns(*maxDatabaseConnections)
sqlConn.SetMaxIdleConns(*maxDatabaseConnections)
err = sqlConn.Ping()
if err != nil {
logger.Fatal("sql-failed-to-connect", err)
}
sqlDB = sqldb.NewSQLDB(sqlConn, *convergenceWorkers, *updateWorkers, format.ENCRYPTED_PROTO, cryptor, guidprovider.DefaultGuidProvider, clock, *databaseDriver)
err = sqlDB.SetIsolationLevel(logger, sqldb.IsolationLevelReadCommitted)
if err != nil {
logger.Fatal("sql-failed-to-set-isolation-level", err)
}
err = sqlDB.CreateConfigurationsTable(logger)
if err != nil {
logger.Fatal("sql-failed-create-configurations-table", err)
}
activeDB = sqlDB
}
if activeDB == nil {
logger.Fatal("no-database-configured", errors.New("no database configured"))
}
encryptor := encryptor.New(logger, activeDB, keyManager, cryptor, clock)
migrationsDone := make(chan struct{})
migrationManager := migration.NewManager(
logger,
etcdDB,
storeClient,
sqlDB,
sqlConn,
cryptor,
migrations.Migrations,
migrationsDone,
clock,
*databaseDriver,
)
desiredHub := events.NewHub()
actualHub := events.NewHub()
repTLSConfig := &rep.TLSConfig{
RequireTLS: *repRequireTLS,
CaCertFile: *repCACert,
CertFile: *repClientCert,
KeyFile: *repClientKey,
ClientCacheSize: *repClientSessionCacheSize,
}
httpClient := cfhttp.NewClient()
repClientFactory, err := rep.NewClientFactory(httpClient, httpClient, repTLSConfig)
if err != nil {
logger.Fatal("new-rep-client-factory-failed", err)
}
auctioneerClient := initializeAuctioneerClient(logger)
exitChan := make(chan struct{})
var accessLogger lager.Logger
if *accessLogPath != "" {
accessLogger = lager.NewLogger("bbs-access")
file, err := os.OpenFile(*accessLogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
logger.Error("invalid-access-log-path", err, lager.Data{"access-log-path": *accessLogPath})
os.Exit(1)
}
accessLogger.RegisterSink(lager.NewWriterSink(file, lager.INFO))
}
var tlsConfig *tls.Config
if *requireSSL {
tlsConfig, err = cfhttp.NewTLSConfig(*certFile, *keyFile, *caFile)
if err != nil {
logger.Fatal("tls-configuration-failed", err)
}
}
cbWorkPool := taskworkpool.New(logger, *taskCallBackWorkers, taskworkpool.HandleCompletedTask, tlsConfig)
handler := handlers.New(
logger,
accessLogger,
*updateWorkers,
*convergenceWorkers,
activeDB,
desiredHub,
actualHub,
cbWorkPool,
serviceClient,
auctioneerClient,
repClientFactory,
migrationsDone,
exitChan,
)
metricsNotifier := metrics.NewPeriodicMetronNotifier(logger)
retirer := controllers.NewActualLRPRetirer(activeDB, actualHub, repClientFactory, serviceClient)
lrpConvergenceController := controllers.NewLRPConvergenceController(logger, activeDB, actualHub, auctioneerClient, serviceClient, retirer, *convergenceWorkers)
taskController := controllers.NewTaskController(activeDB, cbWorkPool, auctioneerClient, serviceClient, repClientFactory)
convergerProcess := converger.New(
logger,
clock,
lrpConvergenceController,
taskController,
serviceClient,
*convergeRepeatInterval,
*kickTaskDuration,
*expirePendingTaskDuration,
*expireCompletedTaskDuration)
var server ifrit.Runner
if tlsConfig != nil {
server = http_server.NewTLSServer(*listenAddress, handler, tlsConfig)
} else {
server = http_server.New(*listenAddress, handler)
}
healthcheckServer := http_server.New(*healthAddress, http.HandlerFunc(healthCheckHandler))
members := grouper.Members{
{"healthcheck", healthcheckServer},
{"lock-maintainer", maintainer},
{"workpool", cbWorkPool},
{"server", server},
{"migration-manager", migrationManager},
{"encryptor", encryptor},
{"hub-maintainer", hubMaintainer(logger, desiredHub, actualHub)},
{"metrics", *metricsNotifier},
{"converger", convergerProcess},
{"registration-runner", registrationRunner},
}
if dbgAddr := debugserver.DebugAddress(flag.CommandLine); dbgAddr != "" {
members = append(grouper.Members{
{"debug-server", debugserver.Runner(dbgAddr, reconfigurableSink)},
}, members...)
}
group := grouper.NewOrdered(os.Interrupt, members)
monitor := ifrit.Invoke(sigmon.New(group))
go func() {
// If a handler writes to this channel, we've hit an unrecoverable error
// and should shut down (cleanly)
<-exitChan
monitor.Signal(os.Interrupt)
}()
logger.Info("started")
err = <-monitor.Wait()
if sqlConn != nil {
sqlConn.Close()
}
if err != nil {
logger.Error("exited-with-failure", err)
os.Exit(1)
}
logger.Info("exited")
}