func (p *Processor) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
close(ready)
httpClient := cf_http.NewClient()
httpClient.Transport = &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: 10 * time.Second,
KeepAlive: 30 * time.Second,
}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: p.skipCertVerify,
MinVersion: tls.VersionTLS10,
},
}
timer := p.clock.NewTimer(p.pollingInterval)
stop := p.sync(signals, httpClient)
for {
if stop {
return nil
}
select {
case <-signals:
return nil
case <-timer.C():
stop = p.sync(signals, httpClient)
timer.Reset(p.pollingInterval)
}
}
}
func initializeServer(logger lager.Logger) ifrit.Runner {
if *staticDirectory == "" {
logger.Fatal("static-directory-missing", nil)
}
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: ccUploadDialTimeout,
KeepAlive: ccUploadKeepAlive,
}).Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: *skipCertVerify,
},
TLSHandshakeTimeout: ccUploadTLSHandshakeTimeout,
}
pollerHttpClient := cf_http.NewClient()
pollerHttpClient.Transport = transport
fileServerHandler, err := handlers.New(*staticDirectory, logger)
if err != nil {
logger.Error("router-building-failed", err)
os.Exit(1)
}
return http_server.New(*serverAddress, fileServerHandler)
}
func NewUaaKeyFetcher(logger lager.Logger, uaaGetKeyEndpoint string) UaaKeyFetcher {
return &uaaKeyFetcher{
uaaGetKeyEndpoint: uaaGetKeyEndpoint,
httpClient: cf_http.NewClient(),
logger: logger,
}
}
func NewETCDMetrics(logger lager.Logger, etcdOptions *ETCDOptions) (*ETCDMetrics, error) {
var tlsConfig *tls.Config
if etcdOptions.CertFile != "" && etcdOptions.KeyFile != "" {
var err error
tlsConfig, err = cf_http.NewTLSConfig(etcdOptions.CertFile, etcdOptions.KeyFile, etcdOptions.CAFile)
if err != nil {
return nil, err
}
tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(etcdOptions.ClientSessionCacheSize)
}
client := cf_http.NewClient()
client.CheckRedirect = func(*http.Request, []*http.Request) error {
return errRedirected
}
if tr, ok := client.Transport.(*http.Transport); ok {
tr.TLSClientConfig = tlsConfig
} else {
return nil, errors.New("Invalid transport")
}
return &ETCDMetrics{
logger: logger,
etcdCluster: etcdOptions.ClusterUrls,
client: client,
}, nil
}
func initializeServer(logger lager.Logger) ifrit.Runner {
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{
Timeout: ccUploadDialTimeout,
KeepAlive: ccUploadKeepAlive,
}).Dial,
TLSClientConfig: &tls.Config{
InsecureSkipVerify: *skipCertVerify,
},
TLSHandshakeTimeout: ccUploadTLSHandshakeTimeout,
}
pollerHttpClient := cf_http.NewClient()
pollerHttpClient.Transport = transport
uploader := ccclient.NewUploader(logger, &http.Client{Transport: transport})
poller := ccclient.NewPoller(logger, pollerHttpClient, *ccJobPollingInterval)
ccUploaderHandler, err := handlers.New(uploader, poller, logger)
if err != nil {
logger.Error("router-building-failed", err)
os.Exit(1)
}
return http_server.New(*serverAddress, ccUploaderHandler)
}
func NewClient(url string) Client {
return &client{
httpClient: cf_http.NewClient(),
streamingHTTPClient: cf_http.NewStreamingClient(),
reqGen: rata.NewRequestGenerator(url, Routes),
}
}
func newTaskWorker(taskQueue <-chan models.Task, receptorBBS bbs.ReceptorBBS, logger lager.Logger) *taskWorker {
return &taskWorker{
taskQueue: taskQueue,
receptorBBS: receptorBBS,
logger: logger,
httpClient: cf_http.NewClient(),
}
}
func newTaskWorker(taskQueue <-chan *models.Task, taskdb bbs.Client, logger lager.Logger) *taskWorker {
return &taskWorker{
taskQueue: taskQueue,
taskdb: taskdb,
logger: logger,
httpClient: cf_http.NewClient(),
}
}
func New(logger lager.Logger, maxWorkers int, cbHandler CompletedTaskHandler) *TaskCompletionWorkPool {
if cbHandler == nil {
panic("callbackHandler cannot be nil")
}
return &TaskCompletionWorkPool{
logger: logger,
maxWorkers: maxWorkers,
callbackHandler: cbHandler,
httpClient: cf_http.NewClient(),
}
}
func initializeEtcdDB(
logger lager.Logger,
cryptor encryption.Cryptor,
storeClient etcddb.StoreClient,
cbClient taskworkpool.TaskCompletionClient,
serviceClient bbs.ServiceClient,
) *etcddb.ETCDDB {
return etcddb.NewETCD(
format.ENCRYPTED_PROTO,
*convergenceWorkers,
*updateWorkers,
cryptor,
storeClient,
initializeAuctioneerClient(logger),
serviceClient,
clock.NewClock(),
rep.NewClientFactory(cf_http.NewClient(), cf_http.NewClient()),
cbClient,
)
}
func NewDiegoExecutor() *DiegoExecutor {
ip, _ := localip.LocalIP()
address := fmt.Sprintf("http://%s:%v", ip, 1800)
repClientFactory := rep.NewClientFactory(cf_http.NewClient())
cellClient := repClientFactory.CreateClient(address)
return &DiegoExecutor{
cellClient: cellClient,
taskStateMap: map[string]mesos.TaskState{},
}
}
func initializeAuctionRunner(logger lager.Logger, cellStateTimeout time.Duration, bbsClient bbs.Client, serviceClient bbs.ServiceClient) auctiontypes.AuctionRunner {
httpClient := cf_http.NewClient()
stateClient := cf_http.NewCustomTimeoutClient(cellStateTimeout)
repClientFactory := rep.NewClientFactory(httpClient, stateClient)
delegate := auctionrunnerdelegate.New(repClientFactory, bbsClient, serviceClient, logger)
metricEmitter := auctionmetricemitterdelegate.New()
workPool, err := workpool.NewWorkPool(*auctionRunnerWorkers)
if err != nil {
logger.Fatal("failed-to-construct-auction-runner-workpool", err, lager.Data{"num-workers": *auctionRunnerWorkers}) // should never happen
}
return auctionrunner.New(
delegate,
metricEmitter,
clock.NewClock(),
workPool,
logger,
)
}
fakeServer.Close()
})
Describe("HandleCompletedTask", func() {
var (
callbackURL string
taskDB *dbfakes.FakeTaskDB
statusCodes chan int
reqCount chan struct{}
task *models.Task
httpClient *http.Client
)
BeforeEach(func() {
httpClient = cf_http.NewClient()
statusCodes = make(chan int)
reqCount = make(chan struct{})
fakeServer.RouteToHandler("POST", "/the-callback/url", func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(<-statusCodes)
})
callbackURL = fakeServer.URL() + "/the-callback/url"
taskDB = new(dbfakes.FakeTaskDB)
taskDB.ResolvingTaskReturns(nil)
taskDB.DeleteTaskReturns(nil)
})
simulateTaskCompleting := func(signals <-chan os.Signal, ready chan<- struct{}) error {
close(ready)
)
var _ = Describe("CfHttp", func() {
var timeout time.Duration
BeforeEach(func() {
timeout = 1 * time.Second
})
JustBeforeEach(func() {
cf_http.Initialize(timeout)
})
Describe("NewClient", func() {
It("returns an http client", func() {
client := cf_http.NewClient()
Expect(client.Timeout).To(Equal(timeout))
transport := client.Transport.(*http.Transport)
Expect(transport.Dial).NotTo(BeNil())
Expect(transport.DisableKeepAlives).To(BeFalse())
})
})
Describe("NewCustomTimeoutClient", func() {
It("returns an http client with specified timeout", func() {
client := cf_http.NewCustomTimeoutClient(5 * time.Second)
Expect(client.Timeout).To(Equal(5 * time.Second))
transport := client.Transport.(*http.Transport)
Expect(transport.Dial).NotTo(BeNil())
Expect(transport.DisableKeepAlives).To(BeFalse())
})
"github.com/cloudfoundry-incubator/bbs/models"
"github.com/cloudfoundry-incubator/cf_http"
"github.com/cloudfoundry-incubator/rep"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/onsi/gomega/ghttp"
)
var _ = Describe("Client", func() {
var fakeServer *ghttp.Server
var client rep.Client
BeforeEach(func() {
fakeServer = ghttp.NewServer()
client = rep.NewClient(cf_http.NewClient(), fakeServer.URL())
})
AfterEach(func() {
fakeServer.Close()
})
Describe("StopLRPInstance", func() {
const cellAddr = "cell.example.com"
var stopErr error
var actualLRP = models.ActualLRP{
ActualLRPKey: models.NewActualLRPKey("some-process-guid", 2, "test-domain"),
ActualLRPInstanceKey: models.NewActualLRPInstanceKey("some-instance-guid", "some-cell-id"),
}
JustBeforeEach(func() {
func configure(logger lager.Logger) (*ssh.ServerConfig, error) {
cf_http.Initialize(*communicationTimeout)
if *diegoAPIURL == "" {
err := errors.New("diegoAPIURL is required")
logger.Fatal("diego-api-url-required", err)
}
url, err := url.Parse(*diegoAPIURL)
if err != nil {
logger.Fatal("failed-to-parse-diego-api-url", err)
}
_, err = url.Parse(*ccAPIURL)
if *ccAPIURL != "" && err != nil {
logger.Fatal("failed-to-parse-cc-api-url", err)
}
var diegoCreds string
if url.User != nil {
diegoCreds = url.User.String()
}
receptorClient := receptor.NewClient(*diegoAPIURL)
authenticatorMap := map[string]authenticators.PasswordAuthenticator{}
if *enableDiegoAuth {
diegoAuthenticator := authenticators.NewDiegoProxyAuthenticator(logger, receptorClient, []byte(diegoCreds))
authenticatorMap[diegoAuthenticator.Realm()] = diegoAuthenticator
}
if *ccAPIURL != "" && *enableCFAuth {
ccClient := cf_http.NewClient()
cfAuthenticator := authenticators.NewCFAuthenticator(logger, ccClient, *ccAPIURL, receptorClient)
authenticatorMap[cfAuthenticator.Realm()] = cfAuthenticator
}
authenticator := authenticators.NewCompositeAuthenticator(authenticatorMap)
sshConfig := &ssh.ServerConfig{
PasswordCallback: authenticator.Authenticate,
AuthLogCallback: func(cmd ssh.ConnMetadata, method string, err error) {
logger.Error("authentication-failed", err, lager.Data{"user": cmd.User()})
},
}
if *hostKey == "" {
err := errors.New("hostKey is required")
logger.Fatal("host-key-required", err)
}
key, err := parsePrivateKey(logger, *hostKey)
if err != nil {
logger.Fatal("failed-to-parse-host-key", err)
}
sshConfig.AddHostKey(key)
return sshConfig, err
}
func NewClient(auctioneerURL string) Client {
return &auctioneerClient{
httpClient: cf_http.NewClient(),
url: auctioneerURL,
}
}
func NewClient() Client {
return &client{
httpClient: cf_http.NewClient(),
}
}
func main() {
cf_debug_server.AddFlags(flag.CommandLine)
cf_lager.AddFlags(flag.CommandLine)
etcdFlags := AddETCDFlags(flag.CommandLine)
encryptionFlags := encryption.AddEncryptionFlags(flag.CommandLine)
flag.Parse()
cf_http.Initialize(*communicationTimeout)
logger, reconfigurableSink := cf_lager.New("bbs")
logger.Info("starting")
initializeDropsonde(logger)
clock := clock.NewClock()
consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
if err != nil {
logger.Fatal("new-consul-client-failed", err)
}
serviceClient := bbs.NewServiceClient(consulClient, clock)
maintainer := initializeLockMaintainer(logger, serviceClient)
_, portString, err := net.SplitHostPort(*listenAddress)
if err != nil {
logger.Fatal("failed-invalid-listen-address", err)
}
portNum, err := net.LookupPort("tcp", portString)
if err != nil {
logger.Fatal("failed-invalid-listen-port", err)
}
_, portString, err = net.SplitHostPort(*healthAddress)
if err != nil {
logger.Fatal("failed-invalid-health-address", err)
}
_, err = net.LookupPort("tcp", portString)
if err != nil {
logger.Fatal("failed-invalid-health-port", err)
}
registrationRunner := initializeRegistrationRunner(logger, consulClient, portNum, clock)
cbWorkPool := taskworkpool.New(logger, *taskCallBackWorkers, taskworkpool.HandleCompletedTask)
var activeDB db.DB
var sqlDB *sqldb.SQLDB
var sqlConn *sql.DB
var storeClient etcddb.StoreClient
var etcdDB *etcddb.ETCDDB
key, keys, err := encryptionFlags.Parse()
if err != nil {
logger.Fatal("cannot-setup-encryption", err)
}
keyManager, err := encryption.NewKeyManager(key, keys)
if err != nil {
logger.Fatal("cannot-setup-encryption", err)
}
cryptor := encryption.NewCryptor(keyManager, rand.Reader)
etcdOptions, err := etcdFlags.Validate()
if err != nil {
logger.Fatal("etcd-validation-failed", err)
}
if etcdOptions.IsConfigured {
storeClient = initializeEtcdStoreClient(logger, etcdOptions)
etcdDB = initializeEtcdDB(logger, cryptor, storeClient, cbWorkPool, serviceClient, *desiredLRPCreationTimeout)
activeDB = etcdDB
}
// If SQL database info is passed in, use SQL instead of ETCD
if *databaseDriver != "" && *databaseConnectionString != "" {
var err error
connectionString := appendSSLConnectionStringParam(logger, *databaseDriver, *databaseConnectionString, *sqlCACertFile)
sqlConn, err = sql.Open(*databaseDriver, connectionString)
if err != nil {
logger.Fatal("failed-to-open-sql", err)
}
defer sqlConn.Close()
sqlConn.SetMaxOpenConns(*maxDatabaseConnections)
sqlConn.SetMaxIdleConns(*maxDatabaseConnections)
err = sqlConn.Ping()
if err != nil {
logger.Fatal("sql-failed-to-connect", err)
}
sqlDB = sqldb.NewSQLDB(sqlConn, *convergenceWorkers, *updateWorkers, format.ENCRYPTED_PROTO, cryptor, guidprovider.DefaultGuidProvider, clock, *databaseDriver)
err = sqlDB.CreateConfigurationsTable(logger)
if err != nil {
logger.Fatal("sql-failed-create-configurations-table", err)
}
activeDB = sqlDB
}
if activeDB == nil {
logger.Fatal("no-database-configured", errors.New("no database configured"))
}
encryptor := encryptor.New(logger, activeDB, keyManager, cryptor, clock)
migrationsDone := make(chan struct{})
migrationManager := migration.NewManager(logger,
etcdDB,
storeClient,
sqlDB,
sqlConn,
cryptor,
migrations.Migrations,
migrationsDone,
clock,
*databaseDriver,
)
desiredHub := events.NewHub()
actualHub := events.NewHub()
repClientFactory := rep.NewClientFactory(cf_http.NewClient(), cf_http.NewClient())
auctioneerClient := initializeAuctioneerClient(logger)
exitChan := make(chan struct{})
handler := handlers.New(
logger,
*updateWorkers,
*convergenceWorkers,
activeDB,
desiredHub,
actualHub,
cbWorkPool,
serviceClient,
auctioneerClient,
repClientFactory,
migrationsDone,
exitChan,
)
metricsNotifier := metrics.NewPeriodicMetronNotifier(
logger,
*reportInterval,
etcdOptions,
clock,
)
var server ifrit.Runner
if *requireSSL {
tlsConfig, err := cf_http.NewTLSConfig(*certFile, *keyFile, *caFile)
if err != nil {
logger.Fatal("tls-configuration-failed", err)
}
server = http_server.NewTLSServer(*listenAddress, handler, tlsConfig)
} else {
server = http_server.New(*listenAddress, handler)
}
healthcheckServer := http_server.New(*healthAddress, http.HandlerFunc(healthCheckHandler))
members := grouper.Members{
{"healthcheck", healthcheckServer},
{"lock-maintainer", maintainer},
{"workpool", cbWorkPool},
{"server", server},
{"migration-manager", migrationManager},
{"encryptor", encryptor},
{"hub-maintainer", hubMaintainer(logger, desiredHub, actualHub)},
{"metrics", *metricsNotifier},
{"registration-runner", registrationRunner},
}
if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
members = append(grouper.Members{
{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
}, members...)
}
group := grouper.NewOrdered(os.Interrupt, members)
monitor := ifrit.Invoke(sigmon.New(group))
go func() {
// If a handler writes to this channel, we've hit an unrecoverable error
// and should shut down (cleanly)
<-exitChan
monitor.Signal(os.Interrupt)
}()
logger.Info("started")
err = <-monitor.Wait()
if sqlConn != nil {
sqlConn.Close()
}
if err != nil {
logger.Error("exited-with-failure", err)
os.Exit(1)
}
logger.Info("exited")
}
"net/http"
"time"
"github.com/cloudfoundry-incubator/cf_http"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
var _ = Describe("CfHttp", func() {
Describe("NewClient", func() {
var timeout time.Duration
BeforeEach(func() {
timeout = 1 * time.Second
})
It("returns an http client", func() {
By("Getting a client before initializaqtion", func() {
Ω(*cf_http.NewClient()).Should(Equal(*http.DefaultClient))
})
cf_http.Initialize(timeout)
Ω(*cf_http.NewClient()).Should(Equal(http.Client{
Timeout: timeout,
}))
})
})
})