// NewClientFromConfigAndTransport creates an new OAuth 2.0 authorized client
// for the given config and transport.
//
// If the transport is nil then util.NewBackOffTransport() is used.
// If local is true then a 3-legged flow is initiated, otherwise the GCE
// Service Account is used.
func NewClientFromConfigAndTransport(local bool, config *oauth2.Config, oauthCacheFile string, transport http.RoundTripper) (*http.Client, error) {
if oauthCacheFile == "" {
oauthCacheFile = "google_storage_token.data"
}
if transport == nil {
transport = util.NewBackOffTransport()
}
var client *http.Client
if local {
tokenClient := &http.Client{
Transport: transport,
Timeout: util.REQUEST_TIMEOUT,
}
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, tokenClient)
tokenSource, err := newCachingTokenSource(oauthCacheFile, ctx, config)
if err != nil {
return nil, fmt.Errorf("NewClientFromConfigAndTransport: Unable to create token source: %s", err)
}
client = &http.Client{
Transport: &oauth2.Transport{
Source: tokenSource,
Base: transport,
},
Timeout: util.REQUEST_TIMEOUT,
}
} else {
// Use compute engine service account.
client = GCEServiceAccountClient(transport)
}
return client, nil
}
// Init initializes the module, the optional http.Client is used to make HTTP
// requests to Google Storage. If nil is supplied then a default client is
// used.
func Init(cl *http.Client) {
if cl != nil {
client = cl
} else {
client = &http.Client{
Transport: util.NewBackOffTransport(),
}
}
}
// IngestersFromConfig creates a list of ingesters from a config struct.
// Usually the struct is created from parsing a config file.
func IngestersFromConfig(config *sharedconfig.Config, client *http.Client) ([]*Ingester, error) {
ret := []*Ingester{}
// Set up the gitinfo object.
var vcs vcsinfo.VCS
var err error
if vcs, err = gitinfo.CloneOrUpdate(config.GitRepoURL, config.GitRepoDir, true); err != nil {
return nil, err
}
// Set up the Google storage client.
if client == nil {
client = &http.Client{
Transport: util.NewBackOffTransport(),
}
}
// for each defined ingester create an instance.
for id, ingesterConf := range config.Ingesters {
processorConstructor, ok := constructors[id]
if !ok {
return nil, fmt.Errorf("Unknow ingester: '%s'", id)
}
// Instantiate the sources
sources := make([]Source, 0, len(ingesterConf.Sources))
for _, dataSource := range ingesterConf.Sources {
oneSource, err := getSource(id, dataSource, client)
if err != nil {
return nil, fmt.Errorf("Error instantiating sources for ingester '%s': %s", id, err)
}
sources = append(sources, oneSource)
}
// instantiate the processor
processor, err := processorConstructor(vcs, ingesterConf)
if err != nil {
return nil, err
}
// create the ingester and add it to the result.
ingester, err := NewIngester(id, ingesterConf, vcs, sources, processor)
if err != nil {
return nil, err
}
ret = append(ret, ingester)
}
return ret, nil
}
// NewJWTServiceAccountClient creates a new http.Client that is loaded by first
// attempting to load the JWT JSON Service Account data from GCE Project Level
// metadata, and if that fails falls back to loading the data from a local
// file.
//
// metadataname - The name of the GCE project level metadata key that holds the JWT JSON. If empty a default is used.
// filename - The name of the local file that holds the JWT JSON. If empty a default is used.
// transport - A transport. If nil then a default is used.
func NewJWTServiceAccountClient(metadataname, filename string, transport http.RoundTripper, scopes ...string) (*http.Client, error) {
if metadataname == "" {
metadataname = metadata.JWT_SERVICE_ACCOUNT
}
if filename == "" {
filename = DEFAULT_JWT_FILENAME
}
var body []byte
jwt, err := metadata.ProjectGet(metadataname)
if err != nil {
body, err = ioutil.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("Couldn't find JWT via metadata or in a local file.")
}
} else {
body = []byte(jwt)
}
if transport == nil {
transport = util.NewBackOffTransport()
}
jwtConfig, err := google.JWTConfigFromJSON(body, scopes...)
if err != nil {
return nil, err
}
tokenClient := &http.Client{
Transport: transport,
Timeout: util.REQUEST_TIMEOUT,
}
ctx := context.WithValue(context.Background(), oauth2.HTTPClient, tokenClient)
return &http.Client{
Transport: &oauth2.Transport{
Source: jwtConfig.TokenSource(ctx),
Base: transport,
},
Timeout: util.REQUEST_TIMEOUT,
}, nil
}
func GetOAuthClient() (*http.Client, error) {
config := auth.OAuthConfig(GSTokenPath, auth.SCOPE_FULL_CONTROL)
return auth.RunFlowWithTransport(config, util.NewBackOffTransport())
}
func main() {
defer common.LogPanic()
// Setup DB flags.
dbConf := db.DBConfigFromFlags()
common.InitWithMetricsCB("ingest", func() string {
common.DecodeTomlFile(*configFilename, &config)
return config.Common.GraphiteServer
})
// Initialize the database. We might not need the oauth dialog if it fails.
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Get a backoff transport.
transport := util.NewBackOffTransport()
// Determine the oauth scopes we are going to use. Storage is used by all
// ingesters.
scopes := []string{storage.CloudPlatformScope}
for _, ingesterConfig := range config.Ingesters {
if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD {
scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope)
}
}
// Initialize the oauth client that is used to access all scopes.
var client *http.Client
var err error
if config.Common.Local {
if config.Common.DoOAuth {
client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile,
config.Common.OAuthClientSecretFile,
transport, scopes...)
if err != nil {
glog.Fatalf("Failed to auth: %s", err)
}
} else {
client = nil
// Add back service account access here when it's fixed.
}
} else {
// Assume we are on a GCE instance.
client = auth.GCEServiceAccountClient(transport)
}
// Initialize the ingester and gold ingester.
ingester.Init(client)
if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok {
if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil {
glog.Fatalf("Unable to initialize GoldIngester: %s", err)
}
}
// TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own
// database connection and they are all instantiated the same way instead of the
// one-off approaches.
if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok {
dbConf := &database.DatabaseConfig{
Host: ingesterConf.DBHost,
Port: ingesterConf.DBPort,
User: database.USER_RW,
Name: ingesterConf.DBName,
MigrationSteps: golddb.MigrationSteps(),
}
if !config.Common.Local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
vdb, err := dbConf.NewVersionedDB()
if err != nil {
glog.Fatalf("Unable to open db connection: %s", err)
}
goldtrybot.Init(vdb)
}
git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false)
if err != nil {
glog.Fatalf("Failed loading Git info: %s\n", err)
}
for dataset, ingesterConfig := range config.Ingesters {
// Get duration equivalent to the number of days.
minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays)
constructorName := ingesterConfig.ConstructorName
if constructorName == "" {
constructorName = dataset
}
constructor := ingester.Constructor(constructorName)
resultIngester := constructor()
glog.Infof("Process name: %s", dataset)
startProcess := NewIngestionProcess(git,
config.Common.TileDir,
dataset,
resultIngester,
ingesterConfig.ExtraParams,
ingesterConfig.RunEvery.Duration,
ingesterConfig.NCommits,
minDuration,
ingesterConfig.StatusDir,
ingesterConfig.MetricName)
startProcess()
}
select {}
}