// NewClientFromConfigAndTransport creates an new OAuth 2.0 authorized client // for the given config and transport. // // If the transport is nil then util.NewBackOffTransport() is used. // If local is true then a 3-legged flow is initiated, otherwise the GCE // Service Account is used. func NewClientFromConfigAndTransport(local bool, config *oauth2.Config, oauthCacheFile string, transport http.RoundTripper) (*http.Client, error) { if oauthCacheFile == "" { oauthCacheFile = "google_storage_token.data" } if transport == nil { transport = util.NewBackOffTransport() } var client *http.Client if local { tokenClient := &http.Client{ Transport: transport, Timeout: util.REQUEST_TIMEOUT, } ctx := context.WithValue(context.Background(), oauth2.HTTPClient, tokenClient) tokenSource, err := newCachingTokenSource(oauthCacheFile, ctx, config) if err != nil { return nil, fmt.Errorf("NewClientFromConfigAndTransport: Unable to create token source: %s", err) } client = &http.Client{ Transport: &oauth2.Transport{ Source: tokenSource, Base: transport, }, Timeout: util.REQUEST_TIMEOUT, } } else { // Use compute engine service account. client = GCEServiceAccountClient(transport) } return client, nil }
// Init initializes the module, the optional http.Client is used to make HTTP // requests to Google Storage. If nil is supplied then a default client is // used. func Init(cl *http.Client) { if cl != nil { client = cl } else { client = &http.Client{ Transport: util.NewBackOffTransport(), } } }
// IngestersFromConfig creates a list of ingesters from a config struct. // Usually the struct is created from parsing a config file. func IngestersFromConfig(config *sharedconfig.Config, client *http.Client) ([]*Ingester, error) { ret := []*Ingester{} // Set up the gitinfo object. var vcs vcsinfo.VCS var err error if vcs, err = gitinfo.CloneOrUpdate(config.GitRepoURL, config.GitRepoDir, true); err != nil { return nil, err } // Set up the Google storage client. if client == nil { client = &http.Client{ Transport: util.NewBackOffTransport(), } } // for each defined ingester create an instance. for id, ingesterConf := range config.Ingesters { processorConstructor, ok := constructors[id] if !ok { return nil, fmt.Errorf("Unknow ingester: '%s'", id) } // Instantiate the sources sources := make([]Source, 0, len(ingesterConf.Sources)) for _, dataSource := range ingesterConf.Sources { oneSource, err := getSource(id, dataSource, client) if err != nil { return nil, fmt.Errorf("Error instantiating sources for ingester '%s': %s", id, err) } sources = append(sources, oneSource) } // instantiate the processor processor, err := processorConstructor(vcs, ingesterConf) if err != nil { return nil, err } // create the ingester and add it to the result. ingester, err := NewIngester(id, ingesterConf, vcs, sources, processor) if err != nil { return nil, err } ret = append(ret, ingester) } return ret, nil }
// NewJWTServiceAccountClient creates a new http.Client that is loaded by first // attempting to load the JWT JSON Service Account data from GCE Project Level // metadata, and if that fails falls back to loading the data from a local // file. // // metadataname - The name of the GCE project level metadata key that holds the JWT JSON. If empty a default is used. // filename - The name of the local file that holds the JWT JSON. If empty a default is used. // transport - A transport. If nil then a default is used. func NewJWTServiceAccountClient(metadataname, filename string, transport http.RoundTripper, scopes ...string) (*http.Client, error) { if metadataname == "" { metadataname = metadata.JWT_SERVICE_ACCOUNT } if filename == "" { filename = DEFAULT_JWT_FILENAME } var body []byte jwt, err := metadata.ProjectGet(metadataname) if err != nil { body, err = ioutil.ReadFile(filename) if err != nil { return nil, fmt.Errorf("Couldn't find JWT via metadata or in a local file.") } } else { body = []byte(jwt) } if transport == nil { transport = util.NewBackOffTransport() } jwtConfig, err := google.JWTConfigFromJSON(body, scopes...) if err != nil { return nil, err } tokenClient := &http.Client{ Transport: transport, Timeout: util.REQUEST_TIMEOUT, } ctx := context.WithValue(context.Background(), oauth2.HTTPClient, tokenClient) return &http.Client{ Transport: &oauth2.Transport{ Source: jwtConfig.TokenSource(ctx), Base: transport, }, Timeout: util.REQUEST_TIMEOUT, }, nil }
func GetOAuthClient() (*http.Client, error) { config := auth.OAuthConfig(GSTokenPath, auth.SCOPE_FULL_CONTROL) return auth.RunFlowWithTransport(config, util.NewBackOffTransport()) }
func main() { defer common.LogPanic() // Setup DB flags. dbConf := db.DBConfigFromFlags() common.InitWithMetricsCB("ingest", func() string { common.DecodeTomlFile(*configFilename, &config) return config.Common.GraphiteServer }) // Initialize the database. We might not need the oauth dialog if it fails. if !config.Common.Local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } if err := dbConf.InitDB(); err != nil { glog.Fatal(err) } // Get a backoff transport. transport := util.NewBackOffTransport() // Determine the oauth scopes we are going to use. Storage is used by all // ingesters. scopes := []string{storage.CloudPlatformScope} for _, ingesterConfig := range config.Ingesters { if ingesterConfig.ConstructorName == gconfig.CONSTRUCTOR_ANDROID_GOLD { scopes = append(scopes, androidbuildinternal.AndroidbuildInternalScope) } } // Initialize the oauth client that is used to access all scopes. var client *http.Client var err error if config.Common.Local { if config.Common.DoOAuth { client, err = auth.InstalledAppClient(config.Common.OAuthCacheFile, config.Common.OAuthClientSecretFile, transport, scopes...) if err != nil { glog.Fatalf("Failed to auth: %s", err) } } else { client = nil // Add back service account access here when it's fixed. } } else { // Assume we are on a GCE instance. client = auth.GCEServiceAccountClient(transport) } // Initialize the ingester and gold ingester. ingester.Init(client) if _, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD]; ok { if err := goldingester.Init(client, filepath.Join(config.Ingesters["gold"].StatusDir, "android-build-info")); err != nil { glog.Fatalf("Unable to initialize GoldIngester: %s", err) } } // TODO(stephana): Cleanup the way trybots are instantiated so that each trybot has it's own // database connection and they are all instantiated the same way instead of the // one-off approaches. if ingesterConf, ok := config.Ingesters[gconfig.CONSTRUCTOR_GOLD_TRYBOT]; ok { dbConf := &database.DatabaseConfig{ Host: ingesterConf.DBHost, Port: ingesterConf.DBPort, User: database.USER_RW, Name: ingesterConf.DBName, MigrationSteps: golddb.MigrationSteps(), } if !config.Common.Local { if err := dbConf.GetPasswordFromMetadata(); err != nil { glog.Fatal(err) } } vdb, err := dbConf.NewVersionedDB() if err != nil { glog.Fatalf("Unable to open db connection: %s", err) } goldtrybot.Init(vdb) } git, err := gitinfo.NewGitInfo(config.Common.GitRepoDir, true, false) if err != nil { glog.Fatalf("Failed loading Git info: %s\n", err) } for dataset, ingesterConfig := range config.Ingesters { // Get duration equivalent to the number of days. minDuration := 24 * time.Hour * time.Duration(ingesterConfig.MinDays) constructorName := ingesterConfig.ConstructorName if constructorName == "" { constructorName = dataset } constructor := ingester.Constructor(constructorName) resultIngester := constructor() glog.Infof("Process name: %s", dataset) startProcess := NewIngestionProcess(git, config.Common.TileDir, dataset, resultIngester, ingesterConfig.ExtraParams, ingesterConfig.RunEvery.Duration, ingesterConfig.NCommits, minDuration, ingesterConfig.StatusDir, ingesterConfig.MetricName) startProcess() } select {} }