func NewDriverGCE(ui packer.Ui, projectId string, c *clientSecrets, key []byte) (Driver, error) {
log.Printf("[INFO] Requesting token...")
log.Printf("[INFO] -- Email: %s", c.Web.ClientEmail)
log.Printf("[INFO] -- Scopes: %s", DriverScopes)
log.Printf("[INFO] -- Private Key Length: %d", len(key))
log.Printf("[INFO] -- Token URL: %s", c.Web.TokenURI)
jwtTok := jwt.NewToken(c.Web.ClientEmail, DriverScopes, key)
jwtTok.ClaimSet.Aud = c.Web.TokenURI
token, err := jwtTok.Assert(new(http.Client))
if err != nil {
return nil, err
}
transport := &oauth.Transport{
Config: &oauth.Config{
ClientId: c.Web.ClientId,
Scope: DriverScopes,
TokenURL: c.Web.TokenURI,
AuthURL: c.Web.AuthURI,
},
Token: token,
}
log.Printf("[INFO] Instantiating client...")
service, err := compute.New(transport.Client())
if err != nil {
return nil, err
}
return &driverGCE{
projectId: projectId,
service: service,
ui: ui,
}, nil
}
func (uploader *Uploader) initGoogleService() error {
token := jwt.NewToken(
uploader.config.ClientEmail,
strings.Join(GOOGLE_SERVICE_SCOPE, " "),
[]byte(uploader.config.PrivateKey),
)
client := &http.Client{}
oauthToken, err := token.Assert(client)
if err != nil {
return err
}
transport := &oauth.Transport{
Token: oauthToken,
}
c := transport.Client()
_, err = oauth2.New(c)
if err != nil {
return err
}
driveService, err := drive.New(c)
if err != nil {
return err
}
uploader.driveService = driveService
return nil
}
func (s *Gcs) getOauthClient() *http.Client {
// Read the secret file bytes into the config.
secret_bytes, err := ioutil.ReadFile(s.SecretFile)
if err != nil {
log.Fatal("error reading secrets file: ", err)
return nil
}
var config struct {
Web struct {
ClientEmail string `json:"client_email"`
ClientID string `json:"client_id"`
TokenURI string `json:"token_uri"`
}
}
err = json.Unmarshal(secret_bytes, &config)
if err != nil {
log.Println(s.SecretFile)
log.Fatal("error unmarshalling secrets: ", err)
return nil
}
// Get the project ID from the client ID.
project_id := strings.SplitN(config.Web.ClientID, "-", 2)[0]
log.Println("project_id: ", project_id)
// Read the pem file bytes for the private key.
key_bytes, err := ioutil.ReadFile(s.PemFile)
if err != nil {
log.Fatal("error reading private key file: ", err)
return nil
}
// Craft the ClaimSet and JWT token.
scope := gcs.DevstorageFull_controlScope
token := jwt.NewToken(config.Web.ClientEmail, scope, key_bytes)
token.ClaimSet.Aud = config.Web.TokenURI
// We need to provide a client.
client := &http.Client{}
// Get the access token.
oauth_token, err := token.Assert(client)
if err != nil {
log.Fatal("assertion error: ", err)
return nil
}
// Refresh token will be missing, but this access_token will be good
// for one hour.
log.Printf("access_token = %v\n", oauth_token.AccessToken)
log.Printf("refresh_token = %v\n", oauth_token.RefreshToken)
log.Printf("expires %v\n", oauth_token.Expiry)
transport := &oauth.Transport{Token: oauth_token}
return transport.Client()
}
func NewShortener(issuer string, key []byte) (s *Shortener, err error) {
s = &Shortener{}
token := jwt.NewToken(issuer, urlshortener.UrlshortenerScope, key)
if s.transport, err = jwt.NewTransport(token); err != nil {
return
}
s.svc, err = urlshortener.New(s.transport.Client())
return
}
func GetServiceAccountToken(config *ServiceAccountConfig) (*oauth.Token, error) {
token := jwt.NewToken(config.ClientEmail, strings.Join(config.Scope, " "), []byte(config.PrivateKey))
client := &http.Client{}
oauthToken, err := token.Assert(client)
if err != nil {
return nil, err
}
return oauthToken, nil
}
func (c *Config) loadAndValidate() error {
var account accountFile
var secrets clientSecretsFile
if err := loadJSON(&account, c.AccountFile); err != nil {
return fmt.Errorf(
"Error loading account file '%s': %s",
c.AccountFile,
err)
}
if err := loadJSON(&secrets, c.ClientSecretsFile); err != nil {
return fmt.Errorf(
"Error loading client secrets file '%s': %s",
c.ClientSecretsFile,
err)
}
// Get the token for use in our requests
log.Printf("[INFO] Requesting Google token...")
log.Printf("[INFO] -- Email: %s", account.ClientEmail)
log.Printf("[INFO] -- Scopes: %s", clientScopes)
log.Printf("[INFO] -- Private Key Length: %d", len(account.PrivateKey))
log.Printf("[INFO] -- Token URL: %s", secrets.Web.TokenURI)
jwtTok := jwt.NewToken(
account.ClientEmail,
clientScopes,
[]byte(account.PrivateKey))
jwtTok.ClaimSet.Aud = secrets.Web.TokenURI
token, err := jwtTok.Assert(new(http.Client))
if err != nil {
return fmt.Errorf("Error retrieving auth token: %s", err)
}
// Instantiate the transport to communicate to Google
transport := &oauth.Transport{
Config: &oauth.Config{
ClientId: account.ClientId,
Scope: clientScopes,
TokenURL: secrets.Web.TokenURI,
AuthURL: secrets.Web.AuthURI,
},
Token: token,
}
log.Printf("[INFO] Instantiating GCE client...")
c.clientCompute, err = compute.New(transport.Client())
if err != nil {
return err
}
return nil
}
func prepareServiceClient(data *oauthConfig, keyBytes []byte) (
*http.Client, error) {
// Craft the ClaimSet and JWT token.
token := jwt.NewToken(data.Web.Client_Email, oauthScope, keyBytes)
token.ClaimSet.Aud = data.Web.Token_Uri
transport, err := jwt.NewTransport(token)
if err != nil {
log.Fatal("Assertion error:", err)
}
return transport.Client(), nil
}
func (c *Client) connect() (*bigquery.Service, error) {
if c.token != nil {
fmt.Println("token expired", c.token.Expired())
fmt.Println("token expiry", c.token.Expiry)
if !c.token.Expired() && c.service != nil {
fmt.Println("REUSE SERVICE")
return c.service, nil
}
}
// generate auth token and create service object
authScope := bigquery.BigqueryScope
pemKeyBytes, err := ioutil.ReadFile(c.pemPath)
if err != nil {
panic(err)
}
t := jwt.NewToken(c.accountEmailAddress, bigquery.BigqueryScope, pemKeyBytes)
httpClient := &http.Client{}
token, err := t.Assert(httpClient)
if err != nil {
return nil, err
}
c.token = token
config := &oauth.Config{
ClientId: c.userAccountClientId,
ClientSecret: c.clientSecret,
Scope: authScope,
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
}
transport := &oauth.Transport{
Token: token,
Config: config,
}
client := transport.Client()
service, err := bigquery.New(client)
if err != nil {
return nil, err
}
c.service = service
return service, nil
}
func NewGCSStorage(gcsClientEmail, gcsTokenURI string, gcsPemBytes []byte) (Storage, error) {
t := jwt.NewToken(gcsClientEmail, storage.DevstorageFull_controlScope, gcsPemBytes)
t.ClaimSet.Aud = gcsTokenURI
transport, err := jwt.NewTransport(t)
if err != nil {
return nil, err
}
client := transport.Client()
gcsService, err := storage.New(client)
if err != nil {
return nil, err
}
return &GoogleCloudStorage{gcsService}, nil
}
// Helper method to create an authenticated connection.
func connect() (*oauth.Token, *bigquery.Service, error) {
if *clientId == "" {
return nil, nil, fmt.Errorf("no client id specified")
}
if *serviceAccount == "" {
return nil, nil, fmt.Errorf("no service account specified")
}
if *projectId == "" {
return nil, nil, fmt.Errorf("no project id specified")
}
authScope := bigquery.BigqueryScope
if *pemFile == "" {
return nil, nil, fmt.Errorf("no credentials specified")
}
pemBytes, err := ioutil.ReadFile(*pemFile)
if err != nil {
return nil, nil, fmt.Errorf("could not access credential file %v - %v", pemFile, err)
}
t := jwt.NewToken(*serviceAccount, authScope, pemBytes)
token, err := t.Assert(&http.Client{})
if err != nil {
fmt.Printf("Invalid token: %v\n", err)
return nil, nil, err
}
config := &oauth.Config{
ClientId: *clientId,
ClientSecret: *clientSecret,
Scope: authScope,
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
}
transport := &oauth.Transport{
Token: token,
Config: config,
}
client := transport.Client()
service, err := bigquery.New(client)
if err != nil {
fmt.Printf("Failed to create new service: %v\n", err)
return nil, nil, err
}
return token, service, nil
}
// GetClient returns an OAuth2 authenticated http.Client that uses the identity
// of the server.
//
// A private key is required to assert the identity.
func (s *ServiceAccount) GetClient(scope string, r http.RoundTripper) (*http.Client, error) {
if r == nil {
r = http.DefaultTransport
}
tok := jwt.NewToken(s.EmailAddress, scope, []byte(s.PrivateKey))
// Get the access token right away by doing an HTTP request.
accessToken, err := tok.Assert(&http.Client{Transport: r})
if err != nil {
return nil, fmt.Errorf("failed to assert new oauth2 token: %s", err)
}
t := &transport{
token: tok,
accessToken: accessToken,
projectID: s.ProjectID,
transport: r,
}
return &http.Client{Transport: t}, nil
}
func (s *gcsStorage) init() error {
jwtToken := jwt.NewToken(os.Getenv(gcsIssEnvVar), gcs.DevstorageRead_writeScope, []byte(os.Getenv(gcsKeyEnvVar)))
oauthToken, err := jwtToken.Assert(http.DefaultClient)
if err != nil {
return err
}
client := (&jwt.Transport{jwtToken, oauthToken, http.DefaultTransport}).Client()
service, err := gcs.New(client)
if err != nil {
return err
}
s.client = client
s.service = service
s.bucket = os.Getenv(gcsBucketEnvVar)
return nil
}
func (c ApiClient) GetToken() *oauth.Token {
keyBytes, err := base64.StdEncoding.DecodeString(c.EncodedKey)
if err != nil {
log.Fatal("Error decoding private key:", err)
}
t := jwt.NewToken(c.ClientId, scope, keyBytes)
t.ClaimSet.Aud = tokenURL
log.Print("Requesting new access token.\n")
httpClient := &http.Client{}
token, err := t.Assert(httpClient)
if err != nil {
log.Fatal("assertion error:", err)
}
log.Printf("New access token acquired.\n")
return token
}
func NewDriverGCE(ui packer.Ui, p string, a *accountFile, c *clientSecretsFile) (Driver, error) {
// Get the token for use in our requests
log.Printf("[INFO] Requesting Google token...")
log.Printf("[INFO] -- Email: %s", a.ClientEmail)
log.Printf("[INFO] -- Scopes: %s", DriverScopes)
log.Printf("[INFO] -- Private Key Length: %d", len(a.PrivateKey))
log.Printf("[INFO] -- Token URL: %s", c.Web.TokenURI)
jwtTok := jwt.NewToken(
a.ClientEmail,
DriverScopes,
[]byte(a.PrivateKey))
jwtTok.ClaimSet.Aud = c.Web.TokenURI
token, err := jwtTok.Assert(new(http.Client))
if err != nil {
return nil, fmt.Errorf("Error retrieving auth token: %s", err)
}
// Instantiate the transport to communicate to Google
transport := &oauth.Transport{
Config: &oauth.Config{
ClientId: a.ClientId,
Scope: DriverScopes,
TokenURL: c.Web.TokenURI,
AuthURL: c.Web.AuthURI,
},
Token: token,
}
log.Printf("[INFO] Instantiating GCE client...")
service, err := compute.New(transport.Client())
if err != nil {
return nil, err
}
return &driverGCE{
projectId: p,
service: service,
ui: ui,
}, nil
}
func main() {
token := jwt.NewToken(Email, Scope, []byte(PEM))
transport, err := jwt.NewTransport(token)
if err != nil {
log.Fatal(err)
}
client := transport.Client()
bq, err := bigquery.New(client)
if err != nil {
log.Fatal(err)
}
rows := make([]*bigquery.TableDataInsertAllRequestRows, 0)
row := &bigquery.TableDataInsertAllRequestRows{
Json: make(map[string]bigquery.JsonValue, 0),
}
row.Json["url"] = "https://github.com"
row.Json["source"] = "example"
row.Json["t"] = time.Now().Unix()
row.Json["http_status"] = 200
rows = append(rows, row)
req := &bigquery.TableDataInsertAllRequest{
Rows: rows,
}
call := bq.Tabledata.InsertAll(App, Dataset, Table, req)
resp, err := call.Do()
if err != nil {
log.Fatal(err)
}
buf, _ := json.Marshal(resp)
log.Print(string(buf))
}
func main() {
goptions.ParseAndFail(&options)
defer options.KeyFile.Close()
pemBytes, err := ioutil.ReadAll(options.KeyFile)
if err != nil {
log.Fatalf("Could not read keyfile: %s", err)
}
token := jwt.NewToken(options.ClientId, storage.DevstorageRead_writeScope, pemBytes)
// token.ClaimSet.Aud = aud
c := &http.Client{}
oauthToken, err := token.Assert(c)
if err != nil {
log.Fatalf("Could not get OAuth token: %s", err)
}
c.Transport = &oauth.Transport{
Token: oauthToken,
}
service, err := storage.New(c)
if err != nil {
log.Fatalf("Could not use storage API: %s", err)
}
objs, err := service.Objects.List(options.Bucket).Do()
if err != nil {
log.Fatalf("Could not list content of bucket %s: %s", options.Bucket, err)
}
for _, obj := range objs.Items {
log.Printf("%s/%s: %s\n", options.Bucket, obj.Name, obj.SelfLink)
}
data := strings.NewReader("Some Data")
newObj, err := service.Objects.Insert(options.Bucket, &storage.Object{}).Name("gcs-test").Media(data).Do()
if err != nil {
log.Fatalf("Could not create new object: %s", err)
}
log.Printf("Uploaded to %s", newObj.SelfLink)
}
func (t *transport) Refresh() error {
// Before we can get a new oauth token, we have to produce a jwt token
//
// the oAuth token returned does not contain a refresh token and expires after one hour,
// which is the same expiration the JWT token provides. Therefore it does not make sense
// to keep the JWT token around, just rebuild it on every refresh
// Read the secret file bytes into the config.
secretFileBytes, err := ioutil.ReadFile(t.ClientSecretFileName)
if err != nil {
return fmt.Errorf("Error reading file %q: %v", t.ClientSecretFileName, err)
}
var config keyConfig
err = json.Unmarshal(secretFileBytes, &config)
if err != nil {
return fmt.Errorf("Failed to unmarshall json in %q: %v", t.ClientSecretFileName, err)
}
// Craft the ClaimSet and JWT token.
jwtToken := jwt.NewToken(config.ClientEmail, strings.Join(t.Scopes, " "), []byte(config.PrivateKey))
//jwtToken.ClaimSet.Aud = config.Web.TokenURI // just in case: assume that TokenURI from secret.json is more current than the default jwt package's
// assert the jwtToken to get the oAuth Token
client := urlfetch.Client(t.Context)
t.Token, err = jwtToken.Assert(client)
if err != nil {
return fmt.Errorf("Assert jwt token error: %v", err)
}
if t.TokenCache != nil {
// Cache the token and ignore error (as we can always get a new one).
t.TokenCache.PutToken(t.Token)
}
return nil
}
// CreateService creates a service with Server auth enabled system
func CreateService(gs *config.GoogleapiServiceAccount) (*drive.Service, error) {
// Settings for authorization.
var configG = &oauth.Config{
ClientId: gs.ClientId,
ClientSecret: gs.ClientSecret,
Scope: "https://www.googleapis.com/auth/drive",
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
AuthURL: "https://accounts.google.com/o/oauth2/auth",
TokenURL: "https://accounts.google.com/o/oauth2/token",
}
// Read the pem file bytes for the private key.
keyBytes, err := ioutil.ReadFile(gs.ServiceAccountKeyFile)
if err != nil {
return nil, err
}
// Craft the ClaimSet and JWT token.
t := jwt.NewToken(gs.ServiceAccountEmail, configG.Scope, keyBytes)
t.ClaimSet.Aud = configG.TokenURL
// Get the access token.
o, err := t.Assert(&http.Client{}) // We need to provide a client.
if err != nil {
return nil, err
}
tr := &oauth.Transport{
Config: configG,
Token: o,
Transport: http.DefaultTransport,
}
// Create a new authorized Drive client.
return drive.New(tr.Client())
}
func main() {
flag.Parse()
if *secretsFile == "" || *pemFile == "" {
flag.Usage()
fmt.Println(usageMsg)
return
}
// Read the secret file bytes into the config.
secretBytes, err := ioutil.ReadFile(*secretsFile)
if err != nil {
log.Fatal("error reading secerets file:", err)
}
var config struct {
Web struct {
ClientEmail string `json:"client_email"`
ClientID string `json:"client_id"`
TokenURI string `json:"token_uri"`
}
}
err = json.Unmarshal(secretBytes, &config)
if err != nil {
log.Fatal("error unmarshalling secerets:", err)
}
// Get the project ID from the client ID.
projectID := strings.SplitN(config.Web.ClientID, "-", 2)[0]
// Read the pem file bytes for the private key.
keyBytes, err := ioutil.ReadFile(*pemFile)
if err != nil {
log.Fatal("error reading private key file:", err)
}
// Craft the ClaimSet and JWT token.
t := jwt.NewToken(config.Web.ClientEmail, scope, keyBytes)
t.ClaimSet.Aud = config.Web.TokenURI
// We need to provide a client.
c := &http.Client{}
// Get the access token.
o, err := t.Assert(c)
if err != nil {
log.Fatal("assertion error:", err)
}
// Refresh token will be missing, but this access_token will be good
// for one hour.
fmt.Printf("access_token = %v\n", o.AccessToken)
fmt.Printf("refresh_token = %v\n", o.RefreshToken)
fmt.Printf("expires %v\n", o.Expiry)
// Form the request to list Google Cloud Storage buckets.
req, err := http.NewRequest("GET", "https://storage.googleapis.com/", nil)
if err != nil {
log.Fatal("http.NewRequest:", err)
}
req.Header.Set("Authorization", "OAuth "+o.AccessToken)
req.Header.Set("x-goog-api-version", "2")
req.Header.Set("x-goog-project-id", projectID)
// Make the request.
r, err := c.Do(req)
if err != nil {
log.Fatal("API request error:", err)
}
defer r.Body.Close()
// Write the response to standard output.
res, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Fatal("error reading API request results:", err)
}
fmt.Printf("\nRESULT:\n%s\n", res)
}
func updatePopularPackagesOnce() error {
const n = 25
c := http.DefaultClient
o, err := jwt.NewToken(
secrets.ServiceAccountSecrets.Web.ClientEmail,
"https://www.googleapis.com/auth/analytics.readonly",
secrets.serviceAccountPEMBytes).Assert(c)
if err != nil {
return err
}
q := url.Values{
"start-date": {time.Now().Add(-8 * 24 * time.Hour).Format("2006-01-02")},
"end-date": {time.Now().Format("2006-01-02")},
"ids": {"ga:58440332"},
"dimensions": {"ga:pagePath"},
"metrics": {"ga:visitors"},
"sort": {"-ga:visitors"},
"filters": {`ga:previousPagePath!=/;ga:pagePath=~^/[a-z][^.?]*\.[^?]+$`},
"max-results": {strconv.Itoa(n + 10)},
"access_token": {o.AccessToken},
}
resp, err := c.Get("https://www.googleapis.com/analytics/v3/data/ga?" + q.Encode())
if err != nil {
return err
}
defer resp.Body.Close()
var data struct {
Rows [][]string `json:"rows"`
}
if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
return err
}
paths := make([]string, len(data.Rows))
for i, r := range data.Rows {
if !strings.HasPrefix(r[0], "/") {
return errors.New("bad path value " + r[0])
}
paths[i] = r[0][1:]
}
pkgs, err := db.Packages(paths)
if err != nil {
return err
}
i := 0
prev := "-"
for _, pkg := range pkgs {
if strings.HasPrefix(pkg.Path, prev) {
continue
}
prev = pkg.Path + "/"
pkgs[i] = pkg
i += 1
if i >= n {
break
}
}
pkgs = pkgs[:i]
popularMutex.Lock()
popularPackages = pkgs
popularMutex.Unlock()
return nil
}
