func MakeClient(token string) *github.Client {
var client *http.Client
var transport http.RoundTripper
if *useMemoryCache {
transport = httpcache.NewMemoryCacheTransport()
} else {
transport = http.DefaultTransport
}
if len(token) > 0 {
rateLimitTransport := &RateLimitRoundTripper{
delegate: transport,
// Global limit is 5000 Q/Hour, try to only use 1800 to make room for other apps
throttle: util.NewTokenBucketRateLimiter(0.5, 10),
}
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
client = &http.Client{
Transport: &oauth2.Transport{
Base: rateLimitTransport,
Source: oauth2.ReuseTokenSource(nil, ts),
},
}
} else {
rateLimitTransport := &RateLimitRoundTripper{
delegate: transport,
throttle: util.NewTokenBucketRateLimiter(0.01, 10),
}
client = &http.Client{
Transport: rateLimitTransport,
}
}
return github.NewClient(client)
}
// New returns a new Amazon Cloud Drive "acd" Client. configFile must exist and must be a valid JSON decodable into Config.
func New(configFile string) (*Client, error) {
config, err := loadConfig(configFile)
if err != nil {
return nil, err
}
ts, err := token.New(config.TokenFile)
if err != nil {
return nil, err
}
c := &Client{
config: config,
cacheFile: config.CacheFile,
httpClient: &http.Client{
Timeout: config.Timeout,
Transport: &oauth2.Transport{
Source: oauth2.ReuseTokenSource(nil, ts),
},
},
}
if err := setEndpoints(c); err != nil {
return nil, err
}
return c, nil
}
// NewRefreshTokenSource returns a token source that obtains its initial token
// based on the provided config and the refresh token.
func NewRefreshTokenSource(config *oauth2.Config, refreshToken string) oauth2.TokenSource {
var noInitialToken *oauth2.Token = nil
return oauth2.ReuseTokenSource(noInitialToken, config.TokenSource(
oauth2.NoContext, // TODO: maybe accept a context later.
&oauth2.Token{RefreshToken: refreshToken},
))
}
// TokenSource returns a TokenSource that returns t until t expires,
// automatically refreshing it as necessary using the provided context and the
// client ID and client secret.
//
// Most users will use Config.Client instead.
func (c *Config) TokenSource(ctx context.Context) oauth2.TokenSource {
source := &tokenSource{
ctx: ctx,
conf: c,
}
return oauth2.ReuseTokenSource(nil, source)
}
// TokenSource returns a ReuseTokenSource.
func (c *Config) TokenSource(ctx context.Context, t *oauth2.Token) oauth2.TokenSource {
tts := &TokenStorageSource{
source: c.Config.TokenSource(ctx, t),
config: c,
}
return oauth2.ReuseTokenSource(t, tts)
}
// tokenSource returns a reusable oauth2.TokenSource.
// When expired, a new token will be obtained using cred.RefreshToken
// and stored in a persistent db.
// The returned TokenSource valid only within provided context c.
func (cred *oauth2Credentials) tokenSource(c context.Context) oauth2.TokenSource {
t := &oauth2.Token{
AccessToken: cred.AccessToken,
Expiry: cred.Expiry,
}
return oauth2.ReuseTokenSource(t, &tokenRefresher{c, cred})
}
// PreExecute will initialize the Config. It MUST be run before the config
// may be used to get information from Github
func (config *Config) PreExecute() error {
if len(config.Org) == 0 {
glog.Fatalf("--organization is required.")
}
if len(config.Project) == 0 {
glog.Fatalf("--project is required.")
}
token := config.Token
if len(token) == 0 && len(config.TokenFile) != 0 {
data, err := ioutil.ReadFile(config.TokenFile)
if err != nil {
glog.Fatalf("error reading token file: %v", err)
}
token = strings.TrimSpace(string(data))
}
// We need to get our Transport/RoundTripper in order based on arguments
// oauth2 Transport // if we have an auth token
// zeroCacheRoundTripper // if we are using the cache want faster timeouts
// webCacheRoundTripper // if we are using the cache
// callLimitRoundTripper ** always
// [http.DefaultTransport] ** always implicit
var transport http.RoundTripper
callLimitTransport := &callLimitRoundTripper{
remaining: tokenLimit + 500, // put in 500 so we at least have a couple to check our real limits
resetTime: time.Now().Add(1 * time.Minute),
}
config.apiLimit = callLimitTransport
transport = callLimitTransport
if config.useMemoryCache {
t := httpcache.NewMemoryCacheTransport()
t.Transport = transport
zeroCacheTransport := &zeroCacheRoundTripper{
delegate: t,
}
transport = zeroCacheTransport
}
if len(token) > 0 {
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
transport = &oauth2.Transport{
Base: transport,
Source: oauth2.ReuseTokenSource(nil, ts),
}
}
client := &http.Client{
Transport: transport,
}
config.client = github.NewClient(client)
config.ResetAPICount()
return nil
}
func newAltTokenSource(tokenURL string) oauth2.TokenSource {
client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
a := &altTokenSource{
oauthClient: client,
tokenURL: tokenURL,
}
return oauth2.ReuseTokenSource(nil, a)
}
func newOauthClient(authTokenPath string) (*http.Client, error) {
token, err := oauth2.ReuseTokenSource(nil, gobSource{path: authTokenPath, base: browserSource{}}).Token()
if err != nil {
log.Infof("problem exchanging code: %s", err)
return nil, err
}
return oauth2Config.Client(context.Background(), token), nil
}
// NewClientFrom returns an http client which will use the provided func
// as a source for oauth tokens. The underlying transport handles automatic
// retries and logging that is useful for integration tests and agents.
func NewClientFrom(src oauth2.TokenSource) *http.Client {
// Wrapping in a ReuseTokenSource will cache the returned token so that src
// is only called when a new token is needed.
return newAuthenticatedClient(
oauth2.ReuseTokenSource(nil, src),
transport(),
)
}
// GetTokenSource builds a new oauth2.TokenSource that uses the ttnctl config to store the token
func GetTokenSource(ctx log.Interface) oauth2.TokenSource {
if tokenSource != nil {
return tokenSource
}
token := getStoredToken(ctx)
source := oauth2.ReuseTokenSource(token, getAccountServerTokenSource(token))
tokenSource = &ttnctlTokenSource{ctx, source}
return tokenSource
}
func newAltTokenSource(tokenURL string) oauth2.TokenSource {
client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
a := &altTokenSource{
oauthClient: client,
tokenURL: tokenURL,
throttle: util.NewTokenBucketRateLimiter(tokenURLQPS, tokenURLBurst),
}
return oauth2.ReuseTokenSource(nil, a)
}
func (c *googinitCmd) RunCommand(args []string) error {
var (
err error
clientId string
clientSecret string
oauthConfig *oauth2.Config
)
if c.storageType != "drive" && c.storageType != "cloud" {
return cmdmain.UsageError("Invalid storage type: must be drive for Google Drive or cloud for Google Cloud Storage.")
}
clientId, clientSecret = getClientInfo()
switch c.storageType {
case "drive":
oauthConfig = &oauth2.Config{
Scopes: []string{drive.Scope},
Endpoint: google.Endpoint,
ClientID: clientId,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
case "cloud":
oauthConfig = &oauth2.Config{
Scopes: []string{storage.ScopeReadWrite},
Endpoint: google.Endpoint,
ClientID: clientId,
ClientSecret: clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
}
token, err := oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{
Config: oauthConfig,
AuthCode: func() string {
fmt.Fprintf(cmdmain.Stdout, "Get auth code from:\n\n")
fmt.Fprintf(cmdmain.Stdout, "%v\n\n", oauthConfig.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
return prompt("Enter auth code:")
},
}).Token()
if err != nil {
return fmt.Errorf("could not acquire token: %v", err)
}
fmt.Fprintf(cmdmain.Stdout, "\nYour Google auth object:\n\n")
enc := json.NewEncoder(cmdmain.Stdout)
authObj := map[string]string{
"client_id": clientId,
"client_secret": clientSecret,
"refresh_token": token.RefreshToken,
}
enc.Encode(authObj)
fmt.Fprint(cmdmain.Stdout, "\n\nFor server-config.json, your 'googlecloudstorage' value (update with your bucket name and path):\n\n")
fmt.Fprintf(cmdmain.Stdout, "%s:%s:%s:bucketName[/optional/dir]\n", clientId, clientSecret, token.RefreshToken)
return nil
}
// ForceRefreshToken forces a refresh of the access token
func ForceRefreshToken(ctx log.Interface) {
tokenSource := GetTokenSource(ctx).(*ttnctlTokenSource)
token, err := tokenSource.Token()
if err != nil {
ctx.WithError(err).Fatal("Could not get access token")
}
token.Expiry = time.Now().Add(-1 * time.Second)
tokenSource.source = oauth2.ReuseTokenSource(token, getAccountServerTokenSource(token))
tokenSource.Token()
}
func tokenSource() oauth2.TokenSource {
var tokensource oauth2.TokenSource
tokenSource, err := google.DefaultTokenSource(oauth2.NoContext)
if err == nil {
return tokenSource
}
oauthConfig := &oauth2.Config{
// The client-id and secret should be for an "Installed Application" when using
// the CLI. Later we'll use a web application with a callback.
ClientID: readFile(stagingPrefix() + "client-id.dat"),
ClientSecret: readFile(stagingPrefix() + "client-secret.dat"),
Endpoint: google.Endpoint,
Scopes: []string{
compute.DevstorageFullControlScope,
compute.ComputeScope,
compute.CloudPlatformScope,
"https://www.googleapis.com/auth/sqlservice",
"https://www.googleapis.com/auth/sqlservice.admin",
},
RedirectURL: "urn:ietf:wg:oauth:2.0:oob",
}
tokenFileName := stagingPrefix() + "token.dat"
tokenFile := tokenCacheFile(tokenFileName)
tokenSource = oauth2.ReuseTokenSource(nil, tokenFile)
token, err := tokenSource.Token()
if err != nil {
log.Printf("Error getting token from %s: %v", tokenFileName, err)
log.Printf("Get auth code from %v", oauthConfig.AuthCodeURL("my-state"))
fmt.Print("\nEnter auth code: ")
sc := bufio.NewScanner(os.Stdin)
sc.Scan()
authCode := strings.TrimSpace(sc.Text())
token, err = oauthConfig.Exchange(oauth2.NoContext, authCode)
if err != nil {
log.Fatalf("Error exchanging auth code for a token: %v", err)
}
if err := tokenFile.WriteToken(token); err != nil {
log.Fatalf("Error writing to %s: %v", tokenFileName, err)
}
tokenSource = oauth2.ReuseTokenSource(token, nil)
}
return tokensource
}
// PreExecute will initialize the GithubConfig. It MUST be run before the config
// may be used to get information from Github
func (config *GithubConfig) PreExecute() error {
if len(config.Org) == 0 {
glog.Fatalf("--organization is required.")
}
if len(config.Project) == 0 {
glog.Fatalf("--project is required.")
}
token := config.Token
if len(token) == 0 && len(config.TokenFile) != 0 {
data, err := ioutil.ReadFile(config.TokenFile)
if err != nil {
glog.Fatalf("error reading token file: %v", err)
}
token = string(data)
}
transport := http.DefaultTransport
if config.useMemoryCache {
transport = httpcache.NewMemoryCacheTransport()
}
// convert from queries per hour to queries per second
config.RateLimit = config.RateLimit / 3600
// ignore the configured rate limit if you don't have a token.
// only get 60 requests per hour!
if len(token) == 0 {
glog.Warningf("Ignoring --rate-limit because no token data available")
config.RateLimit = 0.01
config.RateLimitBurst = 10
}
rateLimitTransport := &rateLimitRoundTripper{
delegate: transport,
throttle: util.NewTokenBucketRateLimiter(config.RateLimit, config.RateLimitBurst),
}
client := &http.Client{
Transport: rateLimitTransport,
}
if len(token) > 0 {
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
client = &http.Client{
Transport: &oauth2.Transport{
Base: rateLimitTransport,
Source: oauth2.ReuseTokenSource(nil, ts),
},
}
}
config.client = github.NewClient(client)
config.analytics.lastAPIReset = time.Now()
return nil
}
func (c *gceCmd) RunCommand(args []string) error {
if c.verbose {
gce.Verbose = true
}
if c.project == "" {
return cmdmain.UsageError("Missing --project flag.")
}
if (c.certFile == "") != (c.keyFile == "") {
return cmdmain.UsageError("--cert and --key must both be given together.")
}
if c.certFile == "" && c.hostname == "" {
return cmdmain.UsageError("Either --hostname, or --cert & --key must provided.")
}
config := gce.NewOAuthConfig(readFile(clientIdDat), readFile(clientSecretDat))
config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
instConf := &gce.InstanceConf{
Name: c.instName,
Project: c.project,
Machine: c.machine,
Zone: c.zone,
CertFile: c.certFile,
KeyFile: c.keyFile,
Hostname: c.hostname,
}
if c.sshPub != "" {
instConf.SSHPub = strings.TrimSpace(readFile(c.sshPub))
}
depl := &gce.Deployer{
Client: oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{
Config: config,
CacheFile: c.project + "-token.json",
AuthCode: func() string {
fmt.Println("Get auth code from:")
fmt.Printf("%v\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
fmt.Println("Enter auth code:")
sc := bufio.NewScanner(os.Stdin)
sc.Scan()
return strings.TrimSpace(sc.Text())
},
})),
Conf: instConf,
}
inst, err := depl.Create(context.TODO())
if err != nil {
return err
}
log.Printf("Instance is up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP)
return nil
}
// sourceForToken returns a TokenSource based on the following rules:
// - if token is not "" it is used as the bearer token for the source
// - if the binary is running on GCE, tokens will be retrieved from the metadata server
// - otherwise, a source is returned which returns an error each time it is used.
func sourceForToken(token string) oauth2.TokenSource {
if token != "" {
return oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
}
tok, err := gceToken()
if err == nil {
return oauth2.ReuseTokenSource(tok, tokenFunc(gceToken))
}
log.Printf("Can't find GCE metadata server: %v", err)
return tokenFunc(func() (*oauth2.Token, error) {
return nil, errors.New("must provide a token on command line or run on GCE")
})
}
func (config *GithubConfig) PreExecute() error {
if len(config.Org) == 0 {
glog.Fatalf("--organization is required.")
}
if len(config.Project) == 0 {
glog.Fatalf("--project is required.")
}
token := config.Token
if len(token) == 0 && len(config.TokenFile) != 0 {
data, err := ioutil.ReadFile(config.TokenFile)
if err != nil {
glog.Fatalf("error reading token file: %v", err)
}
token = string(data)
}
var client *http.Client
var transport http.RoundTripper
if config.useMemoryCache {
transport = httpcache.NewMemoryCacheTransport()
} else {
transport = http.DefaultTransport
}
if len(token) > 0 {
rateLimitTransport := &RateLimitRoundTripper{
delegate: transport,
// Global limit is 5000 Q/Hour, try to only use 1800 to make room for other apps
throttle: util.NewTokenBucketRateLimiter(0.5, 10),
}
ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token})
client = &http.Client{
Transport: &oauth2.Transport{
Base: rateLimitTransport,
Source: oauth2.ReuseTokenSource(nil, ts),
},
}
} else {
rateLimitTransport := &RateLimitRoundTripper{
delegate: transport,
throttle: util.NewTokenBucketRateLimiter(0.01, 10),
}
client = &http.Client{
Transport: rateLimitTransport,
}
}
config.client = github.NewClient(client)
return nil
}
func RefreshFacebookTokens(c *gin.Context) {
if auth.IsAuthenticated(c) {
user := auth.GetUser(c)
if user.LoginType == "facebook" {
aecontext := appengine.NewContext(c.Request)
conf := FBConfig()
toksource := conf.TokenSource(aecontext, &user.Token)
sourceToken := oauth2.ReuseTokenSource(&user.Token, toksource)
client := oauth2.NewClient(aecontext, sourceToken)
client.Get("...")
t, _ := sourceToken.Token()
user.Token = *t
auth.UpdateUser(c, &user)
log.Debugf(aecontext, fmt.Sprintf("refresh tokens %#v", t))
}
}
}
// SetFromGcloud retrieves and sets any missing config values from the gcloud
// configuration if possible possible
func (c *Config) SetFromGcloud() error {
if c.Creds == "" {
c.Creds = os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
if c.Creds == "" {
log.Printf("-creds flag unset, will use gcloud credential")
}
} else {
os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", c.Creds)
}
if c.Project == "" {
log.Printf("-project flag unset, will use gcloud active project")
}
if c.Creds != "" && c.Project != "" {
return nil
}
gcloudCmd := "gcloud"
if runtime.GOOS == "windows" {
gcloudCmd = gcloudCmd + ".cmd"
}
gcloudCmdArgs := []string{"config", "config-helper",
"--format=json(configuration.properties.core.project,credential)"}
gcloudConfig, err := LoadGcloudConfig(gcloudCmd, gcloudCmdArgs)
if err != nil {
return err
}
if c.Project == "" && gcloudConfig.Configuration.Properties.Core.Project != "" {
log.Printf("gcloud active project is \"%s\"",
gcloudConfig.Configuration.Properties.Core.Project)
c.Project = gcloudConfig.Configuration.Properties.Core.Project
}
if c.Creds == "" {
c.TokenSource = oauth2.ReuseTokenSource(
gcloudConfig.Credential.Token(),
&GcloudCmdTokenSource{Command: gcloudCmd, Args: gcloudCmdArgs})
}
return nil
}
// tokenSource creates a new oauth2.TokenSource backed by tokenRefresher,
// using previously stored user credentials if available.
func tokenSource(provider string) (oauth2.TokenSource, error) {
conf := authConfig[provider]
if conf == nil {
return nil, fmt.Errorf("no auth config for %q", provider)
}
t, err := readToken(provider)
if err != nil {
t, err = authorize(conf)
}
if err != nil {
return nil, fmt.Errorf("unable to obtain access token for %q", provider)
}
cache := &cachedTokenSource{
src: conf.TokenSource(context.Background(), t),
provider: provider,
config: conf,
}
return oauth2.ReuseTokenSource(nil, cache), nil
}
// JWTAccessTokenSourceFromJSON uses a Google Developers service account JSON
// key file to read the credentials that authorize and authenticate the
// requests, and returns a TokenSource that does not use any OAuth2 flow but
// instead creates a JWT and sends that as the access token.
// The audience is typically a URL that specifies the scope of the credentials.
func JWTAccessTokenSourceFromJSON(jsonKey []byte, audience string) (oauth2.TokenSource, error) {
cfg, err := JWTConfigFromJSON(jsonKey)
if err != nil {
return nil, fmt.Errorf("google: could not parse JSON key: %v", err)
}
pk, err := internal.ParseKey(cfg.PrivateKey)
if err != nil {
return nil, fmt.Errorf("google: could not parse key: %v", err)
}
ts := &jwtAccessTokenSource{
email: cfg.Email,
audience: audience,
pk: pk,
}
tok, err := ts.Token()
if err != nil {
return nil, err
}
return oauth2.ReuseTokenSource(tok, ts), nil
}
func getEvents(count int64) []*calendar.Event {
sourceToken := oauth2.ReuseTokenSource(nil, &fileTokenSource{})
t, _ := sourceToken.Token()
client := clientConfig.Client(oauth2.NoContext, t)
svc, err := calendar.New(client)
if err != nil {
fmt.Println(err)
}
//c, err := svc.CalendarList.List().Do()
startTime := time.Now().Format(time.RFC3339)
c, err := svc.Events.List(*calendarId).SingleEvents(true).OrderBy("startTime").TimeMin(startTime).MaxResults(count).Do()
//c, err := svc.Events.List(*calendarId).Do()
if err != nil {
fmt.Println(err)
return nil
}
return c.Items
}
// GetClient returns the generated Client.
func GetClient(scopes ...string) *http.Client {
b, err := ioutil.ReadFile("client_secret.json")
if err != nil {
log.Fatalf("Unable to read client secret file: %v", err)
}
config, err := google.ConfigFromJSON(b, scopes...)
if err != nil {
log.Fatalf("Unable to parse client secret file to config: %v", err)
}
tok, err := tokenFromFile(tokenFile)
if err != nil {
tok = getTokenFromWeb(config)
saveToken(tokenFile, tok)
}
ctx := context.Background()
//Very hacky, but somehow works
ts := oauth2.ReuseTokenSource(nil, config.TokenSource(ctx, tok))
t, _ := ts.Token()
return config.Client(ctx, t)
}
// TokenSource returns a JWT TokenSource using the configuration
// in c and the HTTP client from the provided context.
func (c *Config) TokenSource(ctx context.Context) oauth2.TokenSource {
return oauth2.ReuseTokenSource(nil, jwtSource{ctx, c})
}
func testStorage(t *testing.T, bucketDir string) {
if *bucket == "" && *configFile == "" {
t.Skip("Skipping test without --bucket or --config flag")
}
var refreshToken string
if *configFile != "" {
data, err := ioutil.ReadFile(*configFile)
if err != nil {
t.Fatalf("Error reading config file %v: %v", *configFile, err)
}
var conf Config
if err := json.Unmarshal(data, &conf); err != nil {
t.Fatalf("Error decoding config file %v: %v", *configFile, err)
}
*clientID = conf.Auth.ClientID
*clientSecret = conf.Auth.ClientSecret
refreshToken = conf.Auth.RefreshToken
*bucket = conf.Bucket
}
if *bucket == "" {
t.Fatal("bucket not provided in config file or as a flag.")
}
if *clientID == "" {
if !metadata.OnGCE() {
if *clientSecret == "" {
t.Fatal("client ID and client secret required. Obtain from https://console.developers.google.com/ > Project > APIs & Auth > Credentials. Should be a 'native' or 'Installed application'")
}
} else {
*clientID = "auto"
}
}
if *configFile == "" {
config := &oauth2.Config{
Scopes: []string{storage.ScopeReadWrite},
Endpoint: google.Endpoint,
ClientID: *clientID,
ClientSecret: *clientSecret,
RedirectURL: oauthutil.TitleBarRedirectURL,
}
if !metadata.OnGCE() {
token, err := oauth2.ReuseTokenSource(nil,
&oauthutil.TokenSource{
Config: config,
CacheFile: *tokenCache,
AuthCode: func() string {
if *authCode == "" {
t.Skipf("Re-run using --auth_code= with the value obtained from %s",
config.AuthCodeURL("", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
return ""
}
return *authCode
},
}).Token()
if err != nil {
t.Fatalf("could not acquire token: %v", err)
}
refreshToken = token.RefreshToken
}
}
bucketWithDir := path.Join(*bucket, bucketDir)
storagetest.TestOpt(t, storagetest.Opts{
New: func(t *testing.T) (sto blobserver.Storage, cleanup func()) {
sto, err := newFromConfig(nil, jsonconfig.Obj{
"bucket": bucketWithDir,
"auth": map[string]interface{}{
"client_id": *clientID,
"client_secret": *clientSecret,
"refresh_token": refreshToken,
},
})
if err != nil {
t.Fatal(err)
}
if !testing.Short() {
log.Printf("Warning: this test does many serial operations. Without the go test -short flag, this test will be very slow.")
}
// Bail if bucket is not empty
ctx := context.Background()
stor := sto.(*Storage)
objs, err := stor.client.Bucket(stor.bucket).List(ctx, nil)
if err != nil {
t.Fatalf("Error checking if bucket is empty: %v", err)
}
if len(objs.Results) != 0 {
t.Fatalf("Refusing to run test: bucket %v is not empty", *bucket)
}
if bucketWithDir != *bucket {
// Adding "a", and "c" objects in the bucket to make sure objects out of the
// "directory" are not touched and have no influence.
for _, key := range []string{"a", "c"} {
w := stor.client.Bucket(stor.bucket).Object(key).NewWriter(ctx)
if _, err := io.Copy(w, strings.NewReader(key)); err != nil {
t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
}
if err := w.Close(); err != nil {
t.Fatalf("could not insert object %s in bucket %v: %v", key, sto.(*Storage).bucket, err)
}
}
}
clearBucket := func(beforeTests bool) func() {
return func() {
var all []blob.Ref
blobserver.EnumerateAll(context.TODO(), sto, func(sb blob.SizedRef) error {
t.Logf("Deleting: %v", sb.Ref)
all = append(all, sb.Ref)
return nil
})
if err := sto.RemoveBlobs(all); err != nil {
t.Fatalf("Error removing blobs during cleanup: %v", err)
}
if beforeTests {
return
}
if bucketWithDir != *bucket {
// checking that "a" and "c" at the root were left untouched.
for _, key := range []string{"a", "c"} {
rc, err := stor.client.Bucket(stor.bucket).Object(key).NewReader(ctx)
if err != nil {
t.Fatalf("could not find object %s after tests: %v", key, err)
}
if _, err := io.Copy(ioutil.Discard, rc); err != nil {
t.Fatalf("could not find object %s after tests: %v", key, err)
}
if err := stor.client.Bucket(stor.bucket).Object(key).Delete(ctx); err != nil {
t.Fatalf("could not remove object %s after tests: %v", key, err)
}
}
}
}
}
clearBucket(true)()
return sto, clearBucket(false)
},
})
}
func (c *gceCmd) RunCommand(args []string) error {
if c.verbose {
gce.Verbose = true
}
if c.project == "" {
return cmdmain.UsageError("Missing --project flag.")
}
if (c.certFile == "") != (c.keyFile == "") {
return cmdmain.UsageError("--cert and --key must both be given together.")
}
if c.certFile == "" && c.hostname == "" {
return cmdmain.UsageError("Either --hostname, or --cert & --key must provided.")
}
// We embed the client ID and client secret, per
// https://developers.google.com/identity/protocols/OAuth2InstalledApp
// Notably: "The client ID and client secret obtained from the
// Developers Console are embedded in the source code of your
// application. In this context, the client secret is
// obviously not treated as a secret."
//
// These were created at:
// https://console.developers.google.com/apis/credentials?project=camlistore-website
// (Notes for Brad and Mathieu)
const (
clientID = "574004351801-9qqoggh6b5v3jqt722v43ikmgmtv60h3.apps.googleusercontent.com"
clientSecret = "Gf1zwaOcbJnRTE5zD4feKaTI" // NOT a secret, despite name
)
config := gce.NewOAuthConfig(clientID, clientSecret)
config.RedirectURL = "urn:ietf:wg:oauth:2.0:oob"
hc := oauth2.NewClient(oauth2.NoContext, oauth2.ReuseTokenSource(nil, &oauthutil.TokenSource{
Config: config,
CacheFile: c.project + "-token.json",
AuthCode: func() string {
fmt.Println("Get auth code from:")
fmt.Printf("%v\n\n", config.AuthCodeURL("my-state", oauth2.AccessTypeOffline, oauth2.ApprovalForce))
fmt.Print("Enter auth code: ")
sc := bufio.NewScanner(os.Stdin)
sc.Scan()
return strings.TrimSpace(sc.Text())
},
}))
zone := c.zone
if gce.LooksLikeRegion(zone) {
region := zone
zones, err := gce.ZonesOfRegion(hc, c.project, region)
if err != nil {
return err
}
if len(zones) == 0 {
return fmt.Errorf("no zones found in region %q; invalid region?", region)
}
zone = zones[rand.Intn(len(zones))]
}
instConf := &gce.InstanceConf{
Name: c.instName,
Project: c.project,
Machine: c.machine,
Zone: zone,
CertFile: c.certFile,
KeyFile: c.keyFile,
Hostname: c.hostname,
}
if c.sshPub != "" {
instConf.SSHPub = strings.TrimSpace(readFile(c.sshPub))
}
log.Printf("Creating instance %s (in project %s) in zone %s ...", c.instName, c.project, zone)
depl := &gce.Deployer{
Client: hc,
Conf: instConf,
}
inst, err := depl.Create(context.Background())
if err != nil {
return err
}
log.Printf("Instance created; starting up at %s", inst.NetworkInterfaces[0].AccessConfigs[0].NatIP)
return nil
}
// ComputeTokenSource returns a token source that fetches access tokens
// from Google Compute Engine (GCE)'s metadata server. It's only valid to use
// this token source if your program is running on a GCE instance.
// If no account is specified, "default" is used.
// Further information about retrieving access tokens from the GCE metadata
// server can be found at https://cloud.google.com/compute/docs/authentication.
func ComputeTokenSource(account string) oauth2.TokenSource {
return oauth2.ReuseTokenSource(nil, computeSource{account: account})
}
func main() {
flag.Parse()
if *proj == "" {
log.Fatalf("Missing --project flag")
}
prefix := "https://www.googleapis.com/compute/v1/projects/" + *proj
machType := prefix + "/zones/" + *zone + "/machineTypes/" + *mach
const tokenFileName = "token.dat"
tokenFile := tokenCacheFile(tokenFileName)
tokenSource := oauth2.ReuseTokenSource(nil, tokenFile)
token, err := tokenSource.Token()
if err != nil {
if *writeObject != "" {
log.Fatalf("Can't use --write_object without a valid token.dat file already cached.")
}
log.Printf("Error getting token from %s: %v", tokenFileName, err)
log.Printf("Get auth code from %v", config.AuthCodeURL("my-state"))
fmt.Print("\nEnter auth code: ")
sc := bufio.NewScanner(os.Stdin)
sc.Scan()
authCode := strings.TrimSpace(sc.Text())
token, err = config.Exchange(oauth2.NoContext, authCode)
if err != nil {
log.Fatalf("Error exchanging auth code for a token: %v", err)
}
if err := tokenFile.WriteToken(token); err != nil {
log.Fatalf("Error writing to %s: %v", tokenFileName, err)
}
tokenSource = oauth2.ReuseTokenSource(token, nil)
}
oauthClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
if *writeObject != "" {
writeCloudStorageObject(oauthClient)
return
}
computeService, _ := compute.New(oauthClient)
natIP := *staticIP
if natIP == "" {
// Try to find it by name.
aggAddrList, err := computeService.Addresses.AggregatedList(*proj).Do()
if err != nil {
log.Fatal(err)
}
// http://godoc.org/code.google.com/p/google-api-go-client/compute/v1#AddressAggregatedList
IPLoop:
for _, asl := range aggAddrList.Items {
for _, addr := range asl.Addresses {
if addr.Name == *instName+"-ip" && addr.Status == "RESERVED" {
natIP = addr.Address
break IPLoop
}
}
}
}
cloudConfig := baseConfig
if *sshPub != "" {
key := strings.TrimSpace(readFile(*sshPub))
cloudConfig += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", key)
}
if os.Getenv("USER") == "bradfitz" {
cloudConfig += fmt.Sprintf("\nssh_authorized_keys:\n - %s\n", "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAIEAwks9dwWKlRC+73gRbvYtVg0vdCwDSuIlyt4z6xa/YU/jTDynM4R4W10hm2tPjy8iR1k8XhDv4/qdxe6m07NjG/By1tkmGpm1mGwho4Pr5kbAAy/Qg+NLCSdAYnnE00FQEcFOC15GFVMOW2AzDGKisReohwH9eIzHPzdYQNPRWXE= [email protected]")
}
const maxCloudConfig = 32 << 10 // per compute API docs
if len(cloudConfig) > maxCloudConfig {
log.Fatalf("cloud config length of %d bytes is over %d byte limit", len(cloudConfig), maxCloudConfig)
}
instance := &compute.Instance{
Name: *instName,
Description: "Go Builder",
MachineType: machType,
Disks: []*compute.AttachedDisk{instanceDisk(computeService)},
Tags: &compute.Tags{
Items: []string{"http-server", "https-server"},
},
Metadata: &compute.Metadata{
Items: []*compute.MetadataItems{
{
Key: "user-data",
Value: cloudConfig,
},
},
},
NetworkInterfaces: []*compute.NetworkInterface{
&compute.NetworkInterface{
AccessConfigs: []*compute.AccessConfig{
&compute.AccessConfig{
Type: "ONE_TO_ONE_NAT",
Name: "External NAT",
NatIP: natIP,
},
},
Network: prefix + "/global/networks/default",
},
},
ServiceAccounts: []*compute.ServiceAccount{
{
Email: "default",
Scopes: []string{
compute.DevstorageFull_controlScope,
compute.ComputeScope,
},
},
},
}
log.Printf("Creating instance...")
op, err := computeService.Instances.Insert(*proj, *zone, instance).Do()
if err != nil {
log.Fatalf("Failed to create instance: %v", err)
}
opName := op.Name
log.Printf("Created. Waiting on operation %v", opName)
OpLoop:
for {
time.Sleep(2 * time.Second)
op, err := computeService.ZoneOperations.Get(*proj, *zone, opName).Do()
if err != nil {
log.Fatalf("Failed to get op %s: %v", opName, err)
}
switch op.Status {
case "PENDING", "RUNNING":
log.Printf("Waiting on operation %v", opName)
continue
case "DONE":
if op.Error != nil {
for _, operr := range op.Error.Errors {
log.Printf("Error: %+v", operr)
}
log.Fatalf("Failed to start.")
}
log.Printf("Success. %+v", op)
break OpLoop
default:
log.Fatalf("Unknown status %q: %+v", op.Status, op)
}
}
inst, err := computeService.Instances.Get(*proj, *zone, *instName).Do()
if err != nil {
log.Fatalf("Error getting instance after creation: %v", err)
}
ij, _ := json.MarshalIndent(inst, "", " ")
log.Printf("Instance: %s", ij)
}