func maybeRemapCloudSQL(host string) (out string, err error) {
	if !strings.HasSuffix(host, cloudSQLSuffix) {
		return host, nil
	}
	inst := strings.TrimSuffix(host, cloudSQLSuffix)
	if !metadata.OnGCE() {
		return "", errors.New("CloudSQL support only available when running on Google Compute Engine.")
	}
	proj, err := metadata.ProjectID()
	if err != nil {
		return "", fmt.Errorf("Failed to lookup GCE project ID: %v", err)
	}

	admin, _ := sqladmin.New(oauth2.NewClient(context.Background(), google.ComputeTokenSource("")))
	listRes, err := admin.Instances.List(proj).Do()
	if err != nil {
		return "", fmt.Errorf("error enumerating Cloud SQL instances: %v", err)
	}
	for _, it := range listRes.Items {
		if !strings.EqualFold(it.Instance, inst) {
			continue
		}
		js, _ := json.Marshal(it)
		log.Printf("Found Cloud SQL instance %s: %s", inst, js)
		for _, ipm := range it.IpAddresses {
			return ipm.IpAddress, nil
		}
		return "", fmt.Errorf("No external IP address for Cloud SQL instances %s", inst)
	}
	var found []string
	for _, it := range listRes.Items {
		found = append(found, it.Instance)
	}
	return "", fmt.Errorf("Cloud SQL instance %q not found. Found: %q", inst, found)
}
Exemple #2
0
// newGCECloud creates a new instance of GCECloud.
func newGCECloud() (*GCECloud, error) {
	projectID, zone, err := getProjectAndZone()
	if err != nil {
		return nil, err
	}
	// TODO: if we want to use this on a machine that doesn't have the http://metadata server
	// e.g. on a user's machine (not VM) somewhere, we need to have an alternative for
	// instance id lookup.
	instanceID, err := getInstanceID()
	if err != nil {
		return nil, err
	}
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	svc, err := compute.New(client)
	if err != nil {
		return nil, err
	}
	containerSvc, err := container.New(client)
	if err != nil {
		return nil, err
	}
	return &GCECloud{
		service:          svc,
		containerService: containerSvc,
		projectID:        projectID,
		zone:             zone,
		instanceID:       instanceID,
	}, nil
}
Exemple #3
0
// newClient creates http.Client with a jwt service account when
// jsonFile flag is specified, otherwise by obtaining the GCE service
// account's access token.
func newClient(jsonFile string) (*http.Client, error) {
	if jsonFile != "" {
		jsonKey, err := ioutil.ReadFile(jsonFile)
		if err != nil {
			return nil, err
		}
		conf, err := google.JWTConfigFromJSON(jsonKey, pubsub.ScopePubSub)
		if err != nil {
			return nil, err
		}
		return conf.Client(oauth2.NoContext), nil
	}
	if metadata.OnGCE() {
		c := &http.Client{
			Transport: &oauth2.Transport{
				Source: google.ComputeTokenSource(""),
			},
		}
		if *projID == "" {
			projectID, err := metadata.ProjectID()
			if err != nil {
				return nil, fmt.Errorf("ProjectID failed, %v", err)
			}
			*projID = projectID
		}
		return c, nil
	}
	return nil, errors.New("Could not create an authenticated client.")
}
Exemple #4
0
// setupProviderConfig validates and sets up cloudConfig based on framework.TestContext.Provider.
func setupProviderConfig() error {
	switch framework.TestContext.Provider {
	case "":
		glog.Info("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.")

	case "gce", "gke":
		var err error
		framework.Logf("Fetching cloud provider for %q\r\n", framework.TestContext.Provider)
		var tokenSource oauth2.TokenSource
		tokenSource = nil
		if cloudConfig.ServiceAccount != "" {
			// Use specified service account for auth
			framework.Logf("Using service account %q as token source.", cloudConfig.ServiceAccount)
			tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount)
		}
		zone := framework.TestContext.CloudConfig.Zone
		region, err := gcecloud.GetGCERegion(zone)
		if err != nil {
			return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err)
		}
		managedZones := []string{zone} // Only single-zone for now
		cloudConfig.Provider, err = gcecloud.CreateGCECloud(framework.TestContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, nil /* nodeTags */, "" /* nodeInstancePerfix */, tokenSource, false /* useMetadataServer */)
		if err != nil {
			return fmt.Errorf("Error building GCE/GKE provider: %v", err)
		}

	case "aws":
		if cloudConfig.Zone == "" {
			return fmt.Errorf("gce-zone must be specified for AWS")
		}
	}

	return nil
}
Exemple #5
0
func (p *gceTokenProvider) MintToken() (Token, error) {
	src := google.ComputeTokenSource(p.account)
	tok, err := src.Token()
	if err != nil {
		return nil, err
	}
	return makeToken(tok), nil
}
Exemple #6
0
func newAltTokenSource(tokenURL string) oauth2.TokenSource {
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	a := &altTokenSource{
		oauthClient: client,
		tokenURL:    tokenURL,
	}
	return oauth2.ReuseTokenSource(nil, a)
}
Exemple #7
0
// GCEServiceAccountClient creates an oauth client that is uses the auth token
// attached to an instance in GCE. This requires that the necessary scopes are
// attached to the instance upon creation.
// See details here: https://cloud.google.com/compute/docs/authentication
// If transport is nil, the default transport will be used.
func GCEServiceAccountClient(transport http.RoundTripper) *http.Client {
	return &http.Client{
		Transport: &oauth2.Transport{
			Source: google.ComputeTokenSource(""),
			Base:   transport,
		},
		Timeout: util.REQUEST_TIMEOUT,
	}
}
func newAltTokenSource(tokenURL string) oauth2.TokenSource {
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	a := &altTokenSource{
		oauthClient: client,
		tokenURL:    tokenURL,
		throttle:    util.NewTokenBucketRateLimiter(tokenURLQPS, tokenURLBurst),
	}
	return oauth2.ReuseTokenSource(nil, a)
}
Exemple #9
0
// newGCECloud creates a new instance of GCECloud.
func newGCECloud(config io.Reader) (*GCECloud, error) {
	projectID, zone, err := getProjectAndZone()
	if err != nil {
		return nil, err
	}
	// TODO: if we want to use this on a machine that doesn't have the http://metadata server
	// e.g. on a user's machine (not VM) somewhere, we need to have an alternative for
	// instance id lookup.
	instanceID, err := getInstanceID()
	if err != nil {
		return nil, err
	}
	externalID, err := getCurrentExternalID()
	if err != nil {
		return nil, err
	}
	networkName, err := getNetworkName()
	if err != nil {
		return nil, err
	}
	tokenSource := google.ComputeTokenSource("")
	if config != nil {
		var cfg Config
		if err := gcfg.ReadInto(&cfg, config); err != nil {
			glog.Errorf("Couldn't read config: %v", err)
			return nil, err
		}
		if cfg.Global.ProjectID != "" {
			projectID = cfg.Global.ProjectID
		}
		if cfg.Global.NetworkName != "" {
			networkName = cfg.Global.NetworkName
		}
		if cfg.Global.TokenURL != "" {
			tokenSource = newAltTokenSource(cfg.Global.TokenURL)
		}
	}
	client := oauth2.NewClient(oauth2.NoContext, tokenSource)
	svc, err := compute.New(client)
	if err != nil {
		return nil, err
	}
	containerSvc, err := container.New(client)
	if err != nil {
		return nil, err
	}
	return &GCECloud{
		service:          svc,
		containerService: containerSvc,
		projectID:        projectID,
		zone:             zone,
		instanceID:       instanceID,
		externalID:       externalID,
		networkName:      networkName,
		metadataAccess:   getMetadata,
	}, nil
}
func NewDriverGCE(ui packer.Ui, p string, a *accountFile) (Driver, error) {
	var err error

	var client *http.Client

	// Auth with AccountFile first if provided
	if a.PrivateKey != "" {
		log.Printf("[INFO] Requesting Google token via AccountFile...")
		log.Printf("[INFO]   -- Email: %s", a.ClientEmail)
		log.Printf("[INFO]   -- Scopes: %s", DriverScopes)
		log.Printf("[INFO]   -- Private Key Length: %d", len(a.PrivateKey))

		conf := jwt.Config{
			Email:      a.ClientEmail,
			PrivateKey: []byte(a.PrivateKey),
			Scopes:     DriverScopes,
			TokenURL:   "https://accounts.google.com/o/oauth2/token",
		}

		// Initiate an http.Client. The following GET request will be
		// authorized and authenticated on the behalf of
		// your service account.
		client = conf.Client(oauth2.NoContext)
	} else {
		log.Printf("[INFO] Requesting Google token via GCE Service Role...")
		client = &http.Client{
			Transport: &oauth2.Transport{
				// Fetch from Google Compute Engine's metadata server to retrieve
				// an access token for the provided account.
				// If no account is specified, "default" is used.
				Source: google.ComputeTokenSource(""),
			},
		}
	}

	if err != nil {
		return nil, err
	}

	log.Printf("[INFO] Instantiating GCE client...")
	service, err := compute.New(client)
	// Set UserAgent
	versionString := version.FormattedVersion()
	service.UserAgent = fmt.Sprintf(
		"(%s %s) Packer/%s", runtime.GOOS, runtime.GOARCH, versionString)

	if err != nil {
		return nil, err
	}

	return &driverGCE{
		projectId: p,
		service:   service,
		ui:        ui,
	}, nil
}
Exemple #11
0
func ExampleComputeTokenSource() {
	client := &http.Client{
		Transport: &oauth2.Transport{
			// Fetch from Google Compute Engine's metadata server to retrieve
			// an access token for the provided account.
			// If no account is specified, "default" is used.
			Source: google.ComputeTokenSource(""),
		},
	}
	client.Get("...")
}
func newGoogleClient(ctx context.Context, bucket string) (*googleClient, error) {
	client, err := storage.NewClient(
		ctx,
		cloud.WithTokenSource(google.ComputeTokenSource("")),
		cloud.WithScopes(storage.ScopeFullControl),
	)
	if err != nil {
		return nil, err
	}
	return &googleClient{ctx, client.Bucket(bucket)}, nil
}
Exemple #13
0
func CreateGCMSink(uri *url.URL) (core.DataSink, error) {
	if len(uri.Scheme) > 0 {
		return nil, fmt.Errorf("scheme should not be set for GCM sink")
	}
	if len(uri.Host) > 0 {
		return nil, fmt.Errorf("host should not be set for GCM sink")
	}

	opts, err := url.ParseQuery(uri.RawQuery)

	metrics := "all"
	if len(opts["metrics"]) > 0 {
		metrics = opts["metrics"][0]
	}
	var metricFilter MetricFilter = metricsAll
	switch metrics {
	case "all":
		metricFilter = metricsAll
	case "autoscaling":
		metricFilter = metricsOnlyAutoscaling
	default:
		return nil, fmt.Errorf("invalid metrics parameter: %s", metrics)
	}

	if err := gce_util.EnsureOnGCE(); err != nil {
		return nil, err
	}

	// Detect project ID
	projectId, err := gce.ProjectID()
	if err != nil {
		return nil, err
	}

	// Create Google Cloud Monitoring service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	gcmService, err := gcm.New(client)
	if err != nil {
		return nil, err
	}

	sink := &gcmSink{
		registered:   false,
		project:      projectId,
		gcmService:   gcmService,
		metricFilter: metricFilter,
	}
	glog.Infof("created GCM sink")
	if err := sink.registerAllMetrics(); err != nil {
		glog.Warningf("Error during metrics registration: %v", err)
	}
	return sink, nil
}
Exemple #14
0
/*
   The account may be empty or the string "default" to use the instance's main account.
*/
func BuildGCEMetadatTransporter(serviceAccount string) (GoogleOAuthClient, error) {
	client := &http.Client{
		Transport: &oauth2.Transport{

			Source: googleOauth2.ComputeTokenSource(""),
		},
	}

	return &gOAuthClient{
		httpclient: client,
	}, nil
}
// New creates a new APIClient with the provided configuration.
func New(config *Config) (APIClient, error) {
	service, err := cua.New(&http.Client{
		Transport: &oauth2.Transport{Source: google.ComputeTokenSource("")},
	})
	if err != nil {
		return nil, err
	}
	service.BasePath = strings.TrimRight(config.APIBase, "/") + "/projects/"
	service.UserAgent = config.UserAgent
	client := &googleAPIClient{
		config:  config,
		service: service,
	}
	return client, nil
}
Exemple #16
0
// setupProviderConfig validates and sets up cloudConfig based on testContext.Provider.
func setupProviderConfig() error {
	switch testContext.Provider {
	case "":
		glog.Info("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.")

	case "gce", "gke":
		var err error
		Logf("Fetching cloud provider for %q\r\n", testContext.Provider)
		var tokenSource oauth2.TokenSource
		tokenSource = nil
		if cloudConfig.ServiceAccount != "" {
			// Use specified service account for auth
			Logf("Using service account %q as token source.", cloudConfig.ServiceAccount)
			tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount)
		}
		zone := testContext.CloudConfig.Zone
		region, err := gcecloud.GetGCERegion(zone)
		if err != nil {
			return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err)
		}
		managedZones := []string{zone} // Only single-zone for now
		cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, region, zone, managedZones, "" /* networkUrl */, tokenSource, false /* useMetadataServer */)
		if err != nil {
			return fmt.Errorf("Error building GCE/GKE provider: %v", err)
		}

	case "aws":
		awsConfig := "[Global]\n"
		if cloudConfig.Zone == "" {
			return fmt.Errorf("gce-zone must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone)

		if cloudConfig.ClusterTag == "" {
			return fmt.Errorf("--cluster-tag must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag)

		var err error
		cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig))
		if err != nil {
			return fmt.Errorf("Error building AWS provider: %v", err)
		}

	}

	return nil
}
Exemple #17
0
func initGCE() error {
	initGCECalled = true
	if !metadata.OnGCE() {
		return errors.New("not running on GCE; VM support disabled")
	}
	var err error
	projectID, err = metadata.ProjectID()
	if err != nil {
		return fmt.Errorf("failed to get current GCE ProjectID: %v", err)
	}
	tokenSource = google.ComputeTokenSource("default")
	httpClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	serviceCtx = cloud.NewContext(projectID, httpClient)

	projectZone, err = metadata.Get("instance/zone")
	if err != nil || projectZone == "" {
		return fmt.Errorf("failed to get current GCE zone: %v", err)
	}

	// Convert the zone from "projects/1234/zones/us-central1-a" to "us-central1-a".
	projectZone = path.Base(projectZone)
	if !hasComputeScope() {
		return errors.New("The coordinator is not running with access to read and write Compute resources. VM support disabled.")

	}
	projectRegion = projectZone[:strings.LastIndex(projectZone, "-")] // "us-central1"

	externalIP, err = metadata.ExternalIP()
	if err != nil {
		return fmt.Errorf("ExternalIP: %v", err)
	}
	computeService, _ = compute.New(httpClient)
	errTryDeps = checkTryBuildDeps()
	if errTryDeps != nil {
		log.Printf("TryBot builders disabled due to error: %v", errTryDeps)
	} else {
		log.Printf("TryBot builders enabled.")
	}

	inStaging = projectID == "go-dashboard-dev"
	if inStaging {
		log.Printf("Running in staging cluster (%q)", projectID)
	}
	go gcePool.pollQuotaLoop()
	return nil
}
Exemple #18
0
func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())

	client = &http.Client{Transport: &oauth2.Transport{
		Source: google.ComputeTokenSource(""),
	}}

	ctx = cloud.NewContext(projectID, client)

	http.HandleFunc("/healthcheck", func(http.ResponseWriter, *http.Request) {})
	http.HandleFunc("/", notificationHandler)
	log.Println("server listening on", listenAddress)

	if err := http.ListenAndServe(listenAddress, nil); err != nil {
		log.Fatal(err)
	}
}
Exemple #19
0
// CreateGceManager constructs gceManager object.
func CreateGceManager(migs []*config.MigConfig) (*GceManager, error) {
	// Create Google Compute Engine service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}

	manager := &GceManager{
		migs:     migs,
		service:  gceService,
		migCache: map[config.InstanceConfig]*config.MigConfig{},
	}

	go wait.Forever(func() { manager.regenerateCacheIgnoreError() }, time.Hour)

	return manager, nil
}
Exemple #20
0
// CreateGceManager constructs gceManager object.
func CreateGceManager(migs []*config.MigConfig, configReader io.Reader) (*GceManager, error) {
	// Create Google Compute Engine token.
	tokenSource := google.ComputeTokenSource("")
	if configReader != nil {
		var cfg provider_gce.Config
		if err := gcfg.ReadInto(&cfg, configReader); err != nil {
			glog.Errorf("Couldn't read config: %v", err)
			return nil, err
		}
		if cfg.Global.TokenURL == "" {
			glog.Warning("Empty tokenUrl in cloud config")
		} else {
			glog.Infof("Using TokenSource from config %#v", tokenSource)
			tokenSource = provider_gce.NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
		}
	} else {
		glog.Infof("Using default TokenSource %#v", tokenSource)
	}

	// Create Google Compute Engine service.
	client := oauth2.NewClient(oauth2.NoContext, tokenSource)
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}

	migInfos := make([]*migInformation, 0, len(migs))
	for _, mig := range migs {
		migInfos = append(migInfos, &migInformation{
			config: mig,
		})
	}

	manager := &GceManager{
		migs:     migInfos,
		service:  gceService,
		migCache: map[config.InstanceConfig]*config.MigConfig{},
	}

	go wait.Forever(func() { manager.regenerateCacheIgnoreError() }, time.Hour)

	return manager, nil
}
Exemple #21
0
func newGcmSource() (dataSource, error) {
	// Detect project ID
	projectId, err := gce.ProjectID()
	if err != nil {
		return nil, err
	}

	// Create Google Cloud Monitoring service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	s, err := gcm.New(client)
	if err != nil {
		return nil, err
	}

	return &gcmSource{
		project:    projectId,
		gcmService: s,
	}, nil
}
Exemple #22
0
// NewServiceClient returns a Client for use when running on Google
// Compute Engine.  This client can access buckets owned by the same
// project ID as the VM.
func NewServiceClient() (*Client, error) {
	if !metadata.OnGCE() {
		return nil, errors.New("not running on Google Compute Engine")
	}
	scopes, _ := metadata.Scopes("default")
	haveScope := func(scope string) bool {
		for _, x := range scopes {
			if x == scope {
				return true
			}
		}
		return false
	}
	if !haveScope("https://www.googleapis.com/auth/devstorage.full_control") &&
		!haveScope("https://www.googleapis.com/auth/devstorage.read_write") {
		return nil, errors.New("when this Google Compute Engine VM instance was created, it wasn't granted access to Cloud Storage")
	}
	client := oauth2.NewClient(context.Background(), google.ComputeTokenSource(""))
	service, _ := api.New(client)
	return &Client{client: client, service: service}, nil
}
Exemple #23
0
func CreateGCLSink(uri *url.URL) (core.EventSink, error) {
	if err := gce_util.EnsureOnGCE(); err != nil {
		return nil, err
	}

	// Detect project ID
	projectId, err := gce.ProjectID()
	if err != nil {
		return nil, err
	}

	// Create Google Cloud Logging service.
	client := oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource(""))
	gclService, err := gcl.New(client)
	if err != nil {
		return nil, err
	}

	sink := &gclSink{project: projectId, gclService: gclService}
	glog.Info("created GCL sink")
	return sink, nil
}
// CreateGceManager constructs gceManager object.
func CreateGceManager(configReader io.Reader) (*GceManager, error) {
	// Create Google Compute Engine token.
	tokenSource := google.ComputeTokenSource("")
	if configReader != nil {
		var cfg provider_gce.Config
		if err := gcfg.ReadInto(&cfg, configReader); err != nil {
			glog.Errorf("Couldn't read config: %v", err)
			return nil, err
		}
		if cfg.Global.TokenURL == "" {
			glog.Warning("Empty tokenUrl in cloud config")
		} else {
			glog.Infof("Using TokenSource from config %#v", tokenSource)
			tokenSource = provider_gce.NewAltTokenSource(cfg.Global.TokenURL, cfg.Global.TokenBody)
		}
	} else {
		glog.Infof("Using default TokenSource %#v", tokenSource)
	}

	// Create Google Compute Engine service.
	client := oauth2.NewClient(oauth2.NoContext, tokenSource)
	gceService, err := gce.New(client)
	if err != nil {
		return nil, err
	}
	manager := &GceManager{
		migs:     make([]*migInformation, 0),
		service:  gceService,
		migCache: make(map[GceRef]*Mig),
	}
	go wait.Forever(func() {
		manager.cacheMutex.Lock()
		defer manager.cacheMutex.Unlock()
		if err := manager.regenerateCache(); err != nil {
			glog.Errorf("Error while regenerating Mig cache: %v", err)
		}
	}, time.Hour)
	return manager, nil
}
Exemple #25
0
// Client returns an oauth2-based http.Client based on the receiver Config.
func (c *Config) Client(ctx context.Context) (*http.Client, error) {
	switch {
	case c.GCEAccount != "":
		return oauth2.NewClient(ctx, google.ComputeTokenSource(c.GCEAccount)), nil
	case c.ConfigFile != "":
		f, err := vfs.Open(ctx, c.ConfigFile)
		if err != nil {
			return nil, fmt.Errorf("error opening ConfigFile: %v", err)
		}
		data, err := ioutil.ReadAll(f)
		f.Close()
		if err != nil {
			return nil, fmt.Errorf("error reading ConfigFile: %v", err)
		}
		config, err := google.JWTConfigFromJSON(data, c.Scopes...)
		if err != nil {
			return nil, fmt.Errorf("JWT config error: %v", err)
		}
		return config.Client(ctx), nil
	default:
		return nil, errors.New("oauth2 misconfigured")
	}
}
Exemple #26
0
func main() {
	if len(os.Args) < 5 {
		printUsage()
		return
	}

	command := strings.ToLower(os.Args[1])
	if command != "get" && command != "set" && command != "wait" {
		printUsage()
		return
	}
	var value string
	if command == "set" || command == "wait" {
		if len(os.Args) != 6 {
			printUsage()
			return
		}
		value = os.Args[5]
	}

	datasetID := os.Args[2]
	kind := os.Args[3]
	key := os.Args[4]

	ctx := context.Background()
	client, err := datastore.NewClient(ctx, datasetID, cloud.WithTokenSource(google.ComputeTokenSource("")))
	if err != nil {
		log.Fatal(err)
	}

	k := datastore.NewKey(ctx, kind, key, 0, nil)
	e := new(Entity)

	if command == "get" {
		if err := client.Get(ctx, k, e); err != nil {
			if err.Error() != "datastore: no such entity" {
				log.Fatal(err)
			}
		}
		fmt.Println(e.Value)
	} else if command == "set" {
		e.Value = value
		if _, err := client.Put(ctx, k, e); err != nil {
			log.Fatal(err)
		}
	} else if command == "wait" {
		errCount := 0
		for {
			if err := client.Get(ctx, k, e); err != nil {
				if err.Error() != "datastore: no such entity" {
					errCount++
					if errCount > 3 {
						log.Fatal(err)
					}
				}
			} else if e.Value == value {
				break
			} else {
				errCount = 0
			}
			time.Sleep(1 * time.Second)
		}
	}
}
func main() {
	flag.Parse()
	if flag.NArg() != 2 {
		log.Fatalf("Please specify both required arguments." + usage)
	}
	bucket := flag.Arg(0)
	imagePath := flag.Arg(1)
	file, err := os.Open(imagePath)
	if err != nil {
		log.Fatalf("Error opening image file: %v", err)
	}
	fileName := path.Base(imagePath)
	defer file.Close()
	service, err := storage.New(oauth2.NewClient(oauth2.NoContext, google.ComputeTokenSource("")))
	if err != nil {
		log.Fatalf("Failed to create GCS client: %v", err)
	}
	// Insert the image into GCS.
	baseFileName := buildName(0, fileName)
	_, err = service.Objects.Insert(bucket, &storage.Object{Name: baseFileName}).Media(file).Do()
	if err != nil {
		log.Fatalf("Unable to upload initial file to bucket: %v", err)
	}
	c := make(chan *GCSCopyReq, 999)
	f := make(chan string)
	finished := make(chan interface{})
	wg := &sync.WaitGroup{}
	wg.Add(numCopiers)
	for i := 0; i < numCopiers; i++ {
		go func() {
			copyObjects(service, c, f, finished)
			wg.Done()
		}()
	}
	go func() {
		wg.Wait()
		close(f)
		close(finished)
	}()
	go func() {
		i := 0
		for _ = range finished {
			i++
			if i%100 == 0 {
				fmt.Printf("%v/%v copied.\n", i, numFiles)
			}
		}
		fmt.Printf("%v/%v copied.\n", i, numFiles)
	}()
	for i := 1; i < numFiles; i++ {
		c <- &GCSCopyReq{
			SourceBucket: bucket,
			SourceFile:   baseFileName,
			DestBucket:   bucket,
			DestFile:     buildName(i, fileName),
		}
	}
	close(c)
	for errFile := range f {
		fmt.Printf("Could not copy to %v\n", errFile)
	}
}
Exemple #28
0
// NewComputeEngine constructs the PerRPCCredentials that fetches access tokens from
// Google Compute Engine (GCE)'s metadata server. It is only valid to use this
// if your program is running on a GCE instance.
// TODO(dsymonds): Deprecate and remove this.
func NewComputeEngine() credentials.PerRPCCredentials {
	return TokenSource{google.ComputeTokenSource("")}
}
Exemple #29
0
func TestE2E(t *testing.T) {
	util.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()
	if *reportDir != "" {
		if err := os.MkdirAll(*reportDir, 0755); err != nil {
			glog.Errorf("Failed creating report directory: %v", err)
		}
		defer CoreDump(*reportDir)
	}

	if testContext.Provider == "" {
		glog.Info("The --provider flag is not set.  Treating as a conformance test.  Some tests may not be run.")
	}

	if testContext.Provider == "gce" || testContext.Provider == "gke" {
		var err error
		Logf("Fetching cloud provider for %q\r\n", testContext.Provider)
		var tokenSource oauth2.TokenSource
		tokenSource = nil
		if cloudConfig.ServiceAccount != "" {
			// Use specified service account for auth
			Logf("Using service account %q as token source.", cloudConfig.ServiceAccount)
			tokenSource = google.ComputeTokenSource(cloudConfig.ServiceAccount)
		}
		cloudConfig.Provider, err = gcecloud.CreateGCECloud(testContext.CloudConfig.ProjectID, testContext.CloudConfig.Zone, "" /* networkUrl */, tokenSource, false /* useMetadataServer */)
		if err != nil {
			glog.Fatal("Error building GCE provider: ", err)
		}

	}

	if testContext.Provider == "aws" {
		awsConfig := "[Global]\n"
		if cloudConfig.Zone == "" {
			glog.Fatal("gce-zone must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("Zone=%s\n", cloudConfig.Zone)

		if cloudConfig.ClusterTag == "" {
			glog.Fatal("--cluster-tag must be specified for AWS")
		}
		awsConfig += fmt.Sprintf("KubernetesClusterTag=%s\n", cloudConfig.ClusterTag)

		var err error
		cloudConfig.Provider, err = cloudprovider.GetCloudProvider(testContext.Provider, strings.NewReader(awsConfig))
		if err != nil {
			glog.Fatal("Error building AWS provider: ", err)
		}
	}

	// Disable skipped tests unless they are explicitly requested.
	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
		config.GinkgoConfig.SkipString = `\[Skipped\]`
	}
	gomega.RegisterFailHandler(ginkgo.Fail)

	c, err := loadClient()
	if err != nil {
		glog.Fatal("Error loading client: ", err)
	}

	// Delete any namespaces except default and kube-system. This ensures no
	// lingering resources are left over from a previous test run.
	if testContext.CleanStart {
		deleted, err := deleteNamespaces(c, nil /* deleteFilter */, []string{api.NamespaceSystem, api.NamespaceDefault})
		if err != nil {
			t.Errorf("Error deleting orphaned namespaces: %v", err)
		}
		glog.Infof("Waiting for deletion of the following namespaces: %v", deleted)
		if err := waitForNamespacesDeleted(c, deleted, namespaceCleanupTimeout); err != nil {
			glog.Fatalf("Failed to delete orphaned namespaces %v: %v", deleted, err)
		}
	}

	// Ensure all pods are running and ready before starting tests (otherwise,
	// cluster infrastructure pods that are being pulled or started can block
	// test pods from running, and tests that ensure all pods are running and
	// ready will fail).
	if err := waitForPodsRunningReady(api.NamespaceSystem, testContext.MinStartupPods, podStartupTimeout); err != nil {
		t.Errorf("Error waiting for all pods to be running and ready: %v", err)
		return
	}
	// Run tests through the Ginkgo runner with output to console + JUnit for Jenkins
	var r []ginkgo.Reporter
	if *reportDir != "" {
		r = append(r, reporters.NewJUnitReporter(path.Join(*reportDir, fmt.Sprintf("junit_%02d.xml", config.GinkgoConfig.ParallelNode))))
	}
	glog.Infof("Starting e2e run; %q", runId)
	ginkgo.RunSpecsWithDefaultAndCustomReporters(t, "Kubernetes e2e suite", r)
}
Exemple #30
0
func (c *Config) loadAndValidate() error {
	var account accountFile

	// TODO: validation that it isn't blank
	if c.AccountFile == "" {
		c.AccountFile = os.Getenv("GOOGLE_ACCOUNT_FILE")
	}
	if c.Project == "" {
		c.Project = os.Getenv("GOOGLE_PROJECT")
	}
	if c.Region == "" {
		c.Region = os.Getenv("GOOGLE_REGION")
	}

	var client *http.Client

	if c.AccountFile != "" {
		if err := loadJSON(&account, c.AccountFile); err != nil {
			return fmt.Errorf(
				"Error loading account file '%s': %s",
				c.AccountFile,
				err)
		}

		clientScopes := []string{
			"https://www.googleapis.com/auth/compute",
			"https://www.googleapis.com/auth/ndev.clouddns.readwrite",
			"https://www.googleapis.com/auth/devstorage.full_control",
		}

		// Get the token for use in our requests
		log.Printf("[INFO] Requesting Google token...")
		log.Printf("[INFO]   -- Email: %s", account.ClientEmail)
		log.Printf("[INFO]   -- Scopes: %s", clientScopes)
		log.Printf("[INFO]   -- Private Key Length: %d", len(account.PrivateKey))

		conf := jwt.Config{
			Email:      account.ClientEmail,
			PrivateKey: []byte(account.PrivateKey),
			Scopes:     clientScopes,
			TokenURL:   "https://accounts.google.com/o/oauth2/token",
		}

		// Initiate an http.Client. The following GET request will be
		// authorized and authenticated on the behalf of
		// your service account.
		client = conf.Client(oauth2.NoContext)

	} else {
		log.Printf("[INFO] Requesting Google token via GCE Service Role...")
		client = &http.Client{
			Transport: &oauth2.Transport{
				// Fetch from Google Compute Engine's metadata server to retrieve
				// an access token for the provided account.
				// If no account is specified, "default" is used.
				Source: google.ComputeTokenSource(""),
			},
		}

	}

	// Build UserAgent
	versionString := "0.0.0"
	// TODO(dcunnin): Use Terraform's version code from version.go
	// versionString := main.Version
	// if main.VersionPrerelease != "" {
	// 	versionString = fmt.Sprintf("%s-%s", versionString, main.VersionPrerelease)
	// }
	userAgent := fmt.Sprintf(
		"(%s %s) Terraform/%s", runtime.GOOS, runtime.GOARCH, versionString)

	var err error

	log.Printf("[INFO] Instantiating GCE client...")
	c.clientCompute, err = compute.New(client)
	if err != nil {
		return err
	}
	c.clientCompute.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Beta GCE client...")
	c.clientComputeBeta, err = computeBeta.New(client)
	if err != nil {
		return err
	}
	c.clientComputeBeta.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Cloud DNS client...")
	c.clientDns, err = dns.New(client)
	if err != nil {
		return err
	}
	c.clientDns.UserAgent = userAgent

	log.Printf("[INFO] Instantiating Google Storage Client...")
	c.clientStorage, err = storage.New(client)
	if err != nil {
		return err
	}
	c.clientStorage.UserAgent = userAgent

	return nil
}