// NewComputeUtil creates and initializes a ComputeUtil.
func newComputeUtil(driver *Driver) (*ComputeUtil, error) {
client, err := google.DefaultClient(oauth2.NoContext, raw.ComputeScope)
if err != nil {
return nil, err
}
service, err := raw.New(client)
if err != nil {
return nil, err
}
return &ComputeUtil{
zone: driver.Zone,
instanceName: driver.MachineName,
userName: driver.SSHUser,
project: driver.Project,
diskTypeURL: driver.DiskType,
address: driver.Address,
network: driver.Network,
preemptible: driver.Preemptible,
useInternalIP: driver.UseInternalIP,
useInternalIPOnly: driver.UseInternalIPOnly,
service: service,
zoneURL: apiURL + driver.Project + "/zones/" + driver.Zone,
globalURL: apiURL + driver.Project + "/global",
SwarmMaster: driver.SwarmMaster,
SwarmHost: driver.SwarmHost,
}, nil
}
func pubsubContext(ctx context.Context, projectID string) (context.Context, error) {
httpClient, err := google.DefaultClient(ctx, pubsub.ScopePubSub)
if err != nil {
return nil, err
}
return cloud.WithContext(ctx, projectID, httpClient), nil
}
func GetService() (*deploymentmanager.Service, error) {
client, err := google.DefaultClient(oauth2.NoContext, "https://www.googleapis.com/auth/cloud-platform")
if err != nil {
return &deploymentmanager.Service{}, err
}
return deploymentmanager.New(client)
}
func GetStorageService(ctx context.Context) (*storage.Service, error) {
client, err := google.DefaultClient(ctx, storage.DevstorageFullControlScope)
if err != nil {
return nil, err
}
return storage.New(client)
}
func handler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
search, allele, err := requestToSearch(r)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
client, err := google.DefaultClient(c, genomics.GenomicsReadonlyScope)
if err != nil {
http.Error(w, "Invalid server configuration", http.StatusInternalServerError)
}
genomicsService, err := genomics.New(client)
if err != nil {
http.Error(w, "Invalid server configuration", http.StatusInternalServerError)
}
variantsService := genomics.NewVariantsService(genomicsService)
type beaconResponse struct {
XMLName struct{} `xml:"BEACONResponse"`
Exists bool `xml:"exists"`
}
var resp beaconResponse
for {
searchResponse, err := variantsService.Search(search).Do()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
for _, variant := range searchResponse.Variants {
if search.Start != variant.Start {
continue
}
if allele == variant.ReferenceBases {
resp.Exists = true
} else {
for _, base := range variant.AlternateBases {
if base == allele {
resp.Exists = true
break
}
}
}
}
if resp.Exists || searchResponse.NextPageToken == "" {
break
}
search.PageToken = searchResponse.NextPageToken
}
w.Header().Set("Content-Type", "application/xml")
enc := xml.NewEncoder(w)
enc.Indent("", " ")
if err = enc.Encode(resp); err != nil {
http.Error(w, "Failed writing response", http.StatusInternalServerError)
}
}
// ListBuckets returns a slice of all the buckets in the given projectId.
// [START ListBuckets]
func ListBuckets(projectId string) ([]*storage.Bucket, error) {
// Create the client that uses Application Default Credentials
client, err := google.DefaultClient(
oauth2.NoContext,
"https://www.googleapis.com/auth/devstorage.read_only")
if err != nil {
return nil, err
}
// Create the Google Cloud Storage service
service, err := storage.New(client)
if err != nil {
return nil, err
}
// Create the request to list buckets for the project id
request := service.Buckets.List(projectId)
// Execute the request
buckets, err := request.Do()
if err != nil {
return nil, err
}
return buckets.Items, nil
}
// datasets returns a list with the IDs of all the Big Query datasets visible
// with the given context.
func datasets(ctx context.Context) ([]string, error) {
// Create a new authenticated HTTP client over urlfetch.
hc, err := google.DefaultClient(ctx, bigquery.BigqueryScope)
if err != nil {
return nil, fmt.Errorf("could not create http client: %v", err)
}
// Create the BigQuery service.
bq, err := bigquery.New(hc)
if err != nil {
return nil, fmt.Errorf("could not create service: %v", err)
}
// Get the current application ID, which is the same as the project ID.
projectID := appengine.AppID(ctx)
// Return a list of IDs.
var ids []string
datasets, err := bq.Datasets.List(projectID).Do()
if err != nil {
return nil, fmt.Errorf("could not list datasets for %q: %v", projectID, err)
}
for _, d := range datasets.Datasets {
ids = append(ids, d.Id)
}
return ids, nil
}
func handleStoragePut(c context.Context, w http.ResponseWriter, r *http.Request) {
bucket := r.URL.Query().Get("bucket")
name := r.URL.Query().Get("name")
value := r.URL.Query().Get("value")
if bucket == "" || name == "" || value == "" {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Missing bucket, name, or value query parameter."))
return
}
client, err := google.DefaultClient(c, storage.DevstorageReadWriteScope)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Failed to get default google client. " + err.Error()))
return
}
service, err := storage.New(client)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Failed to get storage service. " + err.Error()))
return
}
obj, err := service.Objects.Insert(bucket, &storage.Object{Name: name}).Media(strings.NewReader(value)).Do()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Failed to insert object. " + err.Error()))
return
}
w.Write([]byte(fmt.Sprintf("put succeeded: %v", obj)))
}
func (f *Fetcher) Publish(request *FetchRequest, entries []*FetchResponse) (err error) {
client, err := google.DefaultClient(request.Context, pubsub.CloudPlatformScope)
if err != nil {
return
}
service, err := pubsub.New(client)
if err != nil {
return
}
messages := make([]*pubsub.PubsubMessage, len(entries))
for i := range entries {
messages[i] = &pubsub.PubsubMessage{
Data: base64.StdEncoding.EncodeToString(entries[i].Content),
}
}
pr := pubsub.PublishRequest{
Messages: messages,
}
var topic string
if request.Topic != "" {
topic = request.Topic
} else {
topic = f.Topic
}
if topic == "" {
return fmt.Errorf("fetcher: topic is empty")
}
full := fmt.Sprintf("projects/%s/topics/%s", request.AppID(), topic)
_, err = service.Projects.Topics.Publish(full, &pr).Do()
if err != nil {
return
}
return
}
// NewProtoClient returns a ProtoClient for communicating with a Google cloud service,
// configured with the given ClientOptions.
func NewProtoClient(ctx context.Context, opt ...cloud.ClientOption) (*ProtoClient, error) {
var o opts.DialOpt
for _, opt := range opt {
opt.Resolve(&o)
}
if o.GRPCClient != nil {
return nil, errors.New("unsupported GRPC base transport specified")
}
var client *http.Client
switch {
case o.HTTPClient != nil:
if o.TokenSource != nil {
return nil, errors.New("at most one of WithTokenSource or WithBaseHTTP may be provided")
}
client = o.HTTPClient
case o.TokenSource != nil:
client = oauth2.NewClient(ctx, o.TokenSource)
default:
var err error
client, err = google.DefaultClient(ctx, o.Scopes...)
if err != nil {
return nil, err
}
}
return &ProtoClient{
client: client,
endpoint: o.Endpoint,
userAgent: o.UserAgent,
}, nil
}
func init() {
if !metadata.OnGCE() {
return
}
hc, err := google.DefaultClient(oauth2.NoContext)
if err != nil {
registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
return
}
projID, err := metadata.ProjectID()
if projID == "" || err != nil {
registerBrokenFS(fmt.Errorf("could not get GCE project ID: %v", err))
return
}
ctx := cloud.NewContext(projID, hc)
sc, err := storage.NewClient(ctx)
if err != nil {
registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
return
}
wkfs.RegisterFS("/gcs/", &gcsFS{
ctx: ctx,
sc: sc,
})
}
// If we're not running on GCE (e.g. dev mode on localhost) and have
// no other way to get the info, the error value is is errNoRefresh.
func (h *DeployHandler) authenticatedClient() (project string, hc *http.Client, err error) {
project = os.Getenv("CAMLI_GCE_PROJECT")
accountFile := os.Getenv("CAMLI_GCE_SERVICE_ACCOUNT")
if project != "" && accountFile != "" {
data, errr := ioutil.ReadFile(accountFile)
err = errr
if err != nil {
return
}
jwtConf, errr := google.JWTConfigFromJSON(data, "https://www.googleapis.com/auth/compute.readonly")
err = errr
if err != nil {
return
}
hc = jwtConf.Client(context.Background())
return
}
if !metadata.OnGCE() {
err = errNoRefresh
return
}
project, _ = metadata.ProjectID()
hc, err = google.DefaultClient(oauth2.NoContext)
return project, hc, err
}
func getComputeClient() (*compute.Service, error) {
const retries = 10
const backoff = time.Second * 6
// Setup the gce client for provisioning instances
// Getting credentials on gce jenkins is flaky, so try a couple times
var err error
var cs *compute.Service
for i := 0; i < retries; i++ {
if i > 0 {
time.Sleep(backoff)
}
var client *http.Client
client, err = google.DefaultClient(oauth2.NoContext, compute.ComputeScope)
if err != nil {
continue
}
cs, err = compute.New(client)
if err != nil {
continue
}
return cs, nil
}
return nil, err
}
func handleStorageGet(c context.Context, w http.ResponseWriter, r *http.Request) {
bucket := r.URL.Query().Get("bucket")
name := r.URL.Query().Get("name")
if bucket == "" || name == "" {
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte("Missing bucket or name query parameter."))
return
}
client, err := google.DefaultClient(c, storage.DevstorageReadOnlyScope)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Failed to get default google client. " + err.Error()))
return
}
service, err := storage.New(client)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Failed to get storage service. " + err.Error()))
return
}
res, err := service.Objects.Get(bucket, name).Download()
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("Failed to get object. " + err.Error()))
return
}
w.Header().Set("Content-Type", res.Header.Get("Content-Type"))
_, err = io.Copy(w, res.Body)
if err != nil {
// to late to change status code now
clog.Errorf(c, "io.Copy failed to copy storage get to response. %v", err)
}
}
func init() {
var err error
cloudClient, err = google.DefaultClient(context.Background())
if err != nil {
panic(err)
}
}
// newServiceAccountClient returns a service account authenticated http.Client.
func newServiceAccountClient(c context.Context) (*http.Client, error) {
client, err := google.DefaultClient(c, BigqueryScope)
if err != nil {
return nil, err
}
return client, nil
}
func init() {
gcloudProject = os.Getenv("GCE_PROJECT")
gcloudDomain = os.Getenv("GCE_DOMAIN")
_, err := google.DefaultClient(context.Background(), dns.NdevClouddnsReadwriteScope)
if err == nil && len(gcloudProject) > 0 && len(gcloudDomain) > 0 {
gcloudLiveTest = true
}
}
func NewPubsubService(c context.Context) (s *pubsub.Service, err error) {
client, err := google.DefaultClient(c, pubsub.CloudPlatformScope)
if err != nil {
return
}
s, err = pubsub.New(client)
return
}
func ExampleDefaultClient() {
client, err := google.DefaultClient(oauth2.NoContext,
"https://www.googleapis.com/auth/devstorage.full_control")
if err != nil {
log.Fatal(err)
}
client.Get("...")
}
// InitDefault attempts to initialize the Dial function using application
// default credentials.
func InitDefault(ctx context.Context) error {
cl, err := google.DefaultClient(ctx, "https://www.googleapis.com/auth/sqlservice.admin")
if err != nil {
return err
}
Init(cl, nil, nil)
return nil
}
func NewGlasses() (*Glasses, error) {
client, err := google.DefaultClient(oauth2.NoContext, "https://www.googleapis.com/auth/cloud-platform")
if err != nil {
return nil, err
}
return &Glasses{client}, nil
}
func cloudContext(projectID string) (context.Context, error) {
ctx := context.Background()
httpClient, err := google.DefaultClient(ctx, storage.ScopeFullControl, pubsub.ScopePubSub)
if err != nil {
return nil, err
}
return cloud.WithContext(ctx, projectID, httpClient), nil
}
func NewMonitoringService(c context.Context) (s *cloudmonitoring.Service, err error) {
client, err := google.DefaultClient(c, cloudmonitoring.MonitoringScope)
if err != nil {
return
}
s, err = cloudmonitoring.New(client)
return
}
// New returns a new instance of GceImages
func New(conf *GCEConfig) (*GceImages, error) {
var err error
if conf.ProjectID == "" {
return nil, errors.New("ProjectID is not set. Please check your configuration.")
}
// increase the timeout. Also we need to pass the client with the context itself
timeout := time.Second * 30
ctx := context.WithValue(oauth2.NoContext, oauth2.HTTPClient, &http.Client{
Transport: &http.Transport{TLSHandshakeTimeout: timeout},
Timeout: timeout,
})
var client *http.Client
// allowed scopes
scopes := []string{compute.ComputeScope}
// Recommended way is explicit passing of credentials json which can be
// downloaded from console.developers.google under APIs & Auth/Credentials
// section
if conf.AccountFile != "" {
// expand shell meta character
path, err := homedir.Expand(conf.AccountFile)
if err != nil {
return nil, err
}
jsonContent, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
jtwConfig, err := google.JWTConfigFromJSON(jsonContent, scopes...)
if err != nil {
return nil, err
}
client = jtwConfig.Client(ctx)
} else {
// Look for application default credentials, for more details, see:
// https://developers.google.com/accounts/docs/application-default-credentials
client, err = google.DefaultClient(ctx, scopes...)
if err != nil {
return nil, err
}
}
svc, err := compute.New(client)
if err != nil {
return nil, err
}
return &GceImages{
svc: compute.NewImagesService(svc),
config: conf,
}, nil
}
// NewDefault returns a partial RWVFS implementation backed by Google Cloud Storage.
func NewDefault(bucket string) (rwvfs.FileSystem, error) {
// Authentication is provided by the gcloud tool when running locally, and
// by the associated service account when running on Compute Engine.
client, err := google.DefaultClient(context.Background(), storage.DevstorageFullControlScope)
if err != nil {
return nil, fmt.Errorf("unable to get default client: %v", err)
}
return New(client, bucket)
}
func main() {
flag.Parse()
flagsOk := true
if flag.Lookup("project").Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --project is required\n")
flagsOk = false
}
var sourceFlagCount int
if flag.Lookup("dataset").Value.String() != "" {
sourceFlagCount++
}
if flag.Lookup("jobid").Value.String() != "" {
sourceFlagCount++
}
if sourceFlagCount != 1 {
fmt.Fprintf(os.Stderr, "Exactly one of --dataset or --jobid must be set\n")
flagsOk = false
}
if !flagsOk {
os.Exit(1)
}
tableRE, err := regexp.Compile(*table)
if err != nil {
fmt.Fprintf(os.Stderr, "--table is not a valid regular expression: %q\n", *table)
os.Exit(1)
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
if *jobID != "" {
printQueryResults(client, *jobID)
return
}
ds := client.Dataset(*dataset)
var tables []*bigquery.Table
tables, err = ds.ListTables(context.Background())
if err != nil {
log.Fatalf("Listing tables: %v", err)
}
for _, t := range tables {
if tableRE.MatchString(t.TableID) {
printTable(client, t)
}
}
}
func main() {
flag.Parse()
flagsOk := true
for _, f := range []string{"project", "dataset", "table"} {
if flag.Lookup(f).Value.String() == "" {
fmt.Fprintf(os.Stderr, "Flag --%s is required\n", f)
flagsOk = false
}
}
if !flagsOk {
os.Exit(1)
}
httpClient, err := google.DefaultClient(context.Background(), bigquery.Scope)
if err != nil {
log.Fatalf("Creating http client: %v", err)
}
client, err := bigquery.NewClient(httpClient, *project)
if err != nil {
log.Fatalf("Creating bigquery client: %v", err)
}
it, err := client.Read(&bigquery.Table{
ProjectID: *project,
DatasetID: *dataset,
TableID: *table,
})
if err != nil {
log.Fatalf("Reading: %v", err)
}
// one-space padding.
tw := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
for it.Next(context.Background()) {
var vals bigquery.ValueList
if err := it.Get(&vals); err != nil {
fmt.Printf("err calling get: %v\n", err)
} else {
sep := ""
for _, v := range vals {
fmt.Fprintf(tw, "%s%v", sep, v)
sep = "\t"
}
fmt.Fprintf(tw, "\n")
}
}
tw.Flush()
if err := it.Err(); err != nil {
fmt.Printf("err reading: %v\n")
}
}
func (bs *GCSBackupStorage) authContext() (context.Context, error) {
if bs.authCtx == nil {
client, err := google.DefaultClient(context.TODO())
if err != nil {
return nil, err
}
bs.authCtx = cloud.NewContext(*project, client)
}
return bs.authCtx, nil
}
func createService(ctx context.Context) (*monitoring.Service, error) {
hc, err := google.DefaultClient(ctx, monitoring.MonitoringScope)
if err != nil {
return nil, err
}
s, err := monitoring.New(hc)
if err != nil {
return nil, err
}
return s, nil
}
func newComputeService(ctx context.Context) (*compute.Service, error) {
client, err := google.DefaultClient(ctx, compute.ComputeScope)
if err != nil {
return nil, err
}
computeService, err := compute.New(client)
if err != nil {
return nil, err
}
return computeService, nil
}