func handlePut(res http.ResponseWriter, req *http.Request) { ctx := appengine.NewContext(req) hc := &http.Client{ Transport: &oauth2.Transport{ Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl), Base: &urlfetch.Transport{Context: ctx}, }, } cctx := cloud.NewContext(appengine.AppID(ctx), hc) rdr, hdr, err := req.FormFile("form-upload-file") if err != nil { http.Error(res, "ERROR RECEIVING FILE: "+err.Error(), 500) return } writer := storage.NewWriter(cctx, bucketName, hdr.Filename) io.Copy(writer, rdr) err = writer.Close() if err != nil { http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500) return } }
func newContext(projectID, jsonKey string) (context.Context, error) { if projectID == "" { return nil, errors.New("project id not provided") } if jsonKey == "" { return nil, errors.New("JSON key not provided") } key, err := ioutil.ReadFile(jsonKey) if err != nil { return nil, err } conf, err := google.JWTConfigFromJSON( key, pubsub.ScopeCloudPlatform, pubsub.ScopePubSub, ) if err != nil { return nil, err } ctx := cloud.NewContext(projectID, conf.Client(oauth2.NoContext)) return ctx, nil }
func StartupHandler(w http.ResponseWriter, r *http.Request) { if len(Config) == 0 { c := appengine.NewContext(r) client := &http.Client{ Transport: &oauth2.Transport{ Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly), Base: &urlfetch.Transport{ Context: c, }, }, } bucket, _ := file.DefaultBucketName(c) ctx := cloud.NewContext("davine-web", client) rc, err := storage.NewReader(ctx, bucket, "config.yaml") if err != nil { c.Errorf("error reading config: %v", err.Error()) return } configFile, err := ioutil.ReadAll(rc) rc.Close() if err != nil { c.Errorf("error reading config: %v", err.Error()) return } c.Infof("loaded config file: %v", configFile) yaml.Unmarshal(configFile, &Config) c.Infof("loaded config struct: %v", Config) } }
// This example demonstrates calling the Cloud Pub/Sub API. As of 22 // Oct 2014, the Cloud Pub/Sub API is only available if you're // whitelisted. If you're interested in using it, please apply for the // Limited Preview program at the following form: // http://goo.gl/Wql9HL // // Also, before running this example, be sure to enable Cloud Pub/Sub // service on your project in Developer Console at: // https://console.developers.google.com/ // // Unless you run this sample on Compute Engine instance, please // create a new service account and download a JSON key file for it at // the developer console: https://console.developers.google.com/ // // It has the following subcommands: // // create_topic <name> // delete_topic <name> // create_subscription <name> <linked_topic> // delete_subscription <name> // publish <topic> <message> // pull_messages <subscription> <numworkers> // publish_messages <topic> <numworkers> // // You can choose any names for topic and subscription as long as they // follow the naming rule described at: // https://cloud.google.com/pubsub/overview#names // // You can create/delete topics/subscriptions by self-explanatory // subcommands. // // The "publish" subcommand is for publishing a single message to a // specified Cloud Pub/Sub topic. // // The "pull_messages" subcommand is for continuously pulling messages // from a specified Cloud Pub/Sub subscription with specified number // of workers. // // The "publish_messages" subcommand is for continuously publishing // messages to a specified Cloud Pub/Sub topic with specified number // of workers. func main() { flag.Parse() argv := flag.Args() checkArgs(argv, 1) client, err := newClient(*jsonFile) if err != nil { log.Fatalf("clientAndId failed, %v", err) } if *projID == "" { usageAndExit("Please specify Project ID.") } ctx := cloud.NewContext(*projID, client) m := map[string]func(ctx context.Context, argv []string){ "create_topic": createTopic, "delete_topic": deleteTopic, "create_subscription": createSubscription, "delete_subscription": deleteSubscription, "publish": publish, "pull_messages": pullMessages, "publish_messages": publishMessages, } subcommand := argv[0] f, ok := m[subcommand] if !ok { usageAndExit(fmt.Sprintf("Function not found for %s", subcommand)) } f(ctx, argv) }
func TestNamespaceQuery(t *testing.T) { gotNamespace := make(chan string, 1) ctx := cloud.NewContext("nsQueryTest", &http.Client{Transport: &fakeTransport{ Handler: func(req, resp proto.Message) error { gotNamespace <- req.(*pb.RunQueryRequest).GetPartitionId().GetNamespace() return errors.New("not implemented") }, }}) var gs []Gopher NewQuery("Gopher").GetAll(ctx, &gs) if got, want := <-gotNamespace, ""; got != want { t.Errorf("GetAll: got namespace %q, want %q", got, want) } NewQuery("Gopher").Count(ctx) if got, want := <-gotNamespace, ""; got != want { t.Errorf("Count: got namespace %q, want %q", got, want) } const ns = "not_default" ctx = WithNamespace(ctx, ns) NewQuery("Gopher").GetAll(ctx, &gs) if got, want := <-gotNamespace, ns; got != want { t.Errorf("GetAll: got namespace %q, want %q", got, want) } NewQuery("Gopher").Count(ctx) if got, want := <-gotNamespace, ns; got != want { t.Errorf("Count: got namespace %q, want %q", got, want) } }
func Example_listObjects() { // see the auth example how to initiate a context. ctx := cloud.NewContext("project-id", &http.Client{Transport: nil}) var query *storage.Query for { // If you are using this package on App Engine Managed VMs runtime, // you can init a bucket client with your app's default bucket name. // See http://godoc.org/google.golang.org/appengine/file#DefaultBucketName. objects, err := storage.List(ctx, "bucketname", query) if err != nil { log.Fatal(err) } for _, obj := range objects.Results { log.Printf("object name: %s, size: %v", obj.Name, obj.Size) } // if there are more results, objects.Next // will be non-nil. query = objects.Next if query == nil { break } } log.Println("paginated through all object items in the bucket you specified.") }
func NoAuthContext() context.Context { projID := os.Getenv(envProjID) if projID == "" { log.Fatal("GCLOUD_TESTS_GOLANG_PROJECT_ID must be set. See CONTRIBUTING.md for details.") } return cloud.NewContext(projID, &http.Client{Transport: http.DefaultTransport}) }
func getCloudContext(aeCtx context.Context) (context.Context, error) { data, err := ioutil.ReadFile("gcs.xxjson") if err != nil { return nil, err } conf, err := google.JWTConfigFromJSON( data, storage.ScopeFullControl, ) if err != nil { return nil, err } tokenSource := conf.TokenSource(aeCtx) hc := &http.Client{ Transport: &oauth2.Transport{ Source: tokenSource, Base: &urlfetch.Transport{Context: aeCtx}, }, } return cloud.NewContext(appengine.AppID(aeCtx), hc), nil }
func init() { if !metadata.OnGCE() { return } hc, err := google.DefaultClient(oauth2.NoContext) if err != nil { registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err)) return } projID, err := metadata.ProjectID() if projID == "" || err != nil { registerBrokenFS(fmt.Errorf("could not get GCE project ID: %v", err)) return } ctx := cloud.NewContext(projID, hc) sc, err := storage.NewClient(ctx) if err != nil { registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err)) return } wkfs.RegisterFS("/gcs/", &gcsFS{ ctx: ctx, sc: sc, }) }
func (this *Factory) NewGCSImageStore(conf map[string]string) ImageStore { jsonKey, err := ioutil.ReadFile(conf["KeyFile"]) if err != nil { log.Fatal(err) } cloudConf, err := google.JWTConfigFromJSON( jsonKey, gcs.ScopeFullControl, ) if err != nil { log.Fatal(err) } bucket := conf["BucketName"] ctx := gcloud.NewContext(conf["AppID"], cloudConf.Client(oauth2.NoContext)) mapper := NewNamePathMapper(conf["NamePathRegex"], conf["NamePathMap"]) return NewGCSImageStore( ctx, bucket, conf["StoreRoot"], mapper, ) }
// handler is the main demo entry point that calls the GCS operations. func handler(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { http.NotFound(w, r) return } c := appengine.NewContext(r) if bucket == "" { var err error if bucket, err = file.DefaultBucketName(c); err != nil { log.Errorf(c, "failed to get default GCS bucket name: %v", err) return } } hc := &http.Client{ Transport: &oauth2.Transport{ Source: google.AppEngineTokenSource(c, storage.ScopeFullControl), // Note that the App Engine urlfetch service has a limit of 10MB uploads and // 32MB downloads. // See https://cloud.google.com/appengine/docs/go/urlfetch/#Go_Quotas_and_limits // for more information. Base: &urlfetch.Transport{Context: c}, }, } ctx := cloud.NewContext(appengine.AppID(c), hc) w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c)) fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket) d := &demo{ c: c, w: w, ctx: ctx, } n := "demo-testfile-go" d.createFile(n) d.readFile(n) d.copyFile(n) d.statFile(n) d.createListFiles() d.listBucket() d.listBucketDirMode() d.defaultACL() d.putDefaultACLRule() d.deleteDefaultACLRule() d.bucketACL() d.putBucketACLRule() d.deleteBucketACLRule() d.acl(n) d.putACLRule(n) d.deleteACLRule(n) d.deleteFiles() if d.failed { io.WriteString(w, "\nDemo failed.\n") } else { io.WriteString(w, "\nDemo succeeded.\n") } }
// TODO(djd): Delete this function when it's no longer used. func Context(scopes ...string) context.Context { ctx := oauth2.NoContext ts := TokenSource(ctx, scopes...) if ts == nil { return nil } return cloud.NewContext(ProjID(), oauth2.NewClient(ctx, ts)) }
func TestEmptyAckID(t *testing.T) { ctx := cloud.NewContext("project-id", &http.Client{}) id := []string{"test", ""} err := Ack(ctx, "sub", id...) if err == nil || !strings.Contains(err.Error(), "index 1") { t.Errorf("Ack should report an error indicating the id is empty. Got: %v", err) } }
func getCloudContext(gae context.Context) context.Context { hc := &http.Client{ Transport: &oauth2.Transport{ Source: google.AppEngineTokenSource(gae, storage.ScopeFullControl), Base: &urlfetch.Transport{Context: gae}, }, } return cloud.NewContext(appengine.AppID(gae), hc) }
func (bs *GCSBackupStorage) authContext() (context.Context, error) { if bs.authCtx == nil { client, err := google.DefaultClient(context.TODO()) if err != nil { return nil, err } bs.authCtx = cloud.NewContext(*project, client) } return bs.authCtx, nil }
func auth(r *http.Request) (context.Context, error) { c := appengine.NewContext(r) client, err := google.DefaultClient(c, storage.ScopeFullControl) if err != nil { return nil, err } ctx := cloud.NewContext(appengine.AppID(c), client) return ctx, nil }
func TestErrorOnObjectsInsertCall(t *testing.T) { ctx := cloud.NewContext("project-id", &http.Client{ Transport: &fakeTransport{}}) wc := NewWriter(ctx, "bucketname", "filename1") wc.ContentType = "text/plain" if _, err := wc.Write([]byte("hello world")); err == nil { t.Errorf("expected error on write, got nil") } if err := wc.Close(); err == nil { t.Errorf("expected error on close, got nil") } }
// Cancel removes any written content from this FileWriter. func (w *writer) Cancel() error { w.closed = true err := storageDeleteObject(cloud.NewContext(dummyProjectID, w.client), w.bucket, w.name) if err != nil { if status, ok := err.(*googleapi.Error); ok { if status.Code == http.StatusNotFound { err = nil } } } return err }
func (g Config) contextFromToken(scopes ...string) (context.Context, error) { conf, err := google.JWTConfigFromJSON( []byte(g.Token), scopes..., ) if err != nil { log.Print("probs with token:", g.Token) return nil, err } return cloud.NewContext(g.ProjectID, conf.Client(oauth2.NoContext)), nil }
func getContext(jsonPrivateKeyFilepath string, projectID string) (context.Context, error) { jsonKey, err := ioutil.ReadFile(jsonPrivateKeyFilepath) if err != nil { return nil, err } conf, err := google.JWTConfigFromJSON(jsonKey, storage.ScopeFullControl) if err != nil { return nil, err } ctx := cloud.NewContext(projectID, conf.Client(oauth2.NoContext)) return ctx, nil }
func serviceContext() (context.Context, error) { file := filepath.Join(os.Getenv("HOME"), "keys", "golang-org.service.json") blob, err := ioutil.ReadFile(file) if err != nil { return nil, err } config, err := google.JWTConfigFromJSON(blob, storage.ScopeReadWrite) if err != nil { return nil, err } return cloud.NewContext(projectID, config.Client(context.Background())), nil }
func (ds *Datastore) Init(_ string, project string) { client, err := google.DefaultClient(oauth2.NoContext, "https://www.googleapis.com/auth/devstorage.full_control") if err != nil { x.LogErr(log, err).Fatal("Unable to get client") } ds.ctx = cloud.NewContext(project, client) if ds.ctx == nil { log.Fatal("Failed to get context. context is nil") } log.Info("Connection to Google datastore established") }
func Example_touchObjects() { // see the auth example how to initiate a context. ctx := cloud.NewContext("project-id", &http.Client{Transport: nil}) o, err := storage.Put(ctx, "bucketname", "filename", &storage.Object{ ContentType: "text/plain", }) if err != nil { log.Fatal(err) } log.Println("touched new file:", o) }
// handler is the main demo entry point that calls the GCS operations. func handler(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { http.NotFound(w, r) return } c := appengine.NewContext(r) bucketName, err := file.DefaultBucketName(c) if err != nil { c.Errorf("failed to get default GCS bucket name: %v", err) return } config := google.NewAppEngineConfig(c, storage.ScopeFullControl) ctx := cloud.NewContext(appengine.AppID(c), &http.Client{Transport: config.NewTransport()}) w.Header().Set("Content-Type", "text/plain; charset=utf-8") fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c)) fmt.Fprintf(w, "Using bucket name: %v\n\n", bucketName) d := &demo{ c: c, w: w, ctx: ctx, bucket: bucketName, } n := "demo-testfile-go" d.createFile(n) d.readFile(n) d.copyFile(n) d.statFile(n) d.createListFiles() d.listBucket() d.listBucketDirMode() d.defaultACL() d.putDefaultACLRule() d.deleteDefaultACLRule() d.bucketACL() d.putBucketACLRule() d.deleteBucketACLRule() d.acl(n) d.putACLRule(n) d.deleteACLRule(n) d.deleteFiles() if d.failed { io.WriteString(w, "\nDemo failed.\n") } else { io.WriteString(w, "\nDemo succeeded.\n") } }
func main() { var conf Config if err := envconfig.Process("gcsup", &conf); err != nil { fmt.Printf("Error with configuration [%s]", err) os.Exit(1) } data, err := ioutil.ReadFile(conf.JWTFileLocation) if err != nil { fmt.Printf("Error reading file [%s]\n", err) os.Exit(1) } jwtConf, err := google.JWTConfigFromJSON(data, storage.ScopeFullControl) if err != nil { fmt.Printf("Error creating JWT config [%s]\n", err) os.Exit(1) } fmt.Printf("Uploading %s to gcs://%s/%s\n", conf.LocalFolder, conf.ProjectName, conf.BucketName) ctx := cloud.NewContext(conf.ProjectName, jwtConf.Client(oauth2.NoContext)) client, err := storage.NewClient(ctx) if err != nil { fmt.Printf("Error creating GCS client [%s]\n", err) os.Exit(1) } bucket := client.Bucket(conf.BucketName) files, err := getAllFiles(conf.LocalFolder) if err != nil { fmt.Printf("Error gathering all files [%s]\n", err) os.Exit(1) } var wg sync.WaitGroup for _, file := range files { from := file.AbsolutePath to := strings.TrimPrefix(file.RelativePath, conf.LocalFolder) fmt.Printf("Uploading %s to %s\n", from, to) wg.Add(1) go func(ctx context.Context, bucket *storage.BucketHandle, conf Config, from, to string) { defer wg.Done() if err := upload(ctx, client.Bucket(conf.BucketName), conf, from, to); err != nil { fmt.Printf("Error uploading %s to %s (%s)\n", from, to, err) } }(ctx, bucket, conf, from, to) } fmt.Println("Waiting for all uploads to finish...") wg.Wait() fmt.Println("Done") }
func Example_copyObjects() { // see the auth example how to initiate a context. ctx := cloud.NewContext("project-id", &http.Client{Transport: nil}) o, err := storage.Copy(ctx, "bucketname", "file1", &storage.Object{ Name: "file2", Bucket: "yet-another-bucketname", }) if err != nil { log.Fatal(err) } log.Println("copied file:", o) }
func fileHandler(c *gin.Context) { username := c.Param("username") title := c.Param("title") filename := c.Param("fileName") gaeContext := appengine.NewContext(c.Request) hc := &http.Client{ Transport: &CloudStorageTransport{&oauth2.Transport{ Source: google.AppEngineTokenSource(gaeContext, storage.ScopeFullControl), Base: &urlfetch.Transport{Context: gaeContext}, }}, } bucketName := "balde_de_bits" bucketFile := username + "/" + title + "/" + filename log.Errorf(gaeContext, "ID ->>> %v", appengine.AppID(gaeContext)) log.Errorf(gaeContext, "File name ->>> %v", bucketFile) ctx := cloud.NewContext(appengine.AppID(gaeContext), hc) wc := storage.NewWriter(ctx, bucketName, bucketFile) if strings.Contains(filename, "m3u8") { wc.ContentType = "application/x-mpegURL" wc.CacheControl = "max-age:0" } else if strings.Contains(filename, "ts") { wc.ContentType = "video/MP2T" } else if strings.Contains(filename, "jpg") { wc.ContentType = "image/jpeg" } defer wc.Close() bytesWritten, err := io.Copy(wc, c.Request.Body) if err != nil { log.Errorf(gaeContext, "Writing to cloud storage failed. %v", err.Error()) c.JSON(200, gin.H{ "response": "< FAILED >", }) return } log.Errorf(gaeContext, "Wrote < %v > bytes for file < %v >", bytesWritten, filename) c.JSON(200, gin.H{ "response": "< worked >", }) }
func initGCE() error { initGCECalled = true // Use the staging project if not on GCE. This assumes the DefaultTokenSource // credential used below has access to that project. if !metadata.OnGCE() { projectID = stagingProjectID } var err error projectID, err = metadata.ProjectID() if err != nil { return fmt.Errorf("failed to get current GCE ProjectID: %v", err) } inStaging = projectID == stagingProjectID if inStaging { log.Printf("Running in staging cluster (%q)", projectID) } tokenSource, _ = google.DefaultTokenSource(oauth2.NoContext) httpClient := oauth2.NewClient(oauth2.NoContext, tokenSource) serviceCtx = cloud.NewContext(projectID, httpClient) projectZone, err = metadata.Get("instance/zone") if err != nil || projectZone == "" { return fmt.Errorf("failed to get current GCE zone: %v", err) } // Convert the zone from "projects/1234/zones/us-central1-a" to "us-central1-a". projectZone = path.Base(projectZone) if !hasComputeScope() { return errors.New("The coordinator is not running with access to read and write Compute resources. VM support disabled.") } projectRegion = projectZone[:strings.LastIndex(projectZone, "-")] // "us-central1" externalIP, err = metadata.ExternalIP() if err != nil { return fmt.Errorf("ExternalIP: %v", err) } computeService, _ = compute.New(httpClient) errTryDeps = checkTryBuildDeps() if errTryDeps != nil { log.Printf("TryBot builders disabled due to error: %v", errTryDeps) } else { log.Printf("TryBot builders enabled.") } go gcePool.pollQuotaLoop() return nil }
func connect() (ctx context.Context, err error) { secret, err := ioutil.ReadFile(secretFile) if err != nil { return } config, err := google.JWTConfigFromJSON(secret, datastore.ScopeDatastore, datastore.ScopeUserEmail) if err != nil { return } ctx = cloud.NewContext("coduno", config.Client(oauth2.NoContext)) return }
func (g Config) contextFromJSON(scopes ...string) (context.Context, error) { jsonKey, err := ioutil.ReadFile(g.JSONAuthPath) if err != nil { return nil, err } conf, err := google.JWTConfigFromJSON( jsonKey, scopes..., ) if err != nil { return nil, err } return cloud.NewContext(g.ProjectID, conf.Client(oauth2.NoContext)), nil }