Пример #1
1
func handlePut(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)

	hc := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
			Base:   &urlfetch.Transport{Context: ctx},
		},
	}

	cctx := cloud.NewContext(appengine.AppID(ctx), hc)

	rdr, hdr, err := req.FormFile("form-upload-file")
	if err != nil {
		http.Error(res, "ERROR RECEIVING FILE: "+err.Error(), 500)
		return
	}

	writer := storage.NewWriter(cctx, bucketName, hdr.Filename)
	io.Copy(writer, rdr)

	err = writer.Close()
	if err != nil {
		http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
		return
	}
}
Пример #2
0
func newContext(projectID, jsonKey string) (context.Context, error) {
	if projectID == "" {
		return nil, errors.New("project id not provided")
	}

	if jsonKey == "" {
		return nil, errors.New("JSON key not provided")
	}

	key, err := ioutil.ReadFile(jsonKey)
	if err != nil {
		return nil, err
	}

	conf, err := google.JWTConfigFromJSON(
		key,
		pubsub.ScopeCloudPlatform,
		pubsub.ScopePubSub,
	)
	if err != nil {
		return nil, err
	}

	ctx := cloud.NewContext(projectID, conf.Client(oauth2.NoContext))
	return ctx, nil
}
Пример #3
0
func StartupHandler(w http.ResponseWriter, r *http.Request) {
	if len(Config) == 0 {
		c := appengine.NewContext(r)
		client := &http.Client{
			Transport: &oauth2.Transport{
				Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly),
				Base: &urlfetch.Transport{
					Context: c,
				},
			},
		}
		bucket, _ := file.DefaultBucketName(c)
		ctx := cloud.NewContext("davine-web", client)
		rc, err := storage.NewReader(ctx, bucket, "config.yaml")
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}
		configFile, err := ioutil.ReadAll(rc)
		rc.Close()
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}

		c.Infof("loaded config file: %v", configFile)
		yaml.Unmarshal(configFile, &Config)
		c.Infof("loaded config struct: %v", Config)
	}
}
Пример #4
0
// This example demonstrates calling the Cloud Pub/Sub API. As of 22
// Oct 2014, the Cloud Pub/Sub API is only available if you're
// whitelisted. If you're interested in using it, please apply for the
// Limited Preview program at the following form:
// http://goo.gl/Wql9HL
//
// Also, before running this example, be sure to enable Cloud Pub/Sub
// service on your project in Developer Console at:
// https://console.developers.google.com/
//
// Unless you run this sample on Compute Engine instance, please
// create a new service account and download a JSON key file for it at
// the developer console: https://console.developers.google.com/
//
// It has the following subcommands:
//
//  create_topic <name>
//  delete_topic <name>
//  create_subscription <name> <linked_topic>
//  delete_subscription <name>
//  publish <topic> <message>
//  pull_messages <subscription> <numworkers>
//  publish_messages <topic> <numworkers>
//
// You can choose any names for topic and subscription as long as they
// follow the naming rule described at:
// https://cloud.google.com/pubsub/overview#names
//
// You can create/delete topics/subscriptions by self-explanatory
// subcommands.
//
// The "publish" subcommand is for publishing a single message to a
// specified Cloud Pub/Sub topic.
//
// The "pull_messages" subcommand is for continuously pulling messages
// from a specified Cloud Pub/Sub subscription with specified number
// of workers.
//
// The "publish_messages" subcommand is for continuously publishing
// messages to a specified Cloud Pub/Sub topic with specified number
// of workers.
func main() {
	flag.Parse()
	argv := flag.Args()
	checkArgs(argv, 1)
	client, err := newClient(*jsonFile)
	if err != nil {
		log.Fatalf("clientAndId failed, %v", err)
	}
	if *projID == "" {
		usageAndExit("Please specify Project ID.")
	}
	ctx := cloud.NewContext(*projID, client)
	m := map[string]func(ctx context.Context, argv []string){
		"create_topic":        createTopic,
		"delete_topic":        deleteTopic,
		"create_subscription": createSubscription,
		"delete_subscription": deleteSubscription,
		"publish":             publish,
		"pull_messages":       pullMessages,
		"publish_messages":    publishMessages,
	}
	subcommand := argv[0]
	f, ok := m[subcommand]
	if !ok {
		usageAndExit(fmt.Sprintf("Function not found for %s", subcommand))
	}
	f(ctx, argv)
}
Пример #5
0
func TestNamespaceQuery(t *testing.T) {
	gotNamespace := make(chan string, 1)
	ctx := cloud.NewContext("nsQueryTest", &http.Client{Transport: &fakeTransport{
		Handler: func(req, resp proto.Message) error {
			gotNamespace <- req.(*pb.RunQueryRequest).GetPartitionId().GetNamespace()
			return errors.New("not implemented")
		},
	}})

	var gs []Gopher

	NewQuery("Gopher").GetAll(ctx, &gs)
	if got, want := <-gotNamespace, ""; got != want {
		t.Errorf("GetAll: got namespace %q, want %q", got, want)
	}
	NewQuery("Gopher").Count(ctx)
	if got, want := <-gotNamespace, ""; got != want {
		t.Errorf("Count: got namespace %q, want %q", got, want)
	}

	const ns = "not_default"
	ctx = WithNamespace(ctx, ns)

	NewQuery("Gopher").GetAll(ctx, &gs)
	if got, want := <-gotNamespace, ns; got != want {
		t.Errorf("GetAll: got namespace %q, want %q", got, want)
	}
	NewQuery("Gopher").Count(ctx)
	if got, want := <-gotNamespace, ns; got != want {
		t.Errorf("Count: got namespace %q, want %q", got, want)
	}
}
Пример #6
0
func Example_listObjects() {
	// see the auth example how to initiate a context.
	ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})

	var query *storage.Query
	for {
		// If you are using this package on App Engine Managed VMs runtime,
		// you can init a bucket client with your app's default bucket name.
		// See http://godoc.org/google.golang.org/appengine/file#DefaultBucketName.
		objects, err := storage.List(ctx, "bucketname", query)
		if err != nil {
			log.Fatal(err)
		}
		for _, obj := range objects.Results {
			log.Printf("object name: %s, size: %v", obj.Name, obj.Size)
		}
		// if there are more results, objects.Next
		// will be non-nil.
		query = objects.Next
		if query == nil {
			break
		}
	}

	log.Println("paginated through all object items in the bucket you specified.")
}
Пример #7
0
func NoAuthContext() context.Context {
	projID := os.Getenv(envProjID)
	if projID == "" {
		log.Fatal("GCLOUD_TESTS_GOLANG_PROJECT_ID must be set. See CONTRIBUTING.md for details.")
	}
	return cloud.NewContext(projID, &http.Client{Transport: http.DefaultTransport})
}
Пример #8
0
func getCloudContext(aeCtx context.Context) (context.Context, error) {
	data, err := ioutil.ReadFile("gcs.xxjson")
	if err != nil {
		return nil, err
	}

	conf, err := google.JWTConfigFromJSON(
		data,
		storage.ScopeFullControl,
	)
	if err != nil {
		return nil, err
	}

	tokenSource := conf.TokenSource(aeCtx)

	hc := &http.Client{
		Transport: &oauth2.Transport{
			Source: tokenSource,
			Base:   &urlfetch.Transport{Context: aeCtx},
		},
	}

	return cloud.NewContext(appengine.AppID(aeCtx), hc), nil
}
Пример #9
0
func init() {
	if !metadata.OnGCE() {
		return
	}
	hc, err := google.DefaultClient(oauth2.NoContext)
	if err != nil {
		registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
		return
	}
	projID, err := metadata.ProjectID()
	if projID == "" || err != nil {
		registerBrokenFS(fmt.Errorf("could not get GCE project ID: %v", err))
		return
	}
	ctx := cloud.NewContext(projID, hc)
	sc, err := storage.NewClient(ctx)
	if err != nil {
		registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
		return
	}
	wkfs.RegisterFS("/gcs/", &gcsFS{
		ctx: ctx,
		sc:  sc,
	})
}
Пример #10
0
func (this *Factory) NewGCSImageStore(conf map[string]string) ImageStore {
	jsonKey, err := ioutil.ReadFile(conf["KeyFile"])
	if err != nil {
		log.Fatal(err)
	}
	cloudConf, err := google.JWTConfigFromJSON(
		jsonKey,
		gcs.ScopeFullControl,
	)
	if err != nil {
		log.Fatal(err)
	}

	bucket := conf["BucketName"]

	ctx := gcloud.NewContext(conf["AppID"], cloudConf.Client(oauth2.NoContext))
	mapper := NewNamePathMapper(conf["NamePathRegex"], conf["NamePathMap"])

	return NewGCSImageStore(
		ctx,
		bucket,
		conf["StoreRoot"],
		mapper,
	)
}
Пример #11
0
// handler is the main demo entry point that calls the GCS operations.
func handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	c := appengine.NewContext(r)
	if bucket == "" {
		var err error
		if bucket, err = file.DefaultBucketName(c); err != nil {
			log.Errorf(c, "failed to get default GCS bucket name: %v", err)
			return
		}
	}
	hc := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
			// Note that the App Engine urlfetch service has a limit of 10MB uploads and
			// 32MB downloads.
			// See https://cloud.google.com/appengine/docs/go/urlfetch/#Go_Quotas_and_limits
			// for more information.
			Base: &urlfetch.Transport{Context: c},
		},
	}
	ctx := cloud.NewContext(appengine.AppID(c), hc)
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c))
	fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)

	d := &demo{
		c:   c,
		w:   w,
		ctx: ctx,
	}

	n := "demo-testfile-go"
	d.createFile(n)
	d.readFile(n)
	d.copyFile(n)
	d.statFile(n)
	d.createListFiles()
	d.listBucket()
	d.listBucketDirMode()
	d.defaultACL()
	d.putDefaultACLRule()
	d.deleteDefaultACLRule()
	d.bucketACL()
	d.putBucketACLRule()
	d.deleteBucketACLRule()
	d.acl(n)
	d.putACLRule(n)
	d.deleteACLRule(n)
	d.deleteFiles()

	if d.failed {
		io.WriteString(w, "\nDemo failed.\n")
	} else {
		io.WriteString(w, "\nDemo succeeded.\n")
	}
}
Пример #12
0
// TODO(djd): Delete this function when it's no longer used.
func Context(scopes ...string) context.Context {
	ctx := oauth2.NoContext
	ts := TokenSource(ctx, scopes...)
	if ts == nil {
		return nil
	}
	return cloud.NewContext(ProjID(), oauth2.NewClient(ctx, ts))
}
Пример #13
0
func TestEmptyAckID(t *testing.T) {
	ctx := cloud.NewContext("project-id", &http.Client{})
	id := []string{"test", ""}
	err := Ack(ctx, "sub", id...)

	if err == nil || !strings.Contains(err.Error(), "index 1") {
		t.Errorf("Ack should report an error indicating the id is empty. Got: %v", err)
	}
}
Пример #14
0
func getCloudContext(gae context.Context) context.Context {
	hc := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.AppEngineTokenSource(gae, storage.ScopeFullControl),
			Base:   &urlfetch.Transport{Context: gae},
		},
	}
	return cloud.NewContext(appengine.AppID(gae), hc)
}
Пример #15
0
func (bs *GCSBackupStorage) authContext() (context.Context, error) {
	if bs.authCtx == nil {
		client, err := google.DefaultClient(context.TODO())
		if err != nil {
			return nil, err
		}
		bs.authCtx = cloud.NewContext(*project, client)
	}
	return bs.authCtx, nil
}
Пример #16
0
func auth(r *http.Request) (context.Context, error) {
	c := appengine.NewContext(r)
	client, err := google.DefaultClient(c, storage.ScopeFullControl)
	if err != nil {
		return nil, err
	}

	ctx := cloud.NewContext(appengine.AppID(c), client)

	return ctx, nil
}
Пример #17
0
func TestErrorOnObjectsInsertCall(t *testing.T) {
	ctx := cloud.NewContext("project-id", &http.Client{
		Transport: &fakeTransport{}})
	wc := NewWriter(ctx, "bucketname", "filename1")
	wc.ContentType = "text/plain"
	if _, err := wc.Write([]byte("hello world")); err == nil {
		t.Errorf("expected error on write, got nil")
	}
	if err := wc.Close(); err == nil {
		t.Errorf("expected error on close, got nil")
	}
}
Пример #18
0
// Cancel removes any written content from this FileWriter.
func (w *writer) Cancel() error {
	w.closed = true
	err := storageDeleteObject(cloud.NewContext(dummyProjectID, w.client), w.bucket, w.name)
	if err != nil {
		if status, ok := err.(*googleapi.Error); ok {
			if status.Code == http.StatusNotFound {
				err = nil
			}
		}
	}
	return err
}
Пример #19
0
func (g Config) contextFromToken(scopes ...string) (context.Context, error) {
	conf, err := google.JWTConfigFromJSON(
		[]byte(g.Token),
		scopes...,
	)
	if err != nil {
		log.Print("probs with token:", g.Token)
		return nil, err
	}

	return cloud.NewContext(g.ProjectID, conf.Client(oauth2.NoContext)), nil
}
Пример #20
0
func getContext(jsonPrivateKeyFilepath string, projectID string) (context.Context, error) {
	jsonKey, err := ioutil.ReadFile(jsonPrivateKeyFilepath)
	if err != nil {
		return nil, err
	}
	conf, err := google.JWTConfigFromJSON(jsonKey, storage.ScopeFullControl)
	if err != nil {
		return nil, err
	}
	ctx := cloud.NewContext(projectID, conf.Client(oauth2.NoContext))
	return ctx, nil
}
Пример #21
0
func serviceContext() (context.Context, error) {
	file := filepath.Join(os.Getenv("HOME"), "keys", "golang-org.service.json")
	blob, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, err
	}
	config, err := google.JWTConfigFromJSON(blob, storage.ScopeReadWrite)
	if err != nil {
		return nil, err
	}
	return cloud.NewContext(projectID, config.Client(context.Background())), nil
}
Пример #22
0
func (ds *Datastore) Init(_ string, project string) {
	client, err := google.DefaultClient(oauth2.NoContext,
		"https://www.googleapis.com/auth/devstorage.full_control")
	if err != nil {
		x.LogErr(log, err).Fatal("Unable to get client")
	}
	ds.ctx = cloud.NewContext(project, client)
	if ds.ctx == nil {
		log.Fatal("Failed to get context. context is nil")
	}
	log.Info("Connection to Google datastore established")
}
Пример #23
0
func Example_touchObjects() {
	// see the auth example how to initiate a context.
	ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})

	o, err := storage.Put(ctx, "bucketname", "filename", &storage.Object{
		ContentType: "text/plain",
	})
	if err != nil {
		log.Fatal(err)
	}
	log.Println("touched new file:", o)
}
Пример #24
0
// handler is the main demo entry point that calls the GCS operations.
func handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	c := appengine.NewContext(r)

	bucketName, err := file.DefaultBucketName(c)
	if err != nil {
		c.Errorf("failed to get default GCS bucket name: %v", err)
		return
	}

	config := google.NewAppEngineConfig(c, storage.ScopeFullControl)
	ctx := cloud.NewContext(appengine.AppID(c), &http.Client{Transport: config.NewTransport()})

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c))
	fmt.Fprintf(w, "Using bucket name: %v\n\n", bucketName)

	d := &demo{
		c:      c,
		w:      w,
		ctx:    ctx,
		bucket: bucketName,
	}

	n := "demo-testfile-go"
	d.createFile(n)
	d.readFile(n)
	d.copyFile(n)
	d.statFile(n)
	d.createListFiles()
	d.listBucket()
	d.listBucketDirMode()
	d.defaultACL()
	d.putDefaultACLRule()
	d.deleteDefaultACLRule()
	d.bucketACL()
	d.putBucketACLRule()
	d.deleteBucketACLRule()
	d.acl(n)
	d.putACLRule(n)
	d.deleteACLRule(n)
	d.deleteFiles()

	if d.failed {
		io.WriteString(w, "\nDemo failed.\n")
	} else {
		io.WriteString(w, "\nDemo succeeded.\n")
	}
}
Пример #25
0
func main() {
	var conf Config
	if err := envconfig.Process("gcsup", &conf); err != nil {
		fmt.Printf("Error with configuration [%s]", err)
		os.Exit(1)
	}

	data, err := ioutil.ReadFile(conf.JWTFileLocation)
	if err != nil {
		fmt.Printf("Error reading file [%s]\n", err)
		os.Exit(1)
	}

	jwtConf, err := google.JWTConfigFromJSON(data, storage.ScopeFullControl)
	if err != nil {
		fmt.Printf("Error creating JWT config [%s]\n", err)
		os.Exit(1)
	}

	fmt.Printf("Uploading %s to gcs://%s/%s\n", conf.LocalFolder, conf.ProjectName, conf.BucketName)

	ctx := cloud.NewContext(conf.ProjectName, jwtConf.Client(oauth2.NoContext))
	client, err := storage.NewClient(ctx)
	if err != nil {
		fmt.Printf("Error creating GCS client [%s]\n", err)
		os.Exit(1)
	}
	bucket := client.Bucket(conf.BucketName)

	files, err := getAllFiles(conf.LocalFolder)
	if err != nil {
		fmt.Printf("Error gathering all files [%s]\n", err)
		os.Exit(1)
	}

	var wg sync.WaitGroup
	for _, file := range files {
		from := file.AbsolutePath
		to := strings.TrimPrefix(file.RelativePath, conf.LocalFolder)
		fmt.Printf("Uploading %s to %s\n", from, to)
		wg.Add(1)
		go func(ctx context.Context, bucket *storage.BucketHandle, conf Config, from, to string) {
			defer wg.Done()
			if err := upload(ctx, client.Bucket(conf.BucketName), conf, from, to); err != nil {
				fmt.Printf("Error uploading %s to %s (%s)\n", from, to, err)
			}
		}(ctx, bucket, conf, from, to)
	}
	fmt.Println("Waiting for all uploads to finish...")
	wg.Wait()
	fmt.Println("Done")
}
Пример #26
0
func Example_copyObjects() {
	// see the auth example how to initiate a context.
	ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})

	o, err := storage.Copy(ctx, "bucketname", "file1", &storage.Object{
		Name:   "file2",
		Bucket: "yet-another-bucketname",
	})
	if err != nil {
		log.Fatal(err)
	}
	log.Println("copied file:", o)
}
Пример #27
0
func fileHandler(c *gin.Context) {

	username := c.Param("username")
	title := c.Param("title")
	filename := c.Param("fileName")

	gaeContext := appengine.NewContext(c.Request)

	hc := &http.Client{
		Transport: &CloudStorageTransport{&oauth2.Transport{
			Source: google.AppEngineTokenSource(gaeContext, storage.ScopeFullControl),
			Base:   &urlfetch.Transport{Context: gaeContext},
		}},
	}

	bucketName := "balde_de_bits"
	bucketFile := username + "/" + title + "/" + filename

	log.Errorf(gaeContext, "ID ->>> %v", appengine.AppID(gaeContext))
	log.Errorf(gaeContext, "File name ->>> %v", bucketFile)

	ctx := cloud.NewContext(appengine.AppID(gaeContext), hc)
	wc := storage.NewWriter(ctx, bucketName, bucketFile)

	if strings.Contains(filename, "m3u8") {
		wc.ContentType = "application/x-mpegURL"
		wc.CacheControl = "max-age:0"
	} else if strings.Contains(filename, "ts") {
		wc.ContentType = "video/MP2T"
	} else if strings.Contains(filename, "jpg") {
		wc.ContentType = "image/jpeg"
	}

	defer wc.Close()

	bytesWritten, err := io.Copy(wc, c.Request.Body)

	if err != nil {
		log.Errorf(gaeContext, "Writing to cloud storage failed. %v", err.Error())
		c.JSON(200, gin.H{
			"response": "< FAILED >",
		})
		return
	}

	log.Errorf(gaeContext, "Wrote < %v > bytes for file < %v >", bytesWritten, filename)

	c.JSON(200, gin.H{
		"response": "< worked >",
	})
}
Пример #28
0
func initGCE() error {
	initGCECalled = true
	// Use the staging project if not on GCE. This assumes the DefaultTokenSource
	// credential used below has access to that project.
	if !metadata.OnGCE() {
		projectID = stagingProjectID
	}
	var err error
	projectID, err = metadata.ProjectID()
	if err != nil {
		return fmt.Errorf("failed to get current GCE ProjectID: %v", err)
	}

	inStaging = projectID == stagingProjectID
	if inStaging {
		log.Printf("Running in staging cluster (%q)", projectID)
	}

	tokenSource, _ = google.DefaultTokenSource(oauth2.NoContext)
	httpClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
	serviceCtx = cloud.NewContext(projectID, httpClient)

	projectZone, err = metadata.Get("instance/zone")
	if err != nil || projectZone == "" {
		return fmt.Errorf("failed to get current GCE zone: %v", err)
	}

	// Convert the zone from "projects/1234/zones/us-central1-a" to "us-central1-a".
	projectZone = path.Base(projectZone)
	if !hasComputeScope() {
		return errors.New("The coordinator is not running with access to read and write Compute resources. VM support disabled.")

	}
	projectRegion = projectZone[:strings.LastIndex(projectZone, "-")] // "us-central1"

	externalIP, err = metadata.ExternalIP()
	if err != nil {
		return fmt.Errorf("ExternalIP: %v", err)
	}
	computeService, _ = compute.New(httpClient)
	errTryDeps = checkTryBuildDeps()
	if errTryDeps != nil {
		log.Printf("TryBot builders disabled due to error: %v", errTryDeps)
	} else {
		log.Printf("TryBot builders enabled.")
	}

	go gcePool.pollQuotaLoop()
	return nil
}
Пример #29
0
func connect() (ctx context.Context, err error) {
	secret, err := ioutil.ReadFile(secretFile)
	if err != nil {
		return
	}

	config, err := google.JWTConfigFromJSON(secret, datastore.ScopeDatastore, datastore.ScopeUserEmail)
	if err != nil {
		return
	}

	ctx = cloud.NewContext("coduno", config.Client(oauth2.NoContext))
	return
}
Пример #30
0
func (g Config) contextFromJSON(scopes ...string) (context.Context, error) {
	jsonKey, err := ioutil.ReadFile(g.JSONAuthPath)
	if err != nil {
		return nil, err
	}
	conf, err := google.JWTConfigFromJSON(
		jsonKey,
		scopes...,
	)
	if err != nil {
		return nil, err
	}

	return cloud.NewContext(g.ProjectID, conf.Client(oauth2.NoContext)), nil
}