func handlePut(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: ctx},
},
}
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
rdr, hdr, err := req.FormFile("form-upload-file")
if err != nil {
http.Error(res, "ERROR RECEIVING FILE: "+err.Error(), 500)
return
}
writer := storage.NewWriter(cctx, bucketName, hdr.Filename)
io.Copy(writer, rdr)
err = writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func newContext(projectID, jsonKey string) (context.Context, error) {
if projectID == "" {
return nil, errors.New("project id not provided")
}
if jsonKey == "" {
return nil, errors.New("JSON key not provided")
}
key, err := ioutil.ReadFile(jsonKey)
if err != nil {
return nil, err
}
conf, err := google.JWTConfigFromJSON(
key,
pubsub.ScopeCloudPlatform,
pubsub.ScopePubSub,
)
if err != nil {
return nil, err
}
ctx := cloud.NewContext(projectID, conf.Client(oauth2.NoContext))
return ctx, nil
}
func StartupHandler(w http.ResponseWriter, r *http.Request) {
if len(Config) == 0 {
c := appengine.NewContext(r)
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly),
Base: &urlfetch.Transport{
Context: c,
},
},
}
bucket, _ := file.DefaultBucketName(c)
ctx := cloud.NewContext("davine-web", client)
rc, err := storage.NewReader(ctx, bucket, "config.yaml")
if err != nil {
c.Errorf("error reading config: %v", err.Error())
return
}
configFile, err := ioutil.ReadAll(rc)
rc.Close()
if err != nil {
c.Errorf("error reading config: %v", err.Error())
return
}
c.Infof("loaded config file: %v", configFile)
yaml.Unmarshal(configFile, &Config)
c.Infof("loaded config struct: %v", Config)
}
}
// This example demonstrates calling the Cloud Pub/Sub API. As of 22
// Oct 2014, the Cloud Pub/Sub API is only available if you're
// whitelisted. If you're interested in using it, please apply for the
// Limited Preview program at the following form:
// http://goo.gl/Wql9HL
//
// Also, before running this example, be sure to enable Cloud Pub/Sub
// service on your project in Developer Console at:
// https://console.developers.google.com/
//
// Unless you run this sample on Compute Engine instance, please
// create a new service account and download a JSON key file for it at
// the developer console: https://console.developers.google.com/
//
// It has the following subcommands:
//
// create_topic <name>
// delete_topic <name>
// create_subscription <name> <linked_topic>
// delete_subscription <name>
// publish <topic> <message>
// pull_messages <subscription> <numworkers>
// publish_messages <topic> <numworkers>
//
// You can choose any names for topic and subscription as long as they
// follow the naming rule described at:
// https://cloud.google.com/pubsub/overview#names
//
// You can create/delete topics/subscriptions by self-explanatory
// subcommands.
//
// The "publish" subcommand is for publishing a single message to a
// specified Cloud Pub/Sub topic.
//
// The "pull_messages" subcommand is for continuously pulling messages
// from a specified Cloud Pub/Sub subscription with specified number
// of workers.
//
// The "publish_messages" subcommand is for continuously publishing
// messages to a specified Cloud Pub/Sub topic with specified number
// of workers.
func main() {
flag.Parse()
argv := flag.Args()
checkArgs(argv, 1)
client, err := newClient(*jsonFile)
if err != nil {
log.Fatalf("clientAndId failed, %v", err)
}
if *projID == "" {
usageAndExit("Please specify Project ID.")
}
ctx := cloud.NewContext(*projID, client)
m := map[string]func(ctx context.Context, argv []string){
"create_topic": createTopic,
"delete_topic": deleteTopic,
"create_subscription": createSubscription,
"delete_subscription": deleteSubscription,
"publish": publish,
"pull_messages": pullMessages,
"publish_messages": publishMessages,
}
subcommand := argv[0]
f, ok := m[subcommand]
if !ok {
usageAndExit(fmt.Sprintf("Function not found for %s", subcommand))
}
f(ctx, argv)
}
func TestNamespaceQuery(t *testing.T) {
gotNamespace := make(chan string, 1)
ctx := cloud.NewContext("nsQueryTest", &http.Client{Transport: &fakeTransport{
Handler: func(req, resp proto.Message) error {
gotNamespace <- req.(*pb.RunQueryRequest).GetPartitionId().GetNamespace()
return errors.New("not implemented")
},
}})
var gs []Gopher
NewQuery("Gopher").GetAll(ctx, &gs)
if got, want := <-gotNamespace, ""; got != want {
t.Errorf("GetAll: got namespace %q, want %q", got, want)
}
NewQuery("Gopher").Count(ctx)
if got, want := <-gotNamespace, ""; got != want {
t.Errorf("Count: got namespace %q, want %q", got, want)
}
const ns = "not_default"
ctx = WithNamespace(ctx, ns)
NewQuery("Gopher").GetAll(ctx, &gs)
if got, want := <-gotNamespace, ns; got != want {
t.Errorf("GetAll: got namespace %q, want %q", got, want)
}
NewQuery("Gopher").Count(ctx)
if got, want := <-gotNamespace, ns; got != want {
t.Errorf("Count: got namespace %q, want %q", got, want)
}
}
func Example_listObjects() {
// see the auth example how to initiate a context.
ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})
var query *storage.Query
for {
// If you are using this package on App Engine Managed VMs runtime,
// you can init a bucket client with your app's default bucket name.
// See http://godoc.org/google.golang.org/appengine/file#DefaultBucketName.
objects, err := storage.List(ctx, "bucketname", query)
if err != nil {
log.Fatal(err)
}
for _, obj := range objects.Results {
log.Printf("object name: %s, size: %v", obj.Name, obj.Size)
}
// if there are more results, objects.Next
// will be non-nil.
query = objects.Next
if query == nil {
break
}
}
log.Println("paginated through all object items in the bucket you specified.")
}
func NoAuthContext() context.Context {
projID := os.Getenv(envProjID)
if projID == "" {
log.Fatal("GCLOUD_TESTS_GOLANG_PROJECT_ID must be set. See CONTRIBUTING.md for details.")
}
return cloud.NewContext(projID, &http.Client{Transport: http.DefaultTransport})
}
func getCloudContext(aeCtx context.Context) (context.Context, error) {
data, err := ioutil.ReadFile("gcs.xxjson")
if err != nil {
return nil, err
}
conf, err := google.JWTConfigFromJSON(
data,
storage.ScopeFullControl,
)
if err != nil {
return nil, err
}
tokenSource := conf.TokenSource(aeCtx)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: tokenSource,
Base: &urlfetch.Transport{Context: aeCtx},
},
}
return cloud.NewContext(appengine.AppID(aeCtx), hc), nil
}
func init() {
if !metadata.OnGCE() {
return
}
hc, err := google.DefaultClient(oauth2.NoContext)
if err != nil {
registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
return
}
projID, err := metadata.ProjectID()
if projID == "" || err != nil {
registerBrokenFS(fmt.Errorf("could not get GCE project ID: %v", err))
return
}
ctx := cloud.NewContext(projID, hc)
sc, err := storage.NewClient(ctx)
if err != nil {
registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
return
}
wkfs.RegisterFS("/gcs/", &gcsFS{
ctx: ctx,
sc: sc,
})
}
func (this *Factory) NewGCSImageStore(conf map[string]string) ImageStore {
jsonKey, err := ioutil.ReadFile(conf["KeyFile"])
if err != nil {
log.Fatal(err)
}
cloudConf, err := google.JWTConfigFromJSON(
jsonKey,
gcs.ScopeFullControl,
)
if err != nil {
log.Fatal(err)
}
bucket := conf["BucketName"]
ctx := gcloud.NewContext(conf["AppID"], cloudConf.Client(oauth2.NoContext))
mapper := NewNamePathMapper(conf["NamePathRegex"], conf["NamePathMap"])
return NewGCSImageStore(
ctx,
bucket,
conf["StoreRoot"],
mapper,
)
}
// handler is the main demo entry point that calls the GCS operations.
func handler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
c := appengine.NewContext(r)
if bucket == "" {
var err error
if bucket, err = file.DefaultBucketName(c); err != nil {
log.Errorf(c, "failed to get default GCS bucket name: %v", err)
return
}
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
// Note that the App Engine urlfetch service has a limit of 10MB uploads and
// 32MB downloads.
// See https://cloud.google.com/appengine/docs/go/urlfetch/#Go_Quotas_and_limits
// for more information.
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.NewContext(appengine.AppID(c), hc)
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c))
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)
d := &demo{
c: c,
w: w,
ctx: ctx,
}
n := "demo-testfile-go"
d.createFile(n)
d.readFile(n)
d.copyFile(n)
d.statFile(n)
d.createListFiles()
d.listBucket()
d.listBucketDirMode()
d.defaultACL()
d.putDefaultACLRule()
d.deleteDefaultACLRule()
d.bucketACL()
d.putBucketACLRule()
d.deleteBucketACLRule()
d.acl(n)
d.putACLRule(n)
d.deleteACLRule(n)
d.deleteFiles()
if d.failed {
io.WriteString(w, "\nDemo failed.\n")
} else {
io.WriteString(w, "\nDemo succeeded.\n")
}
}
// TODO(djd): Delete this function when it's no longer used.
func Context(scopes ...string) context.Context {
ctx := oauth2.NoContext
ts := TokenSource(ctx, scopes...)
if ts == nil {
return nil
}
return cloud.NewContext(ProjID(), oauth2.NewClient(ctx, ts))
}
func TestEmptyAckID(t *testing.T) {
ctx := cloud.NewContext("project-id", &http.Client{})
id := []string{"test", ""}
err := Ack(ctx, "sub", id...)
if err == nil || !strings.Contains(err.Error(), "index 1") {
t.Errorf("Ack should report an error indicating the id is empty. Got: %v", err)
}
}
func getCloudContext(gae context.Context) context.Context {
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(gae, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: gae},
},
}
return cloud.NewContext(appengine.AppID(gae), hc)
}
func (bs *GCSBackupStorage) authContext() (context.Context, error) {
if bs.authCtx == nil {
client, err := google.DefaultClient(context.TODO())
if err != nil {
return nil, err
}
bs.authCtx = cloud.NewContext(*project, client)
}
return bs.authCtx, nil
}
func auth(r *http.Request) (context.Context, error) {
c := appengine.NewContext(r)
client, err := google.DefaultClient(c, storage.ScopeFullControl)
if err != nil {
return nil, err
}
ctx := cloud.NewContext(appengine.AppID(c), client)
return ctx, nil
}
func TestErrorOnObjectsInsertCall(t *testing.T) {
ctx := cloud.NewContext("project-id", &http.Client{
Transport: &fakeTransport{}})
wc := NewWriter(ctx, "bucketname", "filename1")
wc.ContentType = "text/plain"
if _, err := wc.Write([]byte("hello world")); err == nil {
t.Errorf("expected error on write, got nil")
}
if err := wc.Close(); err == nil {
t.Errorf("expected error on close, got nil")
}
}
// Cancel removes any written content from this FileWriter.
func (w *writer) Cancel() error {
w.closed = true
err := storageDeleteObject(cloud.NewContext(dummyProjectID, w.client), w.bucket, w.name)
if err != nil {
if status, ok := err.(*googleapi.Error); ok {
if status.Code == http.StatusNotFound {
err = nil
}
}
}
return err
}
func (g Config) contextFromToken(scopes ...string) (context.Context, error) {
conf, err := google.JWTConfigFromJSON(
[]byte(g.Token),
scopes...,
)
if err != nil {
log.Print("probs with token:", g.Token)
return nil, err
}
return cloud.NewContext(g.ProjectID, conf.Client(oauth2.NoContext)), nil
}
func getContext(jsonPrivateKeyFilepath string, projectID string) (context.Context, error) {
jsonKey, err := ioutil.ReadFile(jsonPrivateKeyFilepath)
if err != nil {
return nil, err
}
conf, err := google.JWTConfigFromJSON(jsonKey, storage.ScopeFullControl)
if err != nil {
return nil, err
}
ctx := cloud.NewContext(projectID, conf.Client(oauth2.NoContext))
return ctx, nil
}
func serviceContext() (context.Context, error) {
file := filepath.Join(os.Getenv("HOME"), "keys", "golang-org.service.json")
blob, err := ioutil.ReadFile(file)
if err != nil {
return nil, err
}
config, err := google.JWTConfigFromJSON(blob, storage.ScopeReadWrite)
if err != nil {
return nil, err
}
return cloud.NewContext(projectID, config.Client(context.Background())), nil
}
func (ds *Datastore) Init(_ string, project string) {
client, err := google.DefaultClient(oauth2.NoContext,
"https://www.googleapis.com/auth/devstorage.full_control")
if err != nil {
x.LogErr(log, err).Fatal("Unable to get client")
}
ds.ctx = cloud.NewContext(project, client)
if ds.ctx == nil {
log.Fatal("Failed to get context. context is nil")
}
log.Info("Connection to Google datastore established")
}
func Example_touchObjects() {
// see the auth example how to initiate a context.
ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})
o, err := storage.Put(ctx, "bucketname", "filename", &storage.Object{
ContentType: "text/plain",
})
if err != nil {
log.Fatal(err)
}
log.Println("touched new file:", o)
}
// handler is the main demo entry point that calls the GCS operations.
func handler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
c := appengine.NewContext(r)
bucketName, err := file.DefaultBucketName(c)
if err != nil {
c.Errorf("failed to get default GCS bucket name: %v", err)
return
}
config := google.NewAppEngineConfig(c, storage.ScopeFullControl)
ctx := cloud.NewContext(appengine.AppID(c), &http.Client{Transport: config.NewTransport()})
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c))
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucketName)
d := &demo{
c: c,
w: w,
ctx: ctx,
bucket: bucketName,
}
n := "demo-testfile-go"
d.createFile(n)
d.readFile(n)
d.copyFile(n)
d.statFile(n)
d.createListFiles()
d.listBucket()
d.listBucketDirMode()
d.defaultACL()
d.putDefaultACLRule()
d.deleteDefaultACLRule()
d.bucketACL()
d.putBucketACLRule()
d.deleteBucketACLRule()
d.acl(n)
d.putACLRule(n)
d.deleteACLRule(n)
d.deleteFiles()
if d.failed {
io.WriteString(w, "\nDemo failed.\n")
} else {
io.WriteString(w, "\nDemo succeeded.\n")
}
}
func main() {
var conf Config
if err := envconfig.Process("gcsup", &conf); err != nil {
fmt.Printf("Error with configuration [%s]", err)
os.Exit(1)
}
data, err := ioutil.ReadFile(conf.JWTFileLocation)
if err != nil {
fmt.Printf("Error reading file [%s]\n", err)
os.Exit(1)
}
jwtConf, err := google.JWTConfigFromJSON(data, storage.ScopeFullControl)
if err != nil {
fmt.Printf("Error creating JWT config [%s]\n", err)
os.Exit(1)
}
fmt.Printf("Uploading %s to gcs://%s/%s\n", conf.LocalFolder, conf.ProjectName, conf.BucketName)
ctx := cloud.NewContext(conf.ProjectName, jwtConf.Client(oauth2.NoContext))
client, err := storage.NewClient(ctx)
if err != nil {
fmt.Printf("Error creating GCS client [%s]\n", err)
os.Exit(1)
}
bucket := client.Bucket(conf.BucketName)
files, err := getAllFiles(conf.LocalFolder)
if err != nil {
fmt.Printf("Error gathering all files [%s]\n", err)
os.Exit(1)
}
var wg sync.WaitGroup
for _, file := range files {
from := file.AbsolutePath
to := strings.TrimPrefix(file.RelativePath, conf.LocalFolder)
fmt.Printf("Uploading %s to %s\n", from, to)
wg.Add(1)
go func(ctx context.Context, bucket *storage.BucketHandle, conf Config, from, to string) {
defer wg.Done()
if err := upload(ctx, client.Bucket(conf.BucketName), conf, from, to); err != nil {
fmt.Printf("Error uploading %s to %s (%s)\n", from, to, err)
}
}(ctx, bucket, conf, from, to)
}
fmt.Println("Waiting for all uploads to finish...")
wg.Wait()
fmt.Println("Done")
}
func Example_copyObjects() {
// see the auth example how to initiate a context.
ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})
o, err := storage.Copy(ctx, "bucketname", "file1", &storage.Object{
Name: "file2",
Bucket: "yet-another-bucketname",
})
if err != nil {
log.Fatal(err)
}
log.Println("copied file:", o)
}
func fileHandler(c *gin.Context) {
username := c.Param("username")
title := c.Param("title")
filename := c.Param("fileName")
gaeContext := appengine.NewContext(c.Request)
hc := &http.Client{
Transport: &CloudStorageTransport{&oauth2.Transport{
Source: google.AppEngineTokenSource(gaeContext, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: gaeContext},
}},
}
bucketName := "balde_de_bits"
bucketFile := username + "/" + title + "/" + filename
log.Errorf(gaeContext, "ID ->>> %v", appengine.AppID(gaeContext))
log.Errorf(gaeContext, "File name ->>> %v", bucketFile)
ctx := cloud.NewContext(appengine.AppID(gaeContext), hc)
wc := storage.NewWriter(ctx, bucketName, bucketFile)
if strings.Contains(filename, "m3u8") {
wc.ContentType = "application/x-mpegURL"
wc.CacheControl = "max-age:0"
} else if strings.Contains(filename, "ts") {
wc.ContentType = "video/MP2T"
} else if strings.Contains(filename, "jpg") {
wc.ContentType = "image/jpeg"
}
defer wc.Close()
bytesWritten, err := io.Copy(wc, c.Request.Body)
if err != nil {
log.Errorf(gaeContext, "Writing to cloud storage failed. %v", err.Error())
c.JSON(200, gin.H{
"response": "< FAILED >",
})
return
}
log.Errorf(gaeContext, "Wrote < %v > bytes for file < %v >", bytesWritten, filename)
c.JSON(200, gin.H{
"response": "< worked >",
})
}
func initGCE() error {
initGCECalled = true
// Use the staging project if not on GCE. This assumes the DefaultTokenSource
// credential used below has access to that project.
if !metadata.OnGCE() {
projectID = stagingProjectID
}
var err error
projectID, err = metadata.ProjectID()
if err != nil {
return fmt.Errorf("failed to get current GCE ProjectID: %v", err)
}
inStaging = projectID == stagingProjectID
if inStaging {
log.Printf("Running in staging cluster (%q)", projectID)
}
tokenSource, _ = google.DefaultTokenSource(oauth2.NoContext)
httpClient := oauth2.NewClient(oauth2.NoContext, tokenSource)
serviceCtx = cloud.NewContext(projectID, httpClient)
projectZone, err = metadata.Get("instance/zone")
if err != nil || projectZone == "" {
return fmt.Errorf("failed to get current GCE zone: %v", err)
}
// Convert the zone from "projects/1234/zones/us-central1-a" to "us-central1-a".
projectZone = path.Base(projectZone)
if !hasComputeScope() {
return errors.New("The coordinator is not running with access to read and write Compute resources. VM support disabled.")
}
projectRegion = projectZone[:strings.LastIndex(projectZone, "-")] // "us-central1"
externalIP, err = metadata.ExternalIP()
if err != nil {
return fmt.Errorf("ExternalIP: %v", err)
}
computeService, _ = compute.New(httpClient)
errTryDeps = checkTryBuildDeps()
if errTryDeps != nil {
log.Printf("TryBot builders disabled due to error: %v", errTryDeps)
} else {
log.Printf("TryBot builders enabled.")
}
go gcePool.pollQuotaLoop()
return nil
}
func connect() (ctx context.Context, err error) {
secret, err := ioutil.ReadFile(secretFile)
if err != nil {
return
}
config, err := google.JWTConfigFromJSON(secret, datastore.ScopeDatastore, datastore.ScopeUserEmail)
if err != nil {
return
}
ctx = cloud.NewContext("coduno", config.Client(oauth2.NoContext))
return
}
func (g Config) contextFromJSON(scopes ...string) (context.Context, error) {
jsonKey, err := ioutil.ReadFile(g.JSONAuthPath)
if err != nil {
return nil, err
}
conf, err := google.JWTConfigFromJSON(
jsonKey,
scopes...,
)
if err != nil {
return nil, err
}
return cloud.NewContext(g.ProjectID, conf.Client(oauth2.NoContext)), nil
}