Example #1
0
func (d *demo) listFiles() {
	io.WriteString(d.res, "\nRETRIEVING FILE NAMES\n")

	client, err := storage.NewClient(d.ctx)
	if err != nil {
		log.Errorf(d.ctx, "%v", err)
		return
	}
	defer client.Close()

	// create a query
	q := storage.Query{
		MaxResults: 10,
	}

	objs, err := client.Bucket(gcsBucket).List(d.ctx, &q)
	if err != nil {
		log.Errorf(d.ctx, "%v", err)
		return
	}

	for _, obj := range objs.Results {
		io.WriteString(d.res, obj.Name+"\n")
	}
}
Example #2
0
func main(response http.ResponseWriter, request *http.Request) {
	if request.URL.Path != "/" {
		http.NotFound(response, request)
		return
	}

	ctx := appengine.NewContext(request)
	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "*** ERROR: main NewClient: ", err, " ***")
	}
	defer client.Close()

	a := &app{
		ctx:      ctx,
		response: response,
		request:  request,
		bucket:   client.Bucket(bucket),
		client:   client,
	}

	a.createFiles()
	a.showFiles()
	a.query_delimiter()
}
Example #3
0
func handler(w http.ResponseWriter, r *http.Request) {
	context := appengine.NewContext(r)

	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}

	client, err := storage.NewClient(context)
	if err != nil {
		log.Errorf(context, "error handler newClient: ", err)
		return
	}

	defer client.Close()

	d := &demo{
		context: context,
		w:       w,
		client:  client,
		bucket:  client.Bucket(gcsBucket),
	}

	d.createFiles()
	d.listFiles()
	io.WriteString(d.w, "\nResult From ListDir - With Delimiter\n")
	d.listDir()

}
Example #4
0
// getInstalledTLS returns the TLS certificate and key stored on Google Cloud Storage for the
// instance defined in d.Conf.
//
// If either the TLS keypair doesn't exist, the error is os.ErrNotExist.
func (d *Deployer) getInstalledTLS() (certPEM, keyPEM []byte, err error) {
	ctx := context.Background()
	stoClient, err := cloudstorage.NewClient(ctx, cloud.WithBaseHTTP(d.Client))
	if err != nil {
		return nil, nil, fmt.Errorf("error creating Cloud Storage client to fetch TLS cert & key from new instance: %v", err)
	}
	getFile := func(name string) ([]byte, error) {
		sr, err := stoClient.Bucket(d.Conf.bucketBase()).Object(path.Join(configDir, name)).NewReader(ctx)
		if err == cloudstorage.ErrObjectNotExist {
			return nil, os.ErrNotExist
		}
		if err != nil {
			return nil, err
		}
		defer sr.Close()
		return ioutil.ReadAll(sr)
	}
	var grp syncutil.Group
	grp.Go(func() (err error) {
		certPEM, err = getFile(certFilename())
		return
	})
	grp.Go(func() (err error) {
		keyPEM, err = getFile(keyFilename())
		return
	})
	err = grp.Err()
	return
}
Example #5
0
func handler(res http.ResponseWriter, req *http.Request) {

	if req.URL.Path != "/" {
		http.NotFound(res, req)
		return
	}
	ctx := appengine.NewContext(req)
	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "ERROR handler NewClient: ", err)
		return
	}
	defer client.Close()

	d := &gcsDelimit{
		ctx:    ctx,
		res:    res,
		client: client,
		bucket: client.Bucket(gcsBucket),
	}

	d.createFiles()
	d.listFiles()
	io.WriteString(d.res, "\nResults from list directory **with** delimiter\n")
	d.listDir("", "/", "  ")
	io.WriteString(d.res, "\nResults from list directory **without** delimiter\n")
	d.listDir("", "", "  ")

}
Example #6
0
func (s *Session) listBucket() {

	client, err := storage.NewClient(s.ctx)
	if err != nil {
		log.Errorf(s.ctx, "ERROR listBucket storage.NewClient: %v", err)
		return
	}
	defer client.Close()

	q := &storage.Query{
		Prefix: s.ID,
	}

	objs, err := client.Bucket(gcsBucket).List(s.ctx, q)
	if err != nil {
		log.Errorf(s.ctx, "ERROR listBucket client.Bucket: %v", err)
		return
	}

	for _, obj := range objs.Results {
		s.Pictures[obj.Name] = obj.Name
	}

	s.putSession()
}
Example #7
0
func listBucket(ctx context.Context, bucketName, folder string) ([]string, []string, error) {
	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, nil, err
	}
	defer client.Close()

	var files, folders []string

	query := &storage.Query{
		Delimiter: "/",
		Prefix:    folder,
	}
	// objs is *storage.Objects
	objs, err := client.Bucket(bucketName).List(ctx, query)
	if err != nil {
		return nil, nil, err
	}

	for _, subfolder := range objs.Prefixes {
		folders = append(folders, subfolder[len(folder):])
	}

	for _, obj := range objs.Results {
		files = append(files, obj.Name)
	}

	return files, folders, nil
}
Example #8
0
func handler(res http.ResponseWriter, req *http.Request) {

	// Saving the file
	ctx := appengine.NewContext(req)
	client, err := storage.NewClient(ctx)
	logStorageError(ctx, "Could not create a new client", err)
	defer client.Close()

	writer := client.Bucket(BUCKET_NAME).Object(FILE_NAME).NewWriter(ctx)
	writer.ACL = []storage.ACLRule{{
		storage.AllUsers,
		storage.RoleReader}}

	// Reading the file from disk
	reader, err := os.Open(FILE_NAME)
	logError(err)
	io.Copy(writer, reader)
	writer.Close()

	// Reading the file
	rd, err := client.Bucket(BUCKET_NAME).Object(FILE_NAME).NewReader(ctx)
	logError(err)
	io.Copy(res, rd)
	defer rd.Close()
}
Example #9
0
func Example_auth() {
	// Initialize an authorized context with Google Developers Console
	// JSON key. Read the google package examples to learn more about
	// different authorization flows you can use.
	// http://godoc.org/golang.org/x/oauth2/google
	jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
	if err != nil {
		log.Fatal(err)
	}
	conf, err := google.JWTConfigFromJSON(
		jsonKey,
		storage.ScopeFullControl,
	)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	client, err := storage.NewClient(ctx, cloud.WithTokenSource(conf.TokenSource(ctx)))
	if err != nil {
		log.Fatal(err)
	}

	// Use the client (see other examples)
	doSomething(client)

	// After using the client, free any resources (e.g. network connections).
	client.Close()
}
Example #10
0
func (d *demo) statFiles() {
	io.WriteString(d.res, "\nRETRIEVING FILE STATS\n")

	client, err := storage.NewClient(d.ctx)
	if err != nil {
		log.Errorf(d.ctx, "%v", err)
		return
	}
	defer client.Close()

	// create a query
	q := storage.Query{
		MaxResults: 2,
	}

	// instead of nil
	// now passing in a *storage.Query
	objs, err := client.Bucket(gcsBucket).List(d.ctx, &q)
	if err != nil {
		log.Errorf(d.ctx, "%v", err)
		return
	}

	for _, v := range objs.Results {
		d.statFile(v.Name)
	}
}
Example #11
0
func GetNewStorageClient() (*StorageClient, error) {
	client, err := storage.NewClient(ctx, cloud.WithTokenSource(defaultTokenSource))
	return &StorageClient{
		Client: client,
		Ctx:    ctx,
	}, err
}
Example #12
0
func handler(res http.ResponseWriter, req *http.Request) {

	if req.URL.Path != "/" {
		http.NotFound(res, req)
		return
	}

	ctx := appengine.NewContext(req)

	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "ERROR handler NewClient: ", err)
		return
	}
	defer client.Close()

	d := &demo{
		ctx:    ctx,
		res:    res,
		client: client,
		bucket: client.Bucket(gcsBucket),
	}

	res.Header().Set("Content-Language", "en")

	d.createFiles()
	d.listFiles()
}
Example #13
0
func main() {
	log.SetFlags(0)
	plugin.Param("workspace", &workspace)
	plugin.Param("build", &build)
	plugin.Param("repo", &repo)
	plugin.Param("vargs", &vargs)
	plugin.MustParse()
	sort.Strings(vargs.Gzip) // need for matchGzip

	// context for all clients
	ctx := context.Background()
	// GitHub client
	gts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: vargs.GitHubToken})
	client.ghub = github.NewClient(oauth2.NewClient(ctx, gts))
	// GCS client
	auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
	if err != nil {
		fatalf("auth: %v", err)
	}
	tsrc := auth.TokenSource(ctx)
	client.gcs, err = storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
	if err != nil {
		fatalf("storage client: %v", err)
	}
	// http client with service account authorization
	client.http = oauth2.NewClient(ctx, tsrc)

	run()
	if ecode != 0 {
		msg := fmt.Sprintf("exited with code %d", ecode)
		updateStatus("error", msg, stagingURL)
	}
	os.Exit(ecode)
}
Example #14
0
// Store binary data to GCS
func Store(c context.Context, data []byte, fileName, mimeType, bucketName string) (absFilename string, err error) {
	if bucketName == "" {
		var err error
		if bucketName, err = file.DefaultBucketName(c); err != nil {
			log.Errorf(c, "failed to get default GCS bucket name: %v", err)
			return "", err
		}
	}

	client, err := storage.NewClient(c)
	if err != nil {
		log.Errorf(c, "failed to create storage client: %v", err)
		return "", err
	}
	defer client.Close()

	wc := client.Bucket(bucketName).Object(fileName).NewWriter(c)
	wc.ContentType = mimeType

	if _, err := wc.Write(data); err != nil {
		log.Errorf(c, "upload file: unable to write data to bucket %q, file %q: %v", bucketName, fileName, err)
		return "", err
	}
	if err := wc.Close(); err != nil {
		log.Errorf(c, "upload file: unable to close bucket %q, file %q: %v", bucketName, fileName, err)
		return "", err
	}

	return getAbsFilename(bucketName, fileName), nil
}
Example #15
0
func tlsCertFromGCS(ctx context.Context) (*tls.Certificate, error) {
	sc, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}
	slurp := func(key string) ([]byte, error) {
		const bucket = "camlistore-website-resource"
		rc, err := sc.Bucket(bucket).Object(key).NewReader(ctx)
		if err != nil {
			return nil, fmt.Errorf("Error fetching GCS object %q in bucket %q: %v", key, bucket, err)
		}
		defer rc.Close()
		return ioutil.ReadAll(rc)
	}
	certPem, err := slurp("ssl.crt")
	if err != nil {
		return nil, err
	}
	keyPem, err := slurp("ssl.key")
	if err != nil {
		return nil, err
	}
	cert, err := tls.X509KeyPair(certPem, keyPem)
	if err != nil {
		return nil, err
	}
	return &cert, nil
}
Example #16
0
func newGoogleReplicaAPIServer(ctx context.Context, bucket string) (*googleReplicaAPIServer, error) {
	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}
	return &googleReplicaAPIServer{ctx, client.Bucket(bucket)}, nil
}
Example #17
0
func init() {
	if !metadata.OnGCE() {
		return
	}
	hc, err := google.DefaultClient(oauth2.NoContext)
	if err != nil {
		registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
		return
	}
	projID, err := metadata.ProjectID()
	if projID == "" || err != nil {
		registerBrokenFS(fmt.Errorf("could not get GCE project ID: %v", err))
		return
	}
	ctx := cloud.NewContext(projID, hc)
	sc, err := storage.NewClient(ctx)
	if err != nil {
		registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
		return
	}
	wkfs.RegisterFS("/gcs/", &gcsFS{
		ctx: ctx,
		sc:  sc,
	})
}
Example #18
0
func handler(res http.ResponseWriter, req *http.Request) {

	if req.URL.Path != "/" {
		http.NotFound(res, req)
		return
	}

	ctx := appengine.NewContext(req)

	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "ERROR handler NewClient: ", err)
		return
	}
	defer client.Close()

	d := &demo{
		ctx:    ctx,
		res:    res,
		client: client,
		bucket: client.Bucket(gcsBucket),
	}

	d.createFiles()
	d.listFiles()
	io.WriteString(d.res, "\nFILE NAMES WITH DELIMITER QUERY ( Delimeter: / )\n")
	d.listDelim()

}
Example #19
0
func get(ctx context.Context, fname string) (io.ReadCloser, error) {
	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}
	defer client.Close()
	return client.Bucket(bucket).Object(fname).NewReader(ctx)
}
Example #20
0
func configureStorage(bucketID string) (*storage.BucketHandle, error) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}
	return client.Bucket(bucketID), nil
}
Example #21
0
func upload(srcPath string) {
	if !*flagUpload {
		return
	}
	destName := strings.Replace(filepath.Base(srcPath), "camlistore", "camlistore-"+rev(), 1)
	versionedTarball := "monthly/" + destName

	log.Printf("Uploading %s/%s ...", bucket, versionedTarball)

	ts, err := tokenSource(bucket)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
	if err != nil {
		log.Fatal(err)
	}
	w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
	w.ACL = publicACL(project)
	w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
	contentType := "application/x-gtar"
	if strings.HasSuffix(versionedTarball, ".zip") {
		contentType = "application/zip"
	}
	w.ContentType = contentType
	csw := sha256.New()
	mw := io.MultiWriter(w, csw)

	src, err := os.Open(srcPath)
	if err != nil {
		log.Fatal(err)
	}
	defer src.Close()

	if _, err := io.Copy(mw, src); err != nil {
		log.Fatalf("io.Copy: %v", err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("closing GCS storage writer: %v", err)
	}
	log.Printf("Uploaded monthly tarball to %s", versionedTarball)

	// And upload the corresponding checksum
	checkSumFile := versionedTarball + ".sha256"
	sum := fmt.Sprintf("%x", csw.Sum(nil))
	w = stoClient.Bucket(bucket).Object(checkSumFile).NewWriter(ctx)
	w.ACL = publicACL(project)
	w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
	w.ContentType = "text/plain"
	if _, err := io.Copy(w, strings.NewReader(sum)); err != nil {
		log.Fatalf("error uploading checksum %v: %v", checkSumFile, err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("closing GCS storage writer: %v", err)
	}
	log.Printf("Uploaded monthly tarball checksum to %s", checkSumFile)
}
Example #22
0
func handler(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path != "/" {
		http.NotFound(w, r)
		return
	}
	ctx := appengine.NewContext(r)
	if bucket == "" {
		var err error
		if bucket, err = file.DefaultBucketName(ctx); err != nil {
			log.Errorf(ctx, "error", err)
			return
		}
	}

	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Errorf(ctx, "error", err)
		return
	}
	defer client.Close()

	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(ctx))
	fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)

	d := &demo{
		w:      w,
		ctx:    ctx,
		client: client,
		bucket: client.Bucket(bucket),
	}

	n := "demo-testfile-go"
	d.makeF(n)
	d.rFile(n)
	d.cFile(n)
	d.statFile(n)
	d.createListFiles()
	d.listBucket()
	d.listBucketDirMode()
	d.defaultACL()
	d.putDefaultACLRule()
	d.deleteDefaultACLRule()
	d.bucketACL()
	d.putBucketACLRule()
	d.deleteBucketACLRule()
	d.acl(n)
	d.putACLRule(n)
	d.deleteACLRule(n)
	d.deleteFiles()

	if d.failed {
		io.WriteString(w, "\nDemo failed.\n")
	} else {
		io.WriteString(w, "\nDemo succeeded.\n")
	}
}
Example #23
0
// Retrieve the file stored for the given name
func Retrieve(req *http.Request, userName, fileName string) (*storage.Reader, error) {
	// Creating new context and client.
	ctx := appengine.NewContext(req)
	client, err := storage.NewClient(ctx)
	log.LogErrorWithMsg("Cannot create a new client", err)
	defer client.Close()
	filePath := getFilePath(userName, fileName)
	return client.Bucket(BUCKET_NAME).Object(filePath).NewReader(ctx)
}
Example #24
0
// NewFS creates a new VFS backed by the given Google Cloud Storage bucket.
func NewFS(ctx context.Context, bucket string) (vfs.Interface, error) {
	c, err := storage.NewClient(ctx)
	if err != nil {
		return nil, err
	}
	return &fs{
		client: c,
		bucket: c.Bucket(bucket),
	}, nil
}
Example #25
0
// uploadReleaseTarball uploads the generated tarball of binaries in
// camlistore-release/VERSION/camlistoreVERSION-REV-CONTENTS.EXT. It then makes a copy in
// the same bucket and path, as camlistoreVERSION-CONTENTS.EXT.
func uploadReleaseTarball() {
	proj := "camlistore-website"
	bucket := "camlistore-release"
	tarball := *flagVersion + "/" + filepath.Base(releaseTarball)
	versionedTarball := strings.Replace(tarball, "camlistore"+*flagVersion, "camlistore"+*flagVersion+"-"+rev(), 1)

	log.Printf("Uploading %s/%s ...", bucket, versionedTarball)

	ts, err := tokenSource(bucket)
	if err != nil {
		log.Fatal(err)
	}
	ctx := context.Background()
	stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
	if err != nil {
		log.Fatal(err)
	}
	w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
	w.ACL = publicACL(proj)
	w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
	contentType := "application/x-gtar"
	if *buildOS == "windows" {
		contentType = "application/zip"
	}
	w.ContentType = contentType

	src, err := os.Open(releaseTarball)
	if err != nil {
		log.Fatal(err)
	}
	defer src.Close()

	if _, err := io.Copy(w, src); err != nil {
		log.Fatalf("io.Copy: %v", err)
	}
	if err := w.Close(); err != nil {
		log.Fatalf("closing GCS storage writer: %v", err)
	}
	log.Printf("Uploaded tarball to %s", versionedTarball)
	if !isWIP() {
		log.Printf("Copying tarball to %s/%s ...", bucket, tarball)
		dest := stoClient.Bucket(bucket).Object(tarball)
		if _, err := stoClient.Bucket(bucket).Object(versionedTarball).CopyTo(
			ctx,
			dest,
			&storage.ObjectAttrs{
				ACL:         publicACL(proj),
				ContentType: contentType,
			}); err != nil {
			log.Fatalf("Error uploading %v: %v", tarball, err)
		}
		log.Printf("Uploaded tarball to %s", tarball)
	}
}
Example #26
0
func (s *Session) uploadPhoto() {

	// retrieve the submitted file
	mpf, hdr, err := s.req.FormFile("data")
	if err != nil {
		log.Errorf(s.ctx, "ERROR uploadPhoto s.req.FormFile: %s", err)
		http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
		return
	}
	defer mpf.Close()

	// only allow jpeg or jpg
	ext := hdr.Filename[strings.LastIndex(hdr.Filename, ".")+1:]
	log.Infof(s.ctx, "FILE EXTENSION: %s", ext)
	switch ext {
	case "jpg", "jpeg":
		log.Infof(s.ctx, "GOOD FILE EXTENSION: %s", ext)
	default:
		log.Errorf(s.ctx, "We do not allow files of type %s. We only allow jpg, jpeg extensions.", ext)
		http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
		return
	}

	// make a file name
	h := sha1.New()
	io.Copy(h, mpf)
	name := fmt.Sprintf("%x", h.Sum(nil)) + `.` + ext
	log.Infof(s.ctx, "FILE NAME: %s", name)
	mpf.Seek(0, 0)

	// put file
	client, err := storage.NewClient(s.ctx)
	if err != nil {
		log.Errorf(s.ctx, "ERROR uploadPhoto storage.NewClient: %s", err)
		http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
		return
	}
	defer client.Close()
	writer := client.Bucket(gcsBucket).Object(name).NewWriter(s.ctx)
	writer.ACL = []storage.ACLRule{
		{storage.AllUsers, storage.RoleReader},
	}
	io.Copy(writer, mpf)
	err = writer.Close()
	if err != nil {
		log.Errorf(s.ctx, "ERROR uploadPhoto writer.Close: %s", err)
		http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
		return
	}

	// update session
	s.Pictures[name] = name
	s.putSession()
}
Example #27
0
func handler(res http.ResponseWriter, req *http.Request) {

	ctx := appengine.NewContext(req)
	client, err := storage.NewClient(ctx)
	logStorageError(ctx, "Could not create a new client", err)
	defer client.Close()

	tpl := template.Must(template.ParseFiles("index.html"))
	err = tpl.Execute(res, getPhotoNames(ctx, client))
	logError(err)
}
Example #28
0
func handler(res http.ResponseWriter, req *http.Request) {

	ctx := appengine.NewContext(req)
	client, err := storage.NewClient(ctx)
	StoreLogError(ctx, "unable to create", err)
	defer client.Close()

	tpl := template.Must(template.ParseFiles("index.html"))
	err = tpl.Execute(res, picNames(ctx, client))
	logError(err)
}
Example #29
0
func setupOAuth() error {
	client, err := auth.NewDefaultJWTServiceAccountClient(auth.SCOPE_READ_WRITE)
	if err != nil {
		return fmt.Errorf("Problem setting up client OAuth: %v", err)
	}

	if storageClient, err = storage.NewClient(context.Background(), cloud.WithBaseHTTP(client)); err != nil {
		return fmt.Errorf("Problem authenticating: %v", err)
	}
	return nil
}
Example #30
0
func newGoogleClient(ctx context.Context, bucket string) (*googleClient, error) {
	client, err := storage.NewClient(
		ctx,
		cloud.WithTokenSource(google.ComputeTokenSource("")),
		cloud.WithScopes(storage.ScopeFullControl),
	)
	if err != nil {
		return nil, err
	}
	return &googleClient{ctx, client.Bucket(bucket)}, nil
}