func (d *demo) listFiles() {
io.WriteString(d.res, "\nRETRIEVING FILE NAMES\n")
client, err := storage.NewClient(d.ctx)
if err != nil {
log.Errorf(d.ctx, "%v", err)
return
}
defer client.Close()
// create a query
q := storage.Query{
MaxResults: 10,
}
objs, err := client.Bucket(gcsBucket).List(d.ctx, &q)
if err != nil {
log.Errorf(d.ctx, "%v", err)
return
}
for _, obj := range objs.Results {
io.WriteString(d.res, obj.Name+"\n")
}
}
func main(response http.ResponseWriter, request *http.Request) {
if request.URL.Path != "/" {
http.NotFound(response, request)
return
}
ctx := appengine.NewContext(request)
client, err := storage.NewClient(ctx)
if err != nil {
log.Errorf(ctx, "*** ERROR: main NewClient: ", err, " ***")
}
defer client.Close()
a := &app{
ctx: ctx,
response: response,
request: request,
bucket: client.Bucket(bucket),
client: client,
}
a.createFiles()
a.showFiles()
a.query_delimiter()
}
func handler(w http.ResponseWriter, r *http.Request) {
context := appengine.NewContext(r)
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
client, err := storage.NewClient(context)
if err != nil {
log.Errorf(context, "error handler newClient: ", err)
return
}
defer client.Close()
d := &demo{
context: context,
w: w,
client: client,
bucket: client.Bucket(gcsBucket),
}
d.createFiles()
d.listFiles()
io.WriteString(d.w, "\nResult From ListDir - With Delimiter\n")
d.listDir()
}
// getInstalledTLS returns the TLS certificate and key stored on Google Cloud Storage for the
// instance defined in d.Conf.
//
// If either the TLS keypair doesn't exist, the error is os.ErrNotExist.
func (d *Deployer) getInstalledTLS() (certPEM, keyPEM []byte, err error) {
ctx := context.Background()
stoClient, err := cloudstorage.NewClient(ctx, cloud.WithBaseHTTP(d.Client))
if err != nil {
return nil, nil, fmt.Errorf("error creating Cloud Storage client to fetch TLS cert & key from new instance: %v", err)
}
getFile := func(name string) ([]byte, error) {
sr, err := stoClient.Bucket(d.Conf.bucketBase()).Object(path.Join(configDir, name)).NewReader(ctx)
if err == cloudstorage.ErrObjectNotExist {
return nil, os.ErrNotExist
}
if err != nil {
return nil, err
}
defer sr.Close()
return ioutil.ReadAll(sr)
}
var grp syncutil.Group
grp.Go(func() (err error) {
certPEM, err = getFile(certFilename())
return
})
grp.Go(func() (err error) {
keyPEM, err = getFile(keyFilename())
return
})
err = grp.Err()
return
}
func handler(res http.ResponseWriter, req *http.Request) {
if req.URL.Path != "/" {
http.NotFound(res, req)
return
}
ctx := appengine.NewContext(req)
client, err := storage.NewClient(ctx)
if err != nil {
log.Errorf(ctx, "ERROR handler NewClient: ", err)
return
}
defer client.Close()
d := &gcsDelimit{
ctx: ctx,
res: res,
client: client,
bucket: client.Bucket(gcsBucket),
}
d.createFiles()
d.listFiles()
io.WriteString(d.res, "\nResults from list directory **with** delimiter\n")
d.listDir("", "/", " ")
io.WriteString(d.res, "\nResults from list directory **without** delimiter\n")
d.listDir("", "", " ")
}
func (s *Session) listBucket() {
client, err := storage.NewClient(s.ctx)
if err != nil {
log.Errorf(s.ctx, "ERROR listBucket storage.NewClient: %v", err)
return
}
defer client.Close()
q := &storage.Query{
Prefix: s.ID,
}
objs, err := client.Bucket(gcsBucket).List(s.ctx, q)
if err != nil {
log.Errorf(s.ctx, "ERROR listBucket client.Bucket: %v", err)
return
}
for _, obj := range objs.Results {
s.Pictures[obj.Name] = obj.Name
}
s.putSession()
}
func listBucket(ctx context.Context, bucketName, folder string) ([]string, []string, error) {
client, err := storage.NewClient(ctx)
if err != nil {
return nil, nil, err
}
defer client.Close()
var files, folders []string
query := &storage.Query{
Delimiter: "/",
Prefix: folder,
}
// objs is *storage.Objects
objs, err := client.Bucket(bucketName).List(ctx, query)
if err != nil {
return nil, nil, err
}
for _, subfolder := range objs.Prefixes {
folders = append(folders, subfolder[len(folder):])
}
for _, obj := range objs.Results {
files = append(files, obj.Name)
}
return files, folders, nil
}
func handler(res http.ResponseWriter, req *http.Request) {
// Saving the file
ctx := appengine.NewContext(req)
client, err := storage.NewClient(ctx)
logStorageError(ctx, "Could not create a new client", err)
defer client.Close()
writer := client.Bucket(BUCKET_NAME).Object(FILE_NAME).NewWriter(ctx)
writer.ACL = []storage.ACLRule{{
storage.AllUsers,
storage.RoleReader}}
// Reading the file from disk
reader, err := os.Open(FILE_NAME)
logError(err)
io.Copy(writer, reader)
writer.Close()
// Reading the file
rd, err := client.Bucket(BUCKET_NAME).Object(FILE_NAME).NewReader(ctx)
logError(err)
io.Copy(res, rd)
defer rd.Close()
}
func Example_auth() {
// Initialize an authorized context with Google Developers Console
// JSON key. Read the google package examples to learn more about
// different authorization flows you can use.
// http://godoc.org/golang.org/x/oauth2/google
jsonKey, err := ioutil.ReadFile("/path/to/json/keyfile.json")
if err != nil {
log.Fatal(err)
}
conf, err := google.JWTConfigFromJSON(
jsonKey,
storage.ScopeFullControl,
)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
client, err := storage.NewClient(ctx, cloud.WithTokenSource(conf.TokenSource(ctx)))
if err != nil {
log.Fatal(err)
}
// Use the client (see other examples)
doSomething(client)
// After using the client, free any resources (e.g. network connections).
client.Close()
}
func (d *demo) statFiles() {
io.WriteString(d.res, "\nRETRIEVING FILE STATS\n")
client, err := storage.NewClient(d.ctx)
if err != nil {
log.Errorf(d.ctx, "%v", err)
return
}
defer client.Close()
// create a query
q := storage.Query{
MaxResults: 2,
}
// instead of nil
// now passing in a *storage.Query
objs, err := client.Bucket(gcsBucket).List(d.ctx, &q)
if err != nil {
log.Errorf(d.ctx, "%v", err)
return
}
for _, v := range objs.Results {
d.statFile(v.Name)
}
}
func GetNewStorageClient() (*StorageClient, error) {
client, err := storage.NewClient(ctx, cloud.WithTokenSource(defaultTokenSource))
return &StorageClient{
Client: client,
Ctx: ctx,
}, err
}
func handler(res http.ResponseWriter, req *http.Request) {
if req.URL.Path != "/" {
http.NotFound(res, req)
return
}
ctx := appengine.NewContext(req)
client, err := storage.NewClient(ctx)
if err != nil {
log.Errorf(ctx, "ERROR handler NewClient: ", err)
return
}
defer client.Close()
d := &demo{
ctx: ctx,
res: res,
client: client,
bucket: client.Bucket(gcsBucket),
}
res.Header().Set("Content-Language", "en")
d.createFiles()
d.listFiles()
}
func main() {
log.SetFlags(0)
plugin.Param("workspace", &workspace)
plugin.Param("build", &build)
plugin.Param("repo", &repo)
plugin.Param("vargs", &vargs)
plugin.MustParse()
sort.Strings(vargs.Gzip) // need for matchGzip
// context for all clients
ctx := context.Background()
// GitHub client
gts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: vargs.GitHubToken})
client.ghub = github.NewClient(oauth2.NewClient(ctx, gts))
// GCS client
auth, err := google.JWTConfigFromJSON([]byte(vargs.AuthKey), storage.ScopeFullControl)
if err != nil {
fatalf("auth: %v", err)
}
tsrc := auth.TokenSource(ctx)
client.gcs, err = storage.NewClient(ctx, cloud.WithTokenSource(auth.TokenSource(ctx)))
if err != nil {
fatalf("storage client: %v", err)
}
// http client with service account authorization
client.http = oauth2.NewClient(ctx, tsrc)
run()
if ecode != 0 {
msg := fmt.Sprintf("exited with code %d", ecode)
updateStatus("error", msg, stagingURL)
}
os.Exit(ecode)
}
// Store binary data to GCS
func Store(c context.Context, data []byte, fileName, mimeType, bucketName string) (absFilename string, err error) {
if bucketName == "" {
var err error
if bucketName, err = file.DefaultBucketName(c); err != nil {
log.Errorf(c, "failed to get default GCS bucket name: %v", err)
return "", err
}
}
client, err := storage.NewClient(c)
if err != nil {
log.Errorf(c, "failed to create storage client: %v", err)
return "", err
}
defer client.Close()
wc := client.Bucket(bucketName).Object(fileName).NewWriter(c)
wc.ContentType = mimeType
if _, err := wc.Write(data); err != nil {
log.Errorf(c, "upload file: unable to write data to bucket %q, file %q: %v", bucketName, fileName, err)
return "", err
}
if err := wc.Close(); err != nil {
log.Errorf(c, "upload file: unable to close bucket %q, file %q: %v", bucketName, fileName, err)
return "", err
}
return getAbsFilename(bucketName, fileName), nil
}
func tlsCertFromGCS(ctx context.Context) (*tls.Certificate, error) {
sc, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
slurp := func(key string) ([]byte, error) {
const bucket = "camlistore-website-resource"
rc, err := sc.Bucket(bucket).Object(key).NewReader(ctx)
if err != nil {
return nil, fmt.Errorf("Error fetching GCS object %q in bucket %q: %v", key, bucket, err)
}
defer rc.Close()
return ioutil.ReadAll(rc)
}
certPem, err := slurp("ssl.crt")
if err != nil {
return nil, err
}
keyPem, err := slurp("ssl.key")
if err != nil {
return nil, err
}
cert, err := tls.X509KeyPair(certPem, keyPem)
if err != nil {
return nil, err
}
return &cert, nil
}
func newGoogleReplicaAPIServer(ctx context.Context, bucket string) (*googleReplicaAPIServer, error) {
client, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
return &googleReplicaAPIServer{ctx, client.Bucket(bucket)}, nil
}
func init() {
if !metadata.OnGCE() {
return
}
hc, err := google.DefaultClient(oauth2.NoContext)
if err != nil {
registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
return
}
projID, err := metadata.ProjectID()
if projID == "" || err != nil {
registerBrokenFS(fmt.Errorf("could not get GCE project ID: %v", err))
return
}
ctx := cloud.NewContext(projID, hc)
sc, err := storage.NewClient(ctx)
if err != nil {
registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
return
}
wkfs.RegisterFS("/gcs/", &gcsFS{
ctx: ctx,
sc: sc,
})
}
func handler(res http.ResponseWriter, req *http.Request) {
if req.URL.Path != "/" {
http.NotFound(res, req)
return
}
ctx := appengine.NewContext(req)
client, err := storage.NewClient(ctx)
if err != nil {
log.Errorf(ctx, "ERROR handler NewClient: ", err)
return
}
defer client.Close()
d := &demo{
ctx: ctx,
res: res,
client: client,
bucket: client.Bucket(gcsBucket),
}
d.createFiles()
d.listFiles()
io.WriteString(d.res, "\nFILE NAMES WITH DELIMITER QUERY ( Delimeter: / )\n")
d.listDelim()
}
func get(ctx context.Context, fname string) (io.ReadCloser, error) {
client, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
defer client.Close()
return client.Bucket(bucket).Object(fname).NewReader(ctx)
}
func configureStorage(bucketID string) (*storage.BucketHandle, error) {
ctx := context.Background()
client, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
return client.Bucket(bucketID), nil
}
func upload(srcPath string) {
if !*flagUpload {
return
}
destName := strings.Replace(filepath.Base(srcPath), "camlistore", "camlistore-"+rev(), 1)
versionedTarball := "monthly/" + destName
log.Printf("Uploading %s/%s ...", bucket, versionedTarball)
ts, err := tokenSource(bucket)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
if err != nil {
log.Fatal(err)
}
w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
w.ACL = publicACL(project)
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
contentType := "application/x-gtar"
if strings.HasSuffix(versionedTarball, ".zip") {
contentType = "application/zip"
}
w.ContentType = contentType
csw := sha256.New()
mw := io.MultiWriter(w, csw)
src, err := os.Open(srcPath)
if err != nil {
log.Fatal(err)
}
defer src.Close()
if _, err := io.Copy(mw, src); err != nil {
log.Fatalf("io.Copy: %v", err)
}
if err := w.Close(); err != nil {
log.Fatalf("closing GCS storage writer: %v", err)
}
log.Printf("Uploaded monthly tarball to %s", versionedTarball)
// And upload the corresponding checksum
checkSumFile := versionedTarball + ".sha256"
sum := fmt.Sprintf("%x", csw.Sum(nil))
w = stoClient.Bucket(bucket).Object(checkSumFile).NewWriter(ctx)
w.ACL = publicACL(project)
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
w.ContentType = "text/plain"
if _, err := io.Copy(w, strings.NewReader(sum)); err != nil {
log.Fatalf("error uploading checksum %v: %v", checkSumFile, err)
}
if err := w.Close(); err != nil {
log.Fatalf("closing GCS storage writer: %v", err)
}
log.Printf("Uploaded monthly tarball checksum to %s", checkSumFile)
}
func handler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
ctx := appengine.NewContext(r)
if bucket == "" {
var err error
if bucket, err = file.DefaultBucketName(ctx); err != nil {
log.Errorf(ctx, "error", err)
return
}
}
client, err := storage.NewClient(ctx)
if err != nil {
log.Errorf(ctx, "error", err)
return
}
defer client.Close()
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(ctx))
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)
d := &demo{
w: w,
ctx: ctx,
client: client,
bucket: client.Bucket(bucket),
}
n := "demo-testfile-go"
d.makeF(n)
d.rFile(n)
d.cFile(n)
d.statFile(n)
d.createListFiles()
d.listBucket()
d.listBucketDirMode()
d.defaultACL()
d.putDefaultACLRule()
d.deleteDefaultACLRule()
d.bucketACL()
d.putBucketACLRule()
d.deleteBucketACLRule()
d.acl(n)
d.putACLRule(n)
d.deleteACLRule(n)
d.deleteFiles()
if d.failed {
io.WriteString(w, "\nDemo failed.\n")
} else {
io.WriteString(w, "\nDemo succeeded.\n")
}
}
// Retrieve the file stored for the given name
func Retrieve(req *http.Request, userName, fileName string) (*storage.Reader, error) {
// Creating new context and client.
ctx := appengine.NewContext(req)
client, err := storage.NewClient(ctx)
log.LogErrorWithMsg("Cannot create a new client", err)
defer client.Close()
filePath := getFilePath(userName, fileName)
return client.Bucket(BUCKET_NAME).Object(filePath).NewReader(ctx)
}
// NewFS creates a new VFS backed by the given Google Cloud Storage bucket.
func NewFS(ctx context.Context, bucket string) (vfs.Interface, error) {
c, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
return &fs{
client: c,
bucket: c.Bucket(bucket),
}, nil
}
// uploadReleaseTarball uploads the generated tarball of binaries in
// camlistore-release/VERSION/camlistoreVERSION-REV-CONTENTS.EXT. It then makes a copy in
// the same bucket and path, as camlistoreVERSION-CONTENTS.EXT.
func uploadReleaseTarball() {
proj := "camlistore-website"
bucket := "camlistore-release"
tarball := *flagVersion + "/" + filepath.Base(releaseTarball)
versionedTarball := strings.Replace(tarball, "camlistore"+*flagVersion, "camlistore"+*flagVersion+"-"+rev(), 1)
log.Printf("Uploading %s/%s ...", bucket, versionedTarball)
ts, err := tokenSource(bucket)
if err != nil {
log.Fatal(err)
}
ctx := context.Background()
stoClient, err := storage.NewClient(ctx, cloud.WithTokenSource(ts), cloud.WithBaseHTTP(oauth2.NewClient(ctx, ts)))
if err != nil {
log.Fatal(err)
}
w := stoClient.Bucket(bucket).Object(versionedTarball).NewWriter(ctx)
w.ACL = publicACL(proj)
w.CacheControl = "no-cache" // TODO: remove for non-tip releases? set expirations?
contentType := "application/x-gtar"
if *buildOS == "windows" {
contentType = "application/zip"
}
w.ContentType = contentType
src, err := os.Open(releaseTarball)
if err != nil {
log.Fatal(err)
}
defer src.Close()
if _, err := io.Copy(w, src); err != nil {
log.Fatalf("io.Copy: %v", err)
}
if err := w.Close(); err != nil {
log.Fatalf("closing GCS storage writer: %v", err)
}
log.Printf("Uploaded tarball to %s", versionedTarball)
if !isWIP() {
log.Printf("Copying tarball to %s/%s ...", bucket, tarball)
dest := stoClient.Bucket(bucket).Object(tarball)
if _, err := stoClient.Bucket(bucket).Object(versionedTarball).CopyTo(
ctx,
dest,
&storage.ObjectAttrs{
ACL: publicACL(proj),
ContentType: contentType,
}); err != nil {
log.Fatalf("Error uploading %v: %v", tarball, err)
}
log.Printf("Uploaded tarball to %s", tarball)
}
}
func (s *Session) uploadPhoto() {
// retrieve the submitted file
mpf, hdr, err := s.req.FormFile("data")
if err != nil {
log.Errorf(s.ctx, "ERROR uploadPhoto s.req.FormFile: %s", err)
http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
return
}
defer mpf.Close()
// only allow jpeg or jpg
ext := hdr.Filename[strings.LastIndex(hdr.Filename, ".")+1:]
log.Infof(s.ctx, "FILE EXTENSION: %s", ext)
switch ext {
case "jpg", "jpeg":
log.Infof(s.ctx, "GOOD FILE EXTENSION: %s", ext)
default:
log.Errorf(s.ctx, "We do not allow files of type %s. We only allow jpg, jpeg extensions.", ext)
http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
return
}
// make a file name
h := sha1.New()
io.Copy(h, mpf)
name := fmt.Sprintf("%x", h.Sum(nil)) + `.` + ext
log.Infof(s.ctx, "FILE NAME: %s", name)
mpf.Seek(0, 0)
// put file
client, err := storage.NewClient(s.ctx)
if err != nil {
log.Errorf(s.ctx, "ERROR uploadPhoto storage.NewClient: %s", err)
http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
return
}
defer client.Close()
writer := client.Bucket(gcsBucket).Object(name).NewWriter(s.ctx)
writer.ACL = []storage.ACLRule{
{storage.AllUsers, storage.RoleReader},
}
io.Copy(writer, mpf)
err = writer.Close()
if err != nil {
log.Errorf(s.ctx, "ERROR uploadPhoto writer.Close: %s", err)
http.Redirect(s.res, s.req, `/?id=`+s.ID, http.StatusSeeOther)
return
}
// update session
s.Pictures[name] = name
s.putSession()
}
func handler(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
client, err := storage.NewClient(ctx)
logStorageError(ctx, "Could not create a new client", err)
defer client.Close()
tpl := template.Must(template.ParseFiles("index.html"))
err = tpl.Execute(res, getPhotoNames(ctx, client))
logError(err)
}
func handler(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
client, err := storage.NewClient(ctx)
StoreLogError(ctx, "unable to create", err)
defer client.Close()
tpl := template.Must(template.ParseFiles("index.html"))
err = tpl.Execute(res, picNames(ctx, client))
logError(err)
}
func setupOAuth() error {
client, err := auth.NewDefaultJWTServiceAccountClient(auth.SCOPE_READ_WRITE)
if err != nil {
return fmt.Errorf("Problem setting up client OAuth: %v", err)
}
if storageClient, err = storage.NewClient(context.Background(), cloud.WithBaseHTTP(client)); err != nil {
return fmt.Errorf("Problem authenticating: %v", err)
}
return nil
}
func newGoogleClient(ctx context.Context, bucket string) (*googleClient, error) {
client, err := storage.NewClient(
ctx,
cloud.WithTokenSource(google.ComputeTokenSource("")),
cloud.WithScopes(storage.ScopeFullControl),
)
if err != nil {
return nil, err
}
return &googleClient{ctx, client.Bucket(bucket)}, nil
}