func handlePut(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: ctx},
},
}
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
rdr, hdr, err := req.FormFile("form-upload-file")
if err != nil {
http.Error(res, "ERROR RECEIVING FILE: "+err.Error(), 500)
return
}
writer := storage.NewWriter(cctx, bucketName, hdr.Filename)
io.Copy(writer, rdr)
err = writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
// PutContent stores the []byte content at a location designated by "path".
// This should primarily be used for small objects.
func (d *driver) PutContent(context ctx.Context, path string, contents []byte) error {
return retry(func() error {
wc := storage.NewWriter(d.context(context), d.bucket, d.pathToKey(path))
wc.ContentType = "application/octet-stream"
return putContentsClose(wc, contents)
})
}
func handlePut(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
cctx := getCloudContext(ctx)
if req.Method != "POST" {
http.Error(res, "Must send a file", http.StatusMethodNotAllowed)
return
}
file, hdr, err := req.FormFile("f")
if err != nil {
http.Error(res, err.Error(), 500)
return
}
filename := hdr.Filename
writer := storage.NewWriter(cctx, bucket, prefix+filename)
io.Copy(writer, file)
err = writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
http.Redirect(res, req, "/", http.StatusSeeOther)
}
func (s *googleReplicaAPIServer) PushDiff(pushDiffServer pfs.ReplicaAPI_PushDiffServer) (retErr error) {
defer func() {
if err := pushDiffServer.SendAndClose(google_protobuf.EmptyInstance); err != nil && retErr == nil {
retErr = err
}
}()
request, err := pushDiffServer.Recv()
if err != nil {
return err
}
writer := storage.NewWriter(s.ctx, s.bucket, fmt.Sprintf("%s/%s/%d", request.Commit.Repo.Name, request.Commit.Id, request.Shard))
defer func() {
if err := writer.Close(); err != nil && retErr == nil {
retErr = err
}
}()
reader := &pushDiffReader{
server: pushDiffServer,
}
_, err = reader.buffer.Write(request.Value)
if err != nil {
return err
}
_, err = io.Copy(writer, reader)
return err
}
// uploadFileFromForm uploads a file if it's present in the "image" form field.
func uploadFileFromForm(r *http.Request) (url string, err error) {
f, fh, err := r.FormFile("image")
if err == http.ErrMissingFile {
return "", nil
}
if err != nil {
return "", err
}
if bookshelf.StorageBucket == "" {
return "", errors.New("storage bucket is missing - check config.go")
}
// random filename, retaining existing extension.
name := uuid.NewV4().String() + path.Ext(fh.Filename)
w := storage.NewWriter(bookshelf.StorageCtx, bookshelf.StorageBucket, name)
w.ACL = []storage.ACLRule{{storage.AllUsers, storage.RoleReader}}
w.ContentType = fh.Header.Get("Content-Type")
// Entries are immutable, be aggressive about caching (1 day).
w.CacheControl = "public, max-age=86400"
if _, err := io.Copy(w, f); err != nil {
return "", err
}
if err := w.Close(); err != nil {
return "", err
}
const publicURL = "https://storage.googleapis.com/%s/%s"
return fmt.Sprintf(publicURL, bookshelf.StorageBucket, name), nil
}
func (p provider) Create(ctx context.Context, name string, maxAge time.Duration, contentType string) (storage.Object, error) {
i := strings.Index(name, "/")
if i == -1 {
return nil, errors.New("name invalid, must contain '/'")
}
b := name[:i]
n := name[i:]
w := s.NewWriter(ctx, name, name)
w.ObjectAttrs = s.ObjectAttrs{
ContentType: contentType,
ContentLanguage: "", // TODO(flowlo): Does it make sense to set this?
ContentEncoding: "utf-8",
CacheControl: "max-age=" + strconv.FormatFloat(maxAge.Seconds(), 'f', 0, 64),
ContentDisposition: "attachment; filename=\"" + path.Base(name) + "\"",
}
return &object{
b: b,
n: n,
w: w,
r: nil,
ctx: ctx,
}, nil
}
func (c *GoogleStorage) Post(obj *Object) (*Object, error) {
_, err := c.fetch(obj)
if err != nil {
log.Println("error on read url:", obj.Url, err)
return nil, err
}
// path = obj.path
wc := storage.NewWriter(c.ctx, c.bucket, obj.Path)
wc.ContentType = obj.MimeType
if _, err := wc.Write(obj.Content); err != nil {
log.Printf("error on write data: %s", err)
return nil, err
}
if err := wc.Close(); err != nil {
log.Printf("error on close writer: %s", err)
return nil, err
}
gcsObj := wc.Object()
newUrl := "https://storage.googleapis.com/" + c.bucket + "/" + gcsObj.Name
newObj := &Object{
Filename: obj.Filename,
Bucket: c.bucket,
Path: gcsObj.Name, // without bucket
MimeType: obj.MimeType,
Url: newUrl,
}
return newObj, nil
}
// PutContent stores the []byte content at a location designated by "path".
// This should primarily be used for small objects.
func (d *driver) PutContent(context ctx.Context, path string, contents []byte) error {
wc := storage.NewWriter(d.context(context), d.bucket, d.pathToKey(path))
wc.ContentType = "application/octet-stream"
defer wc.Close()
_, err := wc.Write(contents)
return err
}
// createFile creates a file in Google Cloud Storage.
func (d *demo) createFile(fileName string) {
fmt.Fprintf(d.w, "Creating file /%v/%v\n", d.bucket, fileName)
wc := storage.NewWriter(d.ctx, d.bucket, fileName, &storage.Object{
ContentType: "text/plain",
Metadata: map[string]string{
"x-goog-meta-foo": "foo",
"x-goog-meta-bar": "bar",
},
})
d.cleanUp = append(d.cleanUp, fileName)
if _, err := wc.Write([]byte("abcde\n")); err != nil {
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucket, fileName, err)
return
}
if _, err := wc.Write([]byte(strings.Repeat("f", 1024*4) + "\n")); err != nil {
d.errorf("createFile: unable to write data to bucket %q, file %q: %v", d.bucket, fileName, err)
return
}
if err := wc.Close(); err != nil {
d.errorf("createFile: unable to close bucket %q, file %q: %v", d.bucket, fileName, err)
return
}
// Wait for the file to be fully written.
_, err := wc.Object()
if err != nil {
d.errorf("createFile: unable to finalize file from bucket %q, file %q: %v", d.bucket, fileName, err)
return
}
}
func putFile(ctx context.Context, name string, rdr io.Reader) error {
cctx := getCloudContext(ctx)
writer := storage.NewWriter(cctx, bucketName, name)
writer.ACL = []storage.ACLRule{
{storage.AllUsers, storage.RoleReader},
}
io.Copy(writer, rdr)
return writer.Close()
}
func processImage(bucket, name string) error {
r, err := storage.NewReader(ctx, bucket, name)
if err != nil {
return fmt.Errorf("storage reader: %v", err)
}
img, err := ioutil.ReadAll(r)
r.Close()
if err != nil {
return fmt.Errorf("read image: %v", err)
}
wand := imagick.NewMagickWand()
defer wand.Destroy()
wand.ReadImageBlob(img)
if err := wand.SetImageFormat("WEBP"); err != nil {
return fmt.Errorf("set WEBP format: %v", err)
}
errc := make(chan error, len(sizes))
for suffix, size := range sizes {
go func(wand *imagick.MagickWand, suffix string, x, y uint) {
errc <- func() error {
defer wand.Destroy()
if err := wand.AdaptiveResizeImage(size.x, size.y); err != nil {
return fmt.Errorf("resize: %v", err)
}
target := name
if sep := strings.LastIndex(target, "."); sep >= 0 {
target = target[:sep]
}
target = fmt.Sprintf("%s_%s.webp", target, suffix)
w := storage.NewWriter(ctx, outputBucket, target, nil)
if _, err := w.Write(wand.GetImageBlob()); err != nil {
return fmt.Errorf("new writer: %v", err)
}
if err := w.Close(); err != nil {
return fmt.Errorf("close object writer: %v", err)
}
if _, err := w.Object(); err != nil {
return fmt.Errorf("write op: %v", err)
}
return nil
}()
}(wand.Clone(), suffix, size.x, size.y)
}
for _ = range sizes {
if err := <-errc; err != nil {
return err
}
}
return nil
}
func (bs *GCSBlobStore) OpenWriter(blobpath string) (io.WriteCloser, error) {
if !oflags.IsWriteAllowed(bs.flags) {
return nil, otaru.EPERM
}
ctx := bs.newAuthedContext(context.TODO())
gcsw := storage.NewWriter(ctx, bs.bucketName, blobpath)
gcsw.ContentType = "application/octet-stream"
return &Writer{gcsw}, nil
}
func (o *object) Write(p []byte) (n int, err error) {
if o.r != nil {
return 0, errors.New("object is already opened for reading")
}
if o.w == nil {
o.w = s.NewWriter(o.ctx, o.b, o.n)
}
if o.w == nil {
return 0, errors.New("failed to connect to gcs")
}
return o.w.Write(p)
}
func uploadFile(ctx context.Context, b *Build, version, filename string) error {
file, err := ioutil.ReadFile(filename)
if err != nil {
return err
}
base := filepath.Base(filename)
// Upload the file to Google Cloud Storage.
wr := storage.NewWriter(ctx, storageBucket, base)
wr.ACL = []storage.ACLRule{
{Entity: storage.AllUsers, Role: storage.RoleReader},
}
wr.Write(file)
if err := wr.Close(); err != nil {
return fmt.Errorf("uploading file: %v", err)
}
// Post file details to golang.org.
var kind string
switch {
case b.Source:
kind = "source"
case strings.HasSuffix(base, ".tar.gz"), strings.HasSuffix(base, ".zip"):
kind = "archive"
case strings.HasSuffix(base, ".msi"), strings.HasSuffix(base, ".pkg"):
kind = "installer"
}
req, err := json.Marshal(File{
Filename: base,
Version: version,
OS: b.OS,
Arch: b.Arch,
Checksum: fmt.Sprintf("%x", sha1.Sum(file)),
Size: int64(len(file)),
Kind: kind,
})
if err != nil {
return err
}
v := url.Values{"user": {*user}, "key": []string{userToken()}}
u := fmt.Sprintf("%s?%s", uploadURL, v.Encode())
resp, err := http.Post(u, "application/json", bytes.NewReader(req))
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
b, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("upload failed: %v\n%s", resp.Status, b)
}
return nil
}
func handlePut(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
cctx := getCloudContext(ctx)
writer := storage.NewWriter(cctx, bucketName, "example.txt")
io.WriteString(writer, "Works")
err := writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func fileHandler(c *gin.Context) {
username := c.Param("username")
title := c.Param("title")
filename := c.Param("fileName")
gaeContext := appengine.NewContext(c.Request)
hc := &http.Client{
Transport: &CloudStorageTransport{&oauth2.Transport{
Source: google.AppEngineTokenSource(gaeContext, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: gaeContext},
}},
}
bucketName := "balde_de_bits"
bucketFile := username + "/" + title + "/" + filename
log.Errorf(gaeContext, "ID ->>> %v", appengine.AppID(gaeContext))
log.Errorf(gaeContext, "File name ->>> %v", bucketFile)
ctx := cloud.NewContext(appengine.AppID(gaeContext), hc)
wc := storage.NewWriter(ctx, bucketName, bucketFile)
if strings.Contains(filename, "m3u8") {
wc.ContentType = "application/x-mpegURL"
wc.CacheControl = "max-age:0"
} else if strings.Contains(filename, "ts") {
wc.ContentType = "video/MP2T"
} else if strings.Contains(filename, "jpg") {
wc.ContentType = "image/jpeg"
}
defer wc.Close()
bytesWritten, err := io.Copy(wc, c.Request.Body)
if err != nil {
log.Errorf(gaeContext, "Writing to cloud storage failed. %v", err.Error())
c.JSON(200, gin.H{
"response": "< FAILED >",
})
return
}
log.Errorf(gaeContext, "Wrote < %v > bytes for file < %v >", bytesWritten, filename)
c.JSON(200, gin.H{
"response": "< worked >",
})
}
func ExampleNewWriter() {
ctx := Example_auth()
wc := storage.NewWriter(ctx, "bucketname", "filename1")
wc.ContentType = "text/plain"
wc.ACL = []storage.ACLRule{{storage.AllUsers, storage.RoleReader}}
if _, err := wc.Write([]byte("hello world")); err != nil {
log.Fatal(err)
}
if err := wc.Close(); err != nil {
log.Fatal(err)
}
log.Println("updated object:", wc.Object())
}
func handlePut(res http.ResponseWriter, req *http.Request) {
cctx, err := getCloudContext(req)
if err != nil {
http.Error(res, "ERROR GETTING CCTX: "+err.Error(), 500)
return
}
writer := storage.NewWriter(cctx, gcsBucket, "example999.txt")
io.WriteString(writer, "showing the put get list")
err = writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func handlePut(res http.ResponseWriter, req *http.Request) {
cctx, err := getCloudContext(req)
if err != nil {
http.Error(res, "ERROR GETTING CCTX: "+err.Error(), 500)
return
}
writer := storage.NewWriter(cctx, gcsBucket, "myOffice.txt")
writer.ContentType = "text/plain"
io.WriteString(writer, "in my office")
err = writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func upload(ctx context.Context, bucket, base string, files []*multipart.FileHeader) multiError {
errc := make(chan error)
var errs []error
for _, fh := range files {
go func(fh *multipart.FileHeader) {
f, err := fh.Open()
if err != nil {
errc <- err
return
}
name := base + fh.Filename
wc := storage.NewWriter(ctx, bucket, name)
wc.ObjectAttrs = defaultObjectAttrs(path.Base(fh.Filename))
if _, err := io.Copy(wc, f); err != nil {
errc <- err
return
}
if err := wc.Close(); err != nil {
errc <- err
return
}
if err := f.Close(); err != nil {
errc <- err
return
}
errc <- nil
}(fh)
}
for range files {
err := <-errc
if err != nil {
errs = append(errs, err)
}
}
close(errc)
return errs
}
func Writer(filename string, r *http.Request) (*storage.Writer, error) {
c := appengine.NewContext(r)
bucket, err := file.DefaultBucketName(c)
if err != nil {
log.Errorf(c, "Failed to get default bucket: %v", err)
return nil, err
}
ctx, err := auth(r)
if err != nil {
log.Errorf(c, "Failed to get context: %v", err)
return nil, err
}
w := storage.NewWriter(ctx, bucket, filename)
return w, nil
}
func checkTryBuildDeps() error {
if !hasStorageScope() {
return errors.New("coordinator's GCE instance lacks the storage service scope")
}
wr := storage.NewWriter(serviceCtx, buildLogBucket(), "hello.txt")
fmt.Fprintf(wr, "Hello, world! Coordinator start-up at %v", time.Now())
if err := wr.Close(); err != nil {
return fmt.Errorf("test write of a GCS object to bucket %q failed: %v", buildLogBucket(), err)
}
gobotPass, err := metadata.ProjectAttributeValue("gobot-password")
if err != nil {
return fmt.Errorf("failed to get project metadata 'gobot-password': %v", err)
}
gerritClient = gerrit.NewClient("https://go-review.googlesource.com",
gerrit.BasicAuth("git-gobot.golang.org", strings.TrimSpace(string(gobotPass))))
return nil
}
func handlePut(res http.ResponseWriter, req *http.Request, _ httprouter.Params) {
cctx, err := getCloudContext(req)
if err != nil {
http.Error(res, "ERROR GETTING CCTX: "+err.Error(), 500)
return
}
fileName := "test.txt"
contentToWrite := "hello world!"
writer := storage.NewWriter(cctx, gcsBucket, fileName)
io.WriteString(writer, contentToWrite)
err = writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func WriteFile(c appengine.Context, filename string, data []byte) (gsPath string, err error) {
ctx, err := getGcsContext(c)
if err != nil {
return "", err
}
wc := storage.NewWriter(ctx, gcsBucket, filename)
wc.ContentType = "image/png"
if _, err = wc.Write(data); err != nil {
return "", err
}
if err = wc.Close(); err != nil {
return "", err
}
return "/gs/" + gcsBucket + "/" + filename, nil
}
func upload(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.NewContext(appengine.AppID(c), hc)
err := r.ParseMultipartForm(1048576)
if err != nil {
io.WriteString(w, err.Error())
return
}
file, fileHeader, err := r.FormFile("file")
if err != nil {
io.WriteString(w, err.Error())
return
}
defer file.Close()
data, err := ioutil.ReadAll(file)
if err != nil {
io.WriteString(w, err.Error())
return
}
wc := storage.NewWriter(ctx, bucket, "slide/"+fileHeader.Filename)
wc.ContentType = http.DetectContentType(data)
if _, err := wc.Write(data); err != nil {
io.WriteString(w, err.Error())
return
}
if err := wc.Close(); err != nil {
io.WriteString(w, err.Error())
return
}
}
// cc.ctx and gae are two different context.Context objects
func createFile(cc cloudBucketContext, gae context.Context) error {
var wc = storage.NewWriter(cc.ctx, cc.bucket, cc.filename)
var contentType = DEFAULT_CONTENT_TYPE
if ctype := cc.req.Header.Get("Content-Type"); len(ctype) > 0 {
contentType = ctype
}
wc.ContentType = contentType
// Make sure that signed URLs are downloadable by everyone, thus sharable.
wc.ACL = []storage.ACLRule{{storage.AllUsers, storage.RoleReader}}
var timeout = time.Now().UTC().Add(28 * time.Second)
if bytesCopied, err := ioext.Copy(wc, cc.req.Body, timeout); err != nil {
return err
} else {
log.Infof(gae, "bytesCopied = %d", bytesCopied)
}
if err := wc.Close(); err != nil {
return err
}
return nil
}
func handlePut(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: ctx},
},
}
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
writer := storage.NewWriter(cctx, bucketName, "example2.txt")
io.WriteString(writer, "Another file")
err := writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func Example_writeObjects() {
// see the auth example how to initiate a context.
ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})
wc := storage.NewWriter(ctx, "bucketname", "filename1", &storage.Object{
ContentType: "text/plain",
})
if _, err := wc.Write([]byte("hello world")); err != nil {
log.Fatal(err)
}
if err := wc.Close(); err != nil {
log.Fatal(err)
}
o, err := wc.Object()
if err != nil {
log.Fatal(err)
}
log.Println("updated object:", o)
}
func serveFilesUpload(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
serveAPI(res, req, func() interface{} {
ctx := appengine.NewContext(req)
session, _ := sessionStore.Get(req, "session")
_, ok := session.Values["email"].(string)
if !ok {
return HTTPError{403, "access denied"}
}
bucket, err := file.DefaultBucketName(ctx)
if err != nil {
return err
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: ctx},
},
}
id := uuid.NewV1().String()
ff, _, err := req.FormFile("file")
if err != nil {
return err
}
defer ff.Close()
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
wc := storage.NewWriter(cctx, bucket, id)
io.Copy(wc, ff)
err = wc.Close()
if err != nil {
return err
}
return id
})
}
// Commit flushes all content written to this FileWriter and makes it
// available for future calls to StorageDriver.GetContent and
// StorageDriver.Reader.
func (w *writer) Commit() error {
if err := w.checkClosed(); err != nil {
return err
}
w.closed = true
// no session started yet just perform a simple upload
if w.sessionURI == "" {
err := retry(func() error {
wc := storage.NewWriter(cloud.NewContext(dummyProjectID, w.client), w.bucket, w.name)
wc.ContentType = "application/octet-stream"
return putContentsClose(wc, w.buffer[0:w.buffSize])
})
if err != nil {
return err
}
w.size = w.offset + int64(w.buffSize)
w.buffSize = 0
return nil
}
size := w.offset + int64(w.buffSize)
var nn int
// loop must be performed at least once to ensure the file is committed even when
// the buffer is empty
for {
n, err := putChunk(w.client, w.sessionURI, w.buffer[nn:w.buffSize], w.offset, size)
nn += int(n)
w.offset += n
w.size = w.offset
if err != nil {
w.buffSize = copy(w.buffer, w.buffer[nn:w.buffSize])
return err
}
if nn == w.buffSize {
break
}
}
w.buffSize = 0
return nil
}
