func StartupHandler(w http.ResponseWriter, r *http.Request) {
if len(Config) == 0 {
c := appengine.NewContext(r)
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly),
Base: &urlfetch.Transport{
Context: c,
},
},
}
bucket, _ := file.DefaultBucketName(c)
ctx := cloud.NewContext("davine-web", client)
rc, err := storage.NewReader(ctx, bucket, "config.yaml")
if err != nil {
c.Errorf("error reading config: %v", err.Error())
return
}
configFile, err := ioutil.ReadAll(rc)
rc.Close()
if err != nil {
c.Errorf("error reading config: %v", err.Error())
return
}
c.Infof("loaded config file: %v", configFile)
yaml.Unmarshal(configFile, &Config)
c.Infof("loaded config struct: %v", Config)
}
}
// OpenTestFile is like os.Open but for accessing test files.
func OpenTestFile(name string) (io.ReadCloser, error) {
fn := path.Join(cache, name)
f, err := os.Open(fn)
if err == nil {
return f, nil
}
if err != os.ErrNotExist {
return nil, err
}
rc, err := storage.NewReader(context.Background(), bucket, name)
if err != nil {
return nil, err
}
w, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_EXCL, os.ModeTemporary)
if err != nil {
return nil, err
}
pr, pw := io.Pipe()
go io.Copy(io.MultiWriter(pw, w), rc)
return pr, nil
}
func (a *AppengineStore) Get(name string) (io.ReadCloser, error) {
r, err := storage.NewReader(a.Ctx, a.Cfg.Bucket, name)
if err == storage.ErrObjectNotExist || err == storage.ErrBucketNotExist {
// TODO log an error
return nil, files.ErrNotFound
}
return r, err
}
func processImage(bucket, name string) error {
r, err := storage.NewReader(ctx, bucket, name)
if err != nil {
return fmt.Errorf("storage reader: %v", err)
}
img, err := ioutil.ReadAll(r)
r.Close()
if err != nil {
return fmt.Errorf("read image: %v", err)
}
wand := imagick.NewMagickWand()
defer wand.Destroy()
wand.ReadImageBlob(img)
if err := wand.SetImageFormat("WEBP"); err != nil {
return fmt.Errorf("set WEBP format: %v", err)
}
errc := make(chan error, len(sizes))
for suffix, size := range sizes {
go func(wand *imagick.MagickWand, suffix string, x, y uint) {
errc <- func() error {
defer wand.Destroy()
if err := wand.AdaptiveResizeImage(size.x, size.y); err != nil {
return fmt.Errorf("resize: %v", err)
}
target := name
if sep := strings.LastIndex(target, "."); sep >= 0 {
target = target[:sep]
}
target = fmt.Sprintf("%s_%s.webp", target, suffix)
w := storage.NewWriter(ctx, outputBucket, target, nil)
if _, err := w.Write(wand.GetImageBlob()); err != nil {
return fmt.Errorf("new writer: %v", err)
}
if err := w.Close(); err != nil {
return fmt.Errorf("close object writer: %v", err)
}
if _, err := w.Object(); err != nil {
return fmt.Errorf("write op: %v", err)
}
return nil
}()
}(wand.Clone(), suffix, size.x, size.y)
}
for _ = range sizes {
if err := <-errc; err != nil {
return err
}
}
return nil
}
func ImageHandler(w http.ResponseWriter, r *http.Request) {
val := bone.GetValue(r, "rows")
re := regexp.MustCompile("\\d*")
rowsStr := re.FindString(val)
rows, err := strconv.Atoi(rowsStr)
if err != nil {
fmt.Fprint(w, "Please pass in number of row")
return
}
if rows > 5000 {
fmt.Fprint(w, "Please pass in a row equal or smaller than 5000")
return
}
cachedImage, ok := cache[rows]
if ok {
w.Header().Set("Content-type", "image/png")
w.Header().Set("Cache-control", "public, max-age=259200")
w.Write(cachedImage.Bytes())
return
}
ctx, err := cloudAuthContext(r)
if err == nil {
rc, err := storage.NewReader(ctx, bucket, rowsStr+".png")
if err == nil {
image, err := ioutil.ReadAll(rc)
rc.Close()
if err == nil {
w.Header().Set("Content-type", "image/png")
w.Header().Set("Cache-control", "public, max-age=259200")
w.Write(image)
return
}
}
}
m := image.NewPaletted(image.Rectangle{Min: image.Point{0, 0}, Max: image.Point{rows, rows}}, bwPalette)
m.Pix[m.PixOffset(rows-1, 0)] = 1
for row := 1; row < rows; row++ {
m.Pix[m.PixOffset(rows-1, row)] = 1
for j := rows - row; j < rows-1; j++ {
mid := m.PixOffset(j, row-1)
left, right := mid-1, mid+1
m.Pix[m.PixOffset(j, row)] = rulesUint8[m.Pix[left]][m.Pix[mid]][m.Pix[right]]
}
}
w.Header().Set("Content-type", "image/png")
w.Header().Set("Cache-control", "public, max-age=259200")
buf := new(bytes.Buffer)
png.Encode(buf, m)
cache[rows] = buf
w.Write(buf.Bytes())
}
func downloadFile(
wri http.ResponseWriter,
req *http.Request,
) {
var gae = appengine.NewContext(req)
var bucket = util.GetBucketName()
var vars = mux.Vars(req)
log.Infof(gae, "download file %s from bucket %s", vars["key"], bucket)
var cloudCtx = getCloudContext(gae)
rc, err := storage.NewReader(cloudCtx, bucket, vars["key"])
if err != nil {
log.Infof(gae, "unable to open file %q from bucket %q", vars["key"], bucket)
wri.WriteHeader(http.StatusNotFound)
return
}
defer rc.Close()
// Return an signed URL from which the client can download a file
accessIdFile := os.Getenv("GOOGLE_ACCESS_ID")
if accessIdFile == "" {
log.Errorf(gae, "Missing GOOGLE_ACCESS_ID from app.yaml")
wri.WriteHeader(http.StatusInternalServerError)
return
}
accessId, err := ioutil.ReadFile(accessIdFile)
if err != nil {
log.Errorf(gae, "Cannot read service account access id from %s", accessIdFile)
wri.WriteHeader(http.StatusInternalServerError)
}
privateKeyFile := os.Getenv("GOOGLE_PRIVATE_KEY")
if privateKeyFile == "" {
log.Errorf(gae, "Missing GOOGLE_PRIVATE_KEY from app.yaml")
wri.WriteHeader(http.StatusInternalServerError)
return
}
log.Infof(gae, "Private key file %s", privateKeyFile)
privateKey, err := ioutil.ReadFile(privateKeyFile)
if err != nil {
log.Errorf(gae, "Cannot read service account private key from %s", privateKeyFile)
wri.WriteHeader(http.StatusInternalServerError)
return
}
var opts = storage.SignedURLOptions{
GoogleAccessID: string(accessId),
PrivateKey: privateKey,
Method: "GET",
Expires: time.Now().UTC().Add(300 * time.Second),
}
signedURL, err := storage.SignedURL(bucket, vars["key"], &opts)
if err != nil {
log.Errorf(gae, "Unable to generate a signedURL: %s", err)
wri.WriteHeader(http.StatusInternalServerError)
}
http.Redirect(wri, req, signedURL, http.StatusTemporaryRedirect)
}
func (bs *GCSBlobStore) OpenReader(blobpath string) (io.ReadCloser, error) {
ctx := bs.newAuthedContext(context.TODO())
rc, err := storage.NewReader(ctx, bs.bucketName, blobpath)
if err != nil {
if err == storage.ErrObjectNotExist {
return nil, blobstore.ENOENT
}
return nil, err
}
return rc, nil
}
func handleGet(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
cctx := getCloudContext(ctx)
rdr, err := storage.NewReader(cctx, bucketName, "example.txt")
if err != nil {
http.Error(res, err.Error(), 500)
return
}
defer rdr.Close()
io.Copy(res, rdr)
}
func (o *object) Read(p []byte) (n int, err error) {
if o.w != nil {
return 0, errors.New("object is already opened for writing")
}
if o.r == nil {
rc, err := s.NewReader(o.ctx, o.b, o.n)
if err != nil {
return 0, err
}
o.r = &rc
}
return (*o.r).Read(p)
}
func (s *googleReplicaAPIServer) PullDiff(request *pfs.PullDiffRequest, pullDiffServer pfs.ReplicaAPI_PullDiffServer) (retErr error) {
reader, err := storage.NewReader(s.ctx, s.bucket, fmt.Sprintf("%s/%s/%d", request.Commit.Repo.Name, request.Commit.Id, request.Shard))
if err != nil {
return err
}
defer func() {
if err := reader.Close(); err != nil && retErr == nil {
retErr = err
}
}()
writer := protostream.NewStreamingBytesWriter(pullDiffServer)
_, err = io.Copy(writer, reader)
return err
}
func ExampleNewReader() {
ctx := Example_auth()
rc, err := storage.NewReader(ctx, "bucketname", "filename1")
if err != nil {
log.Fatal(err)
}
slurp, err := ioutil.ReadAll(rc)
rc.Close()
if err != nil {
log.Fatal(err)
}
log.Println("file contents:", slurp)
}
func handleGet(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
cctx := getCloudContext(ctx)
filename := req.FormValue("f")
rdr, err := storage.NewReader(cctx, bucket, prefix+filename)
if err != nil {
http.Error(res, err.Error(), 500)
return
}
defer rdr.Close()
io.Copy(res, rdr)
}
func handleDownload(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
cctx := getCloudContext(ctx)
fileName := req.FormValue("fname")
rdr, err := storage.NewReader(cctx, bucketName, fileName)
if err != nil {
http.Error(res, err.Error(), 500)
return
}
defer rdr.Close()
res.Header().Set("Content-Disposition", `attachment; filename="`+fileName+`"`)
io.Copy(res, rdr)
}
func fromStorage(ctx context.Context, bucket, name string) (io.ReadCloser, error) {
rc, err := storage.NewReader(CloudContext(ctx), bucket, name)
if err != nil {
return nil, err
}
buf := new(bytes.Buffer)
var crc = struct {
io.Reader
io.Closer
}{
io.TeeReader(rc, buf),
cachingCloser{ctx: ctx, rc: rc, buf: buf, key: key(bucket, name)},
}
return crc, nil
}
func Example_readObjects() {
// see the auth example how to initiate a context.
ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})
rc, err := storage.NewReader(ctx, "bucketname", "filename1")
if err != nil {
log.Fatal(err)
}
slurp, err := ioutil.ReadAll(rc)
rc.Close()
if err != nil {
log.Fatal(err)
}
log.Println("file contents:", slurp)
}
func handleGet(res http.ResponseWriter, req *http.Request) {
cctx, err := getCloudContext(req)
if err != nil {
http.Error(res, "ERROR GETTING CCTX: "+err.Error(), 500)
return
}
rdr, err := storage.NewReader(cctx, gcsBucket, "example999.txt")
if err != nil {
http.Error(res, err.Error(), 500)
return
}
defer rdr.Close()
io.Copy(res, rdr)
}
func ReadFile(c appengine.Context, gsPath string) (data []byte, err error) {
ctx, err := getGcsContext(c)
if err != nil {
return nil, err
}
filename := strings.SplitN(gsPath, "/", 4)[3]
rc, err := storage.NewReader(ctx, gcsBucket, filename)
if err != nil {
return nil, err
}
data, err = ioutil.ReadAll(rc)
rc.Close()
return
}
func handleGet(res http.ResponseWriter, req *http.Request, _ httprouter.Params) {
cctx, err := getCloudContext(req)
if err != nil {
http.Error(res, "ERROR GETTING CCTX: "+err.Error(), 500)
return
}
fileName := "test.txt"
rdr, err := storage.NewReader(cctx, gcsBucket, fileName)
if err != nil {
http.Error(res, err.Error(), 500)
return
}
defer rdr.Close()
io.Copy(res, rdr)
}
func (conn GoogleCloudStorageConnection) Get(name string) (string, error) {
log.Debugf("GoogleCloudStorageConnection Get. name: %s", name)
cacheFilepath, err := conn.getCacheFilepath(name)
if err != nil {
log.Errorf("Failed to getCacheFilepath in Get: %s", err)
return cacheFilepath, err
}
cacheDirectory := filepath.Dir(cacheFilepath)
err = os.MkdirAll(cacheDirectory, 0777)
if err != nil {
log.Errorf("Couldn't create cache directory for cacheFilepath %s: %s", cacheFilepath, err)
return cacheFilepath, err
}
if _, err = os.Stat(cacheDirectory); err != nil {
log.Errorf("Cache directory %s doesn't exist!", cacheDirectory)
return cacheFilepath, err
}
w, err := os.Create(cacheFilepath)
if err != nil {
log.Errorf("Couldn't create cache file for cacheFilepath %s: %s", cacheFilepath, err)
return cacheFilepath, err
}
defer w.Close()
wBuffered := bufio.NewWriter(w)
defer wBuffered.Flush()
r, err := storage.NewReader(conn.Context, conn.BucketName, name)
if err != nil {
log.Errorf("Failed to download name %s during initialization: %s", name, err)
defer os.Remove(cacheFilepath)
return cacheFilepath, err
}
defer r.Close()
_, err = io.Copy(wBuffered, r)
time.Sleep(100 * time.Millisecond)
if err != nil {
log.Errorf("Failed to download name %s during download: %s", name, err)
defer os.Remove(cacheFilepath)
return cacheFilepath, err
}
return cacheFilepath, nil
}
// readFile reads the named file in Google Cloud Storage.
func (d *demo) readFile(fileName string) {
io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n")
rc, err := storage.NewReader(d.ctx, d.bucket, fileName)
if err != nil {
d.errorf("readFile: unable to open file from bucket %q, file %q: %v", d.bucket, fileName, err)
return
}
defer rc.Close()
slurp, err := ioutil.ReadAll(rc)
if err != nil {
d.errorf("readFile: unable to read data from bucket %q, file %q: %v", d.bucket, fileName, err)
return
}
fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0])
if len(slurp) > 1024 {
fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:])
} else {
fmt.Fprintf(d.w, "%s\n", slurp)
}
}
// Files
func serveFilesGet(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
serveAPI(res, req, func() interface{} {
ctx := appengine.NewContext(req)
session, _ := sessionStore.Get(req, "session")
_, ok := session.Values["email"].(string)
if !ok {
return HTTPError{403, "access denied"}
}
bucket, err := file.DefaultBucketName(ctx)
if err != nil {
return err
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeReadOnly),
Base: &urlfetch.Transport{Context: ctx},
},
}
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
rc, err := storage.NewReader(cctx, bucket, params.ByName("id"))
if err != nil {
return err
}
name := req.URL.Query().Get("name")
if name == "" {
name = params.ByName("id")
}
name = regexp.MustCompile("[^a-zA-Z-_.]").ReplaceAllString(name, "")
res.Header().Set("Content-Disposition", "inline; filename=\""+name+"\"")
res.Header().Set("Content-Type", "application/octet-stream")
return rc
})
}
// getInstalledTLS returns the TLS certificate and key stored on Google Cloud Storage for the
// instance defined in d.Conf.
func (d *Deployer) getInstalledTLS() (certPEM, keyPEM []byte, err error) {
ctx := cloud.NewContext(d.Conf.Project, d.Client)
getFile := func(name string) ([]byte, error) {
sr, err := cloudstorage.NewReader(ctx, d.Conf.bucketBase(),
path.Join(configDir, name))
if err != nil {
return nil, err
}
defer sr.Close()
return ioutil.ReadAll(sr)
}
var grp syncutil.Group
grp.Go(func() (err error) {
certPEM, err = getFile(certFilename)
return
})
grp.Go(func() (err error) {
keyPEM, err = getFile(keyFilename)
return
})
err = grp.Err()
return
}
// GetContent retrieves the content stored at "path" as a []byte.
// This should primarily be used for small objects.
func (d *driver) GetContent(context ctx.Context, path string) ([]byte, error) {
gcsContext := d.context(context)
name := d.pathToKey(path)
var rc io.ReadCloser
err := retry(func() error {
var err error
rc, err = storage.NewReader(gcsContext, d.bucket, name)
return err
})
if err == storage.ErrObjectNotExist {
return nil, storagedriver.PathNotFoundError{Path: path}
}
if err != nil {
return nil, err
}
defer rc.Close()
p, err := ioutil.ReadAll(rc)
if err != nil {
return nil, err
}
return p, nil
}
func Reader(filename string, r *http.Request) (io.ReadCloser, error) {
c := appengine.NewContext(r)
bucket, err := file.DefaultBucketName(c)
if err != nil {
log.Errorf(c, "Failed to get default bucket: %v", err)
return nil, err
}
ctx, err := auth(r)
if err != nil {
log.Errorf(c, "Failed to get context: %v", err)
return nil, err
}
log.Infof(c, "Retrieving file %v from bucket %v.", filename, bucket)
rc, err := storage.NewReader(ctx, bucket, filename)
if err != nil {
log.Errorf(c, "Failed to open file: %v", err)
return nil, err
}
return rc, nil
}
func getFile(ctx context.Context, name string) (io.ReadCloser, error) {
cctx := getCloudContext(ctx)
return storage.NewReader(cctx, gcsBucket, name)
}
// Open implements part of the VFS interface.
func (s FS) Open(ctx context.Context, path string) (io.ReadCloser, error) {
return storage.NewReader(ctx, s.Bucket, path)
}
func processImage(bucket, name string) error {
r, err := storage.NewReader(ctx, bucket, name)
if err != nil {
return fmt.Errorf("storage reader: %v", err)
}
img, err := ioutil.ReadAll(r)
r.Close()
if err != nil {
return fmt.Errorf("read image: %v", err)
}
wand := imagick.NewMagickWand()
defer wand.Destroy()
wand.ReadImageBlob(img)
if err := wand.SetImageFormat("WEBP"); err != nil {
return fmt.Errorf("set WEBP format: %v", err)
}
errc := make(chan error, len(sizes))
for suffix, size := range sizes {
go func(wand *imagick.MagickWand, suffix string, x, y uint) {
errc <- func() error {
defer wand.Destroy()
width := x
height := y
if x == 0 && y > 0 {
width = uint(float64(wand.GetImageWidth()) / float64(wand.GetImageHeight()) * float64(y))
} else if x > 0 && y == 0 {
height = uint(float64(wand.GetImageHeight()) / float64(wand.GetImageWidth()) * float64(x))
} else if x== 0 && y == 0 {
width = wand.GetImageWidth()
height = wand.GetImageHeight()
}
if err := wand.AdaptiveResizeImage(width, height); err != nil {
return fmt.Errorf("resize: %v", err)
}
target := name
if sep := strings.LastIndex(target, "."); sep >= 0 {
target = target[:sep]
}
s := suffix
if suffix == "thumbnail" {
s = ""
} else {
s = "_" + suffix
}
target = fmt.Sprintf("%s%s.webp", target, s)
w := storage.NewWriter(ctx, outputBucket, target)
w.ContentType = "image/webp"
if _, err := w.Write(wand.GetImageBlob()); err != nil {
return fmt.Errorf("new writer: %v", err)
}
if err := w.Close(); err != nil {
return fmt.Errorf("close object writer: %v", err)
}
return nil
}()
}(wand.Clone(), suffix, size.x, size.y)
}
for _ = range sizes {
if err := <-errc; err != nil {
return err
}
}
return nil
}
func getFile(cctx context.Context, bucket, path string) (io.ReadCloser, error) {
return storage.NewReader(cctx, bucket, path)
}
// ReadFile implements BackupHandle.
func (bh *GCSBackupHandle) ReadFile(filename string) (io.ReadCloser, error) {
if !bh.readOnly {
return nil, fmt.Errorf("ReadFile cannot be called on read-write backup")
}
return storage.NewReader(bh.authCtx, *bucket, objName(bh.dir, bh.name, filename))
}