Exemple #1
0
func StartupHandler(w http.ResponseWriter, r *http.Request) {
	if len(Config) == 0 {
		c := appengine.NewContext(r)
		client := &http.Client{
			Transport: &oauth2.Transport{
				Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly),
				Base: &urlfetch.Transport{
					Context: c,
				},
			},
		}
		bucket, _ := file.DefaultBucketName(c)
		ctx := cloud.NewContext("davine-web", client)
		rc, err := storage.NewReader(ctx, bucket, "config.yaml")
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}
		configFile, err := ioutil.ReadAll(rc)
		rc.Close()
		if err != nil {
			c.Errorf("error reading config: %v", err.Error())
			return
		}

		c.Infof("loaded config file: %v", configFile)
		yaml.Unmarshal(configFile, &Config)
		c.Infof("loaded config struct: %v", Config)
	}
}
Exemple #2
0
// OpenTestFile is like os.Open but for accessing test files.
func OpenTestFile(name string) (io.ReadCloser, error) {
	fn := path.Join(cache, name)

	f, err := os.Open(fn)
	if err == nil {
		return f, nil
	}

	if err != os.ErrNotExist {
		return nil, err
	}

	rc, err := storage.NewReader(context.Background(), bucket, name)
	if err != nil {
		return nil, err
	}

	w, err := os.OpenFile(fn, os.O_CREATE|os.O_WRONLY|os.O_EXCL, os.ModeTemporary)
	if err != nil {
		return nil, err
	}

	pr, pw := io.Pipe()
	go io.Copy(io.MultiWriter(pw, w), rc)

	return pr, nil
}
Exemple #3
0
func (a *AppengineStore) Get(name string) (io.ReadCloser, error) {
	r, err := storage.NewReader(a.Ctx, a.Cfg.Bucket, name)
	if err == storage.ErrObjectNotExist || err == storage.ErrBucketNotExist {
		// TODO log an error
		return nil, files.ErrNotFound
	}
	return r, err
}
Exemple #4
0
func processImage(bucket, name string) error {
	r, err := storage.NewReader(ctx, bucket, name)
	if err != nil {
		return fmt.Errorf("storage reader: %v", err)
	}
	img, err := ioutil.ReadAll(r)
	r.Close()
	if err != nil {
		return fmt.Errorf("read image: %v", err)
	}

	wand := imagick.NewMagickWand()
	defer wand.Destroy()

	wand.ReadImageBlob(img)
	if err := wand.SetImageFormat("WEBP"); err != nil {
		return fmt.Errorf("set WEBP format: %v", err)
	}

	errc := make(chan error, len(sizes))
	for suffix, size := range sizes {
		go func(wand *imagick.MagickWand, suffix string, x, y uint) {
			errc <- func() error {
				defer wand.Destroy()

				if err := wand.AdaptiveResizeImage(size.x, size.y); err != nil {
					return fmt.Errorf("resize: %v", err)
				}

				target := name
				if sep := strings.LastIndex(target, "."); sep >= 0 {
					target = target[:sep]
				}
				target = fmt.Sprintf("%s_%s.webp", target, suffix)

				w := storage.NewWriter(ctx, outputBucket, target, nil)
				if _, err := w.Write(wand.GetImageBlob()); err != nil {
					return fmt.Errorf("new writer: %v", err)
				}
				if err := w.Close(); err != nil {
					return fmt.Errorf("close object writer: %v", err)
				}
				if _, err := w.Object(); err != nil {
					return fmt.Errorf("write op: %v", err)
				}
				return nil
			}()
		}(wand.Clone(), suffix, size.x, size.y)
	}

	for _ = range sizes {
		if err := <-errc; err != nil {
			return err
		}
	}
	return nil
}
Exemple #5
0
func ImageHandler(w http.ResponseWriter, r *http.Request) {
	val := bone.GetValue(r, "rows")
	re := regexp.MustCompile("\\d*")
	rowsStr := re.FindString(val)
	rows, err := strconv.Atoi(rowsStr)
	if err != nil {
		fmt.Fprint(w, "Please pass in number of row")
		return
	}
	if rows > 5000 {
		fmt.Fprint(w, "Please pass in a row equal or smaller than 5000")
		return
	}

	cachedImage, ok := cache[rows]
	if ok {
		w.Header().Set("Content-type", "image/png")
		w.Header().Set("Cache-control", "public, max-age=259200")
		w.Write(cachedImage.Bytes())
		return
	}

	ctx, err := cloudAuthContext(r)
	if err == nil {
		rc, err := storage.NewReader(ctx, bucket, rowsStr+".png")
		if err == nil {
			image, err := ioutil.ReadAll(rc)
			rc.Close()
			if err == nil {
				w.Header().Set("Content-type", "image/png")
				w.Header().Set("Cache-control", "public, max-age=259200")
				w.Write(image)
				return
			}
		}
	}

	m := image.NewPaletted(image.Rectangle{Min: image.Point{0, 0}, Max: image.Point{rows, rows}}, bwPalette)

	m.Pix[m.PixOffset(rows-1, 0)] = 1
	for row := 1; row < rows; row++ {
		m.Pix[m.PixOffset(rows-1, row)] = 1
		for j := rows - row; j < rows-1; j++ {
			mid := m.PixOffset(j, row-1)
			left, right := mid-1, mid+1
			m.Pix[m.PixOffset(j, row)] = rulesUint8[m.Pix[left]][m.Pix[mid]][m.Pix[right]]
		}
	}
	w.Header().Set("Content-type", "image/png")
	w.Header().Set("Cache-control", "public, max-age=259200")
	buf := new(bytes.Buffer)
	png.Encode(buf, m)
	cache[rows] = buf
	w.Write(buf.Bytes())
}
Exemple #6
0
func downloadFile(
	wri http.ResponseWriter,
	req *http.Request,
) {
	var gae = appengine.NewContext(req)
	var bucket = util.GetBucketName()
	var vars = mux.Vars(req)
	log.Infof(gae, "download file %s from bucket %s", vars["key"], bucket)
	var cloudCtx = getCloudContext(gae)
	rc, err := storage.NewReader(cloudCtx, bucket, vars["key"])
	if err != nil {
		log.Infof(gae, "unable to open file %q from bucket %q", vars["key"], bucket)
		wri.WriteHeader(http.StatusNotFound)
		return
	}
	defer rc.Close()
	// Return an signed URL from which the client can download a file
	accessIdFile := os.Getenv("GOOGLE_ACCESS_ID")
	if accessIdFile == "" {
		log.Errorf(gae, "Missing GOOGLE_ACCESS_ID from app.yaml")
		wri.WriteHeader(http.StatusInternalServerError)
		return
	}
	accessId, err := ioutil.ReadFile(accessIdFile)
	if err != nil {
		log.Errorf(gae, "Cannot read service account access id from %s", accessIdFile)
		wri.WriteHeader(http.StatusInternalServerError)
	}
	privateKeyFile := os.Getenv("GOOGLE_PRIVATE_KEY")
	if privateKeyFile == "" {
		log.Errorf(gae, "Missing GOOGLE_PRIVATE_KEY from app.yaml")
		wri.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Infof(gae, "Private key file %s", privateKeyFile)
	privateKey, err := ioutil.ReadFile(privateKeyFile)
	if err != nil {
		log.Errorf(gae, "Cannot read service account private key from %s", privateKeyFile)
		wri.WriteHeader(http.StatusInternalServerError)
		return
	}
	var opts = storage.SignedURLOptions{
		GoogleAccessID: string(accessId),
		PrivateKey:     privateKey,
		Method:         "GET",
		Expires:        time.Now().UTC().Add(300 * time.Second),
	}
	signedURL, err := storage.SignedURL(bucket, vars["key"], &opts)
	if err != nil {
		log.Errorf(gae, "Unable to generate a signedURL: %s", err)
		wri.WriteHeader(http.StatusInternalServerError)
	}
	http.Redirect(wri, req, signedURL, http.StatusTemporaryRedirect)
}
Exemple #7
0
func (bs *GCSBlobStore) OpenReader(blobpath string) (io.ReadCloser, error) {
	ctx := bs.newAuthedContext(context.TODO())
	rc, err := storage.NewReader(ctx, bs.bucketName, blobpath)
	if err != nil {
		if err == storage.ErrObjectNotExist {
			return nil, blobstore.ENOENT
		}
		return nil, err
	}
	return rc, nil
}
func handleGet(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	cctx := getCloudContext(ctx)

	rdr, err := storage.NewReader(cctx, bucketName, "example.txt")
	if err != nil {
		http.Error(res, err.Error(), 500)
		return
	}
	defer rdr.Close()
	io.Copy(res, rdr)
}
Exemple #9
0
func (o *object) Read(p []byte) (n int, err error) {
	if o.w != nil {
		return 0, errors.New("object is already opened for writing")
	}
	if o.r == nil {
		rc, err := s.NewReader(o.ctx, o.b, o.n)
		if err != nil {
			return 0, err
		}
		o.r = &rc
	}
	return (*o.r).Read(p)
}
Exemple #10
0
func (s *googleReplicaAPIServer) PullDiff(request *pfs.PullDiffRequest, pullDiffServer pfs.ReplicaAPI_PullDiffServer) (retErr error) {
	reader, err := storage.NewReader(s.ctx, s.bucket, fmt.Sprintf("%s/%s/%d", request.Commit.Repo.Name, request.Commit.Id, request.Shard))
	if err != nil {
		return err
	}
	defer func() {
		if err := reader.Close(); err != nil && retErr == nil {
			retErr = err
		}
	}()
	writer := protostream.NewStreamingBytesWriter(pullDiffServer)
	_, err = io.Copy(writer, reader)
	return err
}
Exemple #11
0
func ExampleNewReader() {
	ctx := Example_auth()

	rc, err := storage.NewReader(ctx, "bucketname", "filename1")
	if err != nil {
		log.Fatal(err)
	}
	slurp, err := ioutil.ReadAll(rc)
	rc.Close()
	if err != nil {
		log.Fatal(err)
	}

	log.Println("file contents:", slurp)
}
Exemple #12
0
func handleGet(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	cctx := getCloudContext(ctx)

	filename := req.FormValue("f")

	rdr, err := storage.NewReader(cctx, bucket, prefix+filename)
	if err != nil {
		http.Error(res, err.Error(), 500)
		return
	}
	defer rdr.Close()

	io.Copy(res, rdr)
}
func handleDownload(res http.ResponseWriter, req *http.Request) {
	ctx := appengine.NewContext(req)
	cctx := getCloudContext(ctx)

	fileName := req.FormValue("fname")

	rdr, err := storage.NewReader(cctx, bucketName, fileName)
	if err != nil {
		http.Error(res, err.Error(), 500)
		return
	}
	defer rdr.Close()
	res.Header().Set("Content-Disposition", `attachment; filename="`+fileName+`"`)
	io.Copy(res, rdr)
}
Exemple #14
0
func fromStorage(ctx context.Context, bucket, name string) (io.ReadCloser, error) {
	rc, err := storage.NewReader(CloudContext(ctx), bucket, name)
	if err != nil {
		return nil, err
	}
	buf := new(bytes.Buffer)
	var crc = struct {
		io.Reader
		io.Closer
	}{
		io.TeeReader(rc, buf),
		cachingCloser{ctx: ctx, rc: rc, buf: buf, key: key(bucket, name)},
	}
	return crc, nil
}
Exemple #15
0
func Example_readObjects() {
	// see the auth example how to initiate a context.
	ctx := cloud.NewContext("project-id", &http.Client{Transport: nil})

	rc, err := storage.NewReader(ctx, "bucketname", "filename1")
	if err != nil {
		log.Fatal(err)
	}
	slurp, err := ioutil.ReadAll(rc)
	rc.Close()
	if err != nil {
		log.Fatal(err)
	}

	log.Println("file contents:", slurp)
}
Exemple #16
0
func handleGet(res http.ResponseWriter, req *http.Request) {
	cctx, err := getCloudContext(req)
	if err != nil {
		http.Error(res, "ERROR GETTING CCTX: "+err.Error(), 500)
		return
	}

	rdr, err := storage.NewReader(cctx, gcsBucket, "example999.txt")
	if err != nil {
		http.Error(res, err.Error(), 500)
		return
	}
	defer rdr.Close()

	io.Copy(res, rdr)
}
Exemple #17
0
func ReadFile(c appengine.Context, gsPath string) (data []byte, err error) {
	ctx, err := getGcsContext(c)
	if err != nil {
		return nil, err
	}

	filename := strings.SplitN(gsPath, "/", 4)[3]

	rc, err := storage.NewReader(ctx, gcsBucket, filename)
	if err != nil {
		return nil, err
	}

	data, err = ioutil.ReadAll(rc)
	rc.Close()
	return
}
Exemple #18
0
func handleGet(res http.ResponseWriter, req *http.Request, _ httprouter.Params) {
	cctx, err := getCloudContext(req)
	if err != nil {
		http.Error(res, "ERROR GETTING CCTX: "+err.Error(), 500)
		return
	}

	fileName := "test.txt"

	rdr, err := storage.NewReader(cctx, gcsBucket, fileName)
	if err != nil {
		http.Error(res, err.Error(), 500)
		return
	}
	defer rdr.Close()

	io.Copy(res, rdr)
}
func (conn GoogleCloudStorageConnection) Get(name string) (string, error) {
	log.Debugf("GoogleCloudStorageConnection Get. name: %s", name)
	cacheFilepath, err := conn.getCacheFilepath(name)
	if err != nil {
		log.Errorf("Failed to getCacheFilepath in Get: %s", err)
		return cacheFilepath, err
	}
	cacheDirectory := filepath.Dir(cacheFilepath)
	err = os.MkdirAll(cacheDirectory, 0777)
	if err != nil {
		log.Errorf("Couldn't create cache directory for cacheFilepath %s: %s", cacheFilepath, err)
		return cacheFilepath, err
	}
	if _, err = os.Stat(cacheDirectory); err != nil {
		log.Errorf("Cache directory %s doesn't exist!", cacheDirectory)
		return cacheFilepath, err
	}
	w, err := os.Create(cacheFilepath)
	if err != nil {
		log.Errorf("Couldn't create cache file for cacheFilepath %s: %s", cacheFilepath, err)
		return cacheFilepath, err
	}
	defer w.Close()
	wBuffered := bufio.NewWriter(w)
	defer wBuffered.Flush()
	r, err := storage.NewReader(conn.Context, conn.BucketName, name)
	if err != nil {
		log.Errorf("Failed to download name %s during initialization: %s", name, err)
		defer os.Remove(cacheFilepath)
		return cacheFilepath, err
	}
	defer r.Close()
	_, err = io.Copy(wBuffered, r)
	time.Sleep(100 * time.Millisecond)
	if err != nil {
		log.Errorf("Failed to download name %s during download: %s", name, err)
		defer os.Remove(cacheFilepath)
		return cacheFilepath, err
	}
	return cacheFilepath, nil
}
Exemple #20
0
// readFile reads the named file in Google Cloud Storage.
func (d *demo) readFile(fileName string) {
	io.WriteString(d.w, "\nAbbreviated file content (first line and last 1K):\n")

	rc, err := storage.NewReader(d.ctx, d.bucket, fileName)
	if err != nil {
		d.errorf("readFile: unable to open file from bucket %q, file %q: %v", d.bucket, fileName, err)
		return
	}
	defer rc.Close()
	slurp, err := ioutil.ReadAll(rc)
	if err != nil {
		d.errorf("readFile: unable to read data from bucket %q, file %q: %v", d.bucket, fileName, err)
		return
	}

	fmt.Fprintf(d.w, "%s\n", bytes.SplitN(slurp, []byte("\n"), 2)[0])
	if len(slurp) > 1024 {
		fmt.Fprintf(d.w, "...%s\n", slurp[len(slurp)-1024:])
	} else {
		fmt.Fprintf(d.w, "%s\n", slurp)
	}
}
Exemple #21
0
// Files
func serveFilesGet(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
	serveAPI(res, req, func() interface{} {
		ctx := appengine.NewContext(req)
		session, _ := sessionStore.Get(req, "session")

		_, ok := session.Values["email"].(string)
		if !ok {
			return HTTPError{403, "access denied"}
		}

		bucket, err := file.DefaultBucketName(ctx)
		if err != nil {
			return err
		}

		hc := &http.Client{
			Transport: &oauth2.Transport{
				Source: google.AppEngineTokenSource(ctx, storage.ScopeReadOnly),
				Base:   &urlfetch.Transport{Context: ctx},
			},
		}

		cctx := cloud.NewContext(appengine.AppID(ctx), hc)
		rc, err := storage.NewReader(cctx, bucket, params.ByName("id"))
		if err != nil {
			return err
		}

		name := req.URL.Query().Get("name")
		if name == "" {
			name = params.ByName("id")
		}
		name = regexp.MustCompile("[^a-zA-Z-_.]").ReplaceAllString(name, "")

		res.Header().Set("Content-Disposition", "inline; filename=\""+name+"\"")
		res.Header().Set("Content-Type", "application/octet-stream")
		return rc
	})
}
Exemple #22
0
// getInstalledTLS returns the TLS certificate and key stored on Google Cloud Storage for the
// instance defined in d.Conf.
func (d *Deployer) getInstalledTLS() (certPEM, keyPEM []byte, err error) {
	ctx := cloud.NewContext(d.Conf.Project, d.Client)
	getFile := func(name string) ([]byte, error) {
		sr, err := cloudstorage.NewReader(ctx, d.Conf.bucketBase(),
			path.Join(configDir, name))
		if err != nil {
			return nil, err
		}
		defer sr.Close()
		return ioutil.ReadAll(sr)
	}
	var grp syncutil.Group
	grp.Go(func() (err error) {
		certPEM, err = getFile(certFilename)
		return
	})
	grp.Go(func() (err error) {
		keyPEM, err = getFile(keyFilename)
		return
	})
	err = grp.Err()
	return
}
Exemple #23
0
// GetContent retrieves the content stored at "path" as a []byte.
// This should primarily be used for small objects.
func (d *driver) GetContent(context ctx.Context, path string) ([]byte, error) {
	gcsContext := d.context(context)
	name := d.pathToKey(path)
	var rc io.ReadCloser
	err := retry(func() error {
		var err error
		rc, err = storage.NewReader(gcsContext, d.bucket, name)
		return err
	})
	if err == storage.ErrObjectNotExist {
		return nil, storagedriver.PathNotFoundError{Path: path}
	}
	if err != nil {
		return nil, err
	}
	defer rc.Close()

	p, err := ioutil.ReadAll(rc)
	if err != nil {
		return nil, err
	}
	return p, nil
}
Exemple #24
0
func Reader(filename string, r *http.Request) (io.ReadCloser, error) {
	c := appengine.NewContext(r)
	bucket, err := file.DefaultBucketName(c)
	if err != nil {
		log.Errorf(c, "Failed to get default bucket: %v", err)
		return nil, err
	}

	ctx, err := auth(r)
	if err != nil {
		log.Errorf(c, "Failed to get context: %v", err)
		return nil, err
	}

	log.Infof(c, "Retrieving file %v from bucket %v.", filename, bucket)

	rc, err := storage.NewReader(ctx, bucket, filename)
	if err != nil {
		log.Errorf(c, "Failed to open file: %v", err)
		return nil, err
	}

	return rc, nil
}
Exemple #25
0
func getFile(ctx context.Context, name string) (io.ReadCloser, error) {
	cctx := getCloudContext(ctx)

	return storage.NewReader(cctx, gcsBucket, name)
}
Exemple #26
0
// Open implements part of the VFS interface.
func (s FS) Open(ctx context.Context, path string) (io.ReadCloser, error) {
	return storage.NewReader(ctx, s.Bucket, path)
}
Exemple #27
0
func processImage(bucket, name string) error {
	r, err := storage.NewReader(ctx, bucket, name)
	if err != nil {
		return fmt.Errorf("storage reader: %v", err)
	}
	img, err := ioutil.ReadAll(r)
	r.Close()
	if err != nil {
		return fmt.Errorf("read image: %v", err)
	}

	wand := imagick.NewMagickWand()
	defer wand.Destroy()

	wand.ReadImageBlob(img)
	if err := wand.SetImageFormat("WEBP"); err != nil {
		return fmt.Errorf("set WEBP format: %v", err)
	}

	errc := make(chan error, len(sizes))
	for suffix, size := range sizes {
		go func(wand *imagick.MagickWand, suffix string, x, y uint) {
			errc <- func() error {
				defer wand.Destroy()

				width := x
				height := y
				if x == 0 && y > 0 {
					width = uint(float64(wand.GetImageWidth()) / float64(wand.GetImageHeight()) * float64(y))
				} else if x > 0 && y == 0 {
					height = uint(float64(wand.GetImageHeight()) / float64(wand.GetImageWidth()) * float64(x))
				} else if x== 0 && y == 0 {
					width = wand.GetImageWidth()
					height = wand.GetImageHeight()
				}

				if err := wand.AdaptiveResizeImage(width, height); err != nil {
					return fmt.Errorf("resize: %v", err)
				}

				target := name
				if sep := strings.LastIndex(target, "."); sep >= 0 {
					target = target[:sep]
				}
				s := suffix
				if suffix == "thumbnail" {
					s = ""
				} else {
					s = "_" + suffix
				}
				target = fmt.Sprintf("%s%s.webp", target, s)

				w := storage.NewWriter(ctx, outputBucket, target)
				w.ContentType = "image/webp"
				if _, err := w.Write(wand.GetImageBlob()); err != nil {
					return fmt.Errorf("new writer: %v", err)
				}
				if err := w.Close(); err != nil {
					return fmt.Errorf("close object writer: %v", err)
				}
				return nil
			}()
		}(wand.Clone(), suffix, size.x, size.y)
	}

	for _ = range sizes {
		if err := <-errc; err != nil {
			return err
		}
	}
	return nil
}
func getFile(cctx context.Context, bucket, path string) (io.ReadCloser, error) {
	return storage.NewReader(cctx, bucket, path)
}
Exemple #29
0
// ReadFile implements BackupHandle.
func (bh *GCSBackupHandle) ReadFile(filename string) (io.ReadCloser, error) {
	if !bh.readOnly {
		return nil, fmt.Errorf("ReadFile cannot be called on read-write backup")
	}
	return storage.NewReader(bh.authCtx, *bucket, objName(bh.dir, bh.name, filename))
}