func handlePut(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: ctx},
},
}
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
rdr, hdr, err := req.FormFile("form-upload-file")
if err != nil {
http.Error(res, "ERROR RECEIVING FILE: "+err.Error(), 500)
return
}
writer := storage.NewWriter(cctx, bucketName, hdr.Filename)
io.Copy(writer, rdr)
err = writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func StartupHandler(w http.ResponseWriter, r *http.Request) {
if len(Config) == 0 {
c := appengine.NewContext(r)
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeReadOnly),
Base: &urlfetch.Transport{
Context: c,
},
},
}
bucket, _ := file.DefaultBucketName(c)
ctx := cloud.NewContext("davine-web", client)
rc, err := storage.NewReader(ctx, bucket, "config.yaml")
if err != nil {
c.Errorf("error reading config: %v", err.Error())
return
}
configFile, err := ioutil.ReadAll(rc)
rc.Close()
if err != nil {
c.Errorf("error reading config: %v", err.Error())
return
}
c.Infof("loaded config file: %v", configFile)
yaml.Unmarshal(configFile, &Config)
c.Infof("loaded config struct: %v", Config)
}
}
// httpClient returns an authenticated http client, suitable for App Engine.
func httpClient(c context.Context, scopes ...string) *http.Client {
t := &oauth2.Transport{
Source: google.AppEngineTokenSource(c, scopes...),
Base: &urlfetch.Transport{Context: c},
}
return &http.Client{Transport: t}
}
func handler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
transport := &oauth2.Transport{
Source: google.AppEngineTokenSource(c, "https://spreadsheets.google.com/feeds"),
Base: &urlfetch.Transport{Context: c},
}
client := &http.Client{Transport: transport}
spreadsheetsID := "1QwybKPlmYFhFa5CSZqfwsmQDIPTfagMtvQ0C3CA5xwY"
sheetID := "1560292823" // gid=sheetID
resp, err := client.Get("https://docs.google.com/spreadsheets/d/" + spreadsheetsID + "/gviz/tq?gid=" + sheetID)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer resp.Body.Close()
contents, err := ioutil.ReadAll(resp.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
fmt.Fprintf(w, "%s\n", string(contents))
}
func insert(datasetId string, tableName string, ctx context.Context, json map[string]bigquery.JsonValue) {
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, bigquery.BigqueryScope),
Base: &urlfetch.Transport{Context: ctx},
},
}
bq, err := bigquery.New(client)
if err != nil {
fmt.Errorf("could not create service: %v", err)
return
}
appID := appengine.AppID(ctx)
request := new(bigquery.TableDataInsertAllRequest)
rows := make([]*bigquery.TableDataInsertAllRequestRows, 1)
rows[0] = new(bigquery.TableDataInsertAllRequestRows)
rows[0].Json = json
request.Rows = rows
_, errr := bq.Tabledata.InsertAll(appID, datasetId, tableName, request).Do()
if errr != nil {
fmt.Errorf("could not insert: %v", err)
return
}
}
func (cloud *CloudStore) Start() error {
if cloud.Bucket == "" {
return InvalidBucket
}
if cloud.Context == nil {
return InvalidGoogleContext
}
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(cloud.Context, storage.DevstorageFullControlScope),
Base: &urlfetch.Transport{Context: cloud.Context},
},
}
service, err := storage.New(client)
if err != nil {
log.Printf("Unable to get storage service %v", err)
return err
}
cloud.service = service
if _, err := service.Buckets.Get(cloud.Bucket).Do(); err == nil {
log.Printf("Got storage bucket %v %v", cloud.Bucket, err)
} else {
if _, err := service.Buckets.Insert(cloud.Project, &storage.Bucket{Name: cloud.Bucket}).Do(); err == nil {
log.Printf("Created bucket: %v", cloud.Bucket)
} else {
return err
}
}
return nil
}
// datasets returns a list with the ids of all the Big Query datasets visible
// with the given context.
func datasets(ctx context.Context) ([]string, error) {
// create a new authenticated HTTP client over urlfetch.
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, bigquery.BigqueryScope),
Base: &urlfetch.Transport{Context: ctx},
},
}
// create the BigQuery service.
bq, err := bigquery.New(client)
if err != nil {
return nil, fmt.Errorf("could not create service: %v", err)
}
// obtain the current application id, the BigQuery id is the same.
appID := appengine.AppID(ctx)
// prepare the list of ids.
var ids []string
datasets, err := bq.Datasets.List(appID).Do()
if err != nil {
return nil, fmt.Errorf("could not list datasets for %q: %v", appID, err)
}
for _, d := range datasets.Datasets {
ids = append(ids, d.Id)
}
return ids, nil
}
func handlerContainer2(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, bigquery.BigqueryScope),
Base: &urlfetch.Transport{Context: ctx},
},
}
bq, err := bigquery.New(client)
if err != nil {
fmt.Errorf("%v", err)
}
key := datastore.Key{}
c := Container2{
Hoge: Hoge{Name: "hoge", Age: 28},
Key: &key,
}
schema, err := ironmole.BuildTableSchema(&c)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
err = ironmole.CreateTable(bq, "cp300demo1", "go2bq", "Container2", schema)
if err != nil {
log.Errorf(ctx, "%v", err)
}
}
// handler is the main demo entry point that calls the GCS operations.
func handler(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
http.NotFound(w, r)
return
}
c := appengine.NewContext(r)
if bucket == "" {
var err error
if bucket, err = file.DefaultBucketName(c); err != nil {
log.Errorf(c, "failed to get default GCS bucket name: %v", err)
return
}
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
// Note that the App Engine urlfetch service has a limit of 10MB uploads and
// 32MB downloads.
// See https://cloud.google.com/appengine/docs/go/urlfetch/#Go_Quotas_and_limits
// for more information.
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.NewContext(appengine.AppID(c), hc)
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
fmt.Fprintf(w, "Demo GCS Application running from Version: %v\n", appengine.VersionID(c))
fmt.Fprintf(w, "Using bucket name: %v\n\n", bucket)
d := &demo{
c: c,
w: w,
ctx: ctx,
}
n := "demo-testfile-go"
d.createFile(n)
d.readFile(n)
d.copyFile(n)
d.statFile(n)
d.createListFiles()
d.listBucket()
d.listBucketDirMode()
d.defaultACL()
d.putDefaultACLRule()
d.deleteDefaultACLRule()
d.bucketACL()
d.putBucketACLRule()
d.deleteBucketACLRule()
d.acl(n)
d.putACLRule(n)
d.deleteACLRule(n)
d.deleteFiles()
if d.failed {
io.WriteString(w, "\nDemo failed.\n")
} else {
io.WriteString(w, "\nDemo succeeded.\n")
}
}
func getCloudContext(gae context.Context) context.Context {
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(gae, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: gae},
},
}
return cloud.NewContext(appengine.AppID(gae), hc)
}
func (cfg *AppengineStoreConfig) Generate(c context.Context) files.FileStore {
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.WithContext(c, appengine.AppID(c), hc)
return &AppengineStore{cfg, ctx}
}
// newComputeService returns a new Compute Engine API Client,
// to use with Google App Engine.
func newComputeService(c context.Context) (service *compute.Service, err error) {
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, compute.ComputeScope),
Base: &urlfetch.Transport{
Context: c,
},
},
}
return compute.New(client)
}
// Get an auth context for logging RPC.
func cloudAuthContext(r *http.Request) (context.Context, error) {
c := appengine.NewContext(r)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, logging.Scope),
Base: &urlfetch.Transport{Context: c},
},
}
return cloud.WithContext(c, appengine.AppID(c), hc), nil
}
// NewClient returns an http.Client that can be used to create services from the
// Google APIs for Go library https://github.com/google/google-api-go-client.
// The scopes parameter is used to declare the OAuth 2
// scopes, e.g., storage.DevstorageFullControlScope.
func NewClient(c context.Context, scopes ...string) *http.Client {
return &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, scopes...),
// Note that the App Engine urlfetch service has a limit of 10MB uploads and
// 32MB downloads.
// See https://cloud.google.com/appengine/docs/go/urlfetch/#Go_Quotas_and_limits
// for more information.
Base: &urlfetch.Transport{Context: c},
},
}
}
func cloudAuthContext(r *http.Request) (context.Context, error) {
c := appengine.NewContext(r)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: c},
},
}
return cloud.WithContext(c, "rule110-go", hc), nil
}
func ExampleAppEngineTokenSource() {
var req *http.Request // from the ServeHTTP handler
ctx := appengine.NewContext(req)
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, "https://www.googleapis.com/auth/bigquery"),
Base: &urlfetch.Transport{
Context: ctx,
},
},
}
client.Get("...")
}
func fileHandler(c *gin.Context) {
username := c.Param("username")
title := c.Param("title")
filename := c.Param("fileName")
gaeContext := appengine.NewContext(c.Request)
hc := &http.Client{
Transport: &CloudStorageTransport{&oauth2.Transport{
Source: google.AppEngineTokenSource(gaeContext, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: gaeContext},
}},
}
bucketName := "balde_de_bits"
bucketFile := username + "/" + title + "/" + filename
log.Errorf(gaeContext, "ID ->>> %v", appengine.AppID(gaeContext))
log.Errorf(gaeContext, "File name ->>> %v", bucketFile)
ctx := cloud.NewContext(appengine.AppID(gaeContext), hc)
wc := storage.NewWriter(ctx, bucketName, bucketFile)
if strings.Contains(filename, "m3u8") {
wc.ContentType = "application/x-mpegURL"
wc.CacheControl = "max-age:0"
} else if strings.Contains(filename, "ts") {
wc.ContentType = "video/MP2T"
} else if strings.Contains(filename, "jpg") {
wc.ContentType = "image/jpeg"
}
defer wc.Close()
bytesWritten, err := io.Copy(wc, c.Request.Body)
if err != nil {
log.Errorf(gaeContext, "Writing to cloud storage failed. %v", err.Error())
c.JSON(200, gin.H{
"response": "< FAILED >",
})
return
}
log.Errorf(gaeContext, "Wrote < %v > bytes for file < %v >", bytesWritten, filename)
c.JSON(200, gin.H{
"response": "< worked >",
})
}
func handler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.NewContext(appengine.AppID(c), hc)
d := &demo{
c: c,
w: w,
ctx: ctx,
}
d.listBucket()
}
// shortenURL returns a short URL which redirects to the provided url,
// using Google's urlshortener API.
func shortenURL(ctx context.Context, url string) (string, error) {
transport := &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, urlshortener.UrlshortenerScope),
Base: &urlfetch.Transport{Context: ctx},
}
client := &http.Client{Transport: transport}
svc, err := urlshortener.New(client)
if err != nil {
return "", err
}
resp, err := svc.Url.Insert(&urlshortener.Url{LongUrl: url}).Do()
if err != nil {
return "", err
}
return resp.Id, nil
}
func upload(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.NewContext(appengine.AppID(c), hc)
err := r.ParseMultipartForm(1048576)
if err != nil {
io.WriteString(w, err.Error())
return
}
file, fileHeader, err := r.FormFile("file")
if err != nil {
io.WriteString(w, err.Error())
return
}
defer file.Close()
data, err := ioutil.ReadAll(file)
if err != nil {
io.WriteString(w, err.Error())
return
}
wc := storage.NewWriter(ctx, bucket, "slide/"+fileHeader.Filename)
wc.ContentType = http.DetectContentType(data)
if _, err := wc.Write(data); err != nil {
io.WriteString(w, err.Error())
return
}
if err := wc.Close(); err != nil {
io.WriteString(w, err.Error())
return
}
}
func handlePut(res http.ResponseWriter, req *http.Request) {
ctx := appengine.NewContext(req)
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: ctx},
},
}
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
writer := storage.NewWriter(cctx, bucketName, "example2.txt")
io.WriteString(writer, "Another file")
err := writer.Close()
if err != nil {
http.Error(res, "ERROR WRITING TO BUCKET: "+err.Error(), 500)
return
}
}
func serveFilesUpload(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
serveAPI(res, req, func() interface{} {
ctx := appengine.NewContext(req)
session, _ := sessionStore.Get(req, "session")
_, ok := session.Values["email"].(string)
if !ok {
return HTTPError{403, "access denied"}
}
bucket, err := file.DefaultBucketName(ctx)
if err != nil {
return err
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: ctx},
},
}
id := uuid.NewV1().String()
ff, _, err := req.FormFile("file")
if err != nil {
return err
}
defer ff.Close()
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
wc := storage.NewWriter(cctx, bucket, id)
io.Copy(wc, ff)
err = wc.Close()
if err != nil {
return err
}
return id
})
}
func handlerInsert(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, bigquery.BigqueryScope),
Base: &urlfetch.Transport{Context: ctx},
},
}
bq, err := bigquery.New(client)
if err != nil {
fmt.Errorf("%v", err)
}
key := datastore.Key{}
c := Container2{
Hoge: Hoge{Name: "hoge", Age: 28},
Key: &key,
}
jsonValue, err := ironmole.BuildJsonValue(&c)
if err != nil {
log.Errorf(ctx, "%v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
res, err := ironmole.Insert(bq, "cp300demo1", "go2bq", "Container2", jsonValue)
if err != nil {
log.Errorf(ctx, "%v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
for _, insertError := range res.InsertErrors {
for _, err := range insertError.Errors {
log.Errorf(ctx, "Insert Error = %v", err)
}
}
}
func handlerTableMoge(w http.ResponseWriter, r *http.Request) {
ctx := appengine.NewContext(r)
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, bigquery.BigqueryScope),
Base: &urlfetch.Transport{Context: ctx},
},
}
bq, err := bigquery.New(client)
if err != nil {
log.Errorf(ctx, "%v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
table := "Moge"
tableParam := r.FormValue("table")
if len(tableParam) > 0 {
table = tableParam
}
moge := Moge{}
schema, err := ironmole.BuildTableSchemaWithContext(ctx, &moge)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
err = ironmole.CreateTable(bq, "cp300demo1", "go2bq", table, schema)
if err != nil {
log.Errorf(ctx, "%v", err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte("done"))
}
func bigqueryService(c context.Context) (*bigquery.Service, error) {
var client *http.Client
if appengine.IsDevAppServer() {
jsonKey, err := ioutil.ReadFile("service-account.json")
if err != nil {
return nil, err
}
conf, err := google.JWTConfigFromJSON(jsonKey, bigquery.BigqueryScope)
if err != nil {
return nil, err
}
client = conf.Client(c)
} else {
token := google.AppEngineTokenSource(c, bigquery.BigqueryScope)
client = oauth2.NewClient(c, token)
}
service, err := bigquery.New(client)
if err != nil {
return nil, err
}
return service, nil
}
// Files
func serveFilesGet(res http.ResponseWriter, req *http.Request, params httprouter.Params) {
serveAPI(res, req, func() interface{} {
ctx := appengine.NewContext(req)
session, _ := sessionStore.Get(req, "session")
_, ok := session.Values["email"].(string)
if !ok {
return HTTPError{403, "access denied"}
}
bucket, err := file.DefaultBucketName(ctx)
if err != nil {
return err
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, storage.ScopeReadOnly),
Base: &urlfetch.Transport{Context: ctx},
},
}
cctx := cloud.NewContext(appengine.AppID(ctx), hc)
rc, err := storage.NewReader(cctx, bucket, params.ByName("id"))
if err != nil {
return err
}
name := req.URL.Query().Get("name")
if name == "" {
name = params.ByName("id")
}
name = regexp.MustCompile("[^a-zA-Z-_.]").ReplaceAllString(name, "")
res.Header().Set("Content-Disposition", "inline; filename=\""+name+"\"")
res.Header().Set("Content-Type", "application/octet-stream")
return rc
})
}
func dnsUpdate(c context.Context, name, ip string) error {
// name must be a FQDN
if !strings.HasPrefix(name, ".") {
name = name + "."
}
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, "https://www.googleapis.com/auth/ndev.clouddns.readwrite"),
Base: &urlfetch.Transport{
Context: c,
},
},
}
srv, err := dns.New(client)
if err != nil {
return err
}
_, err = createARecord(srv, appengine.AppID(c), "fbcodelab33", name, ip)
return err
}
func listHandler(c *gin.Context) {
gaeContext := appengine.NewContext(c.Request)
fhc := &http.Client{
Transport: &CloudStorageTransport{&oauth2.Transport{
Source: google.AppEngineTokenSource(gaeContext, storage.ScopeFullControl),
Base: &urlfetch.Transport{Context: gaeContext},
}},
}
bucketName := "balde_de_bits"
cloudContext := cloud.NewContext(appengine.AppID(gaeContext), fhc)
objects, _ := storage.ListObjects(cloudContext, bucketName, nil)
usersMap := mapFilesToDictionary(objects)
usersStruct := mapDictionaryToObjects(usersMap)
c.JSON(200, gin.H{
"users": usersStruct,
})
}
func list(datasetId string, tableName string, ctx context.Context, w http.ResponseWriter) ([]*bigquery.TableRow, error) {
client := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(ctx, bigquery.BigqueryScope),
Base: &urlfetch.Transport{Context: ctx},
},
}
bq, err := bigquery.New(client)
if err != nil {
fmt.Errorf("could not create service: %v", err)
return nil, err
}
appID := appengine.AppID(ctx)
result, errr := bq.Tabledata.List(appID, datasetId, tableName).IfNoneMatch("Id=null").Do()
if errr != nil {
fmt.Errorf("could not read: %v", err)
return nil, errr
}
return result.Rows, nil
}
// Store binary data to GCS
func Store(c context.Context, data []byte, fileName, mimeType, bucketName string) (absFilename string, err error) {
if bucketName == "" {
var err error
if bucketName, err = file.DefaultBucketName(c); err != nil {
log.Errorf(c, "failed to get default GCS bucket name: %v", err)
return "", err
}
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
// Note that the App Engine urlfetch service has a limit of 10MB uploads and
// 32MB downloads.
// See https://cloud.google.com/appengine/docs/go/urlfetch/#Go_Quotas_and_limits
// for more information.
Base: &urlfetch.Transport{Context: c},
},
}
ctx := cloud.NewContext(appengine.AppID(c), hc)
wc := storage.NewWriter(ctx, bucketName, fileName)
wc.ContentType = mimeType
if _, err := wc.Write(data); err != nil {
log.Errorf(c, "upload file: unable to write data to bucket %q, file %q: %v", bucketName, fileName, err)
return "", err
}
if err := wc.Close(); err != nil {
log.Errorf(c, "upload file: unable to close bucket %q, file %q: %v", bucketName, fileName, err)
return "", err
}
return getAbsFilename(bucketName, fileName), nil
}