func init() {
gob.Register(time.Time{})
gob.Register(datastore.ByteString{})
gob.Register(&datastore.Key{})
gob.Register(appengine.BlobKey(""))
gob.Register(appengine.GeoPoint{})
}
func (tf *typeFilter) Save() ([]datastore.Property, error) {
props := []datastore.Property{}
for name, propList := range tf.pm {
if len(name) != 0 && name[0] == '$' {
continue
}
multiple := len(propList) > 1
for _, prop := range propList {
toAdd := datastore.Property{
Name: name,
Multiple: multiple,
NoIndex: prop.IndexSetting() == ds.NoIndex,
}
switch prop.Type() {
case ds.PTBytes:
v := prop.Value().([]byte)
if prop.IndexSetting() == ds.ShouldIndex {
toAdd.Value = datastore.ByteString(v)
} else {
toAdd.Value = v
}
case ds.PTKey:
toAdd.Value = dsF2R(prop.Value().(ds.Key))
case ds.PTBlobKey:
toAdd.Value = appengine.BlobKey(prop.Value().(bs.Key))
case ds.PTGeoPoint:
toAdd.Value = appengine.GeoPoint(prop.Value().(ds.GeoPoint))
default:
toAdd.Value = prop.Value()
}
props = append(props, toAdd)
}
}
return props, nil
}
// propValue returns a Go value that combines the raw PropertyValue with a
// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time.
func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) {
switch {
case v.Int64Value != nil:
if m == pb.Property_GD_WHEN {
return fromUnixMicro(*v.Int64Value), nil
} else {
return *v.Int64Value, nil
}
case v.BooleanValue != nil:
return *v.BooleanValue, nil
case v.StringValue != nil:
if m == pb.Property_BLOB {
return []byte(*v.StringValue), nil
} else if m == pb.Property_BLOBKEY {
return appengine.BlobKey(*v.StringValue), nil
} else if m == pb.Property_BYTESTRING {
return ByteString(*v.StringValue), nil
} else {
return *v.StringValue, nil
}
case v.DoubleValue != nil:
return *v.DoubleValue, nil
case v.Referencevalue != nil:
key, err := referenceValueToKey(v.Referencevalue)
if err != nil {
return nil, err
}
return key, nil
case v.Pointvalue != nil:
// NOTE: Strangely, latitude maps to X, longitude to Y.
return appengine.GeoPoint{Lat: v.Pointvalue.GetX(), Lng: v.Pointvalue.GetY()}, nil
}
return nil, nil
}
// propValue returns a Go value that combines the raw PropertyValue with a
// meaning. For example, an Int64Value with GD_WHEN becomes a time.Time.
func propValue(v *pb.PropertyValue, m pb.Property_Meaning) (interface{}, error) {
switch {
case v.Int64Value != nil:
if m == pb.Property_GD_WHEN {
return fromUnixMicro(*v.Int64Value), nil
} else {
return *v.Int64Value, nil
}
case v.BooleanValue != nil:
return *v.BooleanValue, nil
case v.StringValue != nil:
if m == pb.Property_BLOB {
return []byte(*v.StringValue), nil
} else if m == pb.Property_BLOBKEY {
return appengine.BlobKey(*v.StringValue), nil
} else {
return *v.StringValue, nil
}
case v.DoubleValue != nil:
return *v.DoubleValue, nil
case v.Referencevalue != nil:
key, err := referenceValueToKey(v.Referencevalue)
if err != nil {
return nil, err
}
return key, nil
}
return nil, nil
}
// BlobKeyForFile returns a BlobKey for a Google Storage file.
// The filename should be of the form "/gs/bucket_name/object_name".
func BlobKeyForFile(c context.Context, filename string) (appengine.BlobKey, error) {
req := &blobpb.CreateEncodedGoogleStorageKeyRequest{
Filename: &filename,
}
res := &blobpb.CreateEncodedGoogleStorageKeyResponse{}
if err := internal.Call(c, "blobstore", "CreateEncodedGoogleStorageKey", req, res); err != nil {
return "", err
}
return appengine.BlobKey(*res.BlobKey), nil
}
func GetServingUrl(c *gin.Context) {
r := c.Request
ctx := appengine.NewContext(r)
blobKey := appengine.BlobKey(c.Param("blobKey"))
var err_msg string
url, err := image.ServingURL(ctx, blobKey, nil)
if err != nil {
err_msg = "failed"
}
c.JSON(http.StatusOK, gin.H{"err": err_msg, "url": url, "blobKey": url.String()})
}
func serveThumb(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
c := appengine.NewContext(r)
// c := appengine.NewContext(r)
k := appengine.BlobKey(r.FormValue("blobkey"))
var o image.ServingURLOptions = *new(image.ServingURLOptions)
o.Size = 200
o.Crop = true
url, err := image.ServingURL(c, k, &o)
loghttp.E(w, r, err, false)
http.Redirect(w, r, url.String(), http.StatusFound)
}
func handleFilesItem(w http.ResponseWriter, r *http.Request) {
id := r.URL.Path
if id == "" {
w.WriteHeader(404)
return
}
c := appengine.NewContext(r)
key := datastore.NewKey(c, "File", id, 0, nil)
switch r.Method {
case "GET":
var file File
err := datastore.Get(c, key, &file)
if err == datastore.ErrNoSuchEntity {
http.NotFound(w, r)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "query file: %s", err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(&file)
if err != nil {
log.Errorf(c, "json encode: %s", err.Error())
}
case "DELETE":
err := datastore.Delete(c, key)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "delete file: %s", err.Error())
return
}
err = blobstore.Delete(c, appengine.BlobKey(id))
if err != nil {
log.Warningf(c, "delete file '%s': %s", id, err.Error())
}
w.WriteHeader(204)
default:
http.Error(w, "Valid methods are GET and DELETE", http.StatusMethodNotAllowed)
return
}
}
func handleBlobFile(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
key := datastore.NewKey(c, "File", r.URL.Path, 0, nil)
var file File
err := datastore.Get(c, key, &file)
if err != nil {
http.NotFound(w, r)
return
}
w.Header().Set("Content-Type", mime.TypeByExtension(path.Ext(file.Name)))
_, err = io.Copy(w, blobstore.NewReader(c, appengine.BlobKey(r.URL.Path)))
if err != nil {
log.Errorf(c, "stream blob: %s", err.Error())
}
}
func sampleHandler2(w http.ResponseWriter, r *http.Request) {
// [START uploading_a_blob_3]
ctx := appengine.NewContext(r)
blobs, _, err := blobstore.ParseUpload(r)
if err != nil {
serveError(ctx, w, err)
return
}
file := blobs["file"]
if len(file) == 0 {
log.Errorf(ctx, "no file uploaded")
http.Redirect(w, r, "/", http.StatusFound)
return
}
http.Redirect(w, r, "/serve/?blobKey="+string(file[0].BlobKey), http.StatusFound)
// [END uploading_a_blob_3]
// [START serving_a_blob]
blobstore.Send(w, appengine.BlobKey(r.FormValue("blobKey")))
// [END serving_a_blob]
}
func TestReader(t *testing.T) {
for _, rt := range readerTest {
c := aetesting.FakeSingleContext(t, "blobstore", "FetchData", fakeFetchData)
r := NewReader(c, appengine.BlobKey(rt.blobKey))
for i, step := range rt.step {
var (
got string
gotErr error
n int
offset int64
)
switch step.method {
case "LargeReadAt":
p := make([]byte, step.lenp)
n, gotErr = r.ReadAt(p, step.offset)
got = strconv.Itoa(n)
case "Read":
p := make([]byte, step.lenp)
n, gotErr = r.Read(p)
got = string(p[:n])
case "ReadAt":
p := make([]byte, step.lenp)
n, gotErr = r.ReadAt(p, step.offset)
got = string(p[:n])
case "Seek":
offset, gotErr = r.Seek(step.offset, step.whence)
got = strconv.FormatInt(offset, 10)
default:
t.Fatalf("unknown method: %s", step.method)
}
if gotErr != step.wantErr {
t.Fatalf("%s step %d: got error %v want %v", rt.blobKey, i, gotErr, step.wantErr)
}
if got != step.want {
t.Fatalf("%s step %d: got %q want %q", rt.blobKey, i, got, step.want)
}
}
}
}
func dsF2RProp(ctx context.Context, in ds.Property) (datastore.Property, error) {
err := error(nil)
ret := datastore.Property{
NoIndex: in.IndexSetting() == ds.NoIndex,
}
switch in.Type() {
case ds.PTBytes:
v := in.Value().([]byte)
if in.IndexSetting() == ds.ShouldIndex {
ret.Value = datastore.ByteString(v)
} else {
ret.Value = v
}
case ds.PTKey:
ret.Value, err = dsF2R(ctx, in.Value().(*ds.Key))
case ds.PTBlobKey:
ret.Value = appengine.BlobKey(in.Value().(bs.Key))
case ds.PTGeoPoint:
ret.Value = appengine.GeoPoint(in.Value().(ds.GeoPoint))
default:
ret.Value = in.Value()
}
return ret, err
}
]`)
// icon is a sample white png file 16x16,
// dumped as a byte array.
var icon = []byte{
137, 80, 78, 71, 13, 10, 26, 10,
0, 0, 0, 13, 73, 72, 68, 82,
0, 0, 0, 16, 0, 0, 0, 16,
8, 2, 0, 0, 0, 144, 145, 104,
54, 0, 0, 0, 9, 112, 72, 89,
115, 0, 0, 11, 19, 0, 0, 11,
19, 1, 0, 154, 156, 24, 0, 0,
0, 7, 116, 73, 77, 69, 7, 222,
5, 8, 21, 41, 53, 225, 172, 74,
51, 0, 0, 0, 25, 116, 69, 88,
116, 67, 111, 109, 109, 101, 110, 116,
0, 67, 114, 101, 97, 116, 101, 100,
32, 119, 105, 116, 104, 32, 71, 73,
77, 80, 87, 129, 14, 23, 0, 0,
0, 26, 73, 68, 65, 84, 40, 207,
99, 252, 255, 255, 63, 3, 41, 128,
137, 129, 68, 48, 170, 97, 84, 195,
208, 209, 0, 0, 85, 109, 3, 29,
159, 46, 21, 162, 0, 0, 0, 0,
73, 69, 78, 68, 174, 66, 96, 130}
// blobKey is a sample appengine.BlobKey value.
var blobKey = appengine.BlobKey("AMIfv94Ly-gFmdjqsU9IwztyA6jjiChzE8cUSwkP8EE" +
"fo4paIuXmHiwFkoccnayuqcTmkyXfDo8SS9uetO-6h7AhqlKQFYsY1tyGjrhjqmxOYT19CC" +
"tH5tZEL2pxtCBLe6MFProzW1fw1du_vMwPsypKMHnnpZau6F_qJNoc6yoqnYIKGDvroNk")
func handleServe(w http.ResponseWriter, r *http.Request) {
blobstore.Send(w, appengine.BlobKey(r.FormValue("blobKey")))
}
func serveFull(w http.ResponseWriter, r *http.Request, m map[string]interface{}) {
blobstore.Send(w, appengine.BlobKey(r.FormValue("blobkey")))
}
func Serve(c *gin.Context) {
r := c.Request
w := c.Writer
blobstore.Send(w, appengine.BlobKey(r.FormValue("blobKey")))
}
func handleBucketsItem(w http.ResponseWriter, r *http.Request) {
id := r.URL.Path
if id == "" {
w.WriteHeader(404)
return
}
c := appengine.NewContext(r)
key := datastore.NewKey(c, "Bucket", id, 0, nil)
switch r.Method {
case "GET":
var b Bucket
err := datastore.Get(c, key, &b)
if err == datastore.ErrNoSuchEntity {
w.WriteHeader(404)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "query bucket: %s", err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(&b)
if err != nil {
log.Errorf(c, "json encode: %s", err.Error())
}
case "PUT":
var b Bucket
err := json.NewDecoder(r.Body).Decode(&b)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if b.ID != id {
http.Error(w, "Bucket ID must match URL ID", http.StatusBadRequest)
return
}
_, err = datastore.Put(c, key, &b)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "put bucket: %s", err.Error())
return
}
w.WriteHeader(204)
case "DELETE":
err := datastore.RunInTransaction(c, func(c context.Context) error {
metaKey := datastore.NewKey(c, "Meta", "main", 0, nil)
var m Meta
err := datastore.Get(c, metaKey, &m)
if err != nil {
return err
}
b := m.Buckets[:0]
for _, bucket := range m.Buckets {
if bucket == id {
continue
}
b = append(b, bucket)
}
m.Buckets = b
_, err = datastore.Put(c, metaKey, &m)
if err != nil {
return err
}
return err
}, &datastore.TransactionOptions{XG: true})
if err != nil {
log.Warningf(c, "update meta: %s", err.Error())
}
var bk Bucket
err = datastore.Get(c, key, &bk)
if err == nil && bk.Images != nil { // delete images
for _, id := range bk.Images {
err = blobstore.Delete(c, appengine.BlobKey(id))
if err != nil {
log.Warningf(c, "delete image blob '%s': %s", id, err.Error())
}
err = datastore.Delete(c, datastore.NewKey(c, "Image", id, 0, nil))
if err != nil {
log.Warningf(c, "delete image entry '%s': %s", id, err.Error())
}
}
}
err = datastore.Delete(c, key)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "delete bucket: %s", err.Error())
return
}
w.WriteHeader(204)
default:
http.Error(w, "Valid methods are GET, DELETE and PUT", http.StatusMethodNotAllowed)
return
}
}
func decodeProperty(c context.Context, k string, v interface{}, e *Entity) error {
var p datastore.Property
p.Name = k
var err error
switch v.(type) {
// Try to decode property object
case map[string]interface{}:
// Decode custom type
m := v.(map[string]interface{})
t, ok := m["type"]
if !ok {
t = "primitive"
}
if index, ok := m["indexed"]; ok {
if i, ok := index.(bool); ok {
p.NoIndex = !i
}
}
switch t {
case "key":
key, err := decodeKey(c, m["value"])
if err != nil {
return err
}
p.Value = key
case "blobkey":
v, ok := m["value"].(string)
if !ok {
return newDecodePropertyError(k, "blobkey", v)
}
p.Value = appengine.BlobKey(v)
case "blob":
v, ok := m["value"].(string)
if !ok {
return newDecodePropertyError(k, "date", v)
}
p.Value, err = base64.URLEncoding.DecodeString(v)
if err != nil {
return err
}
case "date":
v, ok := m["value"].(string)
if !ok {
return newDecodePropertyError(k, "date", v)
}
var dt time.Time
dt, err = time.Parse(DateTimeFormat, v)
if err != nil {
return newDecodePropertyError(k, "date", err)
}
p.Value = dt.UTC()
default:
if v, ok := m["value"]; ok {
err = decodeJSONPrimitiveValue(v, &p)
} else {
err = fmt.Errorf("aetools: complex property %s without 'value' attribute", k)
}
}
default:
err = decodeJSONPrimitiveValue(v, &p)
}
if err == nil {
e.Properties = append(e.Properties, p)
}
return err
}
func getImage(res http.ResponseWriter, req *http.Request, ps httprouter.Params) {
// requesting an image based on blob key
blobstore.Send(res, appengine.BlobKey(ps.ByName("blobKey")))
}
func TestMarshalUnmarshalPropertyList(t *testing.T) {
c, closeFunc := NewContext(t)
defer closeFunc()
timeVal := time.Now()
timeProp := datastore.Property{Name: "Time",
Value: timeVal, NoIndex: false, Multiple: false}
byteStringVal := datastore.ByteString{0x23}
byteStringProp := datastore.Property{Name: "ByteString",
Value: byteStringVal, NoIndex: false, Multiple: false}
keyVal := datastore.NewKey(c, "Entity", "stringID", 0, nil)
keyProp := datastore.Property{Name: "Key",
Value: keyVal, NoIndex: false, Multiple: false}
blobKeyVal := appengine.BlobKey("blobkey")
blobKeyProp := datastore.Property{Name: "BlobKey",
Value: blobKeyVal, NoIndex: false, Multiple: false}
geoPointVal := appengine.GeoPoint{1, 2}
geoPointProp := datastore.Property{Name: "GeoPoint",
Value: geoPointVal, NoIndex: false, Multiple: false}
pl := datastore.PropertyList{
timeProp,
byteStringProp,
keyProp,
blobKeyProp,
geoPointProp,
}
data, err := nds.MarshalPropertyList(pl)
if err != nil {
t.Fatal(err)
}
testEntity := &struct {
Time time.Time
ByteString datastore.ByteString
Key *datastore.Key
BlobKey appengine.BlobKey
GeoPoint appengine.GeoPoint
}{}
pl = datastore.PropertyList{}
if err := nds.UnmarshalPropertyList(data, &pl); err != nil {
t.Fatal(err)
}
if err := nds.SetValue(reflect.ValueOf(testEntity), pl); err != nil {
t.Fatal(err)
}
if !testEntity.Time.Equal(timeVal) {
t.Fatal("timeVal not equal")
}
if string(testEntity.ByteString) != string(byteStringVal) {
t.Fatal("byteStringVal not equal")
}
if !testEntity.Key.Equal(keyVal) {
t.Fatal("keyVal not equal")
}
if testEntity.BlobKey != blobKeyVal {
t.Fatal("blobKeyVal not equal")
}
if !reflect.DeepEqual(testEntity.GeoPoint, geoPointVal) {
t.Fatal("geoPointVal not equal")
}
}
func handleImages(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
switch r.Method {
case "POST":
blobs, vals, err := blobstore.ParseUpload(r)
if err != nil {
http.Error(w, "bad request", http.StatusBadRequest)
log.Errorf(c, "bad request: %s", err.Error())
return
}
bucketID := vals.Get("BucketID")
if bucketID == "" {
http.Error(w, "BucketID query parameter is required", http.StatusBadRequest)
return
}
url, err := blobstore.UploadURL(c, "/admin/images", nil)
if err == nil {
w.Header().Set("UploadURL", url.String())
}
imgs := make([]Image, 0, 20)
for _, infos := range blobs {
for _, info := range infos {
var img Image
img.Name = info.Filename
imgUrl, err := aimage.ServingURL(c, info.BlobKey, nil)
if err != nil {
log.Errorf(c, "failed to get serving url for blob '%s': %s", info.BlobKey, err.Error())
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
img.URL = imgUrl.String()
img.ID = string(info.BlobKey)
ir := blobstore.NewReader(c, info.BlobKey)
cfg, _, err := image.DecodeConfig(ir)
if err != nil {
log.Warningf(c, "decode image '%s': %s", info.BlobKey, err.Error())
err = blobstore.Delete(c, info.BlobKey)
if err != nil {
log.Errorf(c, "delete blob '%s': %s", info.BlobKey, err.Error())
}
continue
}
img.Height = cfg.Height
img.Width = cfg.Width
imgs = append(imgs, img)
}
}
// add images to bucket and dtore
err = datastore.RunInTransaction(c, func(c context.Context) error {
bucketKey := datastore.NewKey(c, "Bucket", bucketID, 0, nil)
var b Bucket
err := datastore.Get(c, bucketKey, &b)
if err != nil {
return err
}
if b.Images == nil {
b.Images = make([]string, 0, len(imgs))
}
for _, img := range imgs {
_, err = datastore.Put(c, datastore.NewKey(c, "Image", img.ID, 0, nil), &img)
if err != nil {
return err
}
b.Images = append(b.Images, img.ID)
}
_, err = datastore.Put(c, bucketKey, &b)
return err
}, &datastore.TransactionOptions{XG: true})
if err == datastore.ErrNoSuchEntity {
for _, img := range imgs {
err = blobstore.Delete(c, appengine.BlobKey(img.ID))
if err != nil {
log.Warningf(c, "delete blob '%s': %s", img.ID, err.Error())
}
}
http.NotFound(w, r)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "update datastore: %s", err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(&imgs)
if err != nil {
log.Errorf(c, "json encode: %s", err.Error())
}
case "GET":
var imgs []Image
_, err := datastore.NewQuery("Image").GetAll(c, &imgs)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "query images: %s", err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
if imgs == nil {
imgs = []Image{}
}
err = json.NewEncoder(w).Encode(&imgs)
if err != nil {
log.Errorf(c, "json encode: %s", err.Error())
}
default:
http.Error(w, "Valid methods are GET and POST", http.StatusMethodNotAllowed)
return
}
}
func handleImagesItem(w http.ResponseWriter, r *http.Request) {
id := r.URL.Path
if id == "" {
w.WriteHeader(404)
return
}
c := appengine.NewContext(r)
key := datastore.NewKey(c, "Image", id, 0, nil)
switch r.Method {
case "GET":
var img Image
err := datastore.Get(c, key, &img)
if err == datastore.ErrNoSuchEntity {
w.WriteHeader(404)
return
}
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "query image: %s", err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
err = json.NewEncoder(w).Encode(&img)
if err != nil {
log.Errorf(c, "json encode: %s", err.Error())
}
case "DELETE":
err := datastore.Delete(c, key)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "delete image: %s", err.Error())
return
}
err = blobstore.Delete(c, appengine.BlobKey(id))
if err != nil {
log.Warningf(c, "delete image '%s': %s", id, err.Error())
}
w.WriteHeader(204)
case "PUT":
var img Image
err := json.NewDecoder(r.Body).Decode(&img)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if img.ID != id {
http.Error(w, "Image ID must match URL ID", http.StatusBadRequest)
return
}
_, err = datastore.Put(c, key, &img)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
log.Errorf(c, "put image: %s", err.Error())
return
}
w.WriteHeader(204)
default:
http.Error(w, "Valid methods are GET, DELETE and PUT", http.StatusMethodNotAllowed)
return
}
}
// Use of this source code is governed by the Apache 2.0
// license that can be found in the LICENSE file.
package backup
import (
"fmt"
"reflect"
"time"
pb "github.com/sromku/datastore-to-sql/backup/pb"
"google.golang.org/appengine"
)
var (
typeOfBlobKey = reflect.TypeOf(appengine.BlobKey(""))
typeOfByteSlice = reflect.TypeOf([]byte(nil))
typeOfByteString = reflect.TypeOf(ByteString(nil))
typeOfGeoPoint = reflect.TypeOf(appengine.GeoPoint{})
typeOfTime = reflect.TypeOf(time.Time{})
)
// typeMismatchReason returns a string explaining why the property p could not
// be stored in an entity field of type v.Type().
func typeMismatchReason(p Property, v reflect.Value) string {
entityType := "empty"
switch p.Value.(type) {
case int64:
entityType = "int"
case bool:
entityType = "bool"
func idiomPicture(w http.ResponseWriter, r *http.Request) error {
// From https://developers.google.com/appengine/docs/go/blobstore/#Complete_Sample_App
blobstore.Send(w, appengine.BlobKey(r.FormValue("blobKey")))
return nil
}
"github.com/golang/protobuf/proto"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/datastore"
"google.golang.org/appengine/internal"
basepb "google.golang.org/appengine/internal/base"
blobpb "google.golang.org/appengine/internal/blobstore"
)
const (
blobInfoKind = "__BlobInfo__"
blobFileIndexKind = "__BlobFileIndex__"
zeroKey = appengine.BlobKey("")
)
// BlobInfo is the blob metadata that is stored in the datastore.
// Filename may be empty.
type BlobInfo struct {
BlobKey appengine.BlobKey
ContentType string `datastore:"content_type"`
CreationTime time.Time `datastore:"creation"`
Filename string `datastore:"filename"`
Size int64 `datastore:"size"`
MD5 string `datastore:"md5_hash"`
// ObjectName is the Google Cloud Storage name for this blob.
ObjectName string `datastore:"gs_object_name"`
}
// ParseUpload parses the synthetic POST request that your app gets from
// App Engine after a user's successful upload of blobs. Given the request,
// ParseUpload returns a map of the blobs received (keyed by HTML form
// element name) and other non-blob POST parameters.
func ParseUpload(req *http.Request) (blobs map[string][]*BlobInfo, other url.Values, err error) {
_, params, err := mime.ParseMediaType(req.Header.Get("Content-Type"))
if err != nil {
return nil, nil, err
}
boundary := params["boundary"]
if boundary == "" {
return nil, nil, errorf("did not find MIME multipart boundary")
}
blobs = make(map[string][]*BlobInfo)
other = make(url.Values)
mreader := multipart.NewReader(io.MultiReader(req.Body, strings.NewReader("\r\n\r\n")), boundary)
for {
part, perr := mreader.NextPart()
if perr == io.EOF {
break
}
if perr != nil {
return nil, nil, errorf("error reading next mime part with boundary %q (len=%d): %v",
boundary, len(boundary), perr)
}
bi := &BlobInfo{}
ctype, params, err := mime.ParseMediaType(part.Header.Get("Content-Disposition"))
if err != nil {
return nil, nil, err
}
bi.Filename = params["filename"]
formKey := params["name"]
ctype, params, err = mime.ParseMediaType(part.Header.Get("Content-Type"))
if err != nil {
return nil, nil, err
}
bi.BlobKey = appengine.BlobKey(params["blob-key"])
if ctype != "message/external-body" || bi.BlobKey == "" {
if formKey != "" {
slurp, serr := ioutil.ReadAll(part)
if serr != nil {
return nil, nil, errorf("error reading %q MIME part", formKey)
}
other[formKey] = append(other[formKey], string(slurp))
}
continue
}
// App Engine sends a MIME header as the body of each MIME part.
tp := textproto.NewReader(bufio.NewReader(part))
header, mimeerr := tp.ReadMIMEHeader()
if mimeerr != nil {
return nil, nil, mimeerr
}
bi.Size, err = strconv.ParseInt(header.Get("Content-Length"), 10, 64)
if err != nil {
return nil, nil, err
}
bi.ContentType = header.Get("Content-Type")
// Parse the time from the MIME header like:
// X-AppEngine-Upload-Creation: 2011-03-15 21:38:34.712136
createDate := header.Get("X-AppEngine-Upload-Creation")
if createDate == "" {
return nil, nil, errorf("expected to find an X-AppEngine-Upload-Creation header")
}
bi.CreationTime, err = time.Parse("2006-01-02 15:04:05.000000", createDate)
if err != nil {
return nil, nil, errorf("error parsing X-AppEngine-Upload-Creation: %s", err)
}
if hdr := header.Get("Content-MD5"); hdr != "" {
md5, err := base64.URLEncoding.DecodeString(hdr)
if err != nil {
return nil, nil, errorf("bad Content-MD5 %q: %v", hdr, err)
}
bi.MD5 = string(md5)
}
// If the GCS object name was provided, record it.
bi.ObjectName = header.Get("X-AppEngine-Cloud-Storage-Object")
blobs[formKey] = append(blobs[formKey], bi)
}
return
}