// Project can be used to project a Property retrieved from a Projection query
// into a different datatype. For example, if you have a PTInt property, you
// could Project(PTTime) to convert it to a time.Time. The following conversions
// are supported:
// PTXXX <-> PTXXX (i.e. identity)
// PTInt <-> PTTime
// PTString <-> PTBlobKey
// PTString <-> PTBytes
// PTNull <-> Anything
func (p *Property) Project(to PropertyType) (interface{}, error) {
switch {
case to == p.propType:
return p.value, nil
case to == PTInt && p.propType == PTTime:
t := p.value.(time.Time)
v := uint64(t.Unix())*1e6 + uint64(t.Nanosecond()/1e3)
return int64(v), nil
case to == PTTime && p.propType == PTInt:
v := p.value.(int64)
return time.Unix(int64(v/1e6), int64((v%1e6)*1e3)).UTC(), nil
case to == PTString && p.propType == PTBytes:
return string(p.value.([]byte)), nil
case to == PTString && p.propType == PTBlobKey:
return string(p.value.(blobstore.Key)), nil
case to == PTBytes && p.propType == PTString:
return []byte(p.value.(string)), nil
case to == PTBlobKey && p.propType == PTString:
return blobstore.Key(p.value.(string)), nil
case to == PTNull:
return nil, nil
case p.propType == PTNull:
switch to {
case PTInt:
return int64(0), nil
case PTTime:
return time.Time{}, nil
case PTBool:
return false, nil
case PTBytes:
return []byte(nil), nil
case PTString:
return "", nil
case PTFloat:
return float64(0), nil
case PTGeoPoint:
return GeoPoint{}, nil
case PTKey:
return nil, nil
case PTBlobKey:
return blobstore.Key(""), nil
}
fallthrough
default:
return nil, fmt.Errorf("unable to project %s to %s", p.propType, to)
}
}
func dsR2FProp(in datastore.Property) (ds.Property, error) {
val := in.Value
switch x := val.(type) {
case datastore.ByteString:
val = []byte(x)
case *datastore.Key:
val = dsR2F(x)
case appengine.BlobKey:
val = bs.Key(x)
case appengine.GeoPoint:
val = ds.GeoPoint(x)
case time.Time:
// "appengine" layer instantiates with Local timezone.
if x.IsZero() {
val = time.Time{}
} else {
val = x.UTC()
}
default:
val = maybeIndexValue(val)
}
ret := ds.Property{}
is := ds.ShouldIndex
if in.NoIndex {
is = ds.NoIndex
}
err := ret.SetValue(val, is)
return ret, err
}
func (tf *typeFilter) Load(props []datastore.Property) error {
tf.pm = make(ds.PropertyMap, len(props))
for _, p := range props {
val := p.Value
switch x := val.(type) {
case datastore.ByteString:
val = []byte(x)
case *datastore.Key:
val = dsR2F(x)
case appengine.BlobKey:
val = bs.Key(x)
case appengine.GeoPoint:
val = ds.GeoPoint(x)
case time.Time:
// "appengine" layer instantiates with Local timezone.
val = x.UTC()
default:
val = maybeIndexValue(val)
}
prop := ds.Property{}
is := ds.ShouldIndex
if p.NoIndex {
is = ds.NoIndex
}
if err := prop.SetValue(val, is); err != nil {
return err
}
tf.pm[p.Name] = append(tf.pm[p.Name], prop)
}
return nil
}
// Value returns the current value held by this property. It's guaranteed to
// be a valid value type (i.e. `p.SetValue(p.Value(), true)` will never return
// an error).
func (p *Property) Value() interface{} {
switch p.propType {
case PTBytes:
return p.value.(byteSequence).bytes()
case PTString:
return p.value.(byteSequence).string()
case PTBlobKey:
return blobstore.Key(p.value.(byteSequence).string())
default:
return p.value
}
}
// ReadProperty reads a Property from the buffer. `context`, `appid`, and
// `namespace` behave the same way they do for ReadKey, but only have an
// effect if the decoded property has a Key value.
func ReadProperty(buf Buffer, context KeyContext, appid, namespace string) (p ds.Property, err error) {
val := interface{}(nil)
b, err := buf.ReadByte()
if err != nil {
return
}
is := ds.ShouldIndex
if (b & 0x80) == 0 {
is = ds.NoIndex
}
switch ds.PropertyType(b & 0x7f) {
case ds.PTNull:
case ds.PTBool:
b, err = buf.ReadByte()
val = (b != 0)
case ds.PTInt:
val, _, err = cmpbin.ReadInt(buf)
case ds.PTFloat:
val, _, err = cmpbin.ReadFloat64(buf)
case ds.PTString:
val, _, err = cmpbin.ReadString(buf)
case ds.PTBytes:
val, _, err = cmpbin.ReadBytes(buf)
case ds.PTTime:
val, err = ReadTime(buf)
case ds.PTGeoPoint:
val, err = ReadGeoPoint(buf)
case ds.PTKey:
val, err = ReadKey(buf, context, appid, namespace)
case ds.PTBlobKey:
s := ""
if s, _, err = cmpbin.ReadString(buf); err != nil {
break
}
val = blobstore.Key(s)
default:
err = fmt.Errorf("read: unknown type! %v", b)
}
if err == nil {
err = p.SetValue(val, is)
}
return
}
src: &Underspecified{},
plsErr: "non-concrete interface",
},
{
desc: "mismatch (string)",
src: PropertyMap{
"K": {mp(199)},
"S": {mp([]byte("cats"))},
"F": {mp("nurbs")},
},
want: &MismatchTypes{},
loadErr: "type mismatch",
},
{
desc: "mismatch (float)",
src: PropertyMap{"F": {mp(blobstore.Key("wot"))}},
want: &MismatchTypes{},
loadErr: "type mismatch",
},
{
desc: "mismatch (float/overflow)",
src: PropertyMap{"F": {mp(math.MaxFloat64)}},
want: &MismatchTypes{},
loadErr: "overflows",
},
{
desc: "mismatch (key)",
src: PropertyMap{"K": {mp(false)}},
want: &MismatchTypes{},
loadErr: "type mismatch",
},
func TestPropertyMapSerialization(t *testing.T) {
t.Parallel()
tests := []dspmapTC{
{
"basic",
ds.PropertyMap{
"R": {mp(false), mp(2.1), mpNI(3)},
"S": {mp("hello"), mp("world")},
},
},
{
"keys",
ds.PropertyMap{
"DS": {mp(mkKey("appy", "ns", "Foo", 7)), mp(mkKey("other", "", "Yot", "wheeep"))},
"blobstore": {mp(blobstore.Key("sup")), mp(blobstore.Key("nerds"))},
},
},
{
"geo",
ds.PropertyMap{
"G": {mp(ds.GeoPoint{Lat: 1, Lng: 2})},
},
},
{
"data",
ds.PropertyMap{
"S": {mp("sup"), mp("fool"), mp("nerd")},
"D.Foo.Nerd": {mp([]byte("sup")), mp([]byte("fool"))},
},
},
{
"time",
ds.PropertyMap{
"T": {
mp(time.Now().UTC()),
mp(time.Now().Add(time.Second).UTC())},
},
},
{
"empty vals",
ds.PropertyMap{
"T": {mp(true), mp(true)},
"F": {mp(false), mp(false)},
"N": {mp(nil), mp(nil)},
"E": {},
},
},
}
Convey("PropertyMap serialization", t, func() {
Convey("round trip", func() {
for _, tc := range tests {
tc := tc
Convey(tc.name, func() {
data := ToBytesWithContext(tc.props)
dec, err := ReadPropertyMap(mkBuf(data), WithContext, "", "")
So(err, ShouldBeNil)
So(dec, ShouldResemble, tc.props)
})
}
})
})
}
// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
package datastore
import (
"reflect"
"time"
"github.com/luci/gae/service/blobstore"
)
var (
typeOfBool = reflect.TypeOf(true)
typeOfBSKey = reflect.TypeOf(blobstore.Key(""))
typeOfCursorCB = reflect.TypeOf(CursorCB(nil))
typeOfGeoPoint = reflect.TypeOf(GeoPoint{})
typeOfInt64 = reflect.TypeOf(int64(0))
typeOfKey = reflect.TypeOf((*Key)(nil)).Elem()
typeOfPropertyConverter = reflect.TypeOf((*PropertyConverter)(nil)).Elem()
typeOfPropertyLoadSaver = reflect.TypeOf((*PropertyLoadSaver)(nil)).Elem()
typeOfString = reflect.TypeOf("")
typeOfTime = reflect.TypeOf(time.Time{})
typeOfToggle = reflect.TypeOf(Auto)
)
// Project can be used to project a Property retrieved from a Projection query
// into a different datatype. For example, if you have a PTInt property, you
// could Project(PTTime) to convert it to a time.Time. The following conversions
// are supported:
// PTString <-> PTBlobKey
// PTString <-> PTBytes
// PTXXX <-> PTXXX (i.e. identity)
// PTInt <-> PTTime
// PTNull <-> Anything
func (p *Property) Project(to PropertyType) (interface{}, error) {
if to == PTNull {
return nil, nil
}
pt, v := p.propType, p.value
switch pt {
case PTBytes, PTString, PTBlobKey:
v := v.(byteSequence)
switch to {
case PTBytes:
return v.bytes(), nil
case PTString:
return v.string(), nil
case PTBlobKey:
return blobstore.Key(v.string()), nil
}
case PTTime:
switch to {
case PTInt:
return TimeToInt(v.(time.Time)), nil
case PTTime:
return v, nil
}
case PTInt:
switch to {
case PTInt:
return v, nil
case PTTime:
return IntToTime(v.(int64)), nil
}
case to:
return v, nil
case PTNull:
switch to {
case PTInt:
return int64(0), nil
case PTTime:
return time.Time{}, nil
case PTBool:
return false, nil
case PTBytes:
return []byte(nil), nil
case PTString:
return "", nil
case PTFloat:
return float64(0), nil
case PTGeoPoint:
return GeoPoint{}, nil
case PTKey:
return nil, nil
case PTBlobKey:
return blobstore.Key(""), nil
}
}
return nil, fmt.Errorf("unable to project %s to %s", pt, to)
}
func TestProperties(t *testing.T) {
t.Parallel()
Convey("Test Property", t, func() {
Convey("Construction", func() {
Convey("empty", func() {
pv := Property{}
So(pv.Value(), ShouldBeNil)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTNull")
})
Convey("set", func() {
pv := MkPropertyNI(100)
So(pv.Value(), ShouldHaveSameTypeAs, int64(100))
So(pv.Value(), ShouldEqual, 100)
So(pv.IndexSetting(), ShouldEqual, NoIndex)
So(pv.Type().String(), ShouldEqual, "PTInt")
So(pv.SetValue(nil, ShouldIndex), ShouldBeNil)
So(pv.Value(), ShouldBeNil)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTNull")
})
Convey("derived types", func() {
Convey("int", func() {
pv := MkProperty(19)
So(pv.Value(), ShouldHaveSameTypeAs, int64(19))
So(pv.Value(), ShouldEqual, 19)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTInt")
})
Convey("bool (true)", func() {
pv := MkProperty(mybool(true))
So(pv.Value(), ShouldBeTrue)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTBool")
})
Convey("string", func() {
pv := MkProperty(mystring("sup"))
So(pv.Value(), ShouldEqual, "sup")
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTString")
})
Convey("blobstore.Key is distinquished", func() {
pv := MkProperty(blobstore.Key("sup"))
So(pv.Value(), ShouldEqual, blobstore.Key("sup"))
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTBlobKey")
})
Convey("float", func() {
pv := Property{}
So(pv.SetValue(myfloat(19.7), ShouldIndex), ShouldBeNil)
So(pv.Value(), ShouldHaveSameTypeAs, float64(19.7))
So(pv.Value(), ShouldEqual, float32(19.7))
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTFloat")
})
})
Convey("bad type", func() {
pv := Property{}
err := pv.SetValue(complex(100, 29), ShouldIndex)
So(err.Error(), ShouldContainSubstring, "has bad type complex")
So(pv.Value(), ShouldBeNil)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTNull")
})
Convey("invalid GeoPoint", func() {
pv := Property{}
err := pv.SetValue(GeoPoint{-1000, 0}, ShouldIndex)
So(err.Error(), ShouldContainSubstring, "invalid GeoPoint value")
So(pv.Value(), ShouldBeNil)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTNull")
})
Convey("invalid time", func() {
pv := Property{}
loc, err := time.LoadLocation("America/Los_Angeles")
So(err, ShouldBeNil)
t := time.Date(1970, 1, 1, 0, 0, 0, 0, loc)
err = pv.SetValue(t, ShouldIndex)
So(err.Error(), ShouldContainSubstring, "time value has wrong Location")
err = pv.SetValue(time.Unix(math.MaxInt64, 0).UTC(), ShouldIndex)
So(err.Error(), ShouldContainSubstring, "time value out of range")
So(pv.Value(), ShouldBeNil)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTNull")
})
Convey("time gets rounded", func() {
pv := Property{}
now := time.Now().In(time.UTC)
now = now.Round(time.Microsecond).Add(time.Nanosecond * 313)
So(pv.SetValue(now, ShouldIndex), ShouldBeNil)
So(pv.Value(), ShouldHappenBefore, now)
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTTime")
})
Convey("zero time", func() {
now := time.Time{}
So(now.IsZero(), ShouldBeTrue)
pv := Property{}
So(pv.SetValue(now, ShouldIndex), ShouldBeNil)
So(pv.Value(), ShouldResemble, now)
v, err := pv.Project(PTInt)
So(err, ShouldBeNil)
So(v, ShouldEqual, 0)
So(pv.SetValue(0, ShouldIndex), ShouldBeNil)
So(pv.Value(), ShouldEqual, 0)
v, err = pv.Project(PTTime)
So(err, ShouldBeNil)
So(v.(time.Time).IsZero(), ShouldBeTrue)
})
Convey("[]byte allows IndexSetting", func() {
pv := Property{}
So(pv.SetValue([]byte("hello"), ShouldIndex), ShouldBeNil)
So(pv.Value(), ShouldResemble, []byte("hello"))
So(pv.IndexSetting(), ShouldEqual, ShouldIndex)
So(pv.Type().String(), ShouldEqual, "PTBytes")
})
})
Convey("Comparison", func() {
Convey(`A []byte property should equal a string property with the same value.`, func() {
a := MkProperty([]byte("ohaithere"))
b := MkProperty("ohaithere")
So(a.Equal(&b), ShouldBeTrue)
})
})
})
}
var estimateSizeTests = []struct {
pm PropertyMap
expect int
}{
{PropertyMap{"Something": {}}, 9},
{PropertyMap{"Something": mps(100)}, 18},
{PropertyMap{"Something": mps(100.1, "sup")}, 22},
{PropertyMap{
"Something": mps(100, "sup"),
"Keys": mps(MakeKey("aid", "ns", "parent", "something", "kind", int64(20))),
}, 59},
{PropertyMap{
"Null": mps(nil),
"Bool": mps(true, false),
"GP": mps(GeoPoint{23.2, 122.1}),
"bskey": mps(blobstore.Key("hello")),
"[]byte": mps([]byte("sup")),
}, 59},
}
func stablePmString(pm PropertyMap) string {
keys := make([]string, 0, len(pm))
for k := range pm {
keys = append(keys, k)
}
sort.Strings(keys)
buf := &bytes.Buffer{}
_, _ = buf.WriteString("map[")
for i, k := range keys {
if i != 0 {
"Extra", "nuts",
),
}
var collapsedData = []ds.PropertyMap{
// PTTime
pmap("$key", key("Kind", 1), Next,
"Date", time.Date(2000, time.January, 1, 1, 1, 1, 1, time.UTC), Next,
),
pmap("$key", key("Kind", 2), Next,
"Date", time.Date(2000, time.March, 1, 1, 1, 1, 1, time.UTC), Next,
),
// PTBlobKey
pmap("$key", key("Kind", 3), Next,
"Key", blobstore.Key("foo"), Next,
),
pmap("$key", key("Kind", 4), Next,
"Key", blobstore.Key("qux"), Next,
),
// PTBytes
pmap("$key", key("Kind", 5), Next,
"Val", []byte("ohai"), Next,
),
pmap("$key", key("Kind", 6), Next,
"Val", []byte("uwutm8"), Next,
),
}
var queryExecutionTests = []qExTest{