// WritePropertyMap writes an entire PropertyMap to the buffer. `context`
// behaves the same way that it does for WriteKey.
//
// If WritePropertyMapDeterministic is true, then the rows will be sorted by
// property name before they're serialized to buf (mostly useful for testing,
// but also potentially useful if you need to make a hash of the property data).
//
// Write skips metadata keys.
func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err error) {
defer recoverTo(&err)
rows := make(sort.StringSlice, 0, len(pm))
tmpBuf := &bytes.Buffer{}
pm, _ = pm.Save(false)
for name, vals := range pm {
tmpBuf.Reset()
_, e := cmpbin.WriteString(tmpBuf, name)
panicIf(e)
_, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals)))
panicIf(e)
for _, p := range vals {
panicIf(WriteProperty(tmpBuf, context, p))
}
rows = append(rows, tmpBuf.String())
}
if WritePropertyMapDeterministic {
rows.Sort()
}
_, e := cmpbin.WriteUint(buf, uint64(len(pm)))
panicIf(e)
for _, r := range rows {
_, e := buf.WriteString(r)
panicIf(e)
}
return
}
func curs(pairs ...interface{}) queryCursor {
if len(pairs)%2 != 0 {
panic("curs() takes only even pairs")
}
pre := &bytes.Buffer{}
if _, err := cmpbin.WriteUint(pre, uint64(len(pairs)/2)); err != nil {
panic(err)
}
post := serialize.Invertible(&bytes.Buffer{})
for i := 0; i < len(pairs); i += 2 {
k, v := pairs[i].(string), pairs[i+1]
col, err := dstore.ParseIndexColumn(k)
if err != nil {
panic(err)
}
post.SetInvert(col.Descending)
if err := serialize.WriteIndexColumn(pre, col); err != nil {
panic(err)
}
if err := serialize.WriteProperty(post, serialize.WithoutContext, prop(v)); err != nil {
panic(err)
}
}
return queryCursor(serialize.Join(pre.Bytes(), post.Bytes()))
}
// WriteTime writes a time.Time in a byte-sortable way.
//
// This method truncates the time to microseconds and drops the timezone,
// because that's the (undocumented) way that the appengine SDK does it.
func WriteTime(buf Buffer, t time.Time) error {
name, off := t.Zone()
if name != "UTC" || off != 0 {
panic(fmt.Errorf("helper: UTC OR DEATH: %s", t))
}
_, err := cmpbin.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond()/1e3))
return err
}
// cat is a convenience method for concatenating anything with an underlying
// byte representation into a single []byte.
func cat(bytethings ...interface{}) []byte {
err := error(nil)
buf := &bytes.Buffer{}
for _, thing := range bytethings {
switch x := thing.(type) {
case int64:
_, err = cmpbin.WriteInt(buf, x)
case int:
_, err = cmpbin.WriteInt(buf, int64(x))
case uint64:
_, err = cmpbin.WriteUint(buf, x)
case uint:
_, err = cmpbin.WriteUint(buf, uint64(x))
case float64:
_, err = cmpbin.WriteFloat64(buf, x)
case byte:
err = buf.WriteByte(x)
case ds.PropertyType:
err = buf.WriteByte(byte(x))
case string:
_, err = cmpbin.WriteString(buf, x)
case []byte:
_, err = buf.Write(x)
case time.Time:
err = serialize.WriteTime(buf, x)
case *ds.Key:
err = serialize.WriteKey(buf, serialize.WithoutContext, x)
case *ds.IndexDefinition:
err = serialize.WriteIndexDefinition(buf, *x)
case ds.Property:
err = serialize.WriteProperty(buf, serialize.WithoutContext, x)
default:
panic(fmt.Errorf("I don't know how to deal with %T: %#v", thing, thing))
}
die(err)
}
ret := buf.Bytes()
if ret == nil {
ret = []byte{}
}
return ret
}
func curs(pairs ...interface{}) queryCursor {
if len(pairs)%2 != 0 {
panic("curs() takes only even pairs")
}
pre := &bytes.Buffer{}
cmpbin.WriteUint(pre, uint64(len(pairs)/2))
post := serialize.Invertible(&bytes.Buffer{})
for i := 0; i < len(pairs); i += 2 {
k, v := pairs[i].(string), pairs[i+1]
col := dsS.IndexColumn{Property: k}
post.SetInvert(false)
if k[0] == '-' {
post.SetInvert(false)
col.Property = k[1:]
col.Direction = dsS.DESCENDING
}
serialize.WriteIndexColumn(pre, col)
serialize.WriteProperty(post, serialize.WithoutContext, prop(v))
}
return queryCursor(bjoin(pre.Bytes(), post.Bytes()))
}
func TestSerializationReadMisc(t *testing.T) {
t.Parallel()
Convey("Misc Serialization tests", t, func() {
Convey("GeoPoint", func() {
buf := mkBuf(nil)
cmpbin.WriteFloat64(buf, 10)
cmpbin.WriteFloat64(buf, 20)
So(string(ToBytes(ds.GeoPoint{Lat: 10, Lng: 20})), ShouldEqual, buf.String())
})
Convey("IndexColumn", func() {
buf := mkBuf(nil)
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
So(string(ToBytes(ds.IndexColumn{Property: "hi", Direction: ds.DESCENDING})),
ShouldEqual, buf.String())
})
Convey("KeyTok", func() {
buf := mkBuf(nil)
cmpbin.WriteString(buf, "foo")
buf.WriteByte(byte(ds.PTInt))
cmpbin.WriteInt(buf, 20)
So(string(ToBytes(ds.KeyTok{Kind: "foo", IntID: 20})),
ShouldEqual, buf.String())
})
Convey("Property", func() {
buf := mkBuf(nil)
buf.WriteByte(0x80 | byte(ds.PTString))
cmpbin.WriteString(buf, "nerp")
So(string(ToBytes(mp("nerp"))),
ShouldEqual, buf.String())
})
Convey("Time", func() {
tp := mp(time.Now().UTC())
So(string(ToBytes(tp.Value())), ShouldEqual, string(ToBytes(tp)[1:]))
})
Convey("Bad ToBytes", func() {
So(func() { ToBytes(100.7) }, ShouldPanic)
So(func() { ToBytesWithContext(100.7) }, ShouldPanic)
})
Convey("ReadKey", func() {
Convey("good cases", func() {
Convey("w/ ctx decodes normally w/ ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytesWithContext(k)
dk, err := ReadKey(mkBuf(data), WithContext, "", "")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, k)
})
Convey("w/ ctx decodes normally w/o ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytesWithContext(k)
dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
})
Convey("w/o ctx decodes normally w/ ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytes(k)
dk, err := ReadKey(mkBuf(data), WithContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("", "", "knd", "yo", "other", 10))
})
Convey("w/o ctx decodes normally w/o ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytes(k)
dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
})
Convey("IntIDs always sort before StringIDs", func() {
// -1 writes as almost all 1's in the first byte under cmpbin, even
// though it's technically not a valid key.
k := mkKey("aid", "ns", "knd", -1)
data := ToBytes(k)
k = mkKey("aid", "ns", "knd", "hat")
data2 := ToBytes(k)
So(string(data), ShouldBeLessThan, string(data2))
})
})
Convey("err cases", func() {
buf := mkBuf(nil)
Convey("nil", func() {
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("str", func() {
buf.WriteString("sup")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "expected actualCtx")
})
Convey("truncated 1", func() {
buf.WriteByte(1) // actualCtx == 1
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("truncated 2", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("truncated 3", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("huge key", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
for i := 1; i < 60; i++ {
buf.WriteByte(1)
WriteKeyTok(buf, ds.KeyTok{Kind: "sup", IntID: int64(i)})
}
buf.WriteByte(0)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge key")
})
Convey("insufficient tokens", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
cmpbin.WriteUint(buf, 2)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("partial token 1", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("partial token 2", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTString))
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("bad token (invalid type)", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTBlobKey))
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "invalid type PTBlobKey")
})
Convey("bad token (invalid IntID)", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTInt))
cmpbin.WriteInt(buf, -2)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "zero/negative")
})
})
})
Convey("ReadGeoPoint", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
_, err := ReadGeoPoint(buf)
So(err, ShouldEqual, io.EOF)
})
Convey("trunc 2", func() {
cmpbin.WriteFloat64(buf, 100)
_, err := ReadGeoPoint(buf)
So(err, ShouldEqual, io.EOF)
})
Convey("invalid", func() {
cmpbin.WriteFloat64(buf, 100)
cmpbin.WriteFloat64(buf, 1000)
_, err := ReadGeoPoint(buf)
So(err, ShouldErrLike, "invalid GeoPoint")
})
})
Convey("WriteTime", func() {
Convey("in non-UTC!", func() {
pst, err := time.LoadLocation("America/Los_Angeles")
So(err, ShouldBeNil)
So(func() {
WriteTime(mkBuf(nil), time.Now().In(pst))
}, ShouldPanic)
})
})
Convey("ReadTime", func() {
Convey("trunc 1", func() {
_, err := ReadTime(mkBuf(nil))
So(err, ShouldEqual, io.EOF)
})
})
Convey("ReadProperty", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
p, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
So(p.Type(), ShouldEqual, ds.PTNull)
So(p.Value(), ShouldBeNil)
})
Convey("trunc (PTBytes)", func() {
buf.WriteByte(byte(ds.PTBytes))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("trunc (PTBlobKey)", func() {
buf.WriteByte(byte(ds.PTBlobKey))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("invalid type", func() {
buf.WriteByte(byte(ds.PTUnknown + 1))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldErrLike, "unknown type!")
})
})
Convey("ReadPropertyMap", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("too many rows", func() {
cmpbin.WriteUint(buf, 1000000)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge number of rows")
})
Convey("trunc 2", func() {
cmpbin.WriteUint(buf, 10)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("trunc 3", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("too many values", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
cmpbin.WriteUint(buf, 100000)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge number of properties")
})
Convey("trunc 4", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
cmpbin.WriteUint(buf, 10)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
})
Convey("IndexDefinition", func() {
id := ds.IndexDefinition{Kind: "kind"}
data := ToBytes(*id.PrepForIdxTable())
newID, err := ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "prop"})
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "other", Direction: ds.DESCENDING})
id.Ancestor = true
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
// invalid
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "", Direction: ds.DESCENDING})
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
Convey("too many", func() {
id := ds.IndexDefinition{Kind: "wat"}
for i := 0; i < MaxIndexColumns+1; i++ {
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "Hi", Direction: ds.ASCENDING})
}
data := ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldErrLike, "over 64 sort orders")
})
})
})
}
func executeQuery(origQ ds.Query, ns string, isTxn bool, idx, head *memStore, cb ds.RawRunCB) error {
q := origQ.(*queryImpl)
rq, err := q.reduce(ns, isTxn)
if err == errQueryDone {
return nil
}
if err != nil {
return err
}
idxs, err := getIndexes(rq, idx)
if err == errQueryDone {
return nil
}
if err != nil {
return err
}
strategy := pickQueryStrategy(q, rq, cb, head)
if strategy == nil {
// e.g. the normalStrategy found that there were NO entities in the current
// namespace.
return nil
}
offset := q.offset
limit := q.limit
hasLimit := q.limitSet && limit >= 0
cursorPrefix := []byte(nil)
getCursorFn := func(suffix []byte) func() (ds.Cursor, error) {
return func() (ds.Cursor, error) {
if cursorPrefix == nil {
buf := &bytes.Buffer{}
_, err := cmpbin.WriteUint(buf, uint64(len(rq.suffixFormat)))
memoryCorruption(err)
for _, col := range rq.suffixFormat {
err := serialize.WriteIndexColumn(buf, col)
memoryCorruption(err)
}
cursorPrefix = buf.Bytes()
}
// TODO(riannucci): Do we need to decrement suffix instead of increment
// if we're sorting by __key__ DESCENDING?
return queryCursor(bjoin(cursorPrefix, increment(suffix))), nil
}
}
multiIterate(idxs, func(suffix []byte) bool {
if offset > 0 {
offset--
return true
}
if hasLimit {
if limit <= 0 {
return false
}
limit--
}
rawData, decodedProps := parseSuffix(ns, rq.suffixFormat, suffix, -1)
keyProp := decodedProps[len(decodedProps)-1]
if keyProp.Type() != ds.PTKey {
impossible(fmt.Errorf("decoded index row doesn't end with a Key: %#v", keyProp))
}
return strategy.handle(
rawData, decodedProps, keyProp.Value().(ds.Key),
getCursorFn(suffix))
})
return nil
}
func wui(w io.ByteWriter, i uint64) int {
ret, err := cmpbin.WriteUint(w, i)
die(err)
return ret
}