// WritePropertyMap writes an entire PropertyMap to the buffer. `context` // behaves the same way that it does for WriteKey. // // If WritePropertyMapDeterministic is true, then the rows will be sorted by // property name before they're serialized to buf (mostly useful for testing, // but also potentially useful if you need to make a hash of the property data). // // Write skips metadata keys. func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err error) { defer recoverTo(&err) rows := make(sort.StringSlice, 0, len(pm)) tmpBuf := &bytes.Buffer{} pm, _ = pm.Save(false) for name, vals := range pm { tmpBuf.Reset() _, e := cmpbin.WriteString(tmpBuf, name) panicIf(e) _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) panicIf(e) for _, p := range vals { panicIf(WriteProperty(tmpBuf, context, p)) } rows = append(rows, tmpBuf.String()) } if WritePropertyMapDeterministic { rows.Sort() } _, e := cmpbin.WriteUint(buf, uint64(len(pm))) panicIf(e) for _, r := range rows { _, e := buf.WriteString(r) panicIf(e) } return }
func curs(pairs ...interface{}) queryCursor { if len(pairs)%2 != 0 { panic("curs() takes only even pairs") } pre := &bytes.Buffer{} if _, err := cmpbin.WriteUint(pre, uint64(len(pairs)/2)); err != nil { panic(err) } post := serialize.Invertible(&bytes.Buffer{}) for i := 0; i < len(pairs); i += 2 { k, v := pairs[i].(string), pairs[i+1] col, err := dstore.ParseIndexColumn(k) if err != nil { panic(err) } post.SetInvert(col.Descending) if err := serialize.WriteIndexColumn(pre, col); err != nil { panic(err) } if err := serialize.WriteProperty(post, serialize.WithoutContext, prop(v)); err != nil { panic(err) } } return queryCursor(serialize.Join(pre.Bytes(), post.Bytes())) }
// WriteTime writes a time.Time in a byte-sortable way. // // This method truncates the time to microseconds and drops the timezone, // because that's the (undocumented) way that the appengine SDK does it. func WriteTime(buf Buffer, t time.Time) error { name, off := t.Zone() if name != "UTC" || off != 0 { panic(fmt.Errorf("helper: UTC OR DEATH: %s", t)) } _, err := cmpbin.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond()/1e3)) return err }
// cat is a convenience method for concatenating anything with an underlying // byte representation into a single []byte. func cat(bytethings ...interface{}) []byte { err := error(nil) buf := &bytes.Buffer{} for _, thing := range bytethings { switch x := thing.(type) { case int64: _, err = cmpbin.WriteInt(buf, x) case int: _, err = cmpbin.WriteInt(buf, int64(x)) case uint64: _, err = cmpbin.WriteUint(buf, x) case uint: _, err = cmpbin.WriteUint(buf, uint64(x)) case float64: _, err = cmpbin.WriteFloat64(buf, x) case byte: err = buf.WriteByte(x) case ds.PropertyType: err = buf.WriteByte(byte(x)) case string: _, err = cmpbin.WriteString(buf, x) case []byte: _, err = buf.Write(x) case time.Time: err = serialize.WriteTime(buf, x) case *ds.Key: err = serialize.WriteKey(buf, serialize.WithoutContext, x) case *ds.IndexDefinition: err = serialize.WriteIndexDefinition(buf, *x) case ds.Property: err = serialize.WriteProperty(buf, serialize.WithoutContext, x) default: panic(fmt.Errorf("I don't know how to deal with %T: %#v", thing, thing)) } die(err) } ret := buf.Bytes() if ret == nil { ret = []byte{} } return ret }
func curs(pairs ...interface{}) queryCursor { if len(pairs)%2 != 0 { panic("curs() takes only even pairs") } pre := &bytes.Buffer{} cmpbin.WriteUint(pre, uint64(len(pairs)/2)) post := serialize.Invertible(&bytes.Buffer{}) for i := 0; i < len(pairs); i += 2 { k, v := pairs[i].(string), pairs[i+1] col := dsS.IndexColumn{Property: k} post.SetInvert(false) if k[0] == '-' { post.SetInvert(false) col.Property = k[1:] col.Direction = dsS.DESCENDING } serialize.WriteIndexColumn(pre, col) serialize.WriteProperty(post, serialize.WithoutContext, prop(v)) } return queryCursor(bjoin(pre.Bytes(), post.Bytes())) }
func TestSerializationReadMisc(t *testing.T) { t.Parallel() Convey("Misc Serialization tests", t, func() { Convey("GeoPoint", func() { buf := mkBuf(nil) cmpbin.WriteFloat64(buf, 10) cmpbin.WriteFloat64(buf, 20) So(string(ToBytes(ds.GeoPoint{Lat: 10, Lng: 20})), ShouldEqual, buf.String()) }) Convey("IndexColumn", func() { buf := mkBuf(nil) buf.WriteByte(1) cmpbin.WriteString(buf, "hi") So(string(ToBytes(ds.IndexColumn{Property: "hi", Direction: ds.DESCENDING})), ShouldEqual, buf.String()) }) Convey("KeyTok", func() { buf := mkBuf(nil) cmpbin.WriteString(buf, "foo") buf.WriteByte(byte(ds.PTInt)) cmpbin.WriteInt(buf, 20) So(string(ToBytes(ds.KeyTok{Kind: "foo", IntID: 20})), ShouldEqual, buf.String()) }) Convey("Property", func() { buf := mkBuf(nil) buf.WriteByte(0x80 | byte(ds.PTString)) cmpbin.WriteString(buf, "nerp") So(string(ToBytes(mp("nerp"))), ShouldEqual, buf.String()) }) Convey("Time", func() { tp := mp(time.Now().UTC()) So(string(ToBytes(tp.Value())), ShouldEqual, string(ToBytes(tp)[1:])) }) Convey("Bad ToBytes", func() { So(func() { ToBytes(100.7) }, ShouldPanic) So(func() { ToBytesWithContext(100.7) }, ShouldPanic) }) Convey("ReadKey", func() { Convey("good cases", func() { Convey("w/ ctx decodes normally w/ ctx", func() { k := mkKey("aid", "ns", "knd", "yo", "other", 10) data := ToBytesWithContext(k) dk, err := ReadKey(mkBuf(data), WithContext, "", "") So(err, ShouldBeNil) So(dk, ShouldEqualKey, k) }) Convey("w/ ctx decodes normally w/o ctx", func() { k := mkKey("aid", "ns", "knd", "yo", "other", 10) data := ToBytesWithContext(k) dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd") So(err, ShouldBeNil) So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10)) }) Convey("w/o ctx decodes normally w/ ctx", func() { k := mkKey("aid", "ns", "knd", "yo", "other", 10) data := ToBytes(k) dk, err := ReadKey(mkBuf(data), WithContext, "spam", "nerd") So(err, ShouldBeNil) So(dk, ShouldEqualKey, mkKey("", "", "knd", "yo", "other", 10)) }) Convey("w/o ctx decodes normally w/o ctx", func() { k := mkKey("aid", "ns", "knd", "yo", "other", 10) data := ToBytes(k) dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd") So(err, ShouldBeNil) So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10)) }) Convey("IntIDs always sort before StringIDs", func() { // -1 writes as almost all 1's in the first byte under cmpbin, even // though it's technically not a valid key. k := mkKey("aid", "ns", "knd", -1) data := ToBytes(k) k = mkKey("aid", "ns", "knd", "hat") data2 := ToBytes(k) So(string(data), ShouldBeLessThan, string(data2)) }) }) Convey("err cases", func() { buf := mkBuf(nil) Convey("nil", func() { _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("str", func() { buf.WriteString("sup") _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldErrLike, "expected actualCtx") }) Convey("truncated 1", func() { buf.WriteByte(1) // actualCtx == 1 _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("truncated 2", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("truncated 3", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") cmpbin.WriteString(buf, "ns") _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("huge key", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") cmpbin.WriteString(buf, "ns") for i := 1; i < 60; i++ { buf.WriteByte(1) WriteKeyTok(buf, ds.KeyTok{Kind: "sup", IntID: int64(i)}) } buf.WriteByte(0) _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldErrLike, "huge key") }) Convey("insufficient tokens", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") cmpbin.WriteString(buf, "ns") cmpbin.WriteUint(buf, 2) _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("partial token 1", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") cmpbin.WriteString(buf, "ns") buf.WriteByte(1) cmpbin.WriteString(buf, "hi") _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("partial token 2", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") cmpbin.WriteString(buf, "ns") buf.WriteByte(1) cmpbin.WriteString(buf, "hi") buf.WriteByte(byte(ds.PTString)) _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("bad token (invalid type)", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") cmpbin.WriteString(buf, "ns") buf.WriteByte(1) cmpbin.WriteString(buf, "hi") buf.WriteByte(byte(ds.PTBlobKey)) _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldErrLike, "invalid type PTBlobKey") }) Convey("bad token (invalid IntID)", func() { buf.WriteByte(1) // actualCtx == 1 cmpbin.WriteString(buf, "aid") cmpbin.WriteString(buf, "ns") buf.WriteByte(1) cmpbin.WriteString(buf, "hi") buf.WriteByte(byte(ds.PTInt)) cmpbin.WriteInt(buf, -2) _, err := ReadKey(buf, WithContext, "", "") So(err, ShouldErrLike, "zero/negative") }) }) }) Convey("ReadGeoPoint", func() { buf := mkBuf(nil) Convey("trunc 1", func() { _, err := ReadGeoPoint(buf) So(err, ShouldEqual, io.EOF) }) Convey("trunc 2", func() { cmpbin.WriteFloat64(buf, 100) _, err := ReadGeoPoint(buf) So(err, ShouldEqual, io.EOF) }) Convey("invalid", func() { cmpbin.WriteFloat64(buf, 100) cmpbin.WriteFloat64(buf, 1000) _, err := ReadGeoPoint(buf) So(err, ShouldErrLike, "invalid GeoPoint") }) }) Convey("WriteTime", func() { Convey("in non-UTC!", func() { pst, err := time.LoadLocation("America/Los_Angeles") So(err, ShouldBeNil) So(func() { WriteTime(mkBuf(nil), time.Now().In(pst)) }, ShouldPanic) }) }) Convey("ReadTime", func() { Convey("trunc 1", func() { _, err := ReadTime(mkBuf(nil)) So(err, ShouldEqual, io.EOF) }) }) Convey("ReadProperty", func() { buf := mkBuf(nil) Convey("trunc 1", func() { p, err := ReadProperty(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) So(p.Type(), ShouldEqual, ds.PTNull) So(p.Value(), ShouldBeNil) }) Convey("trunc (PTBytes)", func() { buf.WriteByte(byte(ds.PTBytes)) _, err := ReadProperty(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("trunc (PTBlobKey)", func() { buf.WriteByte(byte(ds.PTBlobKey)) _, err := ReadProperty(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("invalid type", func() { buf.WriteByte(byte(ds.PTUnknown + 1)) _, err := ReadProperty(buf, WithContext, "", "") So(err, ShouldErrLike, "unknown type!") }) }) Convey("ReadPropertyMap", func() { buf := mkBuf(nil) Convey("trunc 1", func() { _, err := ReadPropertyMap(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("too many rows", func() { cmpbin.WriteUint(buf, 1000000) _, err := ReadPropertyMap(buf, WithContext, "", "") So(err, ShouldErrLike, "huge number of rows") }) Convey("trunc 2", func() { cmpbin.WriteUint(buf, 10) _, err := ReadPropertyMap(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("trunc 3", func() { cmpbin.WriteUint(buf, 10) cmpbin.WriteString(buf, "ohai") _, err := ReadPropertyMap(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) Convey("too many values", func() { cmpbin.WriteUint(buf, 10) cmpbin.WriteString(buf, "ohai") cmpbin.WriteUint(buf, 100000) _, err := ReadPropertyMap(buf, WithContext, "", "") So(err, ShouldErrLike, "huge number of properties") }) Convey("trunc 4", func() { cmpbin.WriteUint(buf, 10) cmpbin.WriteString(buf, "ohai") cmpbin.WriteUint(buf, 10) _, err := ReadPropertyMap(buf, WithContext, "", "") So(err, ShouldEqual, io.EOF) }) }) Convey("IndexDefinition", func() { id := ds.IndexDefinition{Kind: "kind"} data := ToBytes(*id.PrepForIdxTable()) newID, err := ReadIndexDefinition(mkBuf(data)) So(err, ShouldBeNil) So(newID.Flip(), ShouldResemble, id.Normalize()) id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "prop"}) data = ToBytes(*id.PrepForIdxTable()) newID, err = ReadIndexDefinition(mkBuf(data)) So(err, ShouldBeNil) So(newID.Flip(), ShouldResemble, id.Normalize()) id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "other", Direction: ds.DESCENDING}) id.Ancestor = true data = ToBytes(*id.PrepForIdxTable()) newID, err = ReadIndexDefinition(mkBuf(data)) So(err, ShouldBeNil) So(newID.Flip(), ShouldResemble, id.Normalize()) // invalid id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "", Direction: ds.DESCENDING}) data = ToBytes(*id.PrepForIdxTable()) newID, err = ReadIndexDefinition(mkBuf(data)) So(err, ShouldBeNil) So(newID.Flip(), ShouldResemble, id.Normalize()) Convey("too many", func() { id := ds.IndexDefinition{Kind: "wat"} for i := 0; i < MaxIndexColumns+1; i++ { id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "Hi", Direction: ds.ASCENDING}) } data := ToBytes(*id.PrepForIdxTable()) newID, err = ReadIndexDefinition(mkBuf(data)) So(err, ShouldErrLike, "over 64 sort orders") }) }) }) }
func executeQuery(origQ ds.Query, ns string, isTxn bool, idx, head *memStore, cb ds.RawRunCB) error { q := origQ.(*queryImpl) rq, err := q.reduce(ns, isTxn) if err == errQueryDone { return nil } if err != nil { return err } idxs, err := getIndexes(rq, idx) if err == errQueryDone { return nil } if err != nil { return err } strategy := pickQueryStrategy(q, rq, cb, head) if strategy == nil { // e.g. the normalStrategy found that there were NO entities in the current // namespace. return nil } offset := q.offset limit := q.limit hasLimit := q.limitSet && limit >= 0 cursorPrefix := []byte(nil) getCursorFn := func(suffix []byte) func() (ds.Cursor, error) { return func() (ds.Cursor, error) { if cursorPrefix == nil { buf := &bytes.Buffer{} _, err := cmpbin.WriteUint(buf, uint64(len(rq.suffixFormat))) memoryCorruption(err) for _, col := range rq.suffixFormat { err := serialize.WriteIndexColumn(buf, col) memoryCorruption(err) } cursorPrefix = buf.Bytes() } // TODO(riannucci): Do we need to decrement suffix instead of increment // if we're sorting by __key__ DESCENDING? return queryCursor(bjoin(cursorPrefix, increment(suffix))), nil } } multiIterate(idxs, func(suffix []byte) bool { if offset > 0 { offset-- return true } if hasLimit { if limit <= 0 { return false } limit-- } rawData, decodedProps := parseSuffix(ns, rq.suffixFormat, suffix, -1) keyProp := decodedProps[len(decodedProps)-1] if keyProp.Type() != ds.PTKey { impossible(fmt.Errorf("decoded index row doesn't end with a Key: %#v", keyProp)) } return strategy.handle( rawData, decodedProps, keyProp.Value().(ds.Key), getCursorFn(suffix)) }) return nil }
func wui(w io.ByteWriter, i uint64) int { ret, err := cmpbin.WriteUint(w, i) die(err) return ret }