// WriteProperty writes a Property to the buffer. `context` behaves the same
// way that it does for WriteKey, but only has an effect if `p` contains a
// Key as its Value.
func WriteProperty(buf Buffer, context KeyContext, p ds.Property) (err error) {
defer recoverTo(&err)
typb := byte(p.Type())
if p.IndexSetting() != ds.NoIndex {
typb |= 0x80
}
panicIf(buf.WriteByte(typb))
switch p.Type() {
case ds.PTNull:
case ds.PTBool:
b := p.Value().(bool)
if b {
err = buf.WriteByte(1)
} else {
err = buf.WriteByte(0)
}
case ds.PTInt:
_, err = cmpbin.WriteInt(buf, p.Value().(int64))
case ds.PTFloat:
_, err = cmpbin.WriteFloat64(buf, p.Value().(float64))
case ds.PTString:
_, err = cmpbin.WriteString(buf, p.Value().(string))
case ds.PTBytes:
_, err = cmpbin.WriteBytes(buf, p.Value().([]byte))
case ds.PTTime:
err = WriteTime(buf, p.Value().(time.Time))
case ds.PTGeoPoint:
err = WriteGeoPoint(buf, p.Value().(ds.GeoPoint))
case ds.PTKey:
err = WriteKey(buf, context, p.Value().(ds.Key))
case ds.PTBlobKey:
_, err = cmpbin.WriteString(buf, string(p.Value().(blobstore.Key)))
}
return
}
// WritePropertyMap writes an entire PropertyMap to the buffer. `context`
// behaves the same way that it does for WriteKey.
//
// If WritePropertyMapDeterministic is true, then the rows will be sorted by
// property name before they're serialized to buf (mostly useful for testing,
// but also potentially useful if you need to make a hash of the property data).
//
// Write skips metadata keys.
func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err error) {
defer recoverTo(&err)
rows := make(sort.StringSlice, 0, len(pm))
tmpBuf := &bytes.Buffer{}
pm, _ = pm.Save(false)
for name, vals := range pm {
tmpBuf.Reset()
_, e := cmpbin.WriteString(tmpBuf, name)
panicIf(e)
_, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals)))
panicIf(e)
for _, p := range vals {
panicIf(WriteProperty(tmpBuf, context, p))
}
rows = append(rows, tmpBuf.String())
}
if WritePropertyMapDeterministic {
rows.Sort()
}
_, e := cmpbin.WriteUint(buf, uint64(len(pm)))
panicIf(e)
for _, r := range rows {
_, e := buf.WriteString(r)
panicIf(e)
}
return
}
// writeIndexValue writes the index value of v to buf.
//
// v may be one of the return types from ds.Property's GetIndexTypeAndValue
// method.
func writeIndexValue(buf Buffer, context KeyContext, v interface{}) (err error) {
switch t := v.(type) {
case nil:
case bool:
b := byte(0)
if t {
b = 1
}
err = buf.WriteByte(b)
case int64:
_, err = cmpbin.WriteInt(buf, t)
case float64:
_, err = cmpbin.WriteFloat64(buf, t)
case string:
_, err = cmpbin.WriteString(buf, t)
case []byte:
_, err = cmpbin.WriteBytes(buf, t)
case ds.GeoPoint:
err = WriteGeoPoint(buf, t)
case *ds.Key:
err = WriteKey(buf, context, t)
default:
err = fmt.Errorf("unsupported type: %T", t)
}
return
}
// WriteKeyTok writes a KeyTok to the buffer. You usually want WriteKey
// instead of this.
func WriteKeyTok(buf Buffer, tok ds.KeyTok) (err error) {
// tok.kind ++ typ ++ [tok.stringID || tok.intID]
defer recoverTo(&err)
_, e := cmpbin.WriteString(buf, tok.Kind)
panicIf(e)
if tok.StringID != "" {
panicIf(buf.WriteByte(byte(ds.PTString)))
_, e := cmpbin.WriteString(buf, tok.StringID)
panicIf(e)
} else {
panicIf(buf.WriteByte(byte(ds.PTInt)))
_, e := cmpbin.WriteInt(buf, tok.IntID)
panicIf(e)
}
return nil
}
// WriteKey encodes a key to the buffer. If context is WithContext, then this
// encoded value will include the appid and namespace of the key.
func WriteKey(buf Buffer, context KeyContext, k *ds.Key) (err error) {
// [appid ++ namespace]? ++ [1 ++ token]* ++ NULL
defer recoverTo(&err)
appid, namespace, toks := k.Split()
if context == WithContext {
panicIf(buf.WriteByte(1))
_, e := cmpbin.WriteString(buf, appid)
panicIf(e)
_, e = cmpbin.WriteString(buf, namespace)
panicIf(e)
} else {
panicIf(buf.WriteByte(0))
}
for _, tok := range toks {
panicIf(buf.WriteByte(1))
panicIf(WriteKeyTok(buf, tok))
}
return buf.WriteByte(0)
}
// WriteIndexColumn writes an IndexColumn to the buffer.
func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) {
defer recoverTo(&err)
if !c.Descending {
panicIf(buf.WriteByte(0))
} else {
panicIf(buf.WriteByte(1))
}
_, err = cmpbin.WriteString(buf, c.Property)
return
}
// WriteIndexColumn writes an IndexColumn to the buffer.
func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) {
defer recoverTo(&err)
if c.Direction == ds.ASCENDING {
panicIf(buf.WriteByte(0))
} else {
panicIf(buf.WriteByte(1))
}
_, err = cmpbin.WriteString(buf, c.Property)
return
}
// WriteIndexDefinition writes an IndexDefinition to the buffer
func WriteIndexDefinition(buf Buffer, i ds.IndexDefinition) (err error) {
defer recoverTo(&err)
_, err = cmpbin.WriteString(buf, i.Kind)
panicIf(err)
if !i.Ancestor {
panicIf(buf.WriteByte(0))
} else {
panicIf(buf.WriteByte(1))
}
for _, sb := range i.SortBy {
panicIf(buf.WriteByte(1))
panicIf(WriteIndexColumn(buf, sb))
}
return buf.WriteByte(0)
}
// cat is a convenience method for concatenating anything with an underlying
// byte representation into a single []byte.
func cat(bytethings ...interface{}) []byte {
err := error(nil)
buf := &bytes.Buffer{}
for _, thing := range bytethings {
switch x := thing.(type) {
case int64:
_, err = cmpbin.WriteInt(buf, x)
case int:
_, err = cmpbin.WriteInt(buf, int64(x))
case uint64:
_, err = cmpbin.WriteUint(buf, x)
case uint:
_, err = cmpbin.WriteUint(buf, uint64(x))
case float64:
_, err = cmpbin.WriteFloat64(buf, x)
case byte:
err = buf.WriteByte(x)
case ds.PropertyType:
err = buf.WriteByte(byte(x))
case string:
_, err = cmpbin.WriteString(buf, x)
case []byte:
_, err = buf.Write(x)
case time.Time:
err = serialize.WriteTime(buf, x)
case *ds.Key:
err = serialize.WriteKey(buf, serialize.WithoutContext, x)
case *ds.IndexDefinition:
err = serialize.WriteIndexDefinition(buf, *x)
case ds.Property:
err = serialize.WriteProperty(buf, serialize.WithoutContext, x)
default:
panic(fmt.Errorf("I don't know how to deal with %T: %#v", thing, thing))
}
die(err)
}
ret := buf.Bytes()
if ret == nil {
ret = []byte{}
}
return ret
}
func TestSerializationReadMisc(t *testing.T) {
t.Parallel()
Convey("Misc Serialization tests", t, func() {
Convey("GeoPoint", func() {
buf := mkBuf(nil)
cmpbin.WriteFloat64(buf, 10)
cmpbin.WriteFloat64(buf, 20)
So(string(ToBytes(ds.GeoPoint{Lat: 10, Lng: 20})), ShouldEqual, buf.String())
})
Convey("IndexColumn", func() {
buf := mkBuf(nil)
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
So(string(ToBytes(ds.IndexColumn{Property: "hi", Direction: ds.DESCENDING})),
ShouldEqual, buf.String())
})
Convey("KeyTok", func() {
buf := mkBuf(nil)
cmpbin.WriteString(buf, "foo")
buf.WriteByte(byte(ds.PTInt))
cmpbin.WriteInt(buf, 20)
So(string(ToBytes(ds.KeyTok{Kind: "foo", IntID: 20})),
ShouldEqual, buf.String())
})
Convey("Property", func() {
buf := mkBuf(nil)
buf.WriteByte(0x80 | byte(ds.PTString))
cmpbin.WriteString(buf, "nerp")
So(string(ToBytes(mp("nerp"))),
ShouldEqual, buf.String())
})
Convey("Time", func() {
tp := mp(time.Now().UTC())
So(string(ToBytes(tp.Value())), ShouldEqual, string(ToBytes(tp)[1:]))
})
Convey("Bad ToBytes", func() {
So(func() { ToBytes(100.7) }, ShouldPanic)
So(func() { ToBytesWithContext(100.7) }, ShouldPanic)
})
Convey("ReadKey", func() {
Convey("good cases", func() {
Convey("w/ ctx decodes normally w/ ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytesWithContext(k)
dk, err := ReadKey(mkBuf(data), WithContext, "", "")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, k)
})
Convey("w/ ctx decodes normally w/o ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytesWithContext(k)
dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
})
Convey("w/o ctx decodes normally w/ ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytes(k)
dk, err := ReadKey(mkBuf(data), WithContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("", "", "knd", "yo", "other", 10))
})
Convey("w/o ctx decodes normally w/o ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytes(k)
dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
})
Convey("IntIDs always sort before StringIDs", func() {
// -1 writes as almost all 1's in the first byte under cmpbin, even
// though it's technically not a valid key.
k := mkKey("aid", "ns", "knd", -1)
data := ToBytes(k)
k = mkKey("aid", "ns", "knd", "hat")
data2 := ToBytes(k)
So(string(data), ShouldBeLessThan, string(data2))
})
})
Convey("err cases", func() {
buf := mkBuf(nil)
Convey("nil", func() {
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("str", func() {
buf.WriteString("sup")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "expected actualCtx")
})
Convey("truncated 1", func() {
buf.WriteByte(1) // actualCtx == 1
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("truncated 2", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("truncated 3", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("huge key", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
for i := 1; i < 60; i++ {
buf.WriteByte(1)
WriteKeyTok(buf, ds.KeyTok{Kind: "sup", IntID: int64(i)})
}
buf.WriteByte(0)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge key")
})
Convey("insufficient tokens", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
cmpbin.WriteUint(buf, 2)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("partial token 1", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("partial token 2", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTString))
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("bad token (invalid type)", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTBlobKey))
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "invalid type PTBlobKey")
})
Convey("bad token (invalid IntID)", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTInt))
cmpbin.WriteInt(buf, -2)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "zero/negative")
})
})
})
Convey("ReadGeoPoint", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
_, err := ReadGeoPoint(buf)
So(err, ShouldEqual, io.EOF)
})
Convey("trunc 2", func() {
cmpbin.WriteFloat64(buf, 100)
_, err := ReadGeoPoint(buf)
So(err, ShouldEqual, io.EOF)
})
Convey("invalid", func() {
cmpbin.WriteFloat64(buf, 100)
cmpbin.WriteFloat64(buf, 1000)
_, err := ReadGeoPoint(buf)
So(err, ShouldErrLike, "invalid GeoPoint")
})
})
Convey("WriteTime", func() {
Convey("in non-UTC!", func() {
pst, err := time.LoadLocation("America/Los_Angeles")
So(err, ShouldBeNil)
So(func() {
WriteTime(mkBuf(nil), time.Now().In(pst))
}, ShouldPanic)
})
})
Convey("ReadTime", func() {
Convey("trunc 1", func() {
_, err := ReadTime(mkBuf(nil))
So(err, ShouldEqual, io.EOF)
})
})
Convey("ReadProperty", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
p, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
So(p.Type(), ShouldEqual, ds.PTNull)
So(p.Value(), ShouldBeNil)
})
Convey("trunc (PTBytes)", func() {
buf.WriteByte(byte(ds.PTBytes))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("trunc (PTBlobKey)", func() {
buf.WriteByte(byte(ds.PTBlobKey))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("invalid type", func() {
buf.WriteByte(byte(ds.PTUnknown + 1))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldErrLike, "unknown type!")
})
})
Convey("ReadPropertyMap", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("too many rows", func() {
cmpbin.WriteUint(buf, 1000000)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge number of rows")
})
Convey("trunc 2", func() {
cmpbin.WriteUint(buf, 10)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("trunc 3", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("too many values", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
cmpbin.WriteUint(buf, 100000)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge number of properties")
})
Convey("trunc 4", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
cmpbin.WriteUint(buf, 10)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
})
Convey("IndexDefinition", func() {
id := ds.IndexDefinition{Kind: "kind"}
data := ToBytes(*id.PrepForIdxTable())
newID, err := ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "prop"})
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "other", Direction: ds.DESCENDING})
id.Ancestor = true
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
// invalid
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "", Direction: ds.DESCENDING})
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
Convey("too many", func() {
id := ds.IndexDefinition{Kind: "wat"}
for i := 0; i < MaxIndexColumns+1; i++ {
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "Hi", Direction: ds.ASCENDING})
}
data := ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldErrLike, "over 64 sort orders")
})
})
})
}
func ws(w io.ByteWriter, s string) int {
ret, err := cmpbin.WriteString(w, s)
die(err)
return ret
}