func mkNum(n int64) []byte {
buf := &bytes.Buffer{}
_, err := cmpbin.WriteInt(buf, n)
memoryCorruption(err)
return buf.Bytes()
}
// writeIndexValue writes the index value of v to buf.
//
// v may be one of the return types from ds.Property's GetIndexTypeAndValue
// method.
func writeIndexValue(buf Buffer, context KeyContext, v interface{}) (err error) {
switch t := v.(type) {
case nil:
case bool:
b := byte(0)
if t {
b = 1
}
err = buf.WriteByte(b)
case int64:
_, err = cmpbin.WriteInt(buf, t)
case float64:
_, err = cmpbin.WriteFloat64(buf, t)
case string:
_, err = cmpbin.WriteString(buf, t)
case []byte:
_, err = cmpbin.WriteBytes(buf, t)
case ds.GeoPoint:
err = WriteGeoPoint(buf, t)
case *ds.Key:
err = WriteKey(buf, context, t)
default:
err = fmt.Errorf("unsupported type: %T", t)
}
return
}
// WriteProperty writes a Property to the buffer. `context` behaves the same
// way that it does for WriteKey, but only has an effect if `p` contains a
// Key as its Value.
func WriteProperty(buf Buffer, context KeyContext, p ds.Property) (err error) {
defer recoverTo(&err)
typb := byte(p.Type())
if p.IndexSetting() != ds.NoIndex {
typb |= 0x80
}
panicIf(buf.WriteByte(typb))
switch p.Type() {
case ds.PTNull:
case ds.PTBool:
b := p.Value().(bool)
if b {
err = buf.WriteByte(1)
} else {
err = buf.WriteByte(0)
}
case ds.PTInt:
_, err = cmpbin.WriteInt(buf, p.Value().(int64))
case ds.PTFloat:
_, err = cmpbin.WriteFloat64(buf, p.Value().(float64))
case ds.PTString:
_, err = cmpbin.WriteString(buf, p.Value().(string))
case ds.PTBytes:
_, err = cmpbin.WriteBytes(buf, p.Value().([]byte))
case ds.PTTime:
err = WriteTime(buf, p.Value().(time.Time))
case ds.PTGeoPoint:
err = WriteGeoPoint(buf, p.Value().(ds.GeoPoint))
case ds.PTKey:
err = WriteKey(buf, context, p.Value().(ds.Key))
case ds.PTBlobKey:
_, err = cmpbin.WriteString(buf, string(p.Value().(blobstore.Key)))
}
return
}
func init() {
cb := func(i int64) string {
buf := &bytes.Buffer{}
cmpbin.WriteInt(buf, i)
return buf.String()
}
rs := rand.NewSource(0)
root := datastore.MakeKey("something~else", "", "Parent", 1)
nums := make([]string, 20)
for i := range dataMultiRoot {
id := int64(i + 1)
nums[i] = cb(id)
val := make([]int64, rs.Int63()%20)
for j := range val {
r := rs.Int63()
val[j] = r
}
dataMultiRoot[i] = &Foo{ID: id, Value: val}
dataSingleRoot[i] = &Foo{ID: id, Parent: root, Value: val}
}
for i := range hugeField {
hugeField[i] = byte(i)
}
for i := range hugeData {
hugeData[i] = &Foo{ID: int64(i + 1), ValueNI: hugeField}
}
}
// WriteTime writes a time.Time to the buffer.
//
// The supplied time is rounded via datastore.RoundTime and written as a
// microseconds-since-epoch integer to comform to datastore storage standards.
func WriteTime(buf Buffer, t time.Time) error {
name, off := t.Zone()
if name != "UTC" || off != 0 {
panic(fmt.Errorf("helper: UTC OR DEATH: %s", t))
}
_, err := cmpbin.WriteInt(buf, ds.TimeToInt(t))
return err
}
// cat is a convenience method for concatenating anything with an underlying
// byte representation into a single []byte.
func cat(bytethings ...interface{}) []byte {
err := error(nil)
buf := &bytes.Buffer{}
for _, thing := range bytethings {
switch x := thing.(type) {
case int64:
_, err = cmpbin.WriteInt(buf, x)
case int:
_, err = cmpbin.WriteInt(buf, int64(x))
case uint64:
_, err = cmpbin.WriteUint(buf, x)
case uint:
_, err = cmpbin.WriteUint(buf, uint64(x))
case float64:
_, err = cmpbin.WriteFloat64(buf, x)
case byte:
err = buf.WriteByte(x)
case ds.PropertyType:
err = buf.WriteByte(byte(x))
case string:
_, err = cmpbin.WriteString(buf, x)
case []byte:
_, err = buf.Write(x)
case time.Time:
err = serialize.WriteTime(buf, x)
case *ds.Key:
err = serialize.WriteKey(buf, serialize.WithoutContext, x)
case *ds.IndexDefinition:
err = serialize.WriteIndexDefinition(buf, *x)
case ds.Property:
err = serialize.WriteProperty(buf, serialize.WithoutContext, x)
default:
panic(fmt.Errorf("I don't know how to deal with %T: %#v", thing, thing))
}
die(err)
}
ret := buf.Bytes()
if ret == nil {
ret = []byte{}
}
return ret
}
// WriteKeyTok writes a KeyTok to the buffer. You usually want WriteKey
// instead of this.
func WriteKeyTok(buf Buffer, tok ds.KeyTok) (err error) {
// tok.kind ++ typ ++ [tok.stringID || tok.intID]
defer recoverTo(&err)
_, e := cmpbin.WriteString(buf, tok.Kind)
panicIf(e)
if tok.StringID != "" {
panicIf(buf.WriteByte(byte(ds.PTString)))
_, e := cmpbin.WriteString(buf, tok.StringID)
panicIf(e)
} else {
panicIf(buf.WriteByte(byte(ds.PTInt)))
_, e := cmpbin.WriteInt(buf, tok.IntID)
panicIf(e)
}
return nil
}
func TestSerializationReadMisc(t *testing.T) {
t.Parallel()
Convey("Misc Serialization tests", t, func() {
Convey("GeoPoint", func() {
buf := mkBuf(nil)
cmpbin.WriteFloat64(buf, 10)
cmpbin.WriteFloat64(buf, 20)
So(string(ToBytes(ds.GeoPoint{Lat: 10, Lng: 20})), ShouldEqual, buf.String())
})
Convey("IndexColumn", func() {
buf := mkBuf(nil)
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
So(string(ToBytes(ds.IndexColumn{Property: "hi", Direction: ds.DESCENDING})),
ShouldEqual, buf.String())
})
Convey("KeyTok", func() {
buf := mkBuf(nil)
cmpbin.WriteString(buf, "foo")
buf.WriteByte(byte(ds.PTInt))
cmpbin.WriteInt(buf, 20)
So(string(ToBytes(ds.KeyTok{Kind: "foo", IntID: 20})),
ShouldEqual, buf.String())
})
Convey("Property", func() {
buf := mkBuf(nil)
buf.WriteByte(0x80 | byte(ds.PTString))
cmpbin.WriteString(buf, "nerp")
So(string(ToBytes(mp("nerp"))),
ShouldEqual, buf.String())
})
Convey("Time", func() {
tp := mp(time.Now().UTC())
So(string(ToBytes(tp.Value())), ShouldEqual, string(ToBytes(tp)[1:]))
})
Convey("Bad ToBytes", func() {
So(func() { ToBytes(100.7) }, ShouldPanic)
So(func() { ToBytesWithContext(100.7) }, ShouldPanic)
})
Convey("ReadKey", func() {
Convey("good cases", func() {
Convey("w/ ctx decodes normally w/ ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytesWithContext(k)
dk, err := ReadKey(mkBuf(data), WithContext, "", "")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, k)
})
Convey("w/ ctx decodes normally w/o ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytesWithContext(k)
dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
})
Convey("w/o ctx decodes normally w/ ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytes(k)
dk, err := ReadKey(mkBuf(data), WithContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("", "", "knd", "yo", "other", 10))
})
Convey("w/o ctx decodes normally w/o ctx", func() {
k := mkKey("aid", "ns", "knd", "yo", "other", 10)
data := ToBytes(k)
dk, err := ReadKey(mkBuf(data), WithoutContext, "spam", "nerd")
So(err, ShouldBeNil)
So(dk, ShouldEqualKey, mkKey("spam", "nerd", "knd", "yo", "other", 10))
})
Convey("IntIDs always sort before StringIDs", func() {
// -1 writes as almost all 1's in the first byte under cmpbin, even
// though it's technically not a valid key.
k := mkKey("aid", "ns", "knd", -1)
data := ToBytes(k)
k = mkKey("aid", "ns", "knd", "hat")
data2 := ToBytes(k)
So(string(data), ShouldBeLessThan, string(data2))
})
})
Convey("err cases", func() {
buf := mkBuf(nil)
Convey("nil", func() {
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("str", func() {
buf.WriteString("sup")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "expected actualCtx")
})
Convey("truncated 1", func() {
buf.WriteByte(1) // actualCtx == 1
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("truncated 2", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("truncated 3", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("huge key", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
for i := 1; i < 60; i++ {
buf.WriteByte(1)
WriteKeyTok(buf, ds.KeyTok{Kind: "sup", IntID: int64(i)})
}
buf.WriteByte(0)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge key")
})
Convey("insufficient tokens", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
cmpbin.WriteUint(buf, 2)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("partial token 1", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("partial token 2", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTString))
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("bad token (invalid type)", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTBlobKey))
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "invalid type PTBlobKey")
})
Convey("bad token (invalid IntID)", func() {
buf.WriteByte(1) // actualCtx == 1
cmpbin.WriteString(buf, "aid")
cmpbin.WriteString(buf, "ns")
buf.WriteByte(1)
cmpbin.WriteString(buf, "hi")
buf.WriteByte(byte(ds.PTInt))
cmpbin.WriteInt(buf, -2)
_, err := ReadKey(buf, WithContext, "", "")
So(err, ShouldErrLike, "zero/negative")
})
})
})
Convey("ReadGeoPoint", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
_, err := ReadGeoPoint(buf)
So(err, ShouldEqual, io.EOF)
})
Convey("trunc 2", func() {
cmpbin.WriteFloat64(buf, 100)
_, err := ReadGeoPoint(buf)
So(err, ShouldEqual, io.EOF)
})
Convey("invalid", func() {
cmpbin.WriteFloat64(buf, 100)
cmpbin.WriteFloat64(buf, 1000)
_, err := ReadGeoPoint(buf)
So(err, ShouldErrLike, "invalid GeoPoint")
})
})
Convey("WriteTime", func() {
Convey("in non-UTC!", func() {
pst, err := time.LoadLocation("America/Los_Angeles")
So(err, ShouldBeNil)
So(func() {
WriteTime(mkBuf(nil), time.Now().In(pst))
}, ShouldPanic)
})
})
Convey("ReadTime", func() {
Convey("trunc 1", func() {
_, err := ReadTime(mkBuf(nil))
So(err, ShouldEqual, io.EOF)
})
})
Convey("ReadProperty", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
p, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
So(p.Type(), ShouldEqual, ds.PTNull)
So(p.Value(), ShouldBeNil)
})
Convey("trunc (PTBytes)", func() {
buf.WriteByte(byte(ds.PTBytes))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("trunc (PTBlobKey)", func() {
buf.WriteByte(byte(ds.PTBlobKey))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("invalid type", func() {
buf.WriteByte(byte(ds.PTUnknown + 1))
_, err := ReadProperty(buf, WithContext, "", "")
So(err, ShouldErrLike, "unknown type!")
})
})
Convey("ReadPropertyMap", func() {
buf := mkBuf(nil)
Convey("trunc 1", func() {
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("too many rows", func() {
cmpbin.WriteUint(buf, 1000000)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge number of rows")
})
Convey("trunc 2", func() {
cmpbin.WriteUint(buf, 10)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("trunc 3", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
Convey("too many values", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
cmpbin.WriteUint(buf, 100000)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldErrLike, "huge number of properties")
})
Convey("trunc 4", func() {
cmpbin.WriteUint(buf, 10)
cmpbin.WriteString(buf, "ohai")
cmpbin.WriteUint(buf, 10)
_, err := ReadPropertyMap(buf, WithContext, "", "")
So(err, ShouldEqual, io.EOF)
})
})
Convey("IndexDefinition", func() {
id := ds.IndexDefinition{Kind: "kind"}
data := ToBytes(*id.PrepForIdxTable())
newID, err := ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "prop"})
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "other", Direction: ds.DESCENDING})
id.Ancestor = true
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
// invalid
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "", Direction: ds.DESCENDING})
data = ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldBeNil)
So(newID.Flip(), ShouldResemble, id.Normalize())
Convey("too many", func() {
id := ds.IndexDefinition{Kind: "wat"}
for i := 0; i < MaxIndexColumns+1; i++ {
id.SortBy = append(id.SortBy, ds.IndexColumn{Property: "Hi", Direction: ds.ASCENDING})
}
data := ToBytes(*id.PrepForIdxTable())
newID, err = ReadIndexDefinition(mkBuf(data))
So(err, ShouldErrLike, "over 64 sort orders")
})
})
})
}
func wi(w io.ByteWriter, i int64) int {
ret, err := cmpbin.WriteInt(w, i)
die(err)
return ret
}