func (t *txnBufState) effect() (toPut []datastore.PropertyMap, toPutKeys, toDel []*datastore.Key) {
// TODO(riannucci): preallocate return slices
// need to pull all items out of the in-memory datastore. Fortunately we have
// kindless queries, and we disabled all the special entities, so just
// run a kindless query without any filters and it will return all data
// currently in bufDS :).
fq, err := datastore.NewQuery("").Finalize()
impossible(err)
err = t.bufDS.Run(fq, func(key *datastore.Key, data datastore.PropertyMap, _ datastore.CursorCB) error {
toPutKeys = append(toPutKeys, key)
toPut = append(toPut, data)
return nil
})
memoryCorruption(err)
for keyStr, size := range t.entState.keyToSize {
if size == 0 {
k, err := serialize.ReadKey(bytes.NewBufferString(keyStr), serialize.WithoutContext, t.aid, t.ns)
memoryCorruption(err)
toDel = append(toDel, k)
}
}
return
}
func nq(kindMaybe ...string) *ds.Query {
kind := "Foo"
if len(kindMaybe) == 1 {
kind = kindMaybe[0]
}
return ds.NewQuery(kind)
}
func TestNewDatastore(t *testing.T) {
t.Parallel()
Convey("Can get and use a NewDatastore", t, func() {
ds, err := NewDatastore("aid", "ns")
So(err, ShouldBeNil)
k := ds.MakeKey("Something", 1)
So(k.AppID(), ShouldEqual, "aid")
So(k.Namespace(), ShouldEqual, "ns")
type Model struct {
ID int64 `gae:"$id"`
Value []int64
}
So(ds.Put(&Model{ID: 1, Value: []int64{20, 30}}), ShouldBeNil)
vals := []dsS.PropertyMap{}
So(ds.GetAll(dsS.NewQuery("Model").Project("Value"), &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 2)
So(vals[0]["Value"][0].Value(), ShouldEqual, 20)
So(vals[1]["Value"][0].Value(), ShouldEqual, 30)
})
}
func TestBasicDatastore(t *testing.T) {
t.Parallel()
Convey("basic", t, func() {
inst, err := aetest.NewInstance(&aetest.Options{
StronglyConsistentDatastore: true,
})
So(err, ShouldBeNil)
defer inst.Close()
req, err := inst.NewRequest("GET", "/", nil)
So(err, ShouldBeNil)
ctx := Use(context.Background(), req)
ds := datastore.Get(ctx)
mc := memcache.Get(ctx)
inf := info.Get(ctx)
Convey("logging allows you to tweak the level", func() {
// You have to visually confirm that this actually happens in the stdout
// of the test... yeah I know.
logging.Debugf(ctx, "SHOULD NOT SEE")
logging.Infof(ctx, "SHOULD SEE")
ctx = logging.SetLevel(ctx, logging.Debug)
logging.Debugf(ctx, "SHOULD SEE")
logging.Infof(ctx, "SHOULD SEE (2)")
})
Convey("Can probe/change Namespace", func() {
So(inf.GetNamespace(), ShouldEqual, "")
ctx, err = inf.Namespace("wat")
So(err, ShouldBeNil)
inf = info.Get(ctx)
So(inf.GetNamespace(), ShouldEqual, "wat")
ds = datastore.Get(ctx)
So(ds.MakeKey("Hello", "world").Namespace(), ShouldEqual, "wat")
})
Convey("Can get non-transactional context", func() {
ctx, err := inf.Namespace("foo")
So(err, ShouldBeNil)
ds = datastore.Get(ctx)
inf = info.Get(ctx)
ds.RunInTransaction(func(ctx context.Context) error {
So(ds.MakeKey("Foo", "bar").Namespace(), ShouldEqual, "foo")
So(ds.Put(&TestStruct{ValueI: []int64{100}}), ShouldBeNil)
err = datastore.GetNoTxn(ctx).RunInTransaction(func(ctx context.Context) error {
ds = datastore.Get(ctx)
So(ds.MakeKey("Foo", "bar").Namespace(), ShouldEqual, "foo")
So(ds.Put(&TestStruct{ValueI: []int64{100}}), ShouldBeNil)
return nil
}, nil)
So(err, ShouldBeNil)
return nil
}, nil)
})
Convey("Can Put/Get", func() {
orig := TestStruct{
ValueI: []int64{1, 7, 946688461000000, 996688461000000},
ValueB: []bool{true, false},
ValueS: []string{"hello", "world"},
ValueF: []float64{1.0, 7.0, 946688461000000.0, 996688461000000.0},
ValueBS: [][]byte{
[]byte("allo"),
[]byte("hello"),
[]byte("world"),
[]byte("zurple"),
},
ValueK: []*datastore.Key{
ds.NewKey("Something", "Cool", 0, nil),
ds.NewKey("Something", "", 1, nil),
ds.NewKey("Something", "Recursive", 0,
ds.NewKey("Parent", "", 2, nil)),
},
ValueBK: []blobstore.Key{"bellow", "hello"},
ValueGP: []datastore.GeoPoint{
{Lat: 120.7, Lng: 95.5},
},
}
So(ds.Put(&orig), ShouldBeNil)
ret := TestStruct{ID: orig.ID}
So(ds.Get(&ret), ShouldBeNil)
So(ret, ShouldResemble, orig)
// can't be sure the indexes have caught up... so sleep
time.Sleep(time.Second)
Convey("Can query", func() {
q := datastore.NewQuery("TestStruct")
ds.Run(q, func(ts *TestStruct) {
So(*ts, ShouldResemble, orig)
})
count, err := ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 1)
})
Convey("Can project", func() {
q := datastore.NewQuery("TestStruct").Project("ValueS")
rslts := []datastore.PropertyMap{}
So(ds.GetAll(q, &rslts), ShouldBeNil)
So(rslts, ShouldResemble, []datastore.PropertyMap{
{
"$key": {mpNI(ds.KeyForObj(&orig))},
"ValueS": {mp("hello")},
},
{
"$key": {mpNI(ds.KeyForObj(&orig))},
"ValueS": {mp("world")},
},
})
q = datastore.NewQuery("TestStruct").Project("ValueBS")
rslts = []datastore.PropertyMap{}
So(ds.GetAll(q, &rslts), ShouldBeNil)
So(rslts, ShouldResemble, []datastore.PropertyMap{
{
"$key": {mpNI(ds.KeyForObj(&orig))},
"ValueBS": {mp("allo")},
},
{
"$key": {mpNI(ds.KeyForObj(&orig))},
"ValueBS": {mp("hello")},
},
{
"$key": {mpNI(ds.KeyForObj(&orig))},
"ValueBS": {mp("world")},
},
{
"$key": {mpNI(ds.KeyForObj(&orig))},
"ValueBS": {mp("zurple")},
},
})
count, err := ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 4)
})
})
Convey("Can Put/Get (time)", func() {
// time comparisons in Go are wonky, so this is pulled out
pm := datastore.PropertyMap{
"$key": {mpNI(ds.NewKey("Something", "value", 0, nil))},
"Time": {
mp(time.Date(1938, time.January, 1, 1, 1, 1, 1, time.UTC)),
mp(time.Time{}),
},
}
So(ds.Put(&pm), ShouldBeNil)
rslt := datastore.PropertyMap{}
rslt.SetMeta("key", ds.KeyForObj(pm))
So(ds.Get(&rslt), ShouldBeNil)
So(pm["Time"][0].Value(), ShouldResemble, rslt["Time"][0].Value())
q := datastore.NewQuery("Something").Project("Time")
all := []datastore.PropertyMap{}
So(ds.GetAll(q, &all), ShouldBeNil)
So(len(all), ShouldEqual, 2)
prop := all[0]["Time"][0]
So(prop.Type(), ShouldEqual, datastore.PTInt)
tval, err := prop.Project(datastore.PTTime)
So(err, ShouldBeNil)
So(tval, ShouldResemble, time.Time{})
tval, err = all[1]["Time"][0].Project(datastore.PTTime)
So(err, ShouldBeNil)
So(tval, ShouldResemble, pm["Time"][0].Value())
ent := datastore.PropertyMap{
"$key": {mpNI(ds.MakeKey("Something", "value"))},
}
So(ds.Get(&ent), ShouldBeNil)
So(ent["Time"], ShouldResemble, pm["Time"])
})
Convey("memcache: Set (nil) is the same as Set ([]byte{})", func() {
So(mc.Set(mc.NewItem("bob")), ShouldBeNil) // normally would panic because Value is nil
bob, err := mc.Get("bob")
So(err, ShouldBeNil)
So(bob.Value(), ShouldResemble, []byte{})
})
})
}
func TestQuerySupport(t *testing.T) {
t.Parallel()
Convey("Queries", t, func() {
Convey("Good", func() {
q := datastore.NewQuery("Foo").Ancestor(root)
Convey("normal", func() {
_, _, ds := mkds(dataSingleRoot)
ds.Testable().AddIndexes(&datastore.IndexDefinition{
Kind: "Foo",
Ancestor: true,
SortBy: []datastore.IndexColumn{
{Property: "Value"},
},
})
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
q = q.Lt("Value", 400000000000000000)
vals := []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 8)
count, err := ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 8)
f := &Foo{ID: 1, Parent: root}
So(ds.Get(f), ShouldBeNil)
f.Value = append(f.Value, 100)
So(ds.Put(f), ShouldBeNil)
// Wowee, zowee, merged queries!
vals2 := []*Foo{}
So(ds.GetAll(q, &vals2), ShouldBeNil)
So(len(vals2), ShouldEqual, 9)
So(vals2[0], ShouldResemble, f)
vals2 = []*Foo{}
So(ds.GetAll(q.Limit(2).Offset(1), &vals2), ShouldBeNil)
So(len(vals2), ShouldEqual, 2)
So(vals2, ShouldResemble, vals[:2])
return nil
}, nil), ShouldBeNil)
})
Convey("keysOnly", func() {
_, _, ds := mkds([]*Foo{
{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}},
{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
})
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
q = q.Eq("Value", 1).KeysOnly(true)
vals := []*datastore.Key{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 3)
So(vals[2], ShouldResemble, ds.MakeKey("Parent", 1, "Foo", 5))
// can remove keys
So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 2)), ShouldBeNil)
vals = []*datastore.Key{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 2)
// and add new ones
So(ds.Put(&Foo{ID: 1, Parent: root, Value: []int64{1, 7, 100}}), ShouldBeNil)
So(ds.Put(&Foo{ID: 7, Parent: root, Value: []int64{20, 1}}), ShouldBeNil)
vals = []*datastore.Key{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 4)
So(vals[0].IntID(), ShouldEqual, 1)
So(vals[1].IntID(), ShouldEqual, 4)
So(vals[2].IntID(), ShouldEqual, 5)
So(vals[3].IntID(), ShouldEqual, 7)
return nil
}, nil), ShouldBeNil)
})
Convey("project", func() {
_, _, ds := mkds([]*Foo{
{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}},
{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
})
ds.Testable().AddIndexes(&datastore.IndexDefinition{
Kind: "Foo",
Ancestor: true,
SortBy: []datastore.IndexColumn{
{Property: "Value"},
},
})
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
count, err := ds.Count(q.Project("Value"))
So(err, ShouldBeNil)
So(count, ShouldEqual, 24)
q = q.Project("Value").Offset(4).Limit(10)
vals := []datastore.PropertyMap{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 10)
expect := []struct {
id int64
val int64
}{
{2, 3},
{3, 3},
{4, 3},
{2, 4},
{3, 4},
{2, 5},
{3, 5},
{4, 5},
{2, 6},
{3, 6},
}
for i, pm := range vals {
So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
ds.MakeKey("Parent", 1, "Foo", expect[i].id))
So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
}
// should remove 4 entries, but there are plenty more to fill
So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 2)), ShouldBeNil)
vals = []datastore.PropertyMap{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 10)
expect = []struct {
id int64
val int64
}{
// note (3, 3) and (4, 3) are correctly missing because deleting
// 2 removed two entries which are hidden by the Offset(4).
{3, 4},
{3, 5},
{4, 5},
{3, 6},
{3, 7},
{4, 7},
{3, 8},
{3, 9},
{4, 9},
{4, 11},
}
for i, pm := range vals {
So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
ds.MakeKey("Parent", 1, "Foo", expect[i].id))
So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
}
So(ds.Put(&Foo{ID: 1, Parent: root, Value: []int64{3, 9}}), ShouldBeNil)
vals = []datastore.PropertyMap{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 10)
expect = []struct {
id int64
val int64
}{
// 'invisible' {1, 3} entry bumps the {4, 3} into view.
{4, 3},
{3, 4},
{3, 5},
{4, 5},
{3, 6},
{3, 7},
{4, 7},
{3, 8},
{1, 9},
{3, 9},
{4, 9},
}
for i, pm := range vals {
So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
ds.MakeKey("Parent", 1, "Foo", expect[i].id))
So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
}
return nil
}, nil), ShouldBeNil)
})
Convey("project+distinct", func() {
_, _, ds := mkds([]*Foo{
{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}},
{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
})
ds.Testable().AddIndexes(&datastore.IndexDefinition{
Kind: "Foo",
Ancestor: true,
SortBy: []datastore.IndexColumn{
{Property: "Value"},
},
})
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
q = q.Project("Value").Distinct(true)
vals := []datastore.PropertyMap{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 13)
expect := []struct {
id int64
val int64
}{
{2, 1},
{2, 2},
{2, 3},
{2, 4},
{2, 5},
{2, 6},
{2, 7},
{3, 8},
{3, 9},
{4, 11},
{5, 70},
{4, 100},
{5, 101},
}
for i, pm := range vals {
So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
ds.MakeKey("Parent", 1, "Foo", expect[i].id))
}
return nil
}, nil), ShouldBeNil)
})
Convey("overwrite", func() {
data := []*Foo{
{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1, 2}},
{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
}
_, _, ds := mkds(data)
q = q.Eq("Value", 2, 3)
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
vals := []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 2)
So(vals[0], ShouldResemble, data[0])
So(vals[1], ShouldResemble, data[2])
foo2 := &Foo{ID: 2, Parent: root, Value: []int64{2, 3}}
So(ds.Put(foo2), ShouldBeNil)
vals = []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 2)
So(vals[0], ShouldResemble, foo2)
So(vals[1], ShouldResemble, data[2])
foo1 := &Foo{ID: 1, Parent: root, Value: []int64{2, 3}}
So(ds.Put(foo1), ShouldBeNil)
vals = []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 3)
So(vals[0], ShouldResemble, foo1)
So(vals[1], ShouldResemble, foo2)
So(vals[2], ShouldResemble, data[2])
return nil
}, nil), ShouldBeNil)
})
projectData := []*Foo{
{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}, Sort: []string{"x", "z"}},
{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}, Sort: []string{"b"}},
{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1, 2}, Sort: []string{"aa", "a"}},
{ID: 5, Parent: root, Value: []int64{1, 70, 101}, Sort: []string{"c"}},
}
Convey("project+extra orders", func() {
_, _, ds := mkds(projectData)
ds.Testable().AddIndexes(&datastore.IndexDefinition{
Kind: "Foo",
Ancestor: true,
SortBy: []datastore.IndexColumn{
{Property: "Sort", Descending: true},
{Property: "Value", Descending: true},
},
})
q = q.Project("Value").Order("-Sort", "-Value").Distinct(true)
So(ds.RunInTransaction(func(c context.Context) error {
ds = datastore.Get(c)
So(ds.Put(&Foo{
ID: 1, Parent: root, Value: []int64{0, 1, 1000},
Sort: []string{"zz"}}), ShouldBeNil)
vals := []datastore.PropertyMap{}
So(ds.GetAll(q, &vals), ShouldBeNil)
expect := []struct {
id int64
val int64
}{
{1, 1000},
{1, 1},
{1, 0},
{2, 7},
{2, 6},
{2, 5},
{2, 4},
{2, 3},
{2, 2},
{5, 101},
{5, 70},
{3, 9},
{3, 8},
{4, 100},
{4, 11},
}
for i, pm := range vals {
So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
ds.MakeKey("Parent", 1, "Foo", expect[i].id))
}
return nil
}, nil), ShouldBeNil)
})
Convey("buffered entity sorts before ineq, but after first parent entity", func() {
// If we got this wrong, we'd see Foo,3 come before Foo,2. This might
// happen because we calculate the comparison string for each entity
// based on the whole entity, but we forgot to limit the comparison
// string generation by the inequality criteria.
data := []*Foo{
{ID: 2, Parent: root, Value: []int64{2, 3, 5, 6}, Sort: []string{"z"}},
}
_, _, ds := mkds(data)
ds.Testable().AddIndexes(&datastore.IndexDefinition{
Kind: "Foo",
Ancestor: true,
SortBy: []datastore.IndexColumn{
{Property: "Value"},
},
})
q = q.Gt("Value", 2).Limit(2)
So(ds.RunInTransaction(func(c context.Context) error {
ds = datastore.Get(c)
foo1 := &Foo{ID: 3, Parent: root, Value: []int64{0, 2, 3, 4}}
So(ds.Put(foo1), ShouldBeNil)
vals := []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 2)
So(vals[0], ShouldResemble, data[0])
So(vals[1], ShouldResemble, foo1)
return nil
}, nil), ShouldBeNil)
})
Convey("keysOnly+extra orders", func() {
_, _, ds := mkds(projectData)
ds.Testable().AddIndexes(&datastore.IndexDefinition{
Kind: "Foo",
Ancestor: true,
SortBy: []datastore.IndexColumn{
{Property: "Sort"},
},
})
q = q.Order("Sort").KeysOnly(true)
So(ds.RunInTransaction(func(c context.Context) error {
ds = datastore.Get(c)
So(ds.Put(&Foo{
ID: 1, Parent: root, Value: []int64{0, 1, 1000},
Sort: []string{"x", "zz"}}), ShouldBeNil)
So(ds.Put(&Foo{
ID: 2, Parent: root, Value: []int64{0, 1, 1000},
Sort: []string{"zz", "zzz", "zzzz"}}), ShouldBeNil)
vals := []*datastore.Key{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(len(vals), ShouldEqual, 5)
So(vals, ShouldResemble, []*datastore.Key{
ds.MakeKey("Parent", 1, "Foo", 4),
ds.MakeKey("Parent", 1, "Foo", 3),
ds.MakeKey("Parent", 1, "Foo", 5),
ds.MakeKey("Parent", 1, "Foo", 1),
ds.MakeKey("Parent", 1, "Foo", 2),
})
return nil
}, nil), ShouldBeNil)
})
Convey("query accross nested transactions", func() {
_, _, ds := mkds(projectData)
q = q.Eq("Value", 2, 3)
foo1 := &Foo{ID: 1, Parent: root, Value: []int64{2, 3}}
foo7 := &Foo{ID: 7, Parent: root, Value: []int64{2, 3}}
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
So(ds.Put(foo1), ShouldBeNil)
vals := []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(vals, ShouldResemble, []*Foo{foo1, projectData[0], projectData[2]})
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
vals := []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(vals, ShouldResemble, []*Foo{foo1, projectData[0], projectData[2]})
So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 4)), ShouldBeNil)
So(ds.Put(foo7), ShouldBeNil)
vals = []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7})
return nil
}, nil), ShouldBeNil)
vals = []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(vals, ShouldResembleV, []*Foo{foo1, projectData[0], foo7})
return nil
}, nil), ShouldBeNil)
vals := []*Foo{}
So(ds.GetAll(q, &vals), ShouldBeNil)
So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7})
})
Convey("start transaction from inside query", func() {
_, _, ds := mkds(projectData)
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
q := datastore.NewQuery("Foo").Ancestor(root)
return ds.Run(q, func(pm datastore.PropertyMap) {
So(ds.RunInTransaction(func(c context.Context) error {
ds := datastore.Get(c)
pm["Value"] = append(pm["Value"], datastore.MkProperty("wat"))
return ds.Put(pm)
}, nil), ShouldBeNil)
})
}, &datastore.TransactionOptions{XG: true}), ShouldBeNil)
So(ds.Run(datastore.NewQuery("Foo"), func(pm datastore.PropertyMap) {
val := pm["Value"]
So(val[len(val)-1].Value(), ShouldResemble, "wat")
}), ShouldBeNil)
})
})
})
}
func TestDatastoreSingleReadWriter(t *testing.T) {
t.Parallel()
Convey("Datastore single reads and writes", t, func() {
c := Use(context.Background())
ds := dsS.Get(c)
So(ds, ShouldNotBeNil)
Convey("getting objects that DNE is an error", func() {
So(ds.Get(&Foo{ID: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
})
Convey("bad namespaces fail", func() {
_, err := infoS.Get(c).Namespace("$$blzyall")
So(err.Error(), ShouldContainSubstring, "namespace \"$$blzyall\" does not match")
})
Convey("Can Put stuff", func() {
// with an incomplete key!
f := &Foo{Val: 10}
So(ds.Put(f), ShouldBeNil)
k := ds.KeyForObj(f)
So(k.String(), ShouldEqual, "dev~app::/Foo,1")
Convey("and Get it back", func() {
newFoo := &Foo{ID: 1}
So(ds.Get(newFoo), ShouldBeNil)
So(newFoo, ShouldResemble, f)
Convey("but it's hidden from a different namespace", func() {
c, err := infoS.Get(c).Namespace("whombat")
So(err, ShouldBeNil)
ds = dsS.Get(c)
So(ds.Get(f), ShouldEqual, dsS.ErrNoSuchEntity)
})
Convey("and we can Delete it", func() {
So(ds.Delete(k), ShouldBeNil)
So(ds.Get(newFoo), ShouldEqual, dsS.ErrNoSuchEntity)
})
})
Convey("Deleteing with a bogus key is bad", func() {
So(ds.Delete(ds.NewKey("Foo", "wat", 100, nil)), ShouldEqual, dsS.ErrInvalidKey)
})
Convey("Deleteing a DNE entity is fine", func() {
So(ds.Delete(ds.NewKey("Foo", "wat", 0, nil)), ShouldBeNil)
})
Convey("Deleting entities from a nonexistant namespace works", func() {
aid := infoS.Get(c).FullyQualifiedAppID()
keys := make([]*dsS.Key, 10)
for i := range keys {
keys[i] = ds.MakeKey(aid, "noexist", "Kind", i+1)
}
So(ds.DeleteMulti(keys), ShouldBeNil)
count := 0
So(ds.Raw().DeleteMulti(keys, func(err error) error {
count++
So(err, ShouldBeNil)
return nil
}), ShouldBeNil)
So(count, ShouldEqual, len(keys))
})
Convey("with multiple puts", func() {
So(testGetMeta(c, k), ShouldEqual, 1)
foos := make([]Foo, 10)
for i := range foos {
foos[i].Val = 10
foos[i].Parent = k
}
So(ds.PutMulti(foos), ShouldBeNil)
So(testGetMeta(c, k), ShouldEqual, 11)
keys := make([]*dsS.Key, len(foos))
for i, f := range foos {
keys[i] = ds.KeyForObj(&f)
}
Convey("ensure that group versions persist across deletes", func() {
So(ds.DeleteMulti(append(keys, k)), ShouldBeNil)
ds.Testable().CatchupIndexes()
count := 0
So(ds.Run(dsS.NewQuery(""), func(_ *dsS.Key) {
count++
}), ShouldBeNil)
So(count, ShouldEqual, 3)
So(testGetMeta(c, k), ShouldEqual, 22)
So(ds.Put(&Foo{ID: 1}), ShouldBeNil)
So(testGetMeta(c, k), ShouldEqual, 23)
})
Convey("can Get", func() {
vals := make([]dsS.PropertyMap, len(keys))
for i := range vals {
vals[i] = dsS.PropertyMap{}
So(vals[i].SetMeta("key", keys[i]), ShouldBeTrue)
}
So(ds.GetMulti(vals), ShouldBeNil)
for i, val := range vals {
So(val, ShouldResemble, dsS.PropertyMap{
"Val": {dsS.MkProperty(10)},
"$key": {dsS.MkPropertyNI(keys[i])},
})
}
})
})
Convey("allocating ids prevents their use", func() {
start, err := ds.AllocateIDs(ds.MakeKey("Foo", 0), 100)
So(err, ShouldBeNil)
So(start, ShouldEqual, 2)
f := &Foo{Val: 10}
So(ds.Put(f), ShouldBeNil)
k := ds.KeyForObj(f)
So(k.String(), ShouldEqual, "dev~app::/Foo,102")
})
})
Convey("implements DSTransactioner", func() {
Convey("Put", func() {
f := &Foo{Val: 10}
So(ds.Put(f), ShouldBeNil)
k := ds.KeyForObj(f)
So(k.String(), ShouldEqual, "dev~app::/Foo,1")
Convey("can Put new entity groups", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
f := &Foo{Val: 100}
So(ds.Put(f), ShouldBeNil)
So(f.ID, ShouldEqual, 2)
f.ID = 0
f.Val = 200
So(ds.Put(f), ShouldBeNil)
So(f.ID, ShouldEqual, 3)
return nil
}, &dsS.TransactionOptions{XG: true})
So(err, ShouldBeNil)
f := &Foo{ID: 2}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 100)
f.ID = 3
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 200)
})
Convey("can Put new entities in a current group", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
f := &Foo{Val: 100, Parent: k}
So(ds.Put(f), ShouldBeNil)
So(ds.KeyForObj(f).String(), ShouldEqual, "dev~app::/Foo,1/Foo,1")
f.ID = 0
f.Val = 200
So(ds.Put(f), ShouldBeNil)
So(ds.KeyForObj(f).String(), ShouldEqual, "dev~app::/Foo,1/Foo,2")
return nil
}, nil)
So(err, ShouldBeNil)
f := &Foo{ID: 1, Parent: k}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 100)
f.ID = 2
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 200)
})
Convey("Deletes work too", func() {
err := ds.RunInTransaction(func(c context.Context) error {
return dsS.Get(c).Delete(k)
}, nil)
So(err, ShouldBeNil)
So(ds.Get(&Foo{ID: 1}), ShouldEqual, dsS.ErrNoSuchEntity)
})
Convey("A Get counts against your group count", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
pm := dsS.PropertyMap{}
So(pm.SetMeta("key", ds.NewKey("Foo", "", 20, nil)), ShouldBeTrue)
So(ds.Get(pm), ShouldEqual, dsS.ErrNoSuchEntity)
So(pm.SetMeta("key", k), ShouldBeTrue)
So(ds.Get(pm).Error(), ShouldContainSubstring, "cross-group")
return nil
}, nil)
So(err, ShouldBeNil)
})
Convey("Get takes a snapshot", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 10)
// Don't ever do this in a real program unless you want to guarantee
// a failed transaction :)
f.Val = 11
So(dsS.GetNoTxn(c).Put(f), ShouldBeNil)
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 10)
return nil
}, nil)
So(err, ShouldBeNil)
f := &Foo{ID: 1}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 11)
})
Convey("and snapshots are consistent even after Puts", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
f := &Foo{ID: 1}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 10)
// Don't ever do this in a real program unless you want to guarantee
// a failed transaction :)
f.Val = 11
So(dsS.GetNoTxn(c).Put(f), ShouldBeNil)
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 10)
f.Val = 20
So(ds.Put(f), ShouldBeNil)
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 10) // still gets 10
return nil
}, &dsS.TransactionOptions{Attempts: 1})
So(err.Error(), ShouldContainSubstring, "concurrent")
f := &Foo{ID: 1}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 11)
})
Convey("Reusing a transaction context is bad news", func() {
txnDS := dsS.Interface(nil)
err := ds.RunInTransaction(func(c context.Context) error {
txnDS = dsS.Get(c)
So(txnDS.Get(f), ShouldBeNil)
return nil
}, nil)
So(err, ShouldBeNil)
So(txnDS.Get(f).Error(), ShouldContainSubstring, "expired")
})
Convey("Nested transactions are rejected", func() {
err := ds.RunInTransaction(func(c context.Context) error {
err := dsS.Get(c).RunInTransaction(func(c context.Context) error {
panic("noooo")
}, nil)
So(err.Error(), ShouldContainSubstring, "nested transactions")
return nil
}, nil)
So(err, ShouldBeNil)
})
Convey("Concurrent transactions only accept one set of changes", func() {
// Note: I think this implementation is actually /slightly/ wrong.
// According to my read of the docs for appengine, when you open a
// transaction it actually (essentially) holds a reference to the
// entire datastore. Our implementation takes a snapshot of the
// entity group as soon as something observes/affects it.
//
// That said... I'm not sure if there's really a semantic difference.
err := ds.RunInTransaction(func(c context.Context) error {
So(dsS.Get(c).Put(&Foo{ID: 1, Val: 21}), ShouldBeNil)
err := dsS.GetNoTxn(c).RunInTransaction(func(c context.Context) error {
So(dsS.Get(c).Put(&Foo{ID: 1, Val: 27}), ShouldBeNil)
return nil
}, nil)
So(err, ShouldBeNil)
return nil
}, nil)
So(err.Error(), ShouldContainSubstring, "concurrent")
f := &Foo{ID: 1}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 27)
})
Convey("XG", func() {
Convey("Modifying two groups with XG=false is invalid", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
f := &Foo{ID: 1, Val: 200}
So(ds.Put(f), ShouldBeNil)
f.ID = 2
err := ds.Put(f)
So(err.Error(), ShouldContainSubstring, "cross-group")
return err
}, nil)
So(err.Error(), ShouldContainSubstring, "cross-group")
})
Convey("Modifying >25 groups with XG=true is invald", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
foos := make([]Foo, 25)
for i := int64(1); i < 26; i++ {
foos[i-1].ID = i
foos[i-1].Val = 200
}
So(ds.PutMulti(foos), ShouldBeNil)
err := ds.Put(&Foo{ID: 26})
So(err.Error(), ShouldContainSubstring, "too many entity groups")
return err
}, &dsS.TransactionOptions{XG: true})
So(err.Error(), ShouldContainSubstring, "too many entity groups")
})
})
Convey("Errors and panics", func() {
Convey("returning an error aborts", func() {
err := ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
So(ds.Put(&Foo{ID: 1, Val: 200}), ShouldBeNil)
return fmt.Errorf("thingy")
}, nil)
So(err.Error(), ShouldEqual, "thingy")
f := &Foo{ID: 1}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 10)
})
Convey("panicing aborts", func() {
So(func() {
So(ds.RunInTransaction(func(c context.Context) error {
ds := dsS.Get(c)
So(ds.Put(&Foo{Val: 200}), ShouldBeNil)
panic("wheeeeee")
}, nil), ShouldBeNil)
}, ShouldPanic)
f := &Foo{ID: 1}
So(ds.Get(f), ShouldBeNil)
So(f.Val, ShouldEqual, 10)
})
})
Convey("Transaction retries", func() {
tst := ds.Testable()
Reset(func() { tst.SetTransactionRetryCount(0) })
Convey("SetTransactionRetryCount set to zero", func() {
tst.SetTransactionRetryCount(0)
calls := 0
So(ds.RunInTransaction(func(c context.Context) error {
calls++
return nil
}, nil), ShouldBeNil)
So(calls, ShouldEqual, 1)
})
Convey("default TransactionOptions is 3 attempts", func() {
tst.SetTransactionRetryCount(100) // more than 3
calls := 0
So(ds.RunInTransaction(func(c context.Context) error {
calls++
return nil
}, nil), ShouldEqual, dsS.ErrConcurrentTransaction)
So(calls, ShouldEqual, 3)
})
Convey("non-default TransactionOptions ", func() {
tst.SetTransactionRetryCount(100) // more than 20
calls := 0
So(ds.RunInTransaction(func(c context.Context) error {
calls++
return nil
}, &dsS.TransactionOptions{Attempts: 20}), ShouldEqual, dsS.ErrConcurrentTransaction)
So(calls, ShouldEqual, 20)
})
Convey("SetTransactionRetryCount is respected", func() {
tst.SetTransactionRetryCount(1) // less than 3
calls := 0
So(ds.RunInTransaction(func(c context.Context) error {
calls++
return nil
}, nil), ShouldBeNil)
So(calls, ShouldEqual, 2)
})
Convey("fatal errors are not retried", func() {
tst.SetTransactionRetryCount(1)
calls := 0
So(ds.RunInTransaction(func(c context.Context) error {
calls++
return fmt.Errorf("omg")
}, nil).Error(), ShouldEqual, "omg")
So(calls, ShouldEqual, 1)
})
})
})
})
Convey("Testable.Consistent", func() {
Convey("false", func() {
ds.Testable().Consistent(false) // the default
for i := 0; i < 10; i++ {
So(ds.Put(&Foo{ID: int64(i + 1), Val: i + 1}), ShouldBeNil)
}
q := dsS.NewQuery("Foo").Gt("Val", 3)
count, err := ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 0)
So(ds.Delete(ds.MakeKey("Foo", 4)), ShouldBeNil)
count, err = ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 0)
ds.Testable().Consistent(true)
count, err = ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 6)
})
Convey("true", func() {
ds.Testable().Consistent(true)
for i := 0; i < 10; i++ {
So(ds.Put(&Foo{ID: int64(i + 1), Val: i + 1}), ShouldBeNil)
}
q := dsS.NewQuery("Foo").Gt("Val", 3)
count, err := ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 7)
So(ds.Delete(ds.MakeKey("Foo", 4)), ShouldBeNil)
count, err = ds.Count(q)
So(err, ShouldBeNil)
So(count, ShouldEqual, 6)
})
})
Convey("Testable.DisableSpecialEntities", func() {
ds.Testable().DisableSpecialEntities(true)
So(ds.Put(&Foo{}), ShouldErrLike, "allocateIDs is disabled")
So(ds.Put(&Foo{ID: 1}), ShouldBeNil)
ds.Testable().CatchupIndexes()
count, err := ds.Count(dsS.NewQuery(""))
So(err, ShouldBeNil)
So(count, ShouldEqual, 1) // normally this would include __entity_group__
})
})
}