func (s *supportContext) mkRandKeys(keys []*ds.Key, metas ds.MultiMetaGetter) []string { ret := []string(nil) for i, key := range keys { mg := metas.GetSingle(i) if !ds.GetMetaDefault(mg, CacheEnableMeta, true).(bool) { continue } shards := s.numShards(key) if shards == 0 { continue } if ret == nil { ret = make([]string, len(keys)) } ret[i] = MakeMemcacheKey(s.mr.Intn(shards), key) } return ret }
func (d *dsCache) GetMulti(keys []*ds.Key, metas ds.MultiMetaGetter, cb ds.GetMultiCB) error { lockItems, nonce := d.mkRandLockItems(keys, metas) if len(lockItems) == 0 { return d.RawInterface.GetMulti(keys, metas, cb) } if err := d.mc.AddMulti(lockItems); err != nil { // Ignore this error. Either we couldn't add them because they exist // (so, not an issue), or because memcache is having sad times (in which // case we'll see so in the GetMulti which immediately follows this). } if err := errors.Filter(d.mc.GetMulti(lockItems), memcache.ErrCacheMiss); err != nil { (log.Fields{log.ErrorKey: err}).Warningf( d.c, "dscache: GetMulti: memcache.GetMulti") } p := makeFetchPlan(d.c, d.aid, d.ns, &facts{keys, metas, lockItems, nonce}) if !p.empty() { // looks like we have something to pull from datastore, and maybe some work // to save stuff back to memcache. toCas := []memcache.Item{} j := 0 err := d.RawInterface.GetMulti(p.toGet, p.toGetMeta, func(pm ds.PropertyMap, err error) error { i := p.idxMap[j] toSave := p.toSave[j] j++ data := []byte(nil) // true: save entity to memcache // false: lock entity in memcache forever shouldSave := true if err == nil { p.decoded[i] = pm if toSave != nil { data = encodeItemValue(pm) if len(data) > internalValueSizeLimit { shouldSave = false log.Warningf( d.c, "dscache: encoded entity too big (%d/%d)!", len(data), internalValueSizeLimit) } } } else { p.lme.Assign(i, err) if err != ds.ErrNoSuchEntity { return nil // aka continue to the next entry } } if toSave != nil { if shouldSave { // save mg := metas.GetSingle(i) expSecs := ds.GetMetaDefault(mg, CacheExpirationMeta, CacheTimeSeconds).(int64) toSave.SetFlags(uint32(ItemHasData)) toSave.SetExpiration(time.Duration(expSecs) * time.Second) toSave.SetValue(data) } else { // Set a lock with an infinite timeout. No one else should try to // serialize this item to memcache until something Put/Delete's it. toSave.SetFlags(uint32(ItemHasLock)) toSave.SetExpiration(0) toSave.SetValue(nil) } toCas = append(toCas, toSave) } return nil }) if err != nil { return err } if len(toCas) > 0 { // we have entries to save back to memcache. if err := d.mc.CompareAndSwapMulti(toCas); err != nil { (log.Fields{log.ErrorKey: err}).Warningf( d.c, "dscache: GetMulti: memcache.CompareAndSwapMulti") } } } // finally, run the callback for all of the decoded items and the errors, // if any. for i, dec := range p.decoded { cb(dec, p.lme.GetOne(i)) } return nil }
func TestQuerySupport(t *testing.T) { t.Parallel() Convey("Queries", t, func() { Convey("Good", func() { q := datastore.NewQuery("Foo").Ancestor(root) Convey("normal", func() { _, _, ds := mkds(dataSingleRoot) ds.Testable().AddIndexes(&datastore.IndexDefinition{ Kind: "Foo", Ancestor: true, SortBy: []datastore.IndexColumn{ {Property: "Value"}, }, }) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) q = q.Lt("Value", 400000000000000000) vals := []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 8) count, err := ds.Count(q) So(err, ShouldBeNil) So(count, ShouldEqual, 8) f := &Foo{ID: 1, Parent: root} So(ds.Get(f), ShouldBeNil) f.Value = append(f.Value, 100) So(ds.Put(f), ShouldBeNil) // Wowee, zowee, merged queries! vals2 := []*Foo{} So(ds.GetAll(q, &vals2), ShouldBeNil) So(len(vals2), ShouldEqual, 9) So(vals2[0], ShouldResemble, f) vals2 = []*Foo{} So(ds.GetAll(q.Limit(2).Offset(1), &vals2), ShouldBeNil) So(len(vals2), ShouldEqual, 2) So(vals2, ShouldResemble, vals[:2]) return nil }, nil), ShouldBeNil) }) Convey("keysOnly", func() { _, _, ds := mkds([]*Foo{ {ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}}, {ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}}, {ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}}, {ID: 5, Parent: root, Value: []int64{1, 70, 101}}, }) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) q = q.Eq("Value", 1).KeysOnly(true) vals := []*datastore.Key{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 3) So(vals[2], ShouldResemble, ds.MakeKey("Parent", 1, "Foo", 5)) // can remove keys So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 2)), ShouldBeNil) vals = []*datastore.Key{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 2) // and add new ones So(ds.Put(&Foo{ID: 1, Parent: root, Value: []int64{1, 7, 100}}), ShouldBeNil) So(ds.Put(&Foo{ID: 7, Parent: root, Value: []int64{20, 1}}), ShouldBeNil) vals = []*datastore.Key{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 4) So(vals[0].IntID(), ShouldEqual, 1) So(vals[1].IntID(), ShouldEqual, 4) So(vals[2].IntID(), ShouldEqual, 5) So(vals[3].IntID(), ShouldEqual, 7) return nil }, nil), ShouldBeNil) }) Convey("project", func() { _, _, ds := mkds([]*Foo{ {ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}}, {ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}}, {ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}}, {ID: 5, Parent: root, Value: []int64{1, 70, 101}}, }) ds.Testable().AddIndexes(&datastore.IndexDefinition{ Kind: "Foo", Ancestor: true, SortBy: []datastore.IndexColumn{ {Property: "Value"}, }, }) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) count, err := ds.Count(q.Project("Value")) So(err, ShouldBeNil) So(count, ShouldEqual, 24) q = q.Project("Value").Offset(4).Limit(10) vals := []datastore.PropertyMap{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 10) expect := []struct { id int64 val int64 }{ {2, 3}, {3, 3}, {4, 3}, {2, 4}, {3, 4}, {2, 5}, {3, 5}, {4, 5}, {2, 6}, {3, 6}, } for i, pm := range vals { So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble, ds.MakeKey("Parent", 1, "Foo", expect[i].id)) So(pm["Value"][0].Value(), ShouldEqual, expect[i].val) } // should remove 4 entries, but there are plenty more to fill So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 2)), ShouldBeNil) vals = []datastore.PropertyMap{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 10) expect = []struct { id int64 val int64 }{ // note (3, 3) and (4, 3) are correctly missing because deleting // 2 removed two entries which are hidden by the Offset(4). {3, 4}, {3, 5}, {4, 5}, {3, 6}, {3, 7}, {4, 7}, {3, 8}, {3, 9}, {4, 9}, {4, 11}, } for i, pm := range vals { So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble, ds.MakeKey("Parent", 1, "Foo", expect[i].id)) So(pm["Value"][0].Value(), ShouldEqual, expect[i].val) } So(ds.Put(&Foo{ID: 1, Parent: root, Value: []int64{3, 9}}), ShouldBeNil) vals = []datastore.PropertyMap{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 10) expect = []struct { id int64 val int64 }{ // 'invisible' {1, 3} entry bumps the {4, 3} into view. {4, 3}, {3, 4}, {3, 5}, {4, 5}, {3, 6}, {3, 7}, {4, 7}, {3, 8}, {1, 9}, {3, 9}, {4, 9}, } for i, pm := range vals { So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble, ds.MakeKey("Parent", 1, "Foo", expect[i].id)) So(pm["Value"][0].Value(), ShouldEqual, expect[i].val) } return nil }, nil), ShouldBeNil) }) Convey("project+distinct", func() { _, _, ds := mkds([]*Foo{ {ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}}, {ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}}, {ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}}, {ID: 5, Parent: root, Value: []int64{1, 70, 101}}, }) ds.Testable().AddIndexes(&datastore.IndexDefinition{ Kind: "Foo", Ancestor: true, SortBy: []datastore.IndexColumn{ {Property: "Value"}, }, }) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) q = q.Project("Value").Distinct(true) vals := []datastore.PropertyMap{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 13) expect := []struct { id int64 val int64 }{ {2, 1}, {2, 2}, {2, 3}, {2, 4}, {2, 5}, {2, 6}, {2, 7}, {3, 8}, {3, 9}, {4, 11}, {5, 70}, {4, 100}, {5, 101}, } for i, pm := range vals { So(pm["Value"][0].Value(), ShouldEqual, expect[i].val) So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble, ds.MakeKey("Parent", 1, "Foo", expect[i].id)) } return nil }, nil), ShouldBeNil) }) Convey("overwrite", func() { data := []*Foo{ {ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}}, {ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}}, {ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1, 2}}, {ID: 5, Parent: root, Value: []int64{1, 70, 101}}, } _, _, ds := mkds(data) q = q.Eq("Value", 2, 3) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) vals := []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 2) So(vals[0], ShouldResemble, data[0]) So(vals[1], ShouldResemble, data[2]) foo2 := &Foo{ID: 2, Parent: root, Value: []int64{2, 3}} So(ds.Put(foo2), ShouldBeNil) vals = []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 2) So(vals[0], ShouldResemble, foo2) So(vals[1], ShouldResemble, data[2]) foo1 := &Foo{ID: 1, Parent: root, Value: []int64{2, 3}} So(ds.Put(foo1), ShouldBeNil) vals = []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 3) So(vals[0], ShouldResemble, foo1) So(vals[1], ShouldResemble, foo2) So(vals[2], ShouldResemble, data[2]) return nil }, nil), ShouldBeNil) }) projectData := []*Foo{ {ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}, Sort: []string{"x", "z"}}, {ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}, Sort: []string{"b"}}, {ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1, 2}, Sort: []string{"aa", "a"}}, {ID: 5, Parent: root, Value: []int64{1, 70, 101}, Sort: []string{"c"}}, } Convey("project+extra orders", func() { _, _, ds := mkds(projectData) ds.Testable().AddIndexes(&datastore.IndexDefinition{ Kind: "Foo", Ancestor: true, SortBy: []datastore.IndexColumn{ {Property: "Sort", Descending: true}, {Property: "Value", Descending: true}, }, }) q = q.Project("Value").Order("-Sort", "-Value").Distinct(true) So(ds.RunInTransaction(func(c context.Context) error { ds = datastore.Get(c) So(ds.Put(&Foo{ ID: 1, Parent: root, Value: []int64{0, 1, 1000}, Sort: []string{"zz"}}), ShouldBeNil) vals := []datastore.PropertyMap{} So(ds.GetAll(q, &vals), ShouldBeNil) expect := []struct { id int64 val int64 }{ {1, 1000}, {1, 1}, {1, 0}, {2, 7}, {2, 6}, {2, 5}, {2, 4}, {2, 3}, {2, 2}, {5, 101}, {5, 70}, {3, 9}, {3, 8}, {4, 100}, {4, 11}, } for i, pm := range vals { So(pm["Value"][0].Value(), ShouldEqual, expect[i].val) So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble, ds.MakeKey("Parent", 1, "Foo", expect[i].id)) } return nil }, nil), ShouldBeNil) }) Convey("buffered entity sorts before ineq, but after first parent entity", func() { // If we got this wrong, we'd see Foo,3 come before Foo,2. This might // happen because we calculate the comparison string for each entity // based on the whole entity, but we forgot to limit the comparison // string generation by the inequality criteria. data := []*Foo{ {ID: 2, Parent: root, Value: []int64{2, 3, 5, 6}, Sort: []string{"z"}}, } _, _, ds := mkds(data) ds.Testable().AddIndexes(&datastore.IndexDefinition{ Kind: "Foo", Ancestor: true, SortBy: []datastore.IndexColumn{ {Property: "Value"}, }, }) q = q.Gt("Value", 2).Limit(2) So(ds.RunInTransaction(func(c context.Context) error { ds = datastore.Get(c) foo1 := &Foo{ID: 3, Parent: root, Value: []int64{0, 2, 3, 4}} So(ds.Put(foo1), ShouldBeNil) vals := []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 2) So(vals[0], ShouldResemble, data[0]) So(vals[1], ShouldResemble, foo1) return nil }, nil), ShouldBeNil) }) Convey("keysOnly+extra orders", func() { _, _, ds := mkds(projectData) ds.Testable().AddIndexes(&datastore.IndexDefinition{ Kind: "Foo", Ancestor: true, SortBy: []datastore.IndexColumn{ {Property: "Sort"}, }, }) q = q.Order("Sort").KeysOnly(true) So(ds.RunInTransaction(func(c context.Context) error { ds = datastore.Get(c) So(ds.Put(&Foo{ ID: 1, Parent: root, Value: []int64{0, 1, 1000}, Sort: []string{"x", "zz"}}), ShouldBeNil) So(ds.Put(&Foo{ ID: 2, Parent: root, Value: []int64{0, 1, 1000}, Sort: []string{"zz", "zzz", "zzzz"}}), ShouldBeNil) vals := []*datastore.Key{} So(ds.GetAll(q, &vals), ShouldBeNil) So(len(vals), ShouldEqual, 5) So(vals, ShouldResemble, []*datastore.Key{ ds.MakeKey("Parent", 1, "Foo", 4), ds.MakeKey("Parent", 1, "Foo", 3), ds.MakeKey("Parent", 1, "Foo", 5), ds.MakeKey("Parent", 1, "Foo", 1), ds.MakeKey("Parent", 1, "Foo", 2), }) return nil }, nil), ShouldBeNil) }) Convey("query accross nested transactions", func() { _, _, ds := mkds(projectData) q = q.Eq("Value", 2, 3) foo1 := &Foo{ID: 1, Parent: root, Value: []int64{2, 3}} foo7 := &Foo{ID: 7, Parent: root, Value: []int64{2, 3}} So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) So(ds.Put(foo1), ShouldBeNil) vals := []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(vals, ShouldResemble, []*Foo{foo1, projectData[0], projectData[2]}) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) vals := []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(vals, ShouldResemble, []*Foo{foo1, projectData[0], projectData[2]}) So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 4)), ShouldBeNil) So(ds.Put(foo7), ShouldBeNil) vals = []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7}) return nil }, nil), ShouldBeNil) vals = []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7}) return nil }, nil), ShouldBeNil) vals := []*Foo{} So(ds.GetAll(q, &vals), ShouldBeNil) So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7}) }) Convey("start transaction from inside query", func() { _, _, ds := mkds(projectData) So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) q := datastore.NewQuery("Foo").Ancestor(root) return ds.Run(q, func(pm datastore.PropertyMap) { So(ds.RunInTransaction(func(c context.Context) error { ds := datastore.Get(c) pm["Value"] = append(pm["Value"], datastore.MkProperty("wat")) return ds.Put(pm) }, nil), ShouldBeNil) }) }, &datastore.TransactionOptions{XG: true}), ShouldBeNil) So(ds.Run(datastore.NewQuery("Foo"), func(pm datastore.PropertyMap) { val := pm["Value"] So(val[len(val)-1].Value(), ShouldResemble, "wat") }), ShouldBeNil) }) }) }) }