Esempio n. 1
0
func (s *supportContext) mkRandKeys(keys []*ds.Key, metas ds.MultiMetaGetter) []string {
	ret := []string(nil)
	for i, key := range keys {
		mg := metas.GetSingle(i)
		if !ds.GetMetaDefault(mg, CacheEnableMeta, true).(bool) {
			continue
		}
		shards := s.numShards(key)
		if shards == 0 {
			continue
		}
		if ret == nil {
			ret = make([]string, len(keys))
		}
		ret[i] = MakeMemcacheKey(s.mr.Intn(shards), key)
	}
	return ret
}
Esempio n. 2
0
func (d *dsCache) GetMulti(keys []*ds.Key, metas ds.MultiMetaGetter, cb ds.GetMultiCB) error {
	lockItems, nonce := d.mkRandLockItems(keys, metas)
	if len(lockItems) == 0 {
		return d.RawInterface.GetMulti(keys, metas, cb)
	}

	if err := d.mc.AddMulti(lockItems); err != nil {
		// Ignore this error. Either we couldn't add them because they exist
		// (so, not an issue), or because memcache is having sad times (in which
		// case we'll see so in the GetMulti which immediately follows this).
	}
	if err := errors.Filter(d.mc.GetMulti(lockItems), memcache.ErrCacheMiss); err != nil {
		(log.Fields{log.ErrorKey: err}).Warningf(
			d.c, "dscache: GetMulti: memcache.GetMulti")
	}

	p := makeFetchPlan(d.c, d.aid, d.ns, &facts{keys, metas, lockItems, nonce})

	if !p.empty() {
		// looks like we have something to pull from datastore, and maybe some work
		// to save stuff back to memcache.

		toCas := []memcache.Item{}
		j := 0
		err := d.RawInterface.GetMulti(p.toGet, p.toGetMeta, func(pm ds.PropertyMap, err error) error {
			i := p.idxMap[j]
			toSave := p.toSave[j]
			j++

			data := []byte(nil)

			// true: save entity to memcache
			// false: lock entity in memcache forever
			shouldSave := true
			if err == nil {
				p.decoded[i] = pm
				if toSave != nil {
					data = encodeItemValue(pm)
					if len(data) > internalValueSizeLimit {
						shouldSave = false
						log.Warningf(
							d.c, "dscache: encoded entity too big (%d/%d)!",
							len(data), internalValueSizeLimit)
					}
				}
			} else {
				p.lme.Assign(i, err)
				if err != ds.ErrNoSuchEntity {
					return nil // aka continue to the next entry
				}
			}

			if toSave != nil {
				if shouldSave { // save
					mg := metas.GetSingle(i)
					expSecs := ds.GetMetaDefault(mg, CacheExpirationMeta, CacheTimeSeconds).(int64)
					toSave.SetFlags(uint32(ItemHasData))
					toSave.SetExpiration(time.Duration(expSecs) * time.Second)
					toSave.SetValue(data)
				} else {
					// Set a lock with an infinite timeout. No one else should try to
					// serialize this item to memcache until something Put/Delete's it.
					toSave.SetFlags(uint32(ItemHasLock))
					toSave.SetExpiration(0)
					toSave.SetValue(nil)
				}
				toCas = append(toCas, toSave)
			}
			return nil
		})
		if err != nil {
			return err
		}
		if len(toCas) > 0 {
			// we have entries to save back to memcache.
			if err := d.mc.CompareAndSwapMulti(toCas); err != nil {
				(log.Fields{log.ErrorKey: err}).Warningf(
					d.c, "dscache: GetMulti: memcache.CompareAndSwapMulti")
			}
		}
	}

	// finally, run the callback for all of the decoded items and the errors,
	// if any.
	for i, dec := range p.decoded {
		cb(dec, p.lme.GetOne(i))
	}

	return nil
}
Esempio n. 3
0
func TestQuerySupport(t *testing.T) {
	t.Parallel()

	Convey("Queries", t, func() {
		Convey("Good", func() {
			q := datastore.NewQuery("Foo").Ancestor(root)

			Convey("normal", func() {
				_, _, ds := mkds(dataSingleRoot)
				ds.Testable().AddIndexes(&datastore.IndexDefinition{
					Kind:     "Foo",
					Ancestor: true,
					SortBy: []datastore.IndexColumn{
						{Property: "Value"},
					},
				})

				So(ds.RunInTransaction(func(c context.Context) error {
					ds := datastore.Get(c)

					q = q.Lt("Value", 400000000000000000)

					vals := []*Foo{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 8)

					count, err := ds.Count(q)
					So(err, ShouldBeNil)
					So(count, ShouldEqual, 8)

					f := &Foo{ID: 1, Parent: root}
					So(ds.Get(f), ShouldBeNil)
					f.Value = append(f.Value, 100)
					So(ds.Put(f), ShouldBeNil)

					// Wowee, zowee, merged queries!
					vals2 := []*Foo{}
					So(ds.GetAll(q, &vals2), ShouldBeNil)
					So(len(vals2), ShouldEqual, 9)
					So(vals2[0], ShouldResemble, f)

					vals2 = []*Foo{}
					So(ds.GetAll(q.Limit(2).Offset(1), &vals2), ShouldBeNil)
					So(len(vals2), ShouldEqual, 2)
					So(vals2, ShouldResemble, vals[:2])

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("keysOnly", func() {
				_, _, ds := mkds([]*Foo{
					{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
					{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
					{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}},
					{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
				})

				So(ds.RunInTransaction(func(c context.Context) error {
					ds := datastore.Get(c)

					q = q.Eq("Value", 1).KeysOnly(true)
					vals := []*datastore.Key{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 3)
					So(vals[2], ShouldResemble, ds.MakeKey("Parent", 1, "Foo", 5))

					// can remove keys
					So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 2)), ShouldBeNil)
					vals = []*datastore.Key{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 2)

					// and add new ones
					So(ds.Put(&Foo{ID: 1, Parent: root, Value: []int64{1, 7, 100}}), ShouldBeNil)
					So(ds.Put(&Foo{ID: 7, Parent: root, Value: []int64{20, 1}}), ShouldBeNil)
					vals = []*datastore.Key{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 4)

					So(vals[0].IntID(), ShouldEqual, 1)
					So(vals[1].IntID(), ShouldEqual, 4)
					So(vals[2].IntID(), ShouldEqual, 5)
					So(vals[3].IntID(), ShouldEqual, 7)

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("project", func() {
				_, _, ds := mkds([]*Foo{
					{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
					{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
					{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}},
					{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
				})

				ds.Testable().AddIndexes(&datastore.IndexDefinition{
					Kind:     "Foo",
					Ancestor: true,
					SortBy: []datastore.IndexColumn{
						{Property: "Value"},
					},
				})

				So(ds.RunInTransaction(func(c context.Context) error {
					ds := datastore.Get(c)

					count, err := ds.Count(q.Project("Value"))
					So(err, ShouldBeNil)
					So(count, ShouldEqual, 24)

					q = q.Project("Value").Offset(4).Limit(10)

					vals := []datastore.PropertyMap{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 10)

					expect := []struct {
						id  int64
						val int64
					}{
						{2, 3},
						{3, 3},
						{4, 3},
						{2, 4},
						{3, 4},
						{2, 5},
						{3, 5},
						{4, 5},
						{2, 6},
						{3, 6},
					}

					for i, pm := range vals {
						So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
							ds.MakeKey("Parent", 1, "Foo", expect[i].id))
						So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
					}

					// should remove 4 entries, but there are plenty more to fill
					So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 2)), ShouldBeNil)

					vals = []datastore.PropertyMap{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 10)

					expect = []struct {
						id  int64
						val int64
					}{
						// note (3, 3) and (4, 3) are correctly missing because deleting
						// 2 removed two entries which are hidden by the Offset(4).
						{3, 4},
						{3, 5},
						{4, 5},
						{3, 6},
						{3, 7},
						{4, 7},
						{3, 8},
						{3, 9},
						{4, 9},
						{4, 11},
					}

					for i, pm := range vals {
						So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
							ds.MakeKey("Parent", 1, "Foo", expect[i].id))
						So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
					}

					So(ds.Put(&Foo{ID: 1, Parent: root, Value: []int64{3, 9}}), ShouldBeNil)

					vals = []datastore.PropertyMap{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 10)

					expect = []struct {
						id  int64
						val int64
					}{
						// 'invisible' {1, 3} entry bumps the {4, 3} into view.
						{4, 3},
						{3, 4},
						{3, 5},
						{4, 5},
						{3, 6},
						{3, 7},
						{4, 7},
						{3, 8},
						{1, 9},
						{3, 9},
						{4, 9},
					}

					for i, pm := range vals {
						So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
							ds.MakeKey("Parent", 1, "Foo", expect[i].id))
						So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
					}

					return nil
				}, nil), ShouldBeNil)

			})

			Convey("project+distinct", func() {
				_, _, ds := mkds([]*Foo{
					{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
					{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
					{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1}},
					{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
				})

				ds.Testable().AddIndexes(&datastore.IndexDefinition{
					Kind:     "Foo",
					Ancestor: true,
					SortBy: []datastore.IndexColumn{
						{Property: "Value"},
					},
				})

				So(ds.RunInTransaction(func(c context.Context) error {
					ds := datastore.Get(c)

					q = q.Project("Value").Distinct(true)

					vals := []datastore.PropertyMap{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 13)

					expect := []struct {
						id  int64
						val int64
					}{
						{2, 1},
						{2, 2},
						{2, 3},
						{2, 4},
						{2, 5},
						{2, 6},
						{2, 7},
						{3, 8},
						{3, 9},
						{4, 11},
						{5, 70},
						{4, 100},
						{5, 101},
					}

					for i, pm := range vals {
						So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
						So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
							ds.MakeKey("Parent", 1, "Foo", expect[i].id))
					}

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("overwrite", func() {
				data := []*Foo{
					{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}},
					{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}},
					{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1, 2}},
					{ID: 5, Parent: root, Value: []int64{1, 70, 101}},
				}

				_, _, ds := mkds(data)

				q = q.Eq("Value", 2, 3)

				So(ds.RunInTransaction(func(c context.Context) error {
					ds := datastore.Get(c)

					vals := []*Foo{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 2)

					So(vals[0], ShouldResemble, data[0])
					So(vals[1], ShouldResemble, data[2])

					foo2 := &Foo{ID: 2, Parent: root, Value: []int64{2, 3}}
					So(ds.Put(foo2), ShouldBeNil)

					vals = []*Foo{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 2)

					So(vals[0], ShouldResemble, foo2)
					So(vals[1], ShouldResemble, data[2])

					foo1 := &Foo{ID: 1, Parent: root, Value: []int64{2, 3}}
					So(ds.Put(foo1), ShouldBeNil)

					vals = []*Foo{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 3)

					So(vals[0], ShouldResemble, foo1)
					So(vals[1], ShouldResemble, foo2)
					So(vals[2], ShouldResemble, data[2])

					return nil
				}, nil), ShouldBeNil)
			})

			projectData := []*Foo{
				{ID: 2, Parent: root, Value: []int64{1, 2, 3, 4, 5, 6, 7}, Sort: []string{"x", "z"}},
				{ID: 3, Parent: root, Value: []int64{3, 4, 5, 6, 7, 8, 9}, Sort: []string{"b"}},
				{ID: 4, Parent: root, Value: []int64{3, 5, 7, 9, 11, 100, 1, 2}, Sort: []string{"aa", "a"}},
				{ID: 5, Parent: root, Value: []int64{1, 70, 101}, Sort: []string{"c"}},
			}

			Convey("project+extra orders", func() {

				_, _, ds := mkds(projectData)
				ds.Testable().AddIndexes(&datastore.IndexDefinition{
					Kind:     "Foo",
					Ancestor: true,
					SortBy: []datastore.IndexColumn{
						{Property: "Sort", Descending: true},
						{Property: "Value", Descending: true},
					},
				})

				q = q.Project("Value").Order("-Sort", "-Value").Distinct(true)
				So(ds.RunInTransaction(func(c context.Context) error {
					ds = datastore.Get(c)

					So(ds.Put(&Foo{
						ID: 1, Parent: root, Value: []int64{0, 1, 1000},
						Sort: []string{"zz"}}), ShouldBeNil)

					vals := []datastore.PropertyMap{}
					So(ds.GetAll(q, &vals), ShouldBeNil)

					expect := []struct {
						id  int64
						val int64
					}{
						{1, 1000},
						{1, 1},
						{1, 0},
						{2, 7},
						{2, 6},
						{2, 5},
						{2, 4},
						{2, 3},
						{2, 2},
						{5, 101},
						{5, 70},
						{3, 9},
						{3, 8},
						{4, 100},
						{4, 11},
					}

					for i, pm := range vals {
						So(pm["Value"][0].Value(), ShouldEqual, expect[i].val)
						So(datastore.GetMetaDefault(pm, "key", nil), ShouldResemble,
							ds.MakeKey("Parent", 1, "Foo", expect[i].id))
					}

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("buffered entity sorts before ineq, but after first parent entity", func() {
				// If we got this wrong, we'd see Foo,3 come before Foo,2. This might
				// happen because we calculate the comparison string for each entity
				// based on the whole entity, but we forgot to limit the comparison
				// string generation by the inequality criteria.
				data := []*Foo{
					{ID: 2, Parent: root, Value: []int64{2, 3, 5, 6}, Sort: []string{"z"}},
				}

				_, _, ds := mkds(data)
				ds.Testable().AddIndexes(&datastore.IndexDefinition{
					Kind:     "Foo",
					Ancestor: true,
					SortBy: []datastore.IndexColumn{
						{Property: "Value"},
					},
				})

				q = q.Gt("Value", 2).Limit(2)

				So(ds.RunInTransaction(func(c context.Context) error {
					ds = datastore.Get(c)

					foo1 := &Foo{ID: 3, Parent: root, Value: []int64{0, 2, 3, 4}}
					So(ds.Put(foo1), ShouldBeNil)

					vals := []*Foo{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 2)

					So(vals[0], ShouldResemble, data[0])
					So(vals[1], ShouldResemble, foo1)

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("keysOnly+extra orders", func() {
				_, _, ds := mkds(projectData)
				ds.Testable().AddIndexes(&datastore.IndexDefinition{
					Kind:     "Foo",
					Ancestor: true,
					SortBy: []datastore.IndexColumn{
						{Property: "Sort"},
					},
				})

				q = q.Order("Sort").KeysOnly(true)

				So(ds.RunInTransaction(func(c context.Context) error {
					ds = datastore.Get(c)

					So(ds.Put(&Foo{
						ID: 1, Parent: root, Value: []int64{0, 1, 1000},
						Sort: []string{"x", "zz"}}), ShouldBeNil)

					So(ds.Put(&Foo{
						ID: 2, Parent: root, Value: []int64{0, 1, 1000},
						Sort: []string{"zz", "zzz", "zzzz"}}), ShouldBeNil)

					vals := []*datastore.Key{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(len(vals), ShouldEqual, 5)

					So(vals, ShouldResemble, []*datastore.Key{
						ds.MakeKey("Parent", 1, "Foo", 4),
						ds.MakeKey("Parent", 1, "Foo", 3),
						ds.MakeKey("Parent", 1, "Foo", 5),
						ds.MakeKey("Parent", 1, "Foo", 1),
						ds.MakeKey("Parent", 1, "Foo", 2),
					})

					return nil
				}, nil), ShouldBeNil)
			})

			Convey("query accross nested transactions", func() {
				_, _, ds := mkds(projectData)
				q = q.Eq("Value", 2, 3)

				foo1 := &Foo{ID: 1, Parent: root, Value: []int64{2, 3}}
				foo7 := &Foo{ID: 7, Parent: root, Value: []int64{2, 3}}

				So(ds.RunInTransaction(func(c context.Context) error {
					ds := datastore.Get(c)

					So(ds.Put(foo1), ShouldBeNil)

					vals := []*Foo{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(vals, ShouldResemble, []*Foo{foo1, projectData[0], projectData[2]})

					So(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)

						vals := []*Foo{}
						So(ds.GetAll(q, &vals), ShouldBeNil)
						So(vals, ShouldResemble, []*Foo{foo1, projectData[0], projectData[2]})

						So(ds.Delete(ds.MakeKey("Parent", 1, "Foo", 4)), ShouldBeNil)
						So(ds.Put(foo7), ShouldBeNil)

						vals = []*Foo{}
						So(ds.GetAll(q, &vals), ShouldBeNil)
						So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7})

						return nil
					}, nil), ShouldBeNil)

					vals = []*Foo{}
					So(ds.GetAll(q, &vals), ShouldBeNil)
					So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7})

					return nil
				}, nil), ShouldBeNil)

				vals := []*Foo{}
				So(ds.GetAll(q, &vals), ShouldBeNil)
				So(vals, ShouldResemble, []*Foo{foo1, projectData[0], foo7})

			})

			Convey("start transaction from inside query", func() {
				_, _, ds := mkds(projectData)
				So(ds.RunInTransaction(func(c context.Context) error {
					ds := datastore.Get(c)

					q := datastore.NewQuery("Foo").Ancestor(root)
					return ds.Run(q, func(pm datastore.PropertyMap) {
						So(ds.RunInTransaction(func(c context.Context) error {
							ds := datastore.Get(c)
							pm["Value"] = append(pm["Value"], datastore.MkProperty("wat"))
							return ds.Put(pm)
						}, nil), ShouldBeNil)
					})
				}, &datastore.TransactionOptions{XG: true}), ShouldBeNil)

				So(ds.Run(datastore.NewQuery("Foo"), func(pm datastore.PropertyMap) {
					val := pm["Value"]
					So(val[len(val)-1].Value(), ShouldResemble, "wat")
				}), ShouldBeNil)
			})

		})

	})

}