Esempio n. 1
0
// toEncoded returns a list of all of the serialized versions of these keys,
// plus a stringset of all the encoded root keys that `keys` represents.
func toEncoded(keys []*datastore.Key) (full []string, roots stringset.Set) {
	roots = stringset.New(len(keys))
	full = make([]string, len(keys))
	for i, k := range keys {
		roots.Add(string(serialize.ToBytes(k.Root())))
		full[i] = string(serialize.ToBytes(k))
	}
	return
}
Esempio n. 2
0
func TestDatastoreQueries(t *testing.T) {
	Convey("Datastore Query suport", t, func() {
		c := Use(context.Background())
		ds := dsS.Get(c)
		So(ds, ShouldNotBeNil)

		Convey("can create good queries", func() {
			q := ds.NewQuery("Foo").Filter("farnsworth >", 20).KeysOnly().Limit(10).Offset(39)

			// normally you can only get cursors from inside of the memory
			// implementation, so this construction is just for testing.
			start := queryCursor(bjoin(
				mkNum(2),
				serialize.ToBytes(dsS.IndexColumn{Property: "farnsworth"}),
				serialize.ToBytes(dsS.IndexColumn{Property: "__key__"}),
				serialize.ToBytes(prop(200)),
				serialize.ToBytes(prop(ds.NewKey("Foo", "id", 0, nil)))))

			So(start.String(), ShouldEqual,
				`gYAAZzFdTeeb3d9zOxsAAF-v221Xy32_AIGHyIgAAUc32-AFabMAAA==`)

			end := queryCursor(bjoin(
				mkNum(2),
				serialize.ToBytes(dsS.IndexColumn{Property: "farnsworth"}),
				serialize.ToBytes(dsS.IndexColumn{Property: "__key__"}),
				serialize.ToBytes(prop(3000)),
				serialize.ToBytes(prop(ds.NewKey("Foo", "zeta", 0, nil)))))

			q = q.Start(start).End(end)
			So(q, ShouldNotBeNil)
			So(q.(*queryImpl).err, ShouldBeNil)
			rq, err := q.(*queryImpl).reduce("", false)
			So(rq, ShouldNotBeNil)
			So(err, ShouldBeNil)
		})

		Convey("ensures orders make sense", func() {
			q := ds.NewQuery("Cool")
			q = q.Filter("cat =", 19).Filter("bob =", 10).Order("bob").Order("bob")

			Convey("removes dups and equality orders", func() {
				q = q.Order("wat")
				qi := q.(*queryImpl)
				So(qi.err, ShouldBeNil)
				rq, err := qi.reduce("", false)
				So(err, ShouldBeNil)
				So(rq.suffixFormat, ShouldResemble, []dsS.IndexColumn{
					{Property: "wat"}, {Property: "__key__"}})
			})

			Convey("if we equality-filter on __key__, that's just silly", func() {
				q = q.Order("wat").Filter("__key__ =", ds.NewKey("Foo", "wat", 0, nil))
				_, err := q.(*queryImpl).reduce("", false)
				So(err, ShouldErrLike, "query equality filter on __key__ is silly")
			})

		})

	})
}
Esempio n. 3
0
func addIndex(store *memStore, ns string, compIdx []*ds.IndexDefinition) {
	normalized := make([]*ds.IndexDefinition, len(compIdx))
	idxColl := store.SetCollection("idx", nil)
	for i, idx := range compIdx {
		normalized[i] = idx.Normalize()
		idxColl.Set(serialize.ToBytes(*normalized[i].PrepForIdxTable()), []byte{})
	}

	if allEnts := store.GetCollection("ents:" + ns); allEnts != nil {
		allEnts.VisitItemsAscend(nil, true, func(i *gkvlite.Item) bool {
			pm, err := rpmWoCtx(i.Val, ns)
			memoryCorruption(err)

			prop, err := serialize.ReadProperty(bytes.NewBuffer(i.Key), serialize.WithoutContext, globalAppID, ns)
			memoryCorruption(err)

			k := prop.Value().(ds.Key)

			sip := partiallySerialize(k, pm)

			mergeIndexes(ns, store,
				newMemStore(),
				sip.indexEntries(ns, normalized))
			return true
		})
	}
}
Esempio n. 4
0
// walkCompIdxs walks the table of compound indexes in the store. If `endsWith`
// is provided, this will only walk over compound indexes which match
// Kind, Ancestor, and whose SortBy has `endsWith.SortBy` as a suffix.
func walkCompIdxs(store *memStore, endsWith *ds.IndexDefinition, cb func(*ds.IndexDefinition) bool) {
	idxColl := store.GetCollection("idx")
	if idxColl == nil {
		return
	}
	itrDef := iterDefinition{c: idxColl}

	if endsWith != nil {
		full := serialize.ToBytes(*endsWith.Flip())
		// chop off the null terminating byte
		itrDef.prefix = full[:len(full)-1]
	}

	it := itrDef.mkIter()
	defer it.stop()
	for !it.stopped {
		it.next(nil, func(i *gkvlite.Item) {
			if i == nil {
				return
			}
			qi, err := serialize.ReadIndexDefinition(bytes.NewBuffer(i.Key))
			memoryCorruption(err)
			if !cb(qi.Flip()) {
				it.stop()
			}
		})
	}
}
Esempio n. 5
0
func (d *dataStoreData) putMulti(keys []ds.Key, vals []ds.PropertyMap, cb ds.PutMultiCB) {
	for i, k := range keys {
		pmap, _ := vals[i].Save(false)
		dataBytes := serialize.ToBytes(pmap)

		k, err := func() (ret ds.Key, err error) {
			d.rwlock.Lock()
			defer d.rwlock.Unlock()

			ents, ret := d.entsKeyLocked(k)
			incrementLocked(ents, groupMetaKey(ret))

			old := ents.Get(keyBytes(ret))
			oldPM := ds.PropertyMap(nil)
			if old != nil {
				if oldPM, err = rpmWoCtx(old, ret.Namespace()); err != nil {
					return
				}
			}
			updateIndexes(d.head, ret, oldPM, pmap)
			ents.Set(keyBytes(ret), dataBytes)
			return
		}()
		if cb != nil {
			cb(k, err)
		}
	}
}
Esempio n. 6
0
func incrementLocked(ents *memCollection, key []byte) int64 {
	ret := curVersion(ents, key) + 1
	ents.Set(key, serialize.ToBytes(ds.PropertyMap{
		"__version__": {ds.MkPropertyNI(ret)},
	}))
	return ret
}
Esempio n. 7
0
func HashKey(k datastore.Key) string {
	dgst := sha1.Sum(serialize.ToBytes(k))
	buf := bytes.Buffer{}
	enc := base64.NewEncoder(base64.StdEncoding, &buf)
	_, _ = enc.Write(dgst[:])
	enc.Close()
	return buf.String()[:buf.Len()-Sha1B64Padding]
}
Esempio n. 8
0
func (q *queryImpl) addEqFilt(prop string, p ds.Property) {
	binVal := string(serialize.ToBytes(p))
	if cur, ok := q.eqFilters[prop]; !ok {
		q.eqFilters[prop] = stringSet{binVal: {}}
	} else {
		cur.add(binVal)
	}
}
Esempio n. 9
0
// GetBinaryBounds gets the binary encoding of the upper and lower bounds of
// the inequality filter on fq, if any is defined. If a bound does not exist,
// it is nil.
//
// NOTE: if fq specifies a descending sort order for the inequality, the bounds
// will be inverted, incremented, and flipped.
func GetBinaryBounds(fq *ds.FinalizedQuery) (lower, upper []byte) {
	// Pick up the start/end range from the inequalities, if any.
	//
	// start and end in the reducedQuery are normalized so that `start >=
	// X < end`. Because of that, we need to tweak the inequality filters
	// contained in the query if they use the > or <= operators.
	if ineqProp := fq.IneqFilterProp(); ineqProp != "" {
		_, startOp, startV := fq.IneqFilterLow()
		if startOp != "" {
			lower = serialize.ToBytes(startV)
			if startOp == ">" {
				lower = increment(lower)
			}
		}

		_, endOp, endV := fq.IneqFilterHigh()
		if endOp != "" {
			upper = serialize.ToBytes(endV)
			if endOp == "<=" {
				upper = increment(upper)
			}
		}

		// The inequality is specified in natural (ascending) order in the query's
		// Filter syntax, but the order information may indicate to use a descending
		// index column for it. If that's the case, then we must invert, swap and
		// increment the inequality endpoints.
		//
		// Invert so that the desired numbers are represented correctly in the index.
		// Swap so that our iterators still go from >= start to < end.
		// Increment so that >= and < get correctly bounded (since the iterator is
		// still using natrual bytes ordering)
		if fq.Orders()[0].Descending {
			hi, lo := []byte(nil), []byte(nil)
			if len(lower) > 0 {
				lo = increment(serialize.Invert(lower))
			}
			if len(upper) > 0 {
				hi = increment(serialize.Invert(upper))
			}
			upper, lower = lo, hi
		}
	}
	return
}
Esempio n. 10
0
func partiallySerialize(k ds.Key, pm ds.PropertyMap) (ret serializedIndexablePmap) {
	ret = make(serializedIndexablePmap, len(pm)+2)
	if k == nil {
		impossible(fmt.Errorf("key to partiallySerialize is nil"))
	}
	ret["__key__"] = [][]byte{serialize.ToBytes(ds.MkProperty(k))}
	for k != nil {
		ret["__ancestor__"] = append(ret["__ancestor__"], serialize.ToBytes(ds.MkProperty(k)))
		k = k.Parent()
	}
	for k, vals := range pm {
		newVals := serializeRow(vals)
		if len(newVals) > 0 {
			ret[k] = newVals
		}
	}
	return
}
Esempio n. 11
0
func incrementLocked(ents *memCollection, key []byte, amt int) int64 {
	if amt <= 0 {
		panic(fmt.Errorf("incrementLocked called with bad `amt`: %d", amt))
	}
	ret := curVersion(ents, key) + 1
	ents.Set(key, serialize.ToBytes(ds.PropertyMap{
		"__version__": {ds.MkPropertyNI(ret + int64(amt-1))},
	}))
	return ret
}
Esempio n. 12
0
func TestCompoundIndexes(t *testing.T) {
	t.Parallel()

	idxKey := func(def dsS.IndexDefinition) string {
		So(def, ShouldNotBeNil)
		return "idx::" + string(serialize.ToBytes(*def.PrepForIdxTable()))
	}

	numItms := func(c *memCollection) uint64 {
		ret, _ := c.GetTotals()
		return ret
	}

	Convey("Test Compound indexes", t, func() {
		type Model struct {
			ID int64 `gae:"$id"`

			Field1 []string
			Field2 []int64
		}

		c := Use(context.Background())
		ds := dsS.Get(c)
		t := ds.Testable().(*dsImpl)
		head := t.data.head

		So(ds.Put(&Model{1, []string{"hello", "world"}, []int64{10, 11}}), ShouldBeNil)

		idx := dsS.IndexDefinition{
			Kind: "Model",
			SortBy: []dsS.IndexColumn{
				{Property: "Field2"},
			},
		}

		coll := head.GetCollection(idxKey(idx))
		So(coll, ShouldNotBeNil)
		So(numItms(coll), ShouldEqual, 2)

		idx.SortBy[0].Property = "Field1"
		coll = head.GetCollection(idxKey(idx))
		So(coll, ShouldNotBeNil)
		So(numItms(coll), ShouldEqual, 2)

		idx.SortBy = append(idx.SortBy, dsS.IndexColumn{Property: "Field1"})
		So(head.GetCollection(idxKey(idx)), ShouldBeNil)

		t.AddIndexes(&idx)
		coll = head.GetCollection(idxKey(idx))
		So(coll, ShouldNotBeNil)
		So(numItms(coll), ShouldEqual, 4)
	})
}
Esempio n. 13
0
func serializeRow(vals []ds.Property) serializedPvals {
	dups := map[string]struct{}{}
	ret := make(serializedPvals, 0, len(vals))
	for _, v := range vals {
		if v.IndexSetting() == ds.NoIndex {
			continue
		}
		data := serialize.ToBytes(v.ForIndex())
		dataS := string(data)
		if _, ok := dups[dataS]; ok {
			continue
		}
		dups[dataS] = struct{}{}
		ret = append(ret, data)
	}
	return ret
}
Esempio n. 14
0
func (sip serializedIndexablePmap) indexEntries(ns string, idxs []*ds.IndexDefinition) *memStore {
	ret := newMemStore()
	idxColl := ret.SetCollection("idx", nil)

	mtch := matcher{}
	for _, idx := range idxs {
		idx = idx.Normalize()
		if irg, ok := mtch.match(idx.GetFullSortOrder(), sip); ok {
			idxBin := serialize.ToBytes(*idx.PrepForIdxTable())
			idxColl.Set(idxBin, []byte{})
			coll := ret.SetCollection(fmt.Sprintf("idx:%s:%s", ns, idxBin), nil)
			irg.permute(coll.Set)
		}
	}

	return ret
}
Esempio n. 15
0
func (d *dataStoreData) putMulti(keys []*ds.Key, vals []ds.PropertyMap, cb ds.PutMultiCB) error {
	ns := keys[0].Namespace()

	for i, k := range keys {
		pmap, _ := vals[i].Save(false)
		dataBytes := serialize.ToBytes(pmap)

		k, err := func() (ret *ds.Key, err error) {
			d.Lock()
			defer d.Unlock()

			ents := d.mutableEntsLocked(ns)

			ret, err = d.fixKeyLocked(ents, k)
			if err != nil {
				return
			}
			if !d.disableSpecialEntities {
				incrementLocked(ents, groupMetaKey(ret), 1)
			}

			old := ents.Get(keyBytes(ret))
			oldPM := ds.PropertyMap(nil)
			if old != nil {
				if oldPM, err = rpm(old); err != nil {
					return
				}
			}
			ents.Set(keyBytes(ret), dataBytes)
			updateIndexes(d.head, ret, oldPM, pmap)
			return
		}()
		if cb != nil {
			if err := cb(k, err); err != nil {
				if err == ds.Stop {
					return nil
				}
				return err
			}
		}
	}
	return nil
}
Esempio n. 16
0
func (i *item) getEncKey() string {
	if i.encKey == "" {
		i.encKey = string(serialize.ToBytes(i.key))
	}
	return i.encKey
}
Esempio n. 17
0
// maybeAddDefinition possibly adds a new IndexDefinitionSortable to this slice.
// It's only added if it could be useful in servicing q, otherwise this function
// is a noop.
//
// This returns true iff the proposed index is OK and depletes missingTerms to
// empty.
//
// If the proposed index is PERFECT (e.g. contains enough columns to cover all
// equality filters, and also has the correct suffix), idxs will be replaced
// with JUST that index, and this will return true.
func (idxs *IndexDefinitionSortableSlice) maybeAddDefinition(q *reducedQuery, s *memStore, missingTerms stringSet, id *ds.IndexDefinition) bool {
	// Kindless queries are handled elsewhere.
	if id.Kind != q.kind {
		impossible(
			fmt.Errorf("maybeAddDefinition given index with wrong kind %q v %q", id.Kind, q.kind))
	}

	// If we're an ancestor query, and the index is compound, but doesn't include
	// an Ancestor field, it doesn't work. Builtin indexes can be used for
	// ancestor queries (and have !Ancestor), assuming that it's only equality
	// filters (plus inequality on __key__), or a single inequality.
	if q.eqFilters["__ancestor__"] != nil && !id.Ancestor && !id.Builtin() {
		impossible(
			fmt.Errorf("maybeAddDefinition given compound index with wrong ancestor info: %s %#v", id, q))
	}

	// add __ancestor__ if necessary
	sortBy := id.GetFullSortOrder()

	// If the index has fewer fields than we need for the suffix, it can't
	// possibly help.
	if len(sortBy) < len(q.suffixFormat) {
		return false
	}

	numEqFilts := len(sortBy) - len(q.suffixFormat)
	// make sure the orders are precisely the same
	for i, sb := range sortBy[numEqFilts:] {
		if q.suffixFormat[i] != sb {
			return false
		}
	}

	if id.Builtin() && numEqFilts == 0 {
		if len(q.eqFilters) > 1 || (len(q.eqFilters) == 1 && q.eqFilters["__ancestor__"] == nil) {
			return false
		}
	}

	// Make sure the equalities section doesn't contain any properties we don't
	// want in our query.
	//
	// numByProp && totalEqFilts will be used to see if this is a perfect match
	// later.
	numByProp := make(map[string]int, len(q.eqFilters))
	totalEqFilts := 0

	eqFilts := sortBy[:numEqFilts]
	for _, p := range eqFilts {
		if _, ok := q.eqFilters[p.Property]; !ok {
			return false
		}
		numByProp[p.Property]++
		totalEqFilts++
	}

	// ok, we can actually use this

	// Grab the collection for convenience later. We don't want to invalidate this
	// index's potential just because the collection doesn't exist. If it's
	// a builtin and it doesn't exist, it still needs to be one of the 'possible'
	// indexes... it just means that the user's query will end up with no results.
	coll := s.GetCollection(
		fmt.Sprintf("idx:%s:%s", q.ns, serialize.ToBytes(*id.PrepForIdxTable())))

	// First, see if it's a perfect match. If it is, then our search is over.
	//
	// A perfect match contains ALL the equality filter columns (or more, since
	// we can use residuals to fill in the extras).
	toAdd := IndexDefinitionSortable{coll: coll}
	toAdd.eqFilts = eqFilts
	for _, sb := range toAdd.eqFilts {
		missingTerms.rm(sb.Property)
	}

	perfect := false
	if len(sortBy) == q.numCols {
		perfect = true
		for k, num := range numByProp {
			if num < len(q.eqFilters[k]) {
				perfect = false
				break
			}
		}
	}
	if perfect {
		*idxs = IndexDefinitionSortableSlice{toAdd}
	} else {
		*idxs = append(*idxs, toAdd)
	}
	return len(missingTerms) == 0
}
Esempio n. 18
0
	{"silly inequality (=> v <=)",
		nq().Gte("bob", 10).Lte("bob", 10),
		nil, nil},

	{"cursors get smooshed into the inquality range",
		(nq().Gt("Foo", 3).Lt("Foo", 10).
			Start(curs("Foo", 2, "__key__", key("Something", 1))).
			End(curs("Foo", 20, "__key__", key("Something", 20)))),
		nil,
		&reducedQuery{
			"dev~app", "ns", "Foo", map[string]stringset.Set{}, []dstore.IndexColumn{
				{Property: "Foo"},
				{Property: "__key__"},
			},
			increment(serialize.ToBytes(dstore.MkProperty(3))),
			serialize.ToBytes(dstore.MkProperty(10)),
			2,
		}},

	{"cursors could cause the whole query to be useless",
		(nq().Gt("Foo", 3).Lt("Foo", 10).
			Start(curs("Foo", 200, "__key__", key("Something", 1))).
			End(curs("Foo", 1, "__key__", key("Something", 20)))),
		dstore.ErrNullQuery,
		nil},
}

func TestQueries(t *testing.T) {
	t.Parallel()
Esempio n. 19
0
func (q *queryImpl) Filter(fStr string, val interface{}) ds.Query {
	prop := ""
	op := qInvalid
	p := ds.Property{}
	return q.checkMutateClone(
		func() error {
			var err error
			prop, op, err = parseFilter(fStr)
			if err != nil {
				return err
			}

			if q.kind == "" && prop != "__key__" {
				// https://cloud.google.com/appengine/docs/go/datastore/queries#Go_Kindless_queries
				return fmt.Errorf(
					"kindless queries can only filter on __key__, got %q", fStr)
			}

			err = p.SetValue(val, ds.ShouldIndex)
			if err != nil {
				return err
			}

			if p.Type() == ds.PTKey {
				if !p.Value().(ds.Key).Valid(false, globalAppID, q.ns) {
					return ds.ErrInvalidKey
				}
			}

			if prop == "__key__" {
				if op == qEqual {
					return fmt.Errorf(
						"query equality filter on __key__ is silly: %q", fStr)
				}
				if p.Type() != ds.PTKey {
					return fmt.Errorf("__key__ filter value is not a key: %T", val)
				}
			} else if strings.HasPrefix(prop, "__") && strings.HasSuffix(prop, "__") {
				return fmt.Errorf("filter on reserved property: %q", prop)
			}

			if op != qEqual {
				if q.ineqFilter.prop != "" && q.ineqFilter.prop != prop {
					return fmt.Errorf(
						"inequality filters on multiple properties: %q and %q",
						q.ineqFilter.prop, prop)
				}
				if len(q.order) > 0 && q.order[0].Property != prop {
					return fmt.Errorf(
						"first sort order must match inequality filter: %q v %q",
						q.order[0].Property, prop)
				}
			} else {
				for _, p := range q.project {
					if p == prop {
						return fmt.Errorf(
							"cannot project on field which is used in an equality filter: %q",
							prop)
					}
				}
			}
			return err
		},
		func(q *queryImpl) {
			if op == qEqual {
				// add it to eq filters
				q.addEqFilt(prop, p)

				// remove it from sort orders.
				// https://cloud.google.com/appengine/docs/go/datastore/queries#sort_orders_are_ignored_on_properties_with_equality_filters
				toRm := -1
				for i, o := range q.order {
					if o.Property == prop {
						toRm = i
						break
					}
				}
				if toRm >= 0 {
					q.order = append(q.order[:toRm], q.order[toRm+1:]...)
				}
			} else {
				q.ineqFilter.prop = prop
				if q.ineqFilter.constrain(op, serialize.ToBytes(p)) {
					q.err = errQueryDone
				}
			}
		})
}
Esempio n. 20
0
func keyBytes(key *ds.Key) []byte {
	return serialize.ToBytes(ds.MkProperty(key))
}
Esempio n. 21
0
func TestDSCache(t *testing.T) {
	t.Parallel()

	zeroTime, err := time.Parse("2006-01-02T15:04:05.999999999Z", "2006-01-02T15:04:05.999999999Z")
	if err != nil {
		panic(err)
	}

	Convey("Test dscache", t, func() {
		c := mathrand.Set(context.Background(), rand.New(rand.NewSource(1)))
		clk := testclock.New(zeroTime)
		c = clock.Set(c, clk)
		c = memory.Use(c)

		dsUnder := datastore.Get(c)
		mc := memcache.Get(c)

		shardsForKey := func(k *datastore.Key) int {
			last := k.LastTok()
			if last.Kind == "shardObj" {
				return int(last.IntID)
			}
			if last.Kind == "noCacheObj" {
				return 0
			}
			return DefaultShards
		}

		numMemcacheItems := func() uint64 {
			stats, err := mc.Stats()
			So(err, ShouldBeNil)
			return stats.Items
		}

		Convey("enabled cases", func() {
			c = FilterRDS(c, shardsForKey)
			ds := datastore.Get(c)
			So(dsUnder, ShouldNotBeNil)
			So(ds, ShouldNotBeNil)
			So(mc, ShouldNotBeNil)

			Convey("basically works", func() {
				pm := datastore.PropertyMap{
					"BigData": {datastore.MkProperty([]byte(""))},
					"Value":   {datastore.MkProperty("hi")},
				}
				encoded := append([]byte{0}, serialize.ToBytes(pm)...)

				o := object{ID: 1, Value: "hi"}
				So(ds.Put(&o), ShouldBeNil)

				o = object{ID: 1}
				So(dsUnder.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, "hi")

				itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
				So(err, ShouldEqual, memcache.ErrCacheMiss)

				o = object{ID: 1}
				So(ds.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, "hi")

				itm, err = mc.Get(itm.Key())
				So(err, ShouldBeNil)
				So(itm.Value(), ShouldResemble, encoded)

				Convey("now we don't need the datastore!", func() {
					o := object{ID: 1}

					// delete it, bypassing the cache filter. Don't do this in production
					// unless you want a crappy cache.
					So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
					So(err, ShouldBeNil)
					So(itm.Value(), ShouldResemble, encoded)

					So(ds.Get(&o), ShouldBeNil)
					So(o.Value, ShouldEqual, "hi")
				})

				Convey("deleting it properly records that fact, however", func() {
					o := object{ID: 1}
					So(ds.Delete(ds.KeyForObj(&o)), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)

					itm, err = mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Value(), ShouldResemble, []byte{})

					// this one hits memcache
					So(ds.Get(&o), ShouldEqual, datastore.ErrNoSuchEntity)
				})
			})

			Convey("compression works", func() {
				o := object{ID: 2, Value: `¯\_(ツ)_/¯`}
				data := make([]byte, 4000)
				for i := range data {
					const alpha = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*()"
					data[i] = alpha[i%len(alpha)]
				}
				o.BigData = data

				So(ds.Put(&o), ShouldBeNil)
				So(ds.Get(&o), ShouldBeNil)

				itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&o)))
				So(err, ShouldBeNil)

				So(itm.Value()[0], ShouldEqual, ZlibCompression)
				So(len(itm.Value()), ShouldEqual, 653) // a bit smaller than 4k

				// ensure the next Get comes from the cache
				So(dsUnder.Delete(ds.KeyForObj(&o)), ShouldBeNil)

				o = object{ID: 2}
				So(ds.Get(&o), ShouldBeNil)
				So(o.Value, ShouldEqual, `¯\_(ツ)_/¯`)
				So(o.BigData, ShouldResemble, data)
			})

			Convey("transactions", func() {
				Convey("work", func() {
					// populate an object @ ID1
					So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
					So(ds.Get(&object{ID: 1}), ShouldBeNil)

					So(ds.Put(&object{ID: 2, Value: "nurbs"}), ShouldBeNil)
					So(ds.Get(&object{ID: 2}), ShouldBeNil)

					// memcache now has the wrong value (simulated race)
					So(dsUnder.Put(&object{ID: 1, Value: "else"}), ShouldBeNil)
					So(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						o := &object{ID: 1}
						So(ds.Get(o), ShouldBeNil)
						So(o.Value, ShouldEqual, "else")
						o.Value = "txn"
						So(ds.Put(o), ShouldBeNil)

						So(ds.Delete(ds.KeyForObj(&object{ID: 2})), ShouldBeNil)
						return nil
					}, &datastore.TransactionOptions{XG: true}), ShouldBeNil)

					_, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 1})))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					_, err = mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&object{ID: 2})))
					So(err, ShouldEqual, memcache.ErrCacheMiss)
					o := &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "txn")
				})

				Convey("errors don't invalidate", func() {
					// populate an object @ ID1
					So(ds.Put(&object{ID: 1, Value: "something"}), ShouldBeNil)
					So(ds.Get(&object{ID: 1}), ShouldBeNil)
					So(numMemcacheItems(), ShouldEqual, 1)

					So(ds.RunInTransaction(func(c context.Context) error {
						ds := datastore.Get(c)
						o := &object{ID: 1}
						So(ds.Get(o), ShouldBeNil)
						So(o.Value, ShouldEqual, "something")
						o.Value = "txn"
						So(ds.Put(o), ShouldBeNil)
						return errors.New("OH NOES")
					}, nil).Error(), ShouldContainSubstring, "OH NOES")

					// memcache still has the original
					So(numMemcacheItems(), ShouldEqual, 1)
					So(dsUnder.Delete(ds.KeyForObj(&object{ID: 1})), ShouldBeNil)
					o := &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "something")
				})
			})

			Convey("control", func() {
				Convey("per-model bypass", func() {
					type model struct {
						ID         string           `gae:"$id"`
						UseDSCache datastore.Toggle `gae:"$dscache.enable,false"`

						Value string
					}

					itms := []model{
						{ID: "hi", Value: "something"},
						{ID: "there", Value: "else", UseDSCache: datastore.On},
					}

					So(ds.PutMulti(itms), ShouldBeNil)
					So(ds.GetMulti(itms), ShouldBeNil)

					So(numMemcacheItems(), ShouldEqual, 1)
				})

				Convey("per-key shard count", func() {
					s := &shardObj{ID: 4, Value: "hi"}
					So(ds.Put(s), ShouldBeNil)
					So(ds.Get(s), ShouldBeNil)

					So(numMemcacheItems(), ShouldEqual, 1)
					for i := 0; i < 20; i++ {
						So(ds.Get(s), ShouldBeNil)
					}
					So(numMemcacheItems(), ShouldEqual, 4)
				})

				Convey("per-key cache disablement", func() {
					n := &noCacheObj{ID: "nurbs", Value: true}
					So(ds.Put(n), ShouldBeNil)
					So(ds.Get(n), ShouldBeNil)
					So(numMemcacheItems(), ShouldEqual, 0)
				})

				Convey("per-model expiration", func() {
					type model struct {
						ID         int64 `gae:"$id"`
						DSCacheExp int64 `gae:"$dscache.expiration,7"`

						Value string
					}

					So(ds.Put(&model{ID: 1, Value: "mooo"}), ShouldBeNil)
					So(ds.Get(&model{ID: 1}), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(&model{ID: 1})))
					So(err, ShouldBeNil)

					clk.Add(10 * time.Second)
					_, err = mc.Get(itm.Key())
					So(err, ShouldEqual, memcache.ErrCacheMiss)
				})
			})

			Convey("screw cases", func() {
				Convey("memcache contains bogus value (simulated failed AddMulti)", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("I am a banana")
					itm := mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).SetValue(sekret)
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemUKNONWN)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("memcache contains bogus value (corrupt entry)", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("I am a banana")
					itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
						SetValue(sekret).
						SetFlags(uint32(ItemHasData)))
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasData)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("other entity has the lock", func() {
					o := &object{ID: 1, Value: "spleen"}
					So(ds.Put(o), ShouldBeNil)

					sekret := []byte("r@vmarod!#)%9T")
					itm := (mc.NewItem(MakeMemcacheKey(0, ds.KeyForObj(o))).
						SetValue(sekret).
						SetFlags(uint32(ItemHasLock)))
					So(mc.Set(itm), ShouldBeNil)

					o = &object{ID: 1}
					So(ds.Get(o), ShouldBeNil)
					So(o.Value, ShouldEqual, "spleen")

					itm, err := mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasLock)
					So(itm.Value(), ShouldResemble, sekret)
				})

				Convey("massive entities can't be cached", func() {
					o := &object{ID: 1, Value: "spleen"}
					mr := mathrand.Get(c)
					numRounds := (internalValueSizeLimit / 8) * 2
					buf := bytes.Buffer{}
					for i := 0; i < numRounds; i++ {
						So(binary.Write(&buf, binary.LittleEndian, mr.Int63()), ShouldBeNil)
					}
					o.BigData = buf.Bytes()
					So(ds.Put(o), ShouldBeNil)

					o.BigData = nil
					So(ds.Get(o), ShouldBeNil)

					itm, err := mc.Get(MakeMemcacheKey(0, ds.KeyForObj(o)))
					So(err, ShouldBeNil)

					// Is locked until the next put, forcing all access to the datastore.
					So(itm.Value(), ShouldResemble, []byte{})
					So(itm.Flags(), ShouldEqual, ItemHasLock)

					o.BigData = []byte("hi :)")
					So(ds.Put(o), ShouldBeNil)
					So(ds.Get(o), ShouldBeNil)

					itm, err = mc.Get(itm.Key())
					So(err, ShouldBeNil)
					So(itm.Flags(), ShouldEqual, ItemHasData)
				})

				Convey("failure on Setting memcache locks is a hard stop", func() {
					c, fb := featureBreaker.FilterMC(c, nil)
					fb.BreakFeatures(nil, "SetMulti")
					ds := datastore.Get(c)
					So(ds.Put(&object{ID: 1}).Error(), ShouldContainSubstring, "SetMulti")
				})

				Convey("failure on Setting memcache locks in a transaction is a hard stop", func() {
					c, fb := featureBreaker.FilterMC(c, nil)
					fb.BreakFeatures(nil, "SetMulti")
					ds := datastore.Get(c)
					So(ds.RunInTransaction(func(c context.Context) error {
						So(datastore.Get(c).Put(&object{ID: 1}), ShouldBeNil)
						// no problems here... memcache operations happen after the function
						// body quits.
						return nil
					}, nil).Error(), ShouldContainSubstring, "SetMulti")
				})

			})

			Convey("misc", func() {
				Convey("verify numShards caps at MaxShards", func() {
					sc := supportContext{shardsForKey: shardsForKey}
					So(sc.numShards(ds.KeyForObj(&shardObj{ID: 9001})), ShouldEqual, MaxShards)
				})

				Convey("CompressionType.String", func() {
					So(NoCompression.String(), ShouldEqual, "NoCompression")
					So(ZlibCompression.String(), ShouldEqual, "ZlibCompression")
					So(CompressionType(100).String(), ShouldEqual, "UNKNOWN_CompressionType(100)")
				})
			})
		})

		Convey("disabled cases", func() {
			defer func() {
				globalEnabled = true
			}()

			So(IsGloballyEnabled(c), ShouldBeTrue)

			So(SetGlobalEnable(c, false), ShouldBeNil)
			// twice is a nop
			So(SetGlobalEnable(c, false), ShouldBeNil)

			// but it takes 5 minutes to kick in
			So(IsGloballyEnabled(c), ShouldBeTrue)
			clk.Add(time.Minute*5 + time.Second)
			So(IsGloballyEnabled(c), ShouldBeFalse)

			So(mc.Set(mc.NewItem("test").SetValue([]byte("hi"))), ShouldBeNil)
			So(numMemcacheItems(), ShouldEqual, 1)
			So(SetGlobalEnable(c, true), ShouldBeNil)
			// memcache gets flushed as a side effect
			So(numMemcacheItems(), ShouldEqual, 0)

			// Still takes 5 minutes to kick in
			So(IsGloballyEnabled(c), ShouldBeFalse)
			clk.Add(time.Minute*5 + time.Second)
			So(IsGloballyEnabled(c), ShouldBeTrue)
		})
	})
}
Esempio n. 22
0
func reduce(fq *ds.FinalizedQuery, aid, ns string, isTxn bool) (*reducedQuery, error) {
	if err := fq.Valid(aid, ns); err != nil {
		return nil, err
	}
	if isTxn && fq.Ancestor() == nil {
		return nil, fmt.Errorf("queries within a transaction must include an Ancestor filter")
	}
	if num := numComponents(fq); num > MaxQueryComponents {
		return nil, fmt.Errorf(
			"gae/memory: query is too large. may not have more than "+
				"%d filters + sort orders + ancestor total: had %d",
			MaxQueryComponents, num)
	}

	ret := &reducedQuery{
		aid:          aid,
		ns:           ns,
		kind:         fq.Kind(),
		suffixFormat: fq.Orders(),
	}

	eqFilts := fq.EqFilters()
	ret.eqFilters = make(map[string]stringset.Set, len(eqFilts))
	for prop, vals := range eqFilts {
		sVals := stringset.New(len(vals))
		for _, v := range vals {
			sVals.Add(string(serialize.ToBytes(v)))
		}
		ret.eqFilters[prop] = sVals
	}

	startD, endD := GetBinaryBounds(fq)

	// Now we check the start and end cursors.
	//
	// Cursors are composed of a list of IndexColumns at the beginning, followed
	// by the raw bytes to use for the suffix. The cursor is only valid if all of
	// its IndexColumns match our proposed suffixFormat, as calculated above.
	//
	// Cursors are mutually exclusive with the start/end we picked up from the
	// inequality. In a well formed query, they indicate a subset of results
	// bounded by the inequality. Technically if the start cursor is not >= the
	// low bound, or the end cursor is < the high bound, it's an error, but for
	// simplicity we just cap to the narrowest intersection of the inequality and
	// cursors.
	ret.start = startD
	ret.end = endD
	if start, end := fq.Bounds(); start != nil || end != nil {
		if start != nil {
			if c, ok := start.(queryCursor); ok {
				startCols, startD, err := c.decode()
				if err != nil {
					return nil, err
				}

				if !sortOrdersEqual(startCols, ret.suffixFormat) {
					return nil, errors.New("gae/memory: start cursor is invalid for this query")
				}
				if ret.start == nil || bytes.Compare(ret.start, startD) < 0 {
					ret.start = startD
				}
			} else {
				return nil, errors.New("gae/memory: bad cursor type")
			}
		}

		if end != nil {
			if c, ok := end.(queryCursor); ok {
				endCols, endD, err := c.decode()
				if err != nil {
					return nil, err
				}

				if !sortOrdersEqual(endCols, ret.suffixFormat) {
					return nil, errors.New("gae/memory: end cursor is invalid for this query")
				}
				if ret.end == nil || bytes.Compare(endD, ret.end) < 0 {
					ret.end = endD
				}
			} else {
				return nil, errors.New("gae/memory: bad cursor type")
			}
		}
	}

	// Finally, verify that we could even /potentially/ do work. If we have
	// overlapping range ends, then we don't have anything to do.
	if ret.end != nil && bytes.Compare(ret.start, ret.end) >= 0 {
		return nil, ds.ErrNullQuery
	}

	ret.numCols = len(ret.suffixFormat)
	for prop, vals := range ret.eqFilters {
		if len(ret.suffixFormat) == 1 && prop == "__ancestor__" {
			continue
		}
		ret.numCols += vals.Len()
	}

	return ret, nil
}