func newNormalStrategy(aid, ns string, cb ds.RawRunCB, head *memStore) queryStrategy {
	coll := head.GetCollection("ents:" + ns)
	if coll == nil {
		return nil
	}
	return &normalStrategy{cb, aid, ns, coll, stringset.New(0)}
}
Example #2
0
func withTxnBuf(ctx context.Context, cb func(context.Context) error, opts *datastore.TransactionOptions) error {
	inf := info.Get(ctx)
	ns := inf.GetNamespace()

	parentState, _ := ctx.Value(dsTxnBufParent).(*txnBufState)
	roots := stringset.New(0)
	rootLimit := 1
	if opts != nil && opts.XG {
		rootLimit = XGTransactionGroupLimit
	}
	sizeBudget := DefaultSizeBudget
	if parentState != nil {
		// TODO(riannucci): this is a bit wonky since it means that a child
		// transaction declaring XG=true will only get to modify 25 groups IF
		// they're same groups affected by the parent transactions. So instead of
		// respecting opts.XG for inner transactions, we just dup everything from
		// the parent transaction.
		roots = parentState.roots.Dup()
		rootLimit = parentState.rootLimit

		sizeBudget = parentState.sizeBudget - parentState.entState.total
		if sizeBudget < DefaultSizeThreshold {
			return ErrTransactionTooLarge
		}
	}

	bufDS, err := memory.NewDatastore(inf.FullyQualifiedAppID(), ns)
	if err != nil {
		return err
	}

	state := &txnBufState{
		entState:   &sizeTracker{},
		bufDS:      bufDS.Raw(),
		roots:      roots,
		rootLimit:  rootLimit,
		ns:         ns,
		aid:        inf.AppID(),
		parentDS:   datastore.Get(context.WithValue(ctx, dsTxnBufHaveLock, true)).Raw(),
		sizeBudget: sizeBudget,
	}
	if err = cb(context.WithValue(ctx, dsTxnBufParent, state)); err != nil {
		return err
	}

	// no reason to unlock this ever. At this point it's toast.
	state.Lock()

	if parentState == nil {
		return commitToReal(state)
	}

	if err = parentState.canApplyLocked(state); err != nil {
		return err
	}

	parentState.commitLocked(state)
	return nil
}
Example #3
0
// toEncoded returns a list of all of the serialized versions of these keys,
// plus a stringset of all the encoded root keys that `keys` represents.
func toEncoded(keys []*datastore.Key) (full []string, roots stringset.Set) {
	roots = stringset.New(len(keys))
	full = make([]string, len(keys))
	for i, k := range keys {
		roots.Add(string(serialize.ToBytes(k.Root())))
		full[i] = string(serialize.ToBytes(k))
	}
	return
}
func pickQueryStrategy(fq *ds.FinalizedQuery, rq *reducedQuery, cb ds.RawRunCB, head *memStore) queryStrategy {
	if fq.KeysOnly() {
		return &keysOnlyStrategy{cb, stringset.New(0)}
	}
	if len(fq.Project()) > 0 {
		return newProjectionStrategy(fq, rq, cb)
	}
	return newNormalStrategy(rq.aid, rq.ns, cb, head)
}
Example #5
0
// PropertySlice serializes a single row of a DSProperty map.
func PropertySlice(vals ds.PropertySlice) SerializedPslice {
	dups := stringset.New(0)
	ret := make(SerializedPslice, 0, len(vals))
	for _, v := range vals {
		if v.IndexSetting() == ds.NoIndex {
			continue
		}

		data := ToBytes(v)
		dataS := string(data)
		if !dups.Add(dataS) {
			continue
		}
		ret = append(ret, data)
	}
	return ret
}
Example #6
0
// Project lists one or more field names to project.
func (q *Query) Project(fieldNames ...string) *Query {
	if len(fieldNames) == 0 {
		return q
	}
	return q.mod(func(q *Query) {
		for _, f := range fieldNames {
			if q.reserved(f) {
				return
			}
			if f == "__key__" {
				q.err = fmt.Errorf("cannot project on %q", f)
				return
			}
			if q.project == nil {
				q.project = stringset.New(1)
			}
			q.project.Add(f)
		}
	})
}
Example #7
0
func (t *txnBufState) updateRootsLocked(roots stringset.Set) error {
	curRootLen := t.roots.Len()
	proposedRoots := stringset.New(1)
	roots.Iter(func(root string) bool {
		if !t.roots.Has(root) {
			proposedRoots.Add(root)
		}
		return proposedRoots.Len()+curRootLen <= t.rootLimit
	})
	if proposedRoots.Len()+curRootLen > t.rootLimit {
		return ErrTooManyRoots
	}
	// only need to update the roots if they did something that required updating
	if proposedRoots.Len() > 0 {
		proposedRoots.Iter(func(root string) bool {
			t.roots.Add(root)
			return true
		})
	}
	return nil
}
func newProjectionStrategy(fq *ds.FinalizedQuery, rq *reducedQuery, cb ds.RawRunCB) queryStrategy {
	proj := fq.Project()

	projectionLookups := make([]projectionLookup, len(proj))
	for i, prop := range proj {
		projectionLookups[i].propertyName = prop
		lookupErr := fmt.Errorf("planning a strategy for an unfulfillable query?")
		for j, col := range rq.suffixFormat {
			if col.Property == prop {
				projectionLookups[i].suffixIndex = j
				lookupErr = nil
				break
			}
		}
		impossible(lookupErr)
	}
	ret := &projectionStrategy{cb: cb, project: projectionLookups}
	if fq.Distinct() {
		ret.distinct = stringset.New(0)
	}
	return ret
}
Example #9
0
func reduce(fq *ds.FinalizedQuery, aid, ns string, isTxn bool) (*reducedQuery, error) {
	if err := fq.Valid(aid, ns); err != nil {
		return nil, err
	}
	if isTxn && fq.Ancestor() == nil {
		return nil, fmt.Errorf("queries within a transaction must include an Ancestor filter")
	}
	if num := numComponents(fq); num > MaxQueryComponents {
		return nil, fmt.Errorf(
			"gae/memory: query is too large. may not have more than "+
				"%d filters + sort orders + ancestor total: had %d",
			MaxQueryComponents, num)
	}

	ret := &reducedQuery{
		aid:          aid,
		ns:           ns,
		kind:         fq.Kind(),
		suffixFormat: fq.Orders(),
	}

	eqFilts := fq.EqFilters()
	ret.eqFilters = make(map[string]stringset.Set, len(eqFilts))
	for prop, vals := range eqFilts {
		sVals := stringset.New(len(vals))
		for _, v := range vals {
			sVals.Add(string(serialize.ToBytes(v)))
		}
		ret.eqFilters[prop] = sVals
	}

	startD, endD := GetBinaryBounds(fq)

	// Now we check the start and end cursors.
	//
	// Cursors are composed of a list of IndexColumns at the beginning, followed
	// by the raw bytes to use for the suffix. The cursor is only valid if all of
	// its IndexColumns match our proposed suffixFormat, as calculated above.
	//
	// Cursors are mutually exclusive with the start/end we picked up from the
	// inequality. In a well formed query, they indicate a subset of results
	// bounded by the inequality. Technically if the start cursor is not >= the
	// low bound, or the end cursor is < the high bound, it's an error, but for
	// simplicity we just cap to the narrowest intersection of the inequality and
	// cursors.
	ret.start = startD
	ret.end = endD
	if start, end := fq.Bounds(); start != nil || end != nil {
		if start != nil {
			if c, ok := start.(queryCursor); ok {
				startCols, startD, err := c.decode()
				if err != nil {
					return nil, err
				}

				if !sortOrdersEqual(startCols, ret.suffixFormat) {
					return nil, errors.New("gae/memory: start cursor is invalid for this query")
				}
				if ret.start == nil || bytes.Compare(ret.start, startD) < 0 {
					ret.start = startD
				}
			} else {
				return nil, errors.New("gae/memory: bad cursor type")
			}
		}

		if end != nil {
			if c, ok := end.(queryCursor); ok {
				endCols, endD, err := c.decode()
				if err != nil {
					return nil, err
				}

				if !sortOrdersEqual(endCols, ret.suffixFormat) {
					return nil, errors.New("gae/memory: end cursor is invalid for this query")
				}
				if ret.end == nil || bytes.Compare(endD, ret.end) < 0 {
					ret.end = endD
				}
			} else {
				return nil, errors.New("gae/memory: bad cursor type")
			}
		}
	}

	// Finally, verify that we could even /potentially/ do work. If we have
	// overlapping range ends, then we don't have anything to do.
	if ret.end != nil && bytes.Compare(ret.start, ret.end) >= 0 {
		return nil, ds.ErrNullQuery
	}

	ret.numCols = len(ret.suffixFormat)
	for prop, vals := range ret.eqFilters {
		if len(ret.suffixFormat) == 1 && prop == "__ancestor__" {
			continue
		}
		ret.numCols += vals.Len()
	}

	return ret, nil
}
Example #10
0
// Finalize converts this Query to a FinalizedQuery. If the Query has any
// inconsistencies or violates any of the query rules, that will be returned
// here.
func (q *Query) Finalize() (*FinalizedQuery, error) {
	if q.err != nil || q.finalized != nil {
		return q.finalized, q.err
	}

	ancestor := (*Key)(nil)
	if slice, ok := q.eqFilts["__ancestor__"]; ok {
		ancestor = slice[0].Value().(*Key)
	}

	err := func() error {

		if q.kind == "" { // kindless query checks
			if q.ineqFiltProp != "" && q.ineqFiltProp != "__key__" {
				return fmt.Errorf(
					"kindless queries can only filter on __key__, got %q", q.ineqFiltProp)
			}
			allowedEqs := 0
			if ancestor != nil {
				allowedEqs = 1
			}
			if len(q.eqFilts) > allowedEqs {
				return fmt.Errorf("kindless queries may not have any equality filters")
			}
			for _, o := range q.order {
				if o.Property != "__key__" || o.Descending {
					return fmt.Errorf("invalid order for kindless query: %#v", o)
				}
			}
		}

		if q.keysOnly && q.project != nil && q.project.Len() > 0 {
			return errors.New("cannot project a keysOnly query")
		}

		if q.ineqFiltProp != "" {
			if len(q.order) > 0 && q.order[0].Property != q.ineqFiltProp {
				return fmt.Errorf(
					"first sort order must match inequality filter: %q v %q",
					q.order[0].Property, q.ineqFiltProp)
			}
			if q.ineqFiltLowSet && q.ineqFiltHighSet {
				if q.ineqFiltHigh.Less(&q.ineqFiltLow) ||
					(q.ineqFiltHigh.Equal(&q.ineqFiltLow) &&
						(!q.ineqFiltLowIncl || !q.ineqFiltHighIncl)) {
					return ErrNullQuery
				}
			}
			if q.ineqFiltProp == "__key__" {
				if q.ineqFiltLowSet {
					if ancestor != nil && !q.ineqFiltLow.Value().(*Key).HasAncestor(ancestor) {
						return fmt.Errorf(
							"inequality filters on __key__ must be descendants of the __ancestor__")
					}
				}
				if q.ineqFiltHighSet {
					if ancestor != nil && !q.ineqFiltHigh.Value().(*Key).HasAncestor(ancestor) {
						return fmt.Errorf(
							"inequality filters on __key__ must be descendants of the __ancestor__")
					}
				}
			}
		}

		err := error(nil)
		if q.project != nil {
			q.project.Iter(func(p string) bool {
				if _, iseq := q.eqFilts[p]; iseq {
					err = fmt.Errorf("cannot project on equality filter field: %s", p)
					return false
				}
				return true
			})
		}
		return err
	}()
	if err != nil {
		q.err = err
		return nil, err
	}

	ret := &FinalizedQuery{
		original: q,
		kind:     q.kind,

		keysOnly:             q.keysOnly,
		eventuallyConsistent: q.eventualConsistency || ancestor == nil,
		limit:                q.limit,
		offset:               q.offset,
		start:                q.start,
		end:                  q.end,

		eqFilts: q.eqFilts,

		ineqFiltProp:     q.ineqFiltProp,
		ineqFiltLow:      q.ineqFiltLow,
		ineqFiltLowIncl:  q.ineqFiltLowIncl,
		ineqFiltLowSet:   q.ineqFiltLowSet,
		ineqFiltHigh:     q.ineqFiltHigh,
		ineqFiltHighIncl: q.ineqFiltHighIncl,
		ineqFiltHighSet:  q.ineqFiltHighSet,
	}

	if q.project != nil {
		ret.project = q.project.ToSlice()
		ret.distinct = q.distinct && q.project.Len() > 0

		// If we're DISTINCT && have an inequality filter, we must project that
		// inequality property as well.
		if ret.distinct && ret.ineqFiltProp != "" && !q.project.Has(ret.ineqFiltProp) {
			ret.project = append([]string{ret.ineqFiltProp}, ret.project...)
		}
	}

	seenOrders := stringset.New(len(q.order))

	// if len(q.order) > 0, we already enforce that the first order
	// is the same as the inequality above. Otherwise we need to add it.
	if len(q.order) == 0 && q.ineqFiltProp != "" {
		ret.orders = []IndexColumn{{Property: q.ineqFiltProp}}
		seenOrders.Add(q.ineqFiltProp)
	}

	// drop orders where there's an equality filter
	//   https://cloud.google.com/appengine/docs/go/datastore/queries#sort_orders_are_ignored_on_properties_with_equality_filters
	// Deduplicate orders
	for _, o := range q.order {
		if _, iseq := q.eqFilts[o.Property]; !iseq {
			if seenOrders.Add(o.Property) {
				ret.orders = append(ret.orders, o)
			}
		}
	}

	// Add any projection columns not mentioned in the user-defined order as
	// ASCENDING orders. Technically we could be smart and automatically use
	// a DESCENDING ordered index, if it fit, but the logic gets insane, since all
	// suffixes of all used indexes need to be PRECISELY equal (and so you'd have
	// to hunt/invalidate/something to find the combination of indexes that are
	// compatible with each other as well as the query). If you want to use
	// a DESCENDING column, just add it to the user sort order, and this loop will
	// not synthesize a new suffix entry for it.
	//
	// NOTE: if you want to use an index that sorts by -__key__, you MUST
	// include all of the projected fields for that index in the order explicitly.
	// Otherwise the generated orders will be wacky. So:
	//   Query("Foo").Project("A", "B").Order("A").Order("-__key__")
	//
	// will turn into a orders of:
	//   A, ASCENDING
	//   __key__, DESCENDING
	//   B, ASCENDING
	//   __key__, ASCENDING
	//
	// To prevent this, your query should have another Order("B") clause before
	// the -__key__ clause.
	if len(ret.project) > 0 {
		sort.Strings(ret.project)
		for _, p := range ret.project {
			if !seenOrders.Has(p) {
				ret.orders = append(ret.orders, IndexColumn{Property: p})
			}
		}
	}

	// If the suffix format ends with __key__ already (e.g. .Order("__key__")),
	// then we're good to go. Otherwise we need to add it as the last bit of the
	// suffix, since all indexes implicitly have it as the last column.
	if len(ret.orders) == 0 || ret.orders[len(ret.orders)-1].Property != "__key__" {
		ret.orders = append(ret.orders, IndexColumn{Property: "__key__"})
	}

	q.finalized = ret
	return ret, nil
}
// getRelevantIndexes retrieves the relevant indexes which could be used to
// service q. It returns nil if it's not possible to service q with the current
// indexes.
func getRelevantIndexes(q *reducedQuery, s *memStore) (indexDefinitionSortableSlice, error) {
	missingTerms := stringset.New(len(q.eqFilters))
	for k := range q.eqFilters {
		if k == "__ancestor__" {
			// ancestor is not a prefix which can be satisfied by a single index. It
			// must be satisfied by ALL indexes (and has special logic for this in
			// the addDefinition logic)
			continue
		}
		missingTerms.Add(k)
	}
	idxs := indexDefinitionSortableSlice{}

	// First we add builtins
	// add
	//   idx:KIND
	if idxs.maybeAddDefinition(q, s, missingTerms, &ds.IndexDefinition{
		Kind: q.kind,
	}) {
		return idxs, nil
	}

	// add
	//   idx:KIND:prop
	//   idx:KIND:-prop
	props := stringset.New(len(q.eqFilters) + len(q.suffixFormat))
	for prop := range q.eqFilters {
		props.Add(prop)
	}
	for _, col := range q.suffixFormat[:len(q.suffixFormat)-1] {
		props.Add(col.Property)
	}
	for _, prop := range props.ToSlice() {
		if strings.HasPrefix(prop, "__") && strings.HasSuffix(prop, "__") {
			continue
		}
		if idxs.maybeAddDefinition(q, s, missingTerms, &ds.IndexDefinition{
			Kind: q.kind,
			SortBy: []ds.IndexColumn{
				{Property: prop},
			},
		}) {
			return idxs, nil
		}
		if idxs.maybeAddDefinition(q, s, missingTerms, &ds.IndexDefinition{
			Kind: q.kind,
			SortBy: []ds.IndexColumn{
				{Property: prop, Descending: true},
			},
		}) {
			return idxs, nil
		}
	}

	// Try adding all compound indexes whose suffix matches.
	suffix := &ds.IndexDefinition{
		Kind:     q.kind,
		Ancestor: q.eqFilters["__ancestor__"] != nil,
		SortBy:   q.suffixFormat,
	}
	walkCompIdxs(s, suffix, func(def *ds.IndexDefinition) bool {
		// keep walking until we find a perfect index.
		return !idxs.maybeAddDefinition(q, s, missingTerms, def)
	})

	// this query is impossible to fulfil with the current indexes. Not all the
	// terms (equality + projection) are satisfied.
	if missingTerms.Len() < 0 || len(idxs) == 0 {
		remains := &ds.IndexDefinition{
			Kind:     q.kind,
			Ancestor: q.eqFilters["__ancestor__"] != nil,
		}
		terms := missingTerms.ToSlice()
		if serializationDeterministic {
			sort.Strings(terms)
		}
		for _, term := range terms {
			remains.SortBy = append(remains.SortBy, ds.IndexColumn{Property: term})
		}
		remains.SortBy = append(remains.SortBy, q.suffixFormat...)
		last := remains.SortBy[len(remains.SortBy)-1]
		if !last.Descending {
			// this removes the __key__ column, since it's implicit.
			remains.SortBy = remains.SortBy[:len(remains.SortBy)-1]
		}
		if remains.Builtin() {
			impossible(
				fmt.Errorf("recommended missing index would be a builtin: %s", remains))
		}
		return nil, &ErrMissingIndex{q.ns, remains}
	}

	return idxs, nil
}
Example #12
0
// runMergedQueries executes a user query `fq` against the parent datastore as
// well as the in-memory datastore, calling `cb` with the merged result set.
//
// It's expected that the caller of this function will apply limit and offset
// if the query contains those restrictions. This may convert the query to
// an expanded projection query with more data than the user asked for. It's the
// caller's responsibility to prune away the extra data.
//
// See also `dsTxnBuf.Run()`.
func runMergedQueries(fq *ds.FinalizedQuery, sizes *sizeTracker,
	memDS, parentDS ds.RawInterface, cb func(k *ds.Key, data ds.PropertyMap) error) error {

	toRun, err := adjustQuery(fq)
	if err != nil {
		return err
	}

	cmpLower, cmpUpper := memory.GetBinaryBounds(fq)
	cmpOrder := fq.Orders()
	cmpFn := func(i *item) string {
		return i.getCmpRow(cmpLower, cmpUpper, cmpOrder)
	}

	dedup := stringset.Set(nil)
	distinct := stringset.Set(nil)
	distinctOrder := []ds.IndexColumn(nil)
	if len(fq.Project()) > 0 { // the original query was a projection query
		if fq.Distinct() {
			// it was a distinct projection query, so we need to dedup by distinct
			// options.
			distinct = stringset.New(0)
			proj := fq.Project()
			distinctOrder = make([]ds.IndexColumn, len(proj))
			for i, p := range proj {
				distinctOrder[i].Property = p
			}
		}
	} else {
		// the original was a normal or keys-only query, so we need to dedup by keys.
		dedup = stringset.New(0)
	}

	stopChan := make(chan struct{})

	parIter := queryToIter(stopChan, toRun, parentDS)
	memIter := queryToIter(stopChan, toRun, memDS)

	parItemGet := func() (*item, error) {
		for {
			itm, err := parIter()
			if itm == nil || err != nil {
				return nil, err
			}
			encKey := itm.getEncKey()
			if sizes.has(encKey) || (dedup != nil && dedup.Has(encKey)) {
				continue
			}
			return itm, nil
		}
	}
	memItemGet := func() (*item, error) {
		for {
			itm, err := memIter()
			if itm == nil || err != nil {
				return nil, err
			}
			if dedup != nil && dedup.Has(itm.getEncKey()) {
				continue
			}
			return itm, nil
		}
	}

	defer func() {
		close(stopChan)
		parItemGet()
		memItemGet()
	}()

	pitm, err := parItemGet()
	if err != nil {
		return err
	}

	mitm, err := memItemGet()
	if err != nil {
		return err
	}

	for {
		// the err can be set during the loop below. If we come around the bend and
		// it's set, then we need to return it. We don't check it immediately
		// because it's set after we already have a good result to return to the
		// user.
		if err != nil {
			return err
		}

		usePitm := pitm != nil
		if pitm != nil && mitm != nil {
			usePitm = cmpFn(pitm) < cmpFn(mitm)
		} else if pitm == nil && mitm == nil {
			break
		}

		toUse := (*item)(nil)
		// we check the error at the beginning of the loop.
		if usePitm {
			toUse = pitm
			pitm, err = parItemGet()
		} else {
			toUse = mitm
			mitm, err = memItemGet()
		}

		if dedup != nil {
			if !dedup.Add(toUse.getEncKey()) {
				continue
			}
		}
		if distinct != nil {
			// NOTE: We know that toUse will not be used after this point for
			// comparison purposes, so re-use its cmpRow property for our distinct
			// filter here.
			toUse.cmpRow = ""
			if !distinct.Add(toUse.getCmpRow(nil, nil, distinctOrder)) {
				continue
			}
		}
		if err := cb(toUse.key, toUse.data); err != nil {
			if err == ds.Stop {
				return nil
			}
			return err
		}
	}

	return nil
}