func newNormalStrategy(aid, ns string, cb ds.RawRunCB, head *memStore) queryStrategy {
coll := head.GetCollection("ents:" + ns)
if coll == nil {
return nil
}
return &normalStrategy{cb, aid, ns, coll, stringset.New(0)}
}
func withTxnBuf(ctx context.Context, cb func(context.Context) error, opts *datastore.TransactionOptions) error {
inf := info.Get(ctx)
ns := inf.GetNamespace()
parentState, _ := ctx.Value(dsTxnBufParent).(*txnBufState)
roots := stringset.New(0)
rootLimit := 1
if opts != nil && opts.XG {
rootLimit = XGTransactionGroupLimit
}
sizeBudget := DefaultSizeBudget
if parentState != nil {
// TODO(riannucci): this is a bit wonky since it means that a child
// transaction declaring XG=true will only get to modify 25 groups IF
// they're same groups affected by the parent transactions. So instead of
// respecting opts.XG for inner transactions, we just dup everything from
// the parent transaction.
roots = parentState.roots.Dup()
rootLimit = parentState.rootLimit
sizeBudget = parentState.sizeBudget - parentState.entState.total
if sizeBudget < DefaultSizeThreshold {
return ErrTransactionTooLarge
}
}
bufDS, err := memory.NewDatastore(inf.FullyQualifiedAppID(), ns)
if err != nil {
return err
}
state := &txnBufState{
entState: &sizeTracker{},
bufDS: bufDS.Raw(),
roots: roots,
rootLimit: rootLimit,
ns: ns,
aid: inf.AppID(),
parentDS: datastore.Get(context.WithValue(ctx, dsTxnBufHaveLock, true)).Raw(),
sizeBudget: sizeBudget,
}
if err = cb(context.WithValue(ctx, dsTxnBufParent, state)); err != nil {
return err
}
// no reason to unlock this ever. At this point it's toast.
state.Lock()
if parentState == nil {
return commitToReal(state)
}
if err = parentState.canApplyLocked(state); err != nil {
return err
}
parentState.commitLocked(state)
return nil
}
// toEncoded returns a list of all of the serialized versions of these keys,
// plus a stringset of all the encoded root keys that `keys` represents.
func toEncoded(keys []*datastore.Key) (full []string, roots stringset.Set) {
roots = stringset.New(len(keys))
full = make([]string, len(keys))
for i, k := range keys {
roots.Add(string(serialize.ToBytes(k.Root())))
full[i] = string(serialize.ToBytes(k))
}
return
}
func pickQueryStrategy(fq *ds.FinalizedQuery, rq *reducedQuery, cb ds.RawRunCB, head *memStore) queryStrategy {
if fq.KeysOnly() {
return &keysOnlyStrategy{cb, stringset.New(0)}
}
if len(fq.Project()) > 0 {
return newProjectionStrategy(fq, rq, cb)
}
return newNormalStrategy(rq.aid, rq.ns, cb, head)
}
// PropertySlice serializes a single row of a DSProperty map.
func PropertySlice(vals ds.PropertySlice) SerializedPslice {
dups := stringset.New(0)
ret := make(SerializedPslice, 0, len(vals))
for _, v := range vals {
if v.IndexSetting() == ds.NoIndex {
continue
}
data := ToBytes(v)
dataS := string(data)
if !dups.Add(dataS) {
continue
}
ret = append(ret, data)
}
return ret
}
// Project lists one or more field names to project.
func (q *Query) Project(fieldNames ...string) *Query {
if len(fieldNames) == 0 {
return q
}
return q.mod(func(q *Query) {
for _, f := range fieldNames {
if q.reserved(f) {
return
}
if f == "__key__" {
q.err = fmt.Errorf("cannot project on %q", f)
return
}
if q.project == nil {
q.project = stringset.New(1)
}
q.project.Add(f)
}
})
}
func (t *txnBufState) updateRootsLocked(roots stringset.Set) error {
curRootLen := t.roots.Len()
proposedRoots := stringset.New(1)
roots.Iter(func(root string) bool {
if !t.roots.Has(root) {
proposedRoots.Add(root)
}
return proposedRoots.Len()+curRootLen <= t.rootLimit
})
if proposedRoots.Len()+curRootLen > t.rootLimit {
return ErrTooManyRoots
}
// only need to update the roots if they did something that required updating
if proposedRoots.Len() > 0 {
proposedRoots.Iter(func(root string) bool {
t.roots.Add(root)
return true
})
}
return nil
}
func newProjectionStrategy(fq *ds.FinalizedQuery, rq *reducedQuery, cb ds.RawRunCB) queryStrategy {
proj := fq.Project()
projectionLookups := make([]projectionLookup, len(proj))
for i, prop := range proj {
projectionLookups[i].propertyName = prop
lookupErr := fmt.Errorf("planning a strategy for an unfulfillable query?")
for j, col := range rq.suffixFormat {
if col.Property == prop {
projectionLookups[i].suffixIndex = j
lookupErr = nil
break
}
}
impossible(lookupErr)
}
ret := &projectionStrategy{cb: cb, project: projectionLookups}
if fq.Distinct() {
ret.distinct = stringset.New(0)
}
return ret
}
func reduce(fq *ds.FinalizedQuery, aid, ns string, isTxn bool) (*reducedQuery, error) {
if err := fq.Valid(aid, ns); err != nil {
return nil, err
}
if isTxn && fq.Ancestor() == nil {
return nil, fmt.Errorf("queries within a transaction must include an Ancestor filter")
}
if num := numComponents(fq); num > MaxQueryComponents {
return nil, fmt.Errorf(
"gae/memory: query is too large. may not have more than "+
"%d filters + sort orders + ancestor total: had %d",
MaxQueryComponents, num)
}
ret := &reducedQuery{
aid: aid,
ns: ns,
kind: fq.Kind(),
suffixFormat: fq.Orders(),
}
eqFilts := fq.EqFilters()
ret.eqFilters = make(map[string]stringset.Set, len(eqFilts))
for prop, vals := range eqFilts {
sVals := stringset.New(len(vals))
for _, v := range vals {
sVals.Add(string(serialize.ToBytes(v)))
}
ret.eqFilters[prop] = sVals
}
startD, endD := GetBinaryBounds(fq)
// Now we check the start and end cursors.
//
// Cursors are composed of a list of IndexColumns at the beginning, followed
// by the raw bytes to use for the suffix. The cursor is only valid if all of
// its IndexColumns match our proposed suffixFormat, as calculated above.
//
// Cursors are mutually exclusive with the start/end we picked up from the
// inequality. In a well formed query, they indicate a subset of results
// bounded by the inequality. Technically if the start cursor is not >= the
// low bound, or the end cursor is < the high bound, it's an error, but for
// simplicity we just cap to the narrowest intersection of the inequality and
// cursors.
ret.start = startD
ret.end = endD
if start, end := fq.Bounds(); start != nil || end != nil {
if start != nil {
if c, ok := start.(queryCursor); ok {
startCols, startD, err := c.decode()
if err != nil {
return nil, err
}
if !sortOrdersEqual(startCols, ret.suffixFormat) {
return nil, errors.New("gae/memory: start cursor is invalid for this query")
}
if ret.start == nil || bytes.Compare(ret.start, startD) < 0 {
ret.start = startD
}
} else {
return nil, errors.New("gae/memory: bad cursor type")
}
}
if end != nil {
if c, ok := end.(queryCursor); ok {
endCols, endD, err := c.decode()
if err != nil {
return nil, err
}
if !sortOrdersEqual(endCols, ret.suffixFormat) {
return nil, errors.New("gae/memory: end cursor is invalid for this query")
}
if ret.end == nil || bytes.Compare(endD, ret.end) < 0 {
ret.end = endD
}
} else {
return nil, errors.New("gae/memory: bad cursor type")
}
}
}
// Finally, verify that we could even /potentially/ do work. If we have
// overlapping range ends, then we don't have anything to do.
if ret.end != nil && bytes.Compare(ret.start, ret.end) >= 0 {
return nil, ds.ErrNullQuery
}
ret.numCols = len(ret.suffixFormat)
for prop, vals := range ret.eqFilters {
if len(ret.suffixFormat) == 1 && prop == "__ancestor__" {
continue
}
ret.numCols += vals.Len()
}
return ret, nil
}
// Finalize converts this Query to a FinalizedQuery. If the Query has any
// inconsistencies or violates any of the query rules, that will be returned
// here.
func (q *Query) Finalize() (*FinalizedQuery, error) {
if q.err != nil || q.finalized != nil {
return q.finalized, q.err
}
ancestor := (*Key)(nil)
if slice, ok := q.eqFilts["__ancestor__"]; ok {
ancestor = slice[0].Value().(*Key)
}
err := func() error {
if q.kind == "" { // kindless query checks
if q.ineqFiltProp != "" && q.ineqFiltProp != "__key__" {
return fmt.Errorf(
"kindless queries can only filter on __key__, got %q", q.ineqFiltProp)
}
allowedEqs := 0
if ancestor != nil {
allowedEqs = 1
}
if len(q.eqFilts) > allowedEqs {
return fmt.Errorf("kindless queries may not have any equality filters")
}
for _, o := range q.order {
if o.Property != "__key__" || o.Descending {
return fmt.Errorf("invalid order for kindless query: %#v", o)
}
}
}
if q.keysOnly && q.project != nil && q.project.Len() > 0 {
return errors.New("cannot project a keysOnly query")
}
if q.ineqFiltProp != "" {
if len(q.order) > 0 && q.order[0].Property != q.ineqFiltProp {
return fmt.Errorf(
"first sort order must match inequality filter: %q v %q",
q.order[0].Property, q.ineqFiltProp)
}
if q.ineqFiltLowSet && q.ineqFiltHighSet {
if q.ineqFiltHigh.Less(&q.ineqFiltLow) ||
(q.ineqFiltHigh.Equal(&q.ineqFiltLow) &&
(!q.ineqFiltLowIncl || !q.ineqFiltHighIncl)) {
return ErrNullQuery
}
}
if q.ineqFiltProp == "__key__" {
if q.ineqFiltLowSet {
if ancestor != nil && !q.ineqFiltLow.Value().(*Key).HasAncestor(ancestor) {
return fmt.Errorf(
"inequality filters on __key__ must be descendants of the __ancestor__")
}
}
if q.ineqFiltHighSet {
if ancestor != nil && !q.ineqFiltHigh.Value().(*Key).HasAncestor(ancestor) {
return fmt.Errorf(
"inequality filters on __key__ must be descendants of the __ancestor__")
}
}
}
}
err := error(nil)
if q.project != nil {
q.project.Iter(func(p string) bool {
if _, iseq := q.eqFilts[p]; iseq {
err = fmt.Errorf("cannot project on equality filter field: %s", p)
return false
}
return true
})
}
return err
}()
if err != nil {
q.err = err
return nil, err
}
ret := &FinalizedQuery{
original: q,
kind: q.kind,
keysOnly: q.keysOnly,
eventuallyConsistent: q.eventualConsistency || ancestor == nil,
limit: q.limit,
offset: q.offset,
start: q.start,
end: q.end,
eqFilts: q.eqFilts,
ineqFiltProp: q.ineqFiltProp,
ineqFiltLow: q.ineqFiltLow,
ineqFiltLowIncl: q.ineqFiltLowIncl,
ineqFiltLowSet: q.ineqFiltLowSet,
ineqFiltHigh: q.ineqFiltHigh,
ineqFiltHighIncl: q.ineqFiltHighIncl,
ineqFiltHighSet: q.ineqFiltHighSet,
}
if q.project != nil {
ret.project = q.project.ToSlice()
ret.distinct = q.distinct && q.project.Len() > 0
// If we're DISTINCT && have an inequality filter, we must project that
// inequality property as well.
if ret.distinct && ret.ineqFiltProp != "" && !q.project.Has(ret.ineqFiltProp) {
ret.project = append([]string{ret.ineqFiltProp}, ret.project...)
}
}
seenOrders := stringset.New(len(q.order))
// if len(q.order) > 0, we already enforce that the first order
// is the same as the inequality above. Otherwise we need to add it.
if len(q.order) == 0 && q.ineqFiltProp != "" {
ret.orders = []IndexColumn{{Property: q.ineqFiltProp}}
seenOrders.Add(q.ineqFiltProp)
}
// drop orders where there's an equality filter
// https://cloud.google.com/appengine/docs/go/datastore/queries#sort_orders_are_ignored_on_properties_with_equality_filters
// Deduplicate orders
for _, o := range q.order {
if _, iseq := q.eqFilts[o.Property]; !iseq {
if seenOrders.Add(o.Property) {
ret.orders = append(ret.orders, o)
}
}
}
// Add any projection columns not mentioned in the user-defined order as
// ASCENDING orders. Technically we could be smart and automatically use
// a DESCENDING ordered index, if it fit, but the logic gets insane, since all
// suffixes of all used indexes need to be PRECISELY equal (and so you'd have
// to hunt/invalidate/something to find the combination of indexes that are
// compatible with each other as well as the query). If you want to use
// a DESCENDING column, just add it to the user sort order, and this loop will
// not synthesize a new suffix entry for it.
//
// NOTE: if you want to use an index that sorts by -__key__, you MUST
// include all of the projected fields for that index in the order explicitly.
// Otherwise the generated orders will be wacky. So:
// Query("Foo").Project("A", "B").Order("A").Order("-__key__")
//
// will turn into a orders of:
// A, ASCENDING
// __key__, DESCENDING
// B, ASCENDING
// __key__, ASCENDING
//
// To prevent this, your query should have another Order("B") clause before
// the -__key__ clause.
if len(ret.project) > 0 {
sort.Strings(ret.project)
for _, p := range ret.project {
if !seenOrders.Has(p) {
ret.orders = append(ret.orders, IndexColumn{Property: p})
}
}
}
// If the suffix format ends with __key__ already (e.g. .Order("__key__")),
// then we're good to go. Otherwise we need to add it as the last bit of the
// suffix, since all indexes implicitly have it as the last column.
if len(ret.orders) == 0 || ret.orders[len(ret.orders)-1].Property != "__key__" {
ret.orders = append(ret.orders, IndexColumn{Property: "__key__"})
}
q.finalized = ret
return ret, nil
}
// getRelevantIndexes retrieves the relevant indexes which could be used to
// service q. It returns nil if it's not possible to service q with the current
// indexes.
func getRelevantIndexes(q *reducedQuery, s *memStore) (indexDefinitionSortableSlice, error) {
missingTerms := stringset.New(len(q.eqFilters))
for k := range q.eqFilters {
if k == "__ancestor__" {
// ancestor is not a prefix which can be satisfied by a single index. It
// must be satisfied by ALL indexes (and has special logic for this in
// the addDefinition logic)
continue
}
missingTerms.Add(k)
}
idxs := indexDefinitionSortableSlice{}
// First we add builtins
// add
// idx:KIND
if idxs.maybeAddDefinition(q, s, missingTerms, &ds.IndexDefinition{
Kind: q.kind,
}) {
return idxs, nil
}
// add
// idx:KIND:prop
// idx:KIND:-prop
props := stringset.New(len(q.eqFilters) + len(q.suffixFormat))
for prop := range q.eqFilters {
props.Add(prop)
}
for _, col := range q.suffixFormat[:len(q.suffixFormat)-1] {
props.Add(col.Property)
}
for _, prop := range props.ToSlice() {
if strings.HasPrefix(prop, "__") && strings.HasSuffix(prop, "__") {
continue
}
if idxs.maybeAddDefinition(q, s, missingTerms, &ds.IndexDefinition{
Kind: q.kind,
SortBy: []ds.IndexColumn{
{Property: prop},
},
}) {
return idxs, nil
}
if idxs.maybeAddDefinition(q, s, missingTerms, &ds.IndexDefinition{
Kind: q.kind,
SortBy: []ds.IndexColumn{
{Property: prop, Descending: true},
},
}) {
return idxs, nil
}
}
// Try adding all compound indexes whose suffix matches.
suffix := &ds.IndexDefinition{
Kind: q.kind,
Ancestor: q.eqFilters["__ancestor__"] != nil,
SortBy: q.suffixFormat,
}
walkCompIdxs(s, suffix, func(def *ds.IndexDefinition) bool {
// keep walking until we find a perfect index.
return !idxs.maybeAddDefinition(q, s, missingTerms, def)
})
// this query is impossible to fulfil with the current indexes. Not all the
// terms (equality + projection) are satisfied.
if missingTerms.Len() < 0 || len(idxs) == 0 {
remains := &ds.IndexDefinition{
Kind: q.kind,
Ancestor: q.eqFilters["__ancestor__"] != nil,
}
terms := missingTerms.ToSlice()
if serializationDeterministic {
sort.Strings(terms)
}
for _, term := range terms {
remains.SortBy = append(remains.SortBy, ds.IndexColumn{Property: term})
}
remains.SortBy = append(remains.SortBy, q.suffixFormat...)
last := remains.SortBy[len(remains.SortBy)-1]
if !last.Descending {
// this removes the __key__ column, since it's implicit.
remains.SortBy = remains.SortBy[:len(remains.SortBy)-1]
}
if remains.Builtin() {
impossible(
fmt.Errorf("recommended missing index would be a builtin: %s", remains))
}
return nil, &ErrMissingIndex{q.ns, remains}
}
return idxs, nil
}
// runMergedQueries executes a user query `fq` against the parent datastore as
// well as the in-memory datastore, calling `cb` with the merged result set.
//
// It's expected that the caller of this function will apply limit and offset
// if the query contains those restrictions. This may convert the query to
// an expanded projection query with more data than the user asked for. It's the
// caller's responsibility to prune away the extra data.
//
// See also `dsTxnBuf.Run()`.
func runMergedQueries(fq *ds.FinalizedQuery, sizes *sizeTracker,
memDS, parentDS ds.RawInterface, cb func(k *ds.Key, data ds.PropertyMap) error) error {
toRun, err := adjustQuery(fq)
if err != nil {
return err
}
cmpLower, cmpUpper := memory.GetBinaryBounds(fq)
cmpOrder := fq.Orders()
cmpFn := func(i *item) string {
return i.getCmpRow(cmpLower, cmpUpper, cmpOrder)
}
dedup := stringset.Set(nil)
distinct := stringset.Set(nil)
distinctOrder := []ds.IndexColumn(nil)
if len(fq.Project()) > 0 { // the original query was a projection query
if fq.Distinct() {
// it was a distinct projection query, so we need to dedup by distinct
// options.
distinct = stringset.New(0)
proj := fq.Project()
distinctOrder = make([]ds.IndexColumn, len(proj))
for i, p := range proj {
distinctOrder[i].Property = p
}
}
} else {
// the original was a normal or keys-only query, so we need to dedup by keys.
dedup = stringset.New(0)
}
stopChan := make(chan struct{})
parIter := queryToIter(stopChan, toRun, parentDS)
memIter := queryToIter(stopChan, toRun, memDS)
parItemGet := func() (*item, error) {
for {
itm, err := parIter()
if itm == nil || err != nil {
return nil, err
}
encKey := itm.getEncKey()
if sizes.has(encKey) || (dedup != nil && dedup.Has(encKey)) {
continue
}
return itm, nil
}
}
memItemGet := func() (*item, error) {
for {
itm, err := memIter()
if itm == nil || err != nil {
return nil, err
}
if dedup != nil && dedup.Has(itm.getEncKey()) {
continue
}
return itm, nil
}
}
defer func() {
close(stopChan)
parItemGet()
memItemGet()
}()
pitm, err := parItemGet()
if err != nil {
return err
}
mitm, err := memItemGet()
if err != nil {
return err
}
for {
// the err can be set during the loop below. If we come around the bend and
// it's set, then we need to return it. We don't check it immediately
// because it's set after we already have a good result to return to the
// user.
if err != nil {
return err
}
usePitm := pitm != nil
if pitm != nil && mitm != nil {
usePitm = cmpFn(pitm) < cmpFn(mitm)
} else if pitm == nil && mitm == nil {
break
}
toUse := (*item)(nil)
// we check the error at the beginning of the loop.
if usePitm {
toUse = pitm
pitm, err = parItemGet()
} else {
toUse = mitm
mitm, err = memItemGet()
}
if dedup != nil {
if !dedup.Add(toUse.getEncKey()) {
continue
}
}
if distinct != nil {
// NOTE: We know that toUse will not be used after this point for
// comparison purposes, so re-use its cmpRow property for our distinct
// filter here.
toUse.cmpRow = ""
if !distinct.Add(toUse.getCmpRow(nil, nil, distinctOrder)) {
continue
}
}
if err := cb(toUse.key, toUse.data); err != nil {
if err == ds.Stop {
return nil
}
return err
}
}
return nil
}