func executeItemQuery(con *Context, q *datastore.Query, limit int, cursorStr string) ([]Item, string, error) { if cursor, err := datastore.DecodeCursor(cursorStr); err == nil { q = q.Start(cursor) } var is = make([]Item, 0, limit) var err error t := q.Run(con.C) for { var i Item _, err = t.Next(&i) if err == datastore.Done { break } is = append(is, i) if err != nil { con.Log.Errorf("Error fetching next item: %v", err) return nil, "", err } } var cursor datastore.Cursor if cursor, err = t.Cursor(); err == nil { return is, cursor.String(), nil } return nil, "", err }
func (cdb ComplaintDB) NewIter(q *datastore.Query) *ComplaintIterator { ci := ComplaintIterator{ //CDB: cdb, Query: q, Iter: q.Run(cdb.Ctx()), } return &ci }
// subtreeByPath retrieves a subdirectories of a given directory. // It is relying on an indexed string property "Dir" // containing a string representation of the full path. // // It might be fast for deep, uncached directory subtree, // that have been saved in nested manner. // // However, it might not find recently added directories. // Upon finding nothing, it therefore returns the // "warning" fsi.EmptyQueryResult // // The func could easily be enhanced chunked scanning. // // It is currently used by ReadDir and by the test package. // It is public for the test package. func (fs *dsFileSys) SubtreeByPath(name string, onlyDirectChildren bool) ([]DsDir, error) { dir, bname := fs.SplitX(name) name = dir + common.Filify(bname) if !strings.HasSuffix(name, sep) { name += sep } var q *datastore.Query if onlyDirectChildren { q = datastore.NewQuery(tdir). Filter("Dir=", name). Order("Dir") // Limit(4) } else { pathInc := IncrementString(name) q = datastore.NewQuery(tdir). Filter("Dir>=", name). Filter("Dir<", pathInc). Order("Dir") } // log.Printf("%v", q) var children []DsDir keys, err := q.GetAll(fs.Ctx(), &children) if err != nil { aelog.Errorf(fs.Ctx(), "Error getting all children of %v => %v", dir+bname, err) return children, err } if len(children) < 1 { return children, fsi.EmptyQueryResult } // Very evil: We filter out root node, since it's // has the same dir as the level-1 directories. keyRoot := datastore.NewKey(fs.Ctx(), tdir, fs.RootDir(), 0, nil) idxRoot := -1 for i := 0; i < len(children); i++ { children[i].fSys = fs children[i].Key = keys[i] if keys[i].Equal(keyRoot) { // log.Printf("root idx %v", i) idxRoot = i } } if idxRoot > -1 { children = append(children[:idxRoot], children[idxRoot+1:]...) } return children, nil }
// Snarf down all the keys from the get go. func (cdb *ComplaintDB) NewLongIter(q *datastore.Query) *LongIterator { ctx := cdb.Ctx() keys, err := q.KeysOnly().GetAll(ctx, nil) i := LongIterator{ Ctx: ctx, Keys: keys, err: err, } return &i }
// Snarf down all the keys from the get go. func (cdb *ComplaintDB) NewLongBatchingIter(q *datastore.Query) *LongBatchingIterator { ctx := cdb.Ctx() keys, err := q.KeysOnly().GetAll(ctx, nil) i := LongBatchingIterator{ Ctx: ctx, BatchSize: 100, keys: keys, vals: []*types.Complaint{}, err: err, } return &i }
func fillKeyQuery(c context.Context, q *datastore.Query, results interface{}) error { keys, err := q.GetAll(c, results) if err == nil { rslice := reflect.ValueOf(results).Elem() for i := range keys { if k, ok := rslice.Index(i).Interface().(Keyable); ok { k.setKey(keys[i]) } else if k, ok := rslice.Index(i).Addr().Interface().(Keyable); ok { k.setKey(keys[i]) } else { log.Infof(c, "Warning: %v is not Keyable", rslice.Index(i).Interface()) } } } else { log.Errorf(c, "Error executing query: %v", err) } return err }
func executeCallbackQuery(con *Context, q *datastore.Query, limit int) ([]Callback, error) { var is = make([]Callback, 0, limit) var err error t := q.Run(con.C) for { var i Callback _, err = t.Next(&i) if err == datastore.Done { break } is = append(is, i) if err != nil { con.Log.Errorf("Error fetching next item: %v", err) return nil, err } } return is, nil }
func doQuery(ctx context.Context, query *datastore.Query, resultKey, taskKey *datastore.Key, dst interface{}) ([]*datastore.Key, error) { keys, err := query. Ancestor(resultKey). Filter("Task=", taskKey). Order("-Time"). Limit(1). GetAll(ctx, &dst) if err != nil { return nil, err } return keys, nil }
func (cdb ComplaintDB) getComplaintsByQueryFromDatastore(q *datastore.Query) ([]*datastore.Key, []types.Complaint, error) { cdb.Debugf("gCBQFD_200", "getComplaintsByQueryFromDatastore") var data = []types.Complaint{} cdb.Debugf("gCBQFD_201", "calling GetAll() ...") keys, err := q.GetAll(cdb.Ctx(), &data) cdb.Debugf("gCBQFD_202", "... call done (n=%d)", len(keys)) // We tolerate missing fields, because the DB is full of old objects with dead fields if err != nil { if mismatchErr, ok := err.(*datastore.ErrFieldMismatch); ok { _ = mismatchErr // cdb.Debugf("gCBQFD_203", "missing field: %v", mismatchErr) } else { return nil, nil, fmt.Errorf("gCBQFD: %v", err) } } return keys, data, nil }
// GetAll runs the query and returns all the keys that match the query, as well // as appending the values to dst, setting the goon key fields of dst, and // caching the returned data in local memory. // // For "keys-only" queries dst can be nil, however if it is not, then GetAll // appends zero value structs to dst, only setting the goon key fields. // No data is cached with "keys-only" queries. // // See: https://developers.google.com/appengine/docs/go/datastore/reference#Query.GetAll func (g *Goon) GetAll(q *datastore.Query, dst interface{}) ([]*datastore.Key, error) { v := reflect.ValueOf(dst) vLenBefore := 0 if dst != nil { if v.Kind() != reflect.Ptr { return nil, fmt.Errorf("goon: Expected dst to be a pointer to a slice or nil, got instead: %v", v.Kind()) } v = v.Elem() if v.Kind() != reflect.Slice { return nil, fmt.Errorf("goon: Expected dst to be a pointer to a slice or nil, got instead: %v", v.Kind()) } vLenBefore = v.Len() } keys, err := q.GetAll(g.Context, dst) if err != nil { g.error(err) return nil, err } if dst == nil || len(keys) == 0 { return keys, nil } keysOnly := ((v.Len() - vLenBefore) != len(keys)) updateCache := !g.inTransaction && !keysOnly // If this is a keys-only query, we need to fill the slice with zero value elements if keysOnly { elemType := v.Type().Elem() ptr := false if elemType.Kind() == reflect.Ptr { elemType = elemType.Elem() ptr = true } if elemType.Kind() != reflect.Struct { return keys, fmt.Errorf("goon: Expected struct, got instead: %v", elemType.Kind()) } for i := 0; i < len(keys); i++ { ev := reflect.New(elemType) if !ptr { ev = ev.Elem() } v.Set(reflect.Append(v, ev)) } } if updateCache { g.cacheLock.Lock() defer g.cacheLock.Unlock() } for i, k := range keys { var e interface{} vi := v.Index(vLenBefore + i) if vi.Kind() == reflect.Ptr { e = vi.Interface() } else { e = vi.Addr().Interface() } if err := g.setStructKey(e, k); err != nil { return nil, err } if updateCache { // Cache lock is handled before the for loop g.cache[memkey(k)] = e } } return keys, nil }
// Count returns the number of results for the query. func (g *Goon) Count(q *datastore.Query) (int, error) { return q.Count(g.Context) }
// Run runs the query. func (g *Goon) Run(q *datastore.Query) *Iterator { return &Iterator{ g: g, i: q.Run(g.Context), } }
func (s *Store) GetKeysInQuery(q *datastore.Query) ([]*datastore.Key, error) { return q.KeysOnly().GetAll(s.c, nil) }
func handler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) u := user.Current(c) log.Infof(c, "Got a visitor to the front page!") //keep log in the imports //Check if the request is before or after to create the right query //The GET requests for the stories will be based around the SubmitDateTime //Using "after" will return stories after a certain date from newest to oldest //Using "before" will return stories before a certain date from oldest to newest //Default is to use the latest 3 submissions afterDate := r.FormValue("after") beforeDate := r.FormValue("before") returnLimit := 3 showPrevLink := false var q *datastore.Query if afterDate != "" { showPrevLink = true //Get the results in descending order for newest to oldest afterDate = strings.Replace(afterDate, "%20", " ", -1) //replace all %20 with " " ttime, err := time.Parse(DateTimeDatastoreFormat, afterDate) if err != nil { serveError(c, w, err) return } q = datastore.NewQuery(WebSubmissionEntityName). Filter("SubmitDateTime <", ttime). Order("-SubmitDateTime"). Limit(returnLimit) } else if beforeDate != "" { showPrevLink = true //Get the results is ascending order from oldest to newest beforeDate = strings.Replace(beforeDate, "%20", " ", -1) //replace all %20 with " " ttime, err := time.Parse(DateTimeDatastoreFormat, beforeDate) if err != nil { serveError(c, w, err) return } q = datastore.NewQuery(WebSubmissionEntityName). Filter("SubmitDateTime >", ttime). Order("SubmitDateTime"). Limit(returnLimit) //limit check at the beginning if less than the returnLimit redo from the beginning length, cerr := q.Count(c) if cerr != nil { serveError(c, w, cerr) } //TODO refactor to not duplicate the default query below if length < returnLimit { showPrevLink = false q = datastore.NewQuery(WebSubmissionEntityName). Order("-SubmitDateTime"). Limit(returnLimit) } } else { q = datastore.NewQuery(WebSubmissionEntityName). Order("-SubmitDateTime"). Limit(returnLimit) } //Populate the results struct and store the keys var pageCon PageContainer for t := q.Run(c); ; { var x WebSubmission key, err := t.Next(&x) if err == datastore.Done { break } if err != nil { // serveError(c,w,err) fmt.Fprintf(w, "nope %v", err.Error()) return } if u == nil { pageCon.Stories = append(pageCon.Stories, StoryListData{x, key, false}) } else { pageCon.Stories = append(pageCon.Stories, StoryListData{x, key, u.String() == x.SubmitBy}) } } //if we filled up the page with results there are probably more, build the //next page link length, cerr := q.Count(c) if cerr != nil { serveError(c, w, cerr) } if length == returnLimit { //get the submit datetime of the last story pageCon.AfterLink = pageCon.Stories[returnLimit-1].Story.SubmitDateTime.Format(DateTimeDatastoreFormat) } //If it was a prev page press reverse the result array to get it back into chronological order if length >= 1 && beforeDate != "" { for i, j := 0, len(pageCon.Stories)-1; i < j; i, j = i+1, j-1 { pageCon.Stories[i], pageCon.Stories[j] = pageCon.Stories[j], pageCon.Stories[i] } } //prev page link //check the length because going forward you can have null data if showPrevLink && length >= 1 { pageCon.BeforeLink = pageCon.Stories[0].Story.SubmitDateTime.Format(DateTimeDatastoreFormat) } //build and show the page page := template.Must(template.ParseFiles( "public/templates/_base.html", "public/templates/storylist.html", )) if err := page.Execute(w, pageCon); err != nil { serveError(c, w, err) fmt.Fprintf(w, "\n%v\n%v", err.Error(), pageCon) return } }