func executeItemQuery(con *Context, q *datastore.Query, limit int, cursorStr string) ([]Item, string, error) {
if cursor, err := datastore.DecodeCursor(cursorStr); err == nil {
q = q.Start(cursor)
}
var is = make([]Item, 0, limit)
var err error
t := q.Run(con.C)
for {
var i Item
_, err = t.Next(&i)
if err == datastore.Done {
break
}
is = append(is, i)
if err != nil {
con.Log.Errorf("Error fetching next item: %v", err)
return nil, "", err
}
}
var cursor datastore.Cursor
if cursor, err = t.Cursor(); err == nil {
return is, cursor.String(), nil
}
return nil, "", err
}
func (qm *QueryMarker) UnmarshalJSON(buf []byte) error {
if len(buf) < 2 || buf[0] != '"' || buf[len(buf)-1] != '"' {
return errors.New("QueryMarker: bad cursor value")
}
cursor, err := datastore.DecodeCursor(string(buf[1 : len(buf)-1]))
if err != nil {
return err
}
*qm = QueryMarker{cursor}
return nil
}
func decodeDatastoreCursorValue(d *Decoder, v reflect.Value) error {
s, err := d.DecodeString()
if err != nil {
return err
}
cursor, err := ds.DecodeCursor(s)
if err != nil {
return err
}
v.Set(reflect.ValueOf(cursor))
return nil
}
func init() {
reindexDelayer = delay.Func("reindex-idioms", func(c context.Context, cursorStr string) error {
q := datastore.NewQuery("Idiom")
if cursorStr != "" {
log.Infof(c, "Starting at cursor %v", cursorStr)
cursor, err := datastore.DecodeCursor(cursorStr)
if err != nil {
return err
}
q = q.Start(cursor)
}
iterator := q.Run(c)
reindexedIDs := make([]int, 0, reindexBatchSize)
defer func() {
log.Infof(c, "Reindexed idioms %v", reindexedIDs)
}()
for i := 0; i < reindexBatchSize; i++ {
var idiom Idiom
key, err := iterator.Next(&idiom)
if err == datastore.Done {
log.Infof(c, "Reindexing completed.")
return nil
} else if err != nil {
// ouch :(
return err
}
err = indexIdiomFullText(c, &idiom, key)
if err != nil {
log.Errorf(c, "Reindexing full text idiom %d : %v", idiom.Id, err)
}
err = indexIdiomCheatsheets(c, &idiom)
if err != nil {
log.Errorf(c, "Reindexing cheatsheet of idiom %d : %v", idiom.Id, err)
}
reindexedIDs = append(reindexedIDs, idiom.Id)
}
cursor, err := iterator.Cursor()
if err != nil {
// ouch :(
return err
}
log.Infof(c, "Stopping at cursor %v", cursor.String())
reindexDelayer.Call(c, cursor.String())
return nil
})
}
// MakeDatastoreQuery returns a datastore.Query that generates all namespaces in the range
func (n *NamespaceRange) MakeDatastoreQuery(c context.Context, start string) *datastore.Query {
q := datastore.NewQuery(namespaceKind)
if n.Start != "" {
q = q.Filter("__key__ >=", datastore.NewKey(c, namespaceKind, n.Start, 0, nil))
}
q = q.Filter("__key__ <=", datastore.NewKey(c, namespaceKind, n.End, 0, nil))
q = q.Order("__key__")
q = q.KeysOnly()
if start != "" {
cursor, _ := datastore.DecodeCursor(start)
q = q.Start(cursor)
}
return q
}
// 画像のメタデータ一覧をDSから取得します。
// TODO: 表示する画像数を絞る必要がないなら、Cursor必要ないかも。
func GetImages(c context.Context, cursorStr string) ([]File, string, error) {
parentKey := GetParentKey(c)
q := datastore.NewQuery(KIND_FILE).Ancestor(parentKey).Filter("Type =", IMAGE).Order("-CreatedAt")
if len(cursorStr) != 0 {
cursor, err := datastore.DecodeCursor(cursorStr)
if err != nil {
return []File{}, "", err
}
q = q.Start(cursor)
}
images := []File{}
iter := q.Run(c)
isNext := true
for {
var img File
_, err := iter.Next(&img)
if err == datastore.Done {
isNext = false
break
}
if err != nil {
log.Errorf(c, "fetching next File: %s", err.Error())
break
}
err = img.setThumbnailURL(thumbnailsLongestSide, false)
if err != nil {
log.Errorf(c, "%s", err.Error())
break
}
images = append(images, img)
}
if isNext {
next_cursor, err := iter.Cursor()
if err != nil {
log.Errorf(c, "%s", err.Error())
return []File{}, "", err
}
return images, next_cursor.String(), nil
} else {
return images, "", nil
}
}
func example20() {
var ctx context.Context
// [START cursors]
// Create a query for all Person entities.
q := datastore.NewQuery("Person")
// If the application stored a cursor during a previous request, use it.
item, err := memcache.Get(ctx, "person_cursor")
if err == nil {
cursor, err := datastore.DecodeCursor(string(item.Value))
if err == nil {
q = q.Start(cursor)
}
}
// Iterate over the results.
t := q.Run(ctx)
for {
var p Person
_, err := t.Next(&p)
if err == datastore.Done {
break
}
if err != nil {
log.Errorf(ctx, "fetching next Person: %v", err)
break
}
// Do something with the Person p
}
// Get updated cursor and store it for next time.
if cursor, err := t.Cursor(); err == nil {
memcache.Set(ctx, &memcache.Item{
Key: "person_cursor",
Value: []byte(cursor.String()),
})
}
// [END cursors]
}
func process(c context.Context, processor Processor, start string) error {
// use the full 10 minutes allowed (assuming front-end instance type)
c, _ = context.WithTimeout(c, time.Duration(10)*time.Minute)
// get the query to iterate and the entity slot to load (could be nill for keys_only)
q, e := processor.Start(c)
var cursor *datastore.Cursor
if start != "" {
newCursor, err := datastore.DecodeCursor(start)
if err != nil {
log.Errorf(c, "get start cursor error %s", err.Error())
return err
}
cursor = &newCursor
}
// signal a timeout after 5 minutes
timeout := make(chan bool, 1)
timer := time.AfterFunc(time.Duration(5)*time.Minute, func() {
timeout <- true
})
defer timer.Stop()
// TODO: error handling to retry
Loop:
for {
// check if we've timed out or whether to keep going
select {
case <-timeout:
break Loop
default:
}
processed := 0
if cursor != nil {
q = q.Start(*cursor)
}
it := q.Run(c)
for {
key, err := it.Next(e)
if err == datastore.Done {
break
}
if err != nil {
log.Errorf(c, "get key error %s", err.Error())
return err
}
processor.Process(c, key)
processed++
}
// did we process any?
if processed > 0 {
newCursor, err := it.Cursor()
if err != nil {
log.Errorf(c, "get next cursor error %s", err.Error())
return err
}
cursor = &newCursor
} else {
// otherwise we're finished
cursor = nil
break
}
}
// let the processor write any aggregation entries / tasks etc...
processor.Complete(c)
// if we didn't complete everything then continue from the cursor
if cursor != nil {
processFunc.Call(c, processor, cursor.String())
}
return nil
}
func (it *iterator) iterate(c context.Context, mapper *mapper) (bool, error) {
taskTimeout := time.After(mapper.config.TaskTimeout)
taskRunning := true
// if the query defines the specific namespaces to process
// then we can just process that list directly
if it.Query.selection == selected {
for _, namespace := range it.Query.namespaces {
it.process(c, mapper, namespace)
}
return true, nil
}
q := it.createQuery(c)
var cursor *datastore.Cursor
if it.Cursor != "" {
newCursor, err := datastore.DecodeCursor(it.Cursor)
if err != nil {
log.Errorf(c, "get start cursor error %s", err.Error())
return false, err
}
cursor = &newCursor
}
// main task loop to repeat datastore query with cursor
for taskRunning {
// if cursor is set, start the query at that point
if cursor != nil {
q = q.Start(*cursor)
}
// limit how long the cursor can run before we requery
cursorTimeout := time.After(mapper.config.CursorTimeout)
// datastore cursor context needs to run for the max allowed
cc, _ := context.WithTimeout(c, time.Duration(60)*time.Second)
t := q.Run(cc)
// item loop to iterate cursor
cursorLoop:
for {
key, err := t.Next(nil)
if err == datastore.Done {
// we reached the end
return true, nil
}
if err != nil {
log.Errorf(c, "error %s", err.Error())
return false, err
}
namespace := key.StringID()
if err := it.process(c, mapper, namespace); err != nil {
return false, err
}
select {
case <-taskTimeout:
// clearing the flag breaks us out of the task loop but also lets us update the
// cursor first when we break from the inner cursorLoop
taskRunning = false
break cursorLoop
default:
select {
case <-cursorTimeout:
// this forces a new cursor and query so we don't suffer from datastore timeouts
break cursorLoop
default:
// no timeout so carry on with the current cursor
continue cursorLoop
}
}
}
// we need to get the cursor for where we are upto whether we are requerying
// within this task or scheduling a new continuation slice
newCursor, err := t.Cursor()
if err != nil {
log.Errorf(c, "get next cursor error %s", err.Error())
return false, err
}
cursor = &newCursor
it.Cursor = cursor.String()
}
return false, nil
}
func (d rdsImpl) DecodeCursor(s string) (ds.Cursor, error) {
return datastore.DecodeCursor(s)
}
func (it *Iterator) Next() bool {
if it.offset+1 < len(it.buffer) {
it.offset++
it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
return true
}
if it.done {
return false
}
// Reset buffer and offset
it.offset = 0
it.buffer = make([]string, 0, bufferSize)
// Create query
// TODO (panamafrancis) Keys only query?
q := datastore.NewQuery(it.kind).Limit(bufferSize)
if !it.isAll {
// Filter on the direction {subject,objekt...}
q = q.Filter(it.dir.String()+" =", it.name)
}
// Get last cursor position
cursor, err := datastore.DecodeCursor(it.last)
if err == nil {
q = q.Start(cursor)
}
// Buffer the keys of the next 50 matches
t := q.Run(it.qs.context)
for {
// Quirk of the datastore, you cannot pass a nil value to to Next()
// even if you just want the keys
var k *datastore.Key
skip := false
if it.kind == quadKind {
temp := new(QuadEntry)
k, err = t.Next(temp)
// Skip if quad has been deleted
if len(temp.Added) <= len(temp.Deleted) {
skip = true
}
} else {
temp := new(NodeEntry)
k, err = t.Next(temp)
// Skip if node has been deleted
if temp.Size == 0 {
skip = true
}
}
if err == datastore.Done {
it.done = true
break
}
if err != nil {
clog.Errorf("Error fetching next entry %v", err)
it.err = err
return false
}
if !skip {
it.buffer = append(it.buffer, k.StringID())
}
}
// Save cursor position
cursor, err = t.Cursor()
if err == nil {
it.last = cursor.String()
}
// Protect against bad queries
if it.done && len(it.buffer) == 0 {
clog.Warningf("Query did not return any results")
return false
}
// First result
it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
return true
}
func processPhotos(c context.Context, processor PhotoProcessor) error {
// use the full 10 minutes allowed (assuming front-end instance type)
c, _ = context.WithTimeout(c, time.Duration(10)*time.Minute)
r := processor.Start(c)
log.Debugf(c, "processPhotos from %s to %s cursor %s", r.From.Format(dateFormat), r.To.Format(dateFormat), r.Start)
// TODO: describe pros & cons of different querying + continuation strategies
q := datastore.NewQuery("photo")
q = q.Filter("taken >=", r.From)
q = q.Filter("taken <", r.To)
q = q.Order("taken")
// I use keys only because it saves on cost - entities come from memcache if possible
q = q.KeysOnly()
var cursor *datastore.Cursor
if r.Start != "" {
newCursor, err := datastore.DecodeCursor(r.Start)
if err != nil {
log.Errorf(c, "get start cursor error %s", err.Error())
return err
}
cursor = &newCursor
}
// only one entity is loaded at a time
p := new(Photo)
timeout := make(chan bool, 1)
timer := time.AfterFunc(r.Timeout, func() {
timeout <- true
})
defer timer.Stop()
Loop:
for {
// check if we've timed out or whether to keep going
select {
case <-timeout:
break Loop
default:
}
processed := 0
q = q.Limit(r.Size)
if cursor != nil {
q = q.Start(*cursor)
}
it := q.Run(c)
for {
// if not using keys only then we would load the actual entity here using
// key, err := it.Next(p)
key, err := it.Next(nil)
if err == datastore.Done {
break
}
if err != nil {
log.Errorf(c, "get key error %s", err.Error())
return err
}
// loads the actual entity from memcache / datastore
err = nds.Get(c, key, p)
if err != nil {
log.Errorf(c, "get photo error %s", err.Error())
return err
}
// call the processor with the entity
p.ID = key.IntID()
processor.Process(c, p)
processed++
}
// did we process a full batch? if so, there may be more
if processed == r.Size {
newCursor, err := it.Cursor()
if err != nil {
log.Errorf(c, "get next cursor error %s", err.Error())
return err
}
cursor = &newCursor
} else {
// otherwise we're finished
cursor = nil
break
}
}
// let the processor write any aggregation entries / tasks etc...
processor.Complete(c)
// if we didn't complete everything then continue from the cursor
if cursor != nil {
r.Start = cursor.String()
processPhotosFunc.Call(c, processor)
}
return nil
}
func (s *shard) iterate(c context.Context, mapper *mapper) (bool, error) {
// switch namespace
c, _ = appengine.Namespace(c, s.Namespace)
taskTimeout := time.After(mapper.config.TaskTimeout)
taskRunning := true
jobOutput, useJobOutput := s.job.JobSpec.(JobOutput)
if useJobOutput && s.job.Bucket != "" {
w, err := s.createOutputFile(c)
if err != nil {
return false, err
}
defer w.Close()
jobOutput.Output(w)
}
q := datastore.NewQuery(s.Query.kind)
for _, f := range s.Query.filter {
q = q.Filter(f.FieldName+" "+f.Op.String(), f.Value)
}
var cursor *datastore.Cursor
if s.Cursor != "" {
newCursor, err := datastore.DecodeCursor(s.Cursor)
if err != nil {
log.Errorf(c, "get start cursor error %s", err.Error())
return false, err
}
cursor = &newCursor
}
// what we'll load into if doing full entity loads (i.e. not doing KeysOnly)
var entity interface{}
// is full loading implemented?
jobEntity, useJobEntity := s.job.JobSpec.(JobEntity)
if useJobEntity {
entity = jobEntity.Make()
} else {
q = q.KeysOnly()
}
// main task loop to repeat datastore query with cursor
for taskRunning {
// if cursor is set, start the query at that point
if cursor != nil {
q = q.Start(*cursor)
}
// limit how long the cursor can run before we requery
cursorTimeout := time.After(mapper.config.CursorTimeout)
// datastore cursor context needs to run for the max allowed
cc, _ := context.WithTimeout(c, time.Duration(60)*time.Second)
it := q.Run(cc)
// item loop to iterate cursor
cursorLoop:
for {
key, err := it.Next(entity)
if err == datastore.Done {
// we reached the end
return true, nil
}
// TODO: option to fail or continue on individual errors
// or add error handling logic to job to give it a chance (?)
if err != nil {
log.Errorf(c, "key %v error %v", key, err)
// return false, err
continue cursorLoop
}
if err := s.job.JobSpec.Next(c, s.Counters, key); err != nil {
// TODO: instead of failing the entire slice, try to figure
// out if it's possible to continue from this point or maybe
// the last cursor position to avoid re-processing entities.
// NOTE: this would need to truncate any output file being
// written so entries weren't doubled up but maybe possible.
return false, err
}
s.Count++
select {
case <-taskTimeout:
// clearing the flag breaks us out of the task loop but also lets us update the
// cursor first when we break from the inner cursorLoop
taskRunning = false
break cursorLoop
default:
select {
case <-cursorTimeout:
// this forces a new cursor and query so we don't suffer from datastore timeouts
break cursorLoop
default:
// no timeout so carry on with the current cursor
continue cursorLoop
}
}
}
// we need to get the cursor for where we are upto whether we are requerying
// within this task or scheduling a new continuation slice
newCursor, err := it.Cursor()
if err != nil {
log.Errorf(c, "get next cursor error %s", err.Error())
return false, err
}
cursor = &newCursor
s.Cursor = cursor.String()
}
return false, nil
}