Example #1
0
func executeItemQuery(con *Context, q *datastore.Query, limit int, cursorStr string) ([]Item, string, error) {
	if cursor, err := datastore.DecodeCursor(cursorStr); err == nil {
		q = q.Start(cursor)
	}

	var is = make([]Item, 0, limit)
	var err error
	t := q.Run(con.C)
	for {
		var i Item
		_, err = t.Next(&i)
		if err == datastore.Done {
			break
		}

		is = append(is, i)
		if err != nil {
			con.Log.Errorf("Error fetching next item: %v", err)
			return nil, "", err
		}
	}

	var cursor datastore.Cursor
	if cursor, err = t.Cursor(); err == nil {
		return is, cursor.String(), nil
	}

	return nil, "", err
}
Example #2
0
func (qm *QueryMarker) UnmarshalJSON(buf []byte) error {
	if len(buf) < 2 || buf[0] != '"' || buf[len(buf)-1] != '"' {
		return errors.New("QueryMarker: bad cursor value")
	}
	cursor, err := datastore.DecodeCursor(string(buf[1 : len(buf)-1]))
	if err != nil {
		return err
	}
	*qm = QueryMarker{cursor}
	return nil
}
Example #3
0
func decodeDatastoreCursorValue(d *Decoder, v reflect.Value) error {
	s, err := d.DecodeString()
	if err != nil {
		return err
	}
	cursor, err := ds.DecodeCursor(s)
	if err != nil {
		return err
	}
	v.Set(reflect.ValueOf(cursor))
	return nil
}
func init() {
	reindexDelayer = delay.Func("reindex-idioms", func(c context.Context, cursorStr string) error {
		q := datastore.NewQuery("Idiom")
		if cursorStr != "" {
			log.Infof(c, "Starting at cursor %v", cursorStr)
			cursor, err := datastore.DecodeCursor(cursorStr)
			if err != nil {
				return err
			}
			q = q.Start(cursor)
		}
		iterator := q.Run(c)

		reindexedIDs := make([]int, 0, reindexBatchSize)
		defer func() {
			log.Infof(c, "Reindexed idioms %v", reindexedIDs)
		}()

		for i := 0; i < reindexBatchSize; i++ {
			var idiom Idiom
			key, err := iterator.Next(&idiom)
			if err == datastore.Done {
				log.Infof(c, "Reindexing completed.")
				return nil
			} else if err != nil {
				// ouch :(
				return err
			}

			err = indexIdiomFullText(c, &idiom, key)
			if err != nil {
				log.Errorf(c, "Reindexing full text idiom %d : %v", idiom.Id, err)
			}
			err = indexIdiomCheatsheets(c, &idiom)
			if err != nil {
				log.Errorf(c, "Reindexing cheatsheet of idiom %d : %v", idiom.Id, err)
			}

			reindexedIDs = append(reindexedIDs, idiom.Id)
		}

		cursor, err := iterator.Cursor()
		if err != nil {
			// ouch :(
			return err
		}
		log.Infof(c, "Stopping at cursor %v", cursor.String())
		reindexDelayer.Call(c, cursor.String())
		return nil
	})
}
// MakeDatastoreQuery returns a datastore.Query that generates all namespaces in the range
func (n *NamespaceRange) MakeDatastoreQuery(c context.Context, start string) *datastore.Query {
	q := datastore.NewQuery(namespaceKind)
	if n.Start != "" {
		q = q.Filter("__key__ >=", datastore.NewKey(c, namespaceKind, n.Start, 0, nil))
	}
	q = q.Filter("__key__ <=", datastore.NewKey(c, namespaceKind, n.End, 0, nil))
	q = q.Order("__key__")
	q = q.KeysOnly()
	if start != "" {
		cursor, _ := datastore.DecodeCursor(start)
		q = q.Start(cursor)
	}
	return q
}
// 画像のメタデータ一覧をDSから取得します。
// TODO: 表示する画像数を絞る必要がないなら、Cursor必要ないかも。
func GetImages(c context.Context, cursorStr string) ([]File, string, error) {
	parentKey := GetParentKey(c)
	q := datastore.NewQuery(KIND_FILE).Ancestor(parentKey).Filter("Type =", IMAGE).Order("-CreatedAt")

	if len(cursorStr) != 0 {
		cursor, err := datastore.DecodeCursor(cursorStr)
		if err != nil {
			return []File{}, "", err
		}

		q = q.Start(cursor)
	}

	images := []File{}
	iter := q.Run(c)
	isNext := true
	for {
		var img File
		_, err := iter.Next(&img)
		if err == datastore.Done {
			isNext = false
			break
		}
		if err != nil {
			log.Errorf(c, "fetching next File: %s", err.Error())
			break
		}

		err = img.setThumbnailURL(thumbnailsLongestSide, false)
		if err != nil {
			log.Errorf(c, "%s", err.Error())
			break
		}
		images = append(images, img)
	}

	if isNext {
		next_cursor, err := iter.Cursor()
		if err != nil {
			log.Errorf(c, "%s", err.Error())
			return []File{}, "", err
		}
		return images, next_cursor.String(), nil
	} else {
		return images, "", nil
	}
}
Example #7
0
func example20() {
	var ctx context.Context

	// [START cursors]
	// Create a query for all Person entities.
	q := datastore.NewQuery("Person")

	// If the application stored a cursor during a previous request, use it.
	item, err := memcache.Get(ctx, "person_cursor")
	if err == nil {
		cursor, err := datastore.DecodeCursor(string(item.Value))
		if err == nil {
			q = q.Start(cursor)
		}
	}

	// Iterate over the results.
	t := q.Run(ctx)
	for {
		var p Person
		_, err := t.Next(&p)
		if err == datastore.Done {
			break
		}
		if err != nil {
			log.Errorf(ctx, "fetching next Person: %v", err)
			break
		}
		// Do something with the Person p
	}

	// Get updated cursor and store it for next time.
	if cursor, err := t.Cursor(); err == nil {
		memcache.Set(ctx, &memcache.Item{
			Key:   "person_cursor",
			Value: []byte(cursor.String()),
		})
	}
	// [END cursors]
}
func process(c context.Context, processor Processor, start string) error {
	// use the full 10 minutes allowed (assuming front-end instance type)
	c, _ = context.WithTimeout(c, time.Duration(10)*time.Minute)

	// get the query to iterate and the entity slot to load (could be nill for keys_only)
	q, e := processor.Start(c)

	var cursor *datastore.Cursor
	if start != "" {
		newCursor, err := datastore.DecodeCursor(start)
		if err != nil {
			log.Errorf(c, "get start cursor error %s", err.Error())
			return err
		}
		cursor = &newCursor
	}

	// signal a timeout after 5 minutes
	timeout := make(chan bool, 1)
	timer := time.AfterFunc(time.Duration(5)*time.Minute, func() {
		timeout <- true
	})
	defer timer.Stop()

	// TODO: error handling to retry
Loop:
	for {
		// check if we've timed out or whether to keep going
		select {
		case <-timeout:
			break Loop
		default:
		}

		processed := 0

		if cursor != nil {
			q = q.Start(*cursor)
		}
		it := q.Run(c)
		for {
			key, err := it.Next(e)
			if err == datastore.Done {
				break
			}
			if err != nil {
				log.Errorf(c, "get key error %s", err.Error())
				return err
			}

			processor.Process(c, key)
			processed++
		}

		// did we process any?
		if processed > 0 {
			newCursor, err := it.Cursor()
			if err != nil {
				log.Errorf(c, "get next cursor error %s", err.Error())
				return err
			}
			cursor = &newCursor
		} else {
			// otherwise we're finished
			cursor = nil
			break
		}
	}

	// let the processor write any aggregation entries / tasks etc...
	processor.Complete(c)

	// if we didn't complete everything then continue from the cursor
	if cursor != nil {
		processFunc.Call(c, processor, cursor.String())
	}

	return nil
}
func (it *iterator) iterate(c context.Context, mapper *mapper) (bool, error) {
	taskTimeout := time.After(mapper.config.TaskTimeout)
	taskRunning := true

	// if the query defines the specific namespaces to process
	// then we can just process that list directly
	if it.Query.selection == selected {
		for _, namespace := range it.Query.namespaces {
			it.process(c, mapper, namespace)
		}
		return true, nil
	}

	q := it.createQuery(c)

	var cursor *datastore.Cursor
	if it.Cursor != "" {
		newCursor, err := datastore.DecodeCursor(it.Cursor)
		if err != nil {
			log.Errorf(c, "get start cursor error %s", err.Error())
			return false, err
		}
		cursor = &newCursor
	}

	// main task loop to repeat datastore query with cursor
	for taskRunning {

		// if cursor is set, start the query at that point
		if cursor != nil {
			q = q.Start(*cursor)
		}

		// limit how long the cursor can run before we requery
		cursorTimeout := time.After(mapper.config.CursorTimeout)
		// datastore cursor context needs to run for the max allowed
		cc, _ := context.WithTimeout(c, time.Duration(60)*time.Second)
		t := q.Run(cc)

		// item loop to iterate cursor
	cursorLoop:
		for {
			key, err := t.Next(nil)
			if err == datastore.Done {
				// we reached the end
				return true, nil
			}

			if err != nil {
				log.Errorf(c, "error %s", err.Error())
				return false, err
			}

			namespace := key.StringID()
			if err := it.process(c, mapper, namespace); err != nil {
				return false, err
			}

			select {
			case <-taskTimeout:
				// clearing the flag breaks us out of the task loop but also lets us update the
				// cursor first when we break from the inner cursorLoop
				taskRunning = false
				break cursorLoop
			default:
				select {
				case <-cursorTimeout:
					// this forces a new cursor and query so we don't suffer from datastore timeouts
					break cursorLoop
				default:
					// no timeout so carry on with the current cursor
					continue cursorLoop
				}
			}
		}

		// we need to get the cursor for where we are upto whether we are requerying
		// within this task or scheduling a new continuation slice
		newCursor, err := t.Cursor()
		if err != nil {
			log.Errorf(c, "get next cursor error %s", err.Error())
			return false, err
		}
		cursor = &newCursor
		it.Cursor = cursor.String()
	}

	return false, nil
}
Example #10
0
func (d rdsImpl) DecodeCursor(s string) (ds.Cursor, error) {
	return datastore.DecodeCursor(s)
}
Example #11
0
func (it *Iterator) Next() bool {
	if it.offset+1 < len(it.buffer) {
		it.offset++
		it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
		return true
	}
	if it.done {
		return false
	}
	// Reset buffer and offset
	it.offset = 0
	it.buffer = make([]string, 0, bufferSize)
	// Create query
	// TODO (panamafrancis) Keys only query?
	q := datastore.NewQuery(it.kind).Limit(bufferSize)
	if !it.isAll {
		// Filter on the direction {subject,objekt...}
		q = q.Filter(it.dir.String()+" =", it.name)
	}
	// Get last cursor position
	cursor, err := datastore.DecodeCursor(it.last)
	if err == nil {
		q = q.Start(cursor)
	}
	// Buffer the keys of the next 50 matches
	t := q.Run(it.qs.context)
	for {
		// Quirk of the datastore, you cannot pass a nil value to to Next()
		// even if you just want the keys
		var k *datastore.Key
		skip := false
		if it.kind == quadKind {
			temp := new(QuadEntry)
			k, err = t.Next(temp)
			// Skip if quad has been deleted
			if len(temp.Added) <= len(temp.Deleted) {
				skip = true
			}
		} else {
			temp := new(NodeEntry)
			k, err = t.Next(temp)
			// Skip if node has been deleted
			if temp.Size == 0 {
				skip = true
			}
		}
		if err == datastore.Done {
			it.done = true
			break
		}
		if err != nil {
			clog.Errorf("Error fetching next entry %v", err)
			it.err = err
			return false
		}
		if !skip {
			it.buffer = append(it.buffer, k.StringID())
		}
	}
	// Save cursor position
	cursor, err = t.Cursor()
	if err == nil {
		it.last = cursor.String()
	}
	// Protect against bad queries
	if it.done && len(it.buffer) == 0 {
		clog.Warningf("Query did not return any results")
		return false
	}
	// First result
	it.result = &Token{Kind: it.kind, Hash: it.buffer[it.offset]}
	return true
}
func processPhotos(c context.Context, processor PhotoProcessor) error {
	// use the full 10 minutes allowed (assuming front-end instance type)
	c, _ = context.WithTimeout(c, time.Duration(10)*time.Minute)

	r := processor.Start(c)
	log.Debugf(c, "processPhotos from %s to %s cursor %s", r.From.Format(dateFormat), r.To.Format(dateFormat), r.Start)

	// TODO: describe pros & cons of different querying + continuation strategies

	q := datastore.NewQuery("photo")
	q = q.Filter("taken >=", r.From)
	q = q.Filter("taken <", r.To)
	q = q.Order("taken")

	// I use keys only because it saves on cost - entities come from memcache if possible
	q = q.KeysOnly()

	var cursor *datastore.Cursor
	if r.Start != "" {
		newCursor, err := datastore.DecodeCursor(r.Start)
		if err != nil {
			log.Errorf(c, "get start cursor error %s", err.Error())
			return err
		}
		cursor = &newCursor
	}

	// only one entity is loaded at a time
	p := new(Photo)

	timeout := make(chan bool, 1)
	timer := time.AfterFunc(r.Timeout, func() {
		timeout <- true
	})
	defer timer.Stop()

Loop:
	for {
		// check if we've timed out or whether to keep going
		select {
		case <-timeout:
			break Loop
		default:
		}

		processed := 0

		q = q.Limit(r.Size)
		if cursor != nil {
			q = q.Start(*cursor)
		}
		it := q.Run(c)
		for {
			// if not using keys only then we would load the actual entity here using
			// key, err := it.Next(p)
			key, err := it.Next(nil)
			if err == datastore.Done {
				break
			}
			if err != nil {
				log.Errorf(c, "get key error %s", err.Error())
				return err
			}

			// loads the actual entity from memcache / datastore
			err = nds.Get(c, key, p)
			if err != nil {
				log.Errorf(c, "get photo error %s", err.Error())
				return err
			}

			// call the processor with the entity
			p.ID = key.IntID()
			processor.Process(c, p)

			processed++
		}

		// did we process a full batch? if so, there may be more
		if processed == r.Size {
			newCursor, err := it.Cursor()
			if err != nil {
				log.Errorf(c, "get next cursor error %s", err.Error())
				return err
			}
			cursor = &newCursor
		} else {
			// otherwise we're finished
			cursor = nil
			break
		}
	}

	// let the processor write any aggregation entries / tasks etc...
	processor.Complete(c)

	// if we didn't complete everything then continue from the cursor
	if cursor != nil {
		r.Start = cursor.String()
		processPhotosFunc.Call(c, processor)
	}

	return nil
}
Example #13
0
func (s *shard) iterate(c context.Context, mapper *mapper) (bool, error) {
	// switch namespace
	c, _ = appengine.Namespace(c, s.Namespace)

	taskTimeout := time.After(mapper.config.TaskTimeout)
	taskRunning := true

	jobOutput, useJobOutput := s.job.JobSpec.(JobOutput)
	if useJobOutput && s.job.Bucket != "" {
		w, err := s.createOutputFile(c)
		if err != nil {
			return false, err
		}
		defer w.Close()
		jobOutput.Output(w)
	}

	q := datastore.NewQuery(s.Query.kind)
	for _, f := range s.Query.filter {
		q = q.Filter(f.FieldName+" "+f.Op.String(), f.Value)
	}

	var cursor *datastore.Cursor
	if s.Cursor != "" {
		newCursor, err := datastore.DecodeCursor(s.Cursor)
		if err != nil {
			log.Errorf(c, "get start cursor error %s", err.Error())
			return false, err
		}
		cursor = &newCursor
	}

	// what we'll load into if doing full entity loads (i.e. not doing KeysOnly)
	var entity interface{}

	// is full loading implemented?
	jobEntity, useJobEntity := s.job.JobSpec.(JobEntity)
	if useJobEntity {
		entity = jobEntity.Make()
	} else {
		q = q.KeysOnly()
	}

	// main task loop to repeat datastore query with cursor
	for taskRunning {

		// if cursor is set, start the query at that point
		if cursor != nil {
			q = q.Start(*cursor)
		}

		// limit how long the cursor can run before we requery
		cursorTimeout := time.After(mapper.config.CursorTimeout)
		// datastore cursor context needs to run for the max allowed
		cc, _ := context.WithTimeout(c, time.Duration(60)*time.Second)
		it := q.Run(cc)

		// item loop to iterate cursor
	cursorLoop:
		for {
			key, err := it.Next(entity)
			if err == datastore.Done {
				// we reached the end
				return true, nil
			}

			// TODO: option to fail or continue on individual errors
			// or add error handling logic to job to give it a chance (?)
			if err != nil {
				log.Errorf(c, "key %v error %v", key, err)
				// return false, err
				continue cursorLoop
			}

			if err := s.job.JobSpec.Next(c, s.Counters, key); err != nil {
				// TODO: instead of failing the entire slice, try to figure
				// out if it's possible to continue from this point or maybe
				// the last cursor position to avoid re-processing entities.
				// NOTE: this would need to truncate any output file being
				// written so entries weren't doubled up but maybe possible.
				return false, err
			}
			s.Count++

			select {
			case <-taskTimeout:
				// clearing the flag breaks us out of the task loop but also lets us update the
				// cursor first when we break from the inner cursorLoop
				taskRunning = false
				break cursorLoop
			default:
				select {
				case <-cursorTimeout:
					// this forces a new cursor and query so we don't suffer from datastore timeouts
					break cursorLoop
				default:
					// no timeout so carry on with the current cursor
					continue cursorLoop
				}
			}
		}

		// we need to get the cursor for where we are upto whether we are requerying
		// within this task or scheduling a new continuation slice
		newCursor, err := it.Cursor()
		if err != nil {
			log.Errorf(c, "get next cursor error %s", err.Error())
			return false, err
		}
		cursor = &newCursor
		s.Cursor = cursor.String()
	}

	return false, nil
}