Esempio n. 1
0
// HasParent returns true if key or any of its parents equals
// parent (recursively), otherwise false.
func HasParent(parent, key *datastore.Key) bool {
	for key != nil {
		if key.Equal(parent) {
			return true
		}
		key = key.Parent()
	}
	return false
}
Esempio n. 2
0
// SyncEntityHandler synchronizes a range of entity keys. This handler
// expects the same parameters as SyncKindHandler, except for "kind".
// Instead of kind, you have to specify a mandatory "startKey", and optionally
// an "endKey".
//
// If specified, both startKey and endKey must be URL Encoded complete datastore
// keys. The start is inclusive and all entities up to end (exclusive) will be
// synced. If end is empty, or an invalid key, all entities following start are
// synced, until there is no more entities. If start and end are equal, then only
// one entity is synced.
//
// To optimize memory usage, this handler processes up to bigqueysync.BatchSize
// entities, and if the query is not done, reschedule itself with the last
// processed key as start. Also, in order to keep the payload sent to Bigquery
// under the URLFetch limit, the entities are processed in small chuncks of 9
// entities.
func SyncEntityHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	err := r.ParseForm()
	if err != nil {
		errorf(c, w, 400, "Invalid request: %v", err)
		return
	}
	var (
		start = decodeKey(r.Form.Get("startKey"))
		end   = decodeKey(r.Form.Get("endKey"))
		p     = r.Form.Get("project")
		d     = r.Form.Get("dataset")
		e     = r.Form.Get("exclude")
		q     = r.Form.Get("queue")
		last  *datastore.Key
	)
	if start == nil {
		errorf(c, w, http.StatusBadRequest, "Start key can't be nil.")
		return
	}
	if p == "" || d == "" {
		errorf(c, w, http.StatusBadRequest, "Invalid project/dataset: %s/%d", p, d)
		return
	}
	tpl := page{resp: w, req: r}

	count, last, err := bigquerysync.SyncKeyRange(c, p, d, start, end, e)
	// Error running
	if err != nil {
		err := fmt.Errorf("bundle: error in SyncKeyRange(%s, %s): %d, %s:\n%v", start, end, count, last, err)
		tpl.ServerError(err)
		return
	}
	// Range is not done, let's reschedule from last key.
	if !last.Equal(end) {
		err := scheduleRangeSync(c, w, last, end, p, d, e, q)
		if err != nil {
			errorf(c, w, 500, "Error in schedule next range: %v", err)
		}
		return
	}

	infof(c, w, "Range synced sucessfully [%s,%s[. %d entities synced.", start, end, count)
}
Esempio n. 3
0
// createQuery builds a range query using start and end. It works
// for [start,end[, [start,nil] and [start,start] intervals. The
// returned query is sorted by __key__ and limited to BatchSize.
func createQuery(start, end *datastore.Key, cur datastore.Cursor) *datastore.Query {
	q := datastore.NewQuery(start.Kind())

	if start.Equal(end) {
		q = q.Filter("__key__ =", start)
	} else {
		q = q.Filter("__key__ >=", start)
		if end != nil {
			q = q.Filter("__key__ <", end)
		}
	}

	if cur.String() != "" {
		q = q.Start(cur)
	}

	q = q.Order("__key__")
	return q
}