// HasParent returns true if key or any of its parents equals
// parent (recursively), otherwise false.
func HasParent(parent, key *datastore.Key) bool {
for key != nil {
if key.Equal(parent) {
return true
}
key = key.Parent()
}
return false
}
// SyncEntityHandler synchronizes a range of entity keys. This handler
// expects the same parameters as SyncKindHandler, except for "kind".
// Instead of kind, you have to specify a mandatory "startKey", and optionally
// an "endKey".
//
// If specified, both startKey and endKey must be URL Encoded complete datastore
// keys. The start is inclusive and all entities up to end (exclusive) will be
// synced. If end is empty, or an invalid key, all entities following start are
// synced, until there is no more entities. If start and end are equal, then only
// one entity is synced.
//
// To optimize memory usage, this handler processes up to bigqueysync.BatchSize
// entities, and if the query is not done, reschedule itself with the last
// processed key as start. Also, in order to keep the payload sent to Bigquery
// under the URLFetch limit, the entities are processed in small chuncks of 9
// entities.
func SyncEntityHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
err := r.ParseForm()
if err != nil {
errorf(c, w, 400, "Invalid request: %v", err)
return
}
var (
start = decodeKey(r.Form.Get("startKey"))
end = decodeKey(r.Form.Get("endKey"))
p = r.Form.Get("project")
d = r.Form.Get("dataset")
e = r.Form.Get("exclude")
q = r.Form.Get("queue")
last *datastore.Key
)
if start == nil {
errorf(c, w, http.StatusBadRequest, "Start key can't be nil.")
return
}
if p == "" || d == "" {
errorf(c, w, http.StatusBadRequest, "Invalid project/dataset: %s/%d", p, d)
return
}
tpl := page{resp: w, req: r}
count, last, err := bigquerysync.SyncKeyRange(c, p, d, start, end, e)
// Error running
if err != nil {
err := fmt.Errorf("bundle: error in SyncKeyRange(%s, %s): %d, %s:\n%v", start, end, count, last, err)
tpl.ServerError(err)
return
}
// Range is not done, let's reschedule from last key.
if !last.Equal(end) {
err := scheduleRangeSync(c, w, last, end, p, d, e, q)
if err != nil {
errorf(c, w, 500, "Error in schedule next range: %v", err)
}
return
}
infof(c, w, "Range synced sucessfully [%s,%s[. %d entities synced.", start, end, count)
}
// createQuery builds a range query using start and end. It works
// for [start,end[, [start,nil] and [start,start] intervals. The
// returned query is sorted by __key__ and limited to BatchSize.
func createQuery(start, end *datastore.Key, cur datastore.Cursor) *datastore.Query {
q := datastore.NewQuery(start.Kind())
if start.Equal(end) {
q = q.Filter("__key__ =", start)
} else {
q = q.Filter("__key__ >=", start)
if end != nil {
q = q.Filter("__key__ <", end)
}
}
if cur.String() != "" {
q = q.Start(cur)
}
q = q.Order("__key__")
return q
}
