func TestSyncInvalidKeyIntervals(t *testing.T) {
c := SetupEnv(t)
defer c.Close()
var start, end *datastore.Key
ingested, last, err := bigquerysync.SyncKeyRange(c, "project", "dataset", start, end, "")
if err == nil {
t.Errorf("Missing error for nil start key")
}
if ingested != 0 {
t.Errorf("Ingested %d entities for nil start and end keys", ingested)
}
if last != nil {
t.Errorf("Unexpected last %s (expected nil)", last)
}
}
func TestSyncSingleEntity(t *testing.T) {
c := SetupEnv(t)
defer c.Close()
start := datastore.NewKey(c, "Sample", "", 2, nil)
end := datastore.NewKey(c, "Sample", "", 2, nil)
ingested, last, err := bigquerysync.SyncKeyRange(c, "project", "dataset", start, end, "")
if err != nil {
t.Errorf("Unexpected failure: %s", err.Error())
}
if ingested != 1 {
t.Errorf("More than one entities ingested: %d", ingested)
}
if !last.Equal(end) {
t.Errorf("Unexpected last %s (expected %s)", last, end)
}
}
func TestSyncKeyRangeWithOpenEnd(t *testing.T) {
c := SetupEnv(t)
defer c.Close()
var start, end *datastore.Key
start = datastore.NewKey(c, "Sample", "", 1, nil)
ingested, last, err := bigquerysync.SyncKeyRange(c, "project", "dataset", start, end, "")
if err != nil {
t.Errorf("Unexpected failure: %s", err.Error())
}
if ingested != 3 {
t.Errorf("Unexpected ammount of ingested entities: %d, expected: %d", ingested, 3)
}
if !last.Equal(end) {
t.Errorf("Unexpected last %s (expected %s)", last, end)
}
}
func TestSyncExplicitKeyRange(t *testing.T) {
c := SetupEnv(t)
defer c.Close()
start := datastore.NewKey(c, "Sample", "", 1, nil)
end := datastore.NewKey(c, "Sample", "", 3, nil)
ingested, last, err := bigquerysync.SyncKeyRange(c, "project", "dataset", start, end, "")
if err != nil {
t.Errorf("Unexpected failure: %s", err.Error())
}
if ingested != 2 {
t.Errorf("Unexpected ammount of ingested entities: %d, expected %d", ingested, 2)
}
if !last.Equal(end) {
t.Errorf("Unexpected last %s (expected %s)", last, end)
}
}
// SyncEntityHandler synchronizes a range of entity keys. This handler
// expects the same parameters as SyncKindHandler, except for "kind".
// Instead of kind, you have to specify a mandatory "startKey", and optionally
// an "endKey".
//
// If specified, both startKey and endKey must be URL Encoded complete datastore
// keys. The start is inclusive and all entities up to end (exclusive) will be
// synced. If end is empty, or an invalid key, all entities following start are
// synced, until there is no more entities. If start and end are equal, then only
// one entity is synced.
//
// To optimize memory usage, this handler processes up to bigqueysync.BatchSize
// entities, and if the query is not done, reschedule itself with the last
// processed key as start. Also, in order to keep the payload sent to Bigquery
// under the URLFetch limit, the entities are processed in small chuncks of 9
// entities.
func SyncEntityHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
err := r.ParseForm()
if err != nil {
errorf(c, w, 400, "Invalid request: %v", err)
return
}
var (
start = decodeKey(r.Form.Get("startKey"))
end = decodeKey(r.Form.Get("endKey"))
p = r.Form.Get("project")
d = r.Form.Get("dataset")
e = r.Form.Get("exclude")
q = r.Form.Get("queue")
last *datastore.Key
)
if start == nil {
errorf(c, w, http.StatusBadRequest, "Start key can't be nil.")
return
}
if p == "" || d == "" {
errorf(c, w, http.StatusBadRequest, "Invalid project/dataset: %s/%d", p, d)
return
}
tpl := page{resp: w, req: r}
count, last, err := bigquerysync.SyncKeyRange(c, p, d, start, end, e)
// Error running
if err != nil {
err := fmt.Errorf("bundle: error in SyncKeyRange(%s, %s): %d, %s:\n%v", start, end, count, last, err)
tpl.ServerError(err)
return
}
// Range is not done, let's reschedule from last key.
if !last.Equal(end) {
err := scheduleRangeSync(c, w, last, end, p, d, e, q)
if err != nil {
errorf(c, w, 500, "Error in schedule next range: %v", err)
}
return
}
infof(c, w, "Range synced sucessfully [%s,%s[. %d entities synced.", start, end, count)
}