// SyncKindHandler spawn task queues for paralell synchronization of all entities
// in a specific Kind. This handler requires the form parameters "project" and
// "dataset", as well as the "kind" parameter.
//
// The following path synchronizes all entities with kind "Baz", into a
// table named "Baz", under the dataset "bar" in the "foo" project:
//
// /bq/sync/kind?project=foo&dataset=bar&kind=Baz
//
// This handler also supports the "exclude" optional parameter, that is a regular
// expression to exclude field names. For example, the following path will
// do the same as the previous one, and also will filter the properties starting
// with lowercase "a", "b" or that contains "foo" in their names:
//
// /bq/sync/kind?project=foo&dataset=bar&kind=Baz&exclude=^a.*|^b.*|foo
//
// This is particulary usefull on large entities, to skip long text fields
// and keep the request of each row under the Bigquery limit of 20Kb.
//
// Finally, the "queue" parameter can be specified to target a specific task
// queue to run all sync jobs.
func SyncKindHandler(w http.ResponseWriter, r *http.Request) {
c := appengine.NewContext(r)
err := r.ParseForm()
if err != nil {
errorf(c, w, 400, "Invalid request: %v", err)
return
}
var (
p = r.Form.Get("project")
d = r.Form.Get("dataset")
k = r.Form.Get("kind")
e = r.Form.Get("exclude")
q = r.Form.Get("queue")
)
if p == "" || d == "" || k == "" {
errorf(c, w, 400, "Invalid parameters: project='%s', dataset='%s', kind='%s'", p, d, k)
}
ranges := bigquerysync.KeyRangesForKind(c, k)
infof(c, w, "Ranges: %v\n", ranges)
for _, r := range ranges {
scheduleRangeSync(c, w, r.Start, r.End, p, d, e, q)
}
}
func TestKeyRangeForKind(t *testing.T) {
c, clean, err := aetest.NewContext()
if err != nil {
t.Fatal(err)
}
defer clean()
// No entities: empty range
ranges := bigquerysync.KeyRangesForKind(c, "RangeTest")
if len(ranges) != 0 {
t.Errorf("Unexpected ranges returned: %d, expected 0: %#v", len(ranges), ranges)
}
// Setup datastore - __scatter__ is replaced with _scatter__ for testing
aetools.LoadJSON(c, SampleEntities, aetools.LoadSync)
// No scatter, single range
ranges = bigquerysync.KeyRangesForKind(c, "Sample")
if len(ranges) != 1 {
t.Errorf("Unexpected ranges without scatters: %v, expected length 1", ranges)
} else {
if ranges[0].Start == nil {
t.Errorf("Unexpected nil start for Sample")
} else {
if ranges[0].Start.IntID() != 1 || ranges[0].Start.Kind() != "Sample" {
t.Errorf("Unexpected start key for Sample: %#v", ranges[0].Start)
}
if ranges[0].End != nil {
t.Errorf("Unexpected end key for Sample: %#v, expected nil", ranges[0].End)
}
}
}
// Scattered entities: sorted key ranges expected
ranges = bigquerysync.KeyRangesForKind(c, "RangeTest")
expected := []struct {
Start int64
End int64
}{
{1, 30},
{30, 50},
{50, 1000},
{1000, 0},
}
if len(ranges) != len(expected) {
t.Errorf("Unexpected ranges with scatter: %d, expected 3", len(ranges))
} else {
for i, e := range expected {
r := ranges[i]
if r.Start == nil {
t.Errorf("Unexpected nil start at range %d", i)
} else if r.Start.IntID() != e.Start {
t.Errorf("Unexpected start at range %d: %#v, expected %d", i, r.Start.IntID(), e.Start)
}
if e.End == 0 {
if r.End != nil {
t.Errorf("Unexpected end at range %d: %#v, expected nil", i, r.End)
}
} else if r.End == nil {
t.Errorf("Unexpected nil end at range %d, expected %d", i, r.End, e.End)
} else if r.End.IntID() != e.End {
t.Errorf("Unexpected end at range %d: %#v, expected %d", i, r.End.IntID(), e.End)
}
}
// Check if all entity keys match
for i, r := range ranges {
if r.Start != nil {
if r.Start.Kind() != "RangeTest" {
t.Errorf("Unexpected kind at range %d: %s", i, r.Start.Kind())
}
}
if r.End != nil {
if r.End.Kind() != "RangeTest" {
t.Errorf("Unexpected kind at range %d: %s", i, r.End.Kind())
}
}
}
}
}