// SyncKindHandler spawn task queues for paralell synchronization of all entities // in a specific Kind. This handler requires the form parameters "project" and // "dataset", as well as the "kind" parameter. // // The following path synchronizes all entities with kind "Baz", into a // table named "Baz", under the dataset "bar" in the "foo" project: // // /bq/sync/kind?project=foo&dataset=bar&kind=Baz // // This handler also supports the "exclude" optional parameter, that is a regular // expression to exclude field names. For example, the following path will // do the same as the previous one, and also will filter the properties starting // with lowercase "a", "b" or that contains "foo" in their names: // // /bq/sync/kind?project=foo&dataset=bar&kind=Baz&exclude=^a.*|^b.*|foo // // This is particulary usefull on large entities, to skip long text fields // and keep the request of each row under the Bigquery limit of 20Kb. // // Finally, the "queue" parameter can be specified to target a specific task // queue to run all sync jobs. func SyncKindHandler(w http.ResponseWriter, r *http.Request) { c := appengine.NewContext(r) err := r.ParseForm() if err != nil { errorf(c, w, 400, "Invalid request: %v", err) return } var ( p = r.Form.Get("project") d = r.Form.Get("dataset") k = r.Form.Get("kind") e = r.Form.Get("exclude") q = r.Form.Get("queue") ) if p == "" || d == "" || k == "" { errorf(c, w, 400, "Invalid parameters: project='%s', dataset='%s', kind='%s'", p, d, k) } ranges := bigquerysync.KeyRangesForKind(c, k) infof(c, w, "Ranges: %v\n", ranges) for _, r := range ranges { scheduleRangeSync(c, w, r.Start, r.End, p, d, e, q) } }
func TestKeyRangeForKind(t *testing.T) { c, clean, err := aetest.NewContext() if err != nil { t.Fatal(err) } defer clean() // No entities: empty range ranges := bigquerysync.KeyRangesForKind(c, "RangeTest") if len(ranges) != 0 { t.Errorf("Unexpected ranges returned: %d, expected 0: %#v", len(ranges), ranges) } // Setup datastore - __scatter__ is replaced with _scatter__ for testing aetools.LoadJSON(c, SampleEntities, aetools.LoadSync) // No scatter, single range ranges = bigquerysync.KeyRangesForKind(c, "Sample") if len(ranges) != 1 { t.Errorf("Unexpected ranges without scatters: %v, expected length 1", ranges) } else { if ranges[0].Start == nil { t.Errorf("Unexpected nil start for Sample") } else { if ranges[0].Start.IntID() != 1 || ranges[0].Start.Kind() != "Sample" { t.Errorf("Unexpected start key for Sample: %#v", ranges[0].Start) } if ranges[0].End != nil { t.Errorf("Unexpected end key for Sample: %#v, expected nil", ranges[0].End) } } } // Scattered entities: sorted key ranges expected ranges = bigquerysync.KeyRangesForKind(c, "RangeTest") expected := []struct { Start int64 End int64 }{ {1, 30}, {30, 50}, {50, 1000}, {1000, 0}, } if len(ranges) != len(expected) { t.Errorf("Unexpected ranges with scatter: %d, expected 3", len(ranges)) } else { for i, e := range expected { r := ranges[i] if r.Start == nil { t.Errorf("Unexpected nil start at range %d", i) } else if r.Start.IntID() != e.Start { t.Errorf("Unexpected start at range %d: %#v, expected %d", i, r.Start.IntID(), e.Start) } if e.End == 0 { if r.End != nil { t.Errorf("Unexpected end at range %d: %#v, expected nil", i, r.End) } } else if r.End == nil { t.Errorf("Unexpected nil end at range %d, expected %d", i, r.End, e.End) } else if r.End.IntID() != e.End { t.Errorf("Unexpected end at range %d: %#v, expected %d", i, r.End.IntID(), e.End) } } // Check if all entity keys match for i, r := range ranges { if r.Start != nil { if r.Start.Kind() != "RangeTest" { t.Errorf("Unexpected kind at range %d: %s", i, r.Start.Kind()) } } if r.End != nil { if r.End.Kind() != "RangeTest" { t.Errorf("Unexpected kind at range %d: %s", i, r.End.Kind()) } } } } }