コード例 #1
0
ファイル: bundle.go プロジェクト: ronoaldo/aetools
// SyncKindHandler spawn task queues for paralell synchronization of all entities
// in a specific Kind. This handler requires the form parameters "project" and
// "dataset", as well as the "kind" parameter.
//
// The following path synchronizes all entities with kind "Baz", into a
// table named "Baz", under the dataset "bar" in the "foo" project:
//
//	/bq/sync/kind?project=foo&dataset=bar&kind=Baz
//
// This handler also supports the "exclude" optional parameter, that is a regular
// expression to exclude field names. For example, the following path will
// do the same as the previous one, and also will filter the properties starting
// with lowercase "a", "b" or that contains "foo" in their names:
//
//	/bq/sync/kind?project=foo&dataset=bar&kind=Baz&exclude=^a.*|^b.*|foo
//
// This is particulary usefull on large entities, to skip long text fields
// and keep the request of each row under the Bigquery limit of 20Kb.
//
// Finally, the "queue" parameter can be specified to target a specific task
// queue to run all sync jobs.
func SyncKindHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	err := r.ParseForm()
	if err != nil {
		errorf(c, w, 400, "Invalid request: %v", err)
		return
	}
	var (
		p = r.Form.Get("project")
		d = r.Form.Get("dataset")
		k = r.Form.Get("kind")
		e = r.Form.Get("exclude")
		q = r.Form.Get("queue")
	)
	if p == "" || d == "" || k == "" {
		errorf(c, w, 400, "Invalid parameters: project='%s', dataset='%s', kind='%s'", p, d, k)
	}
	ranges := bigquerysync.KeyRangesForKind(c, k)
	infof(c, w, "Ranges: %v\n", ranges)
	for _, r := range ranges {
		scheduleRangeSync(c, w, r.Start, r.End, p, d, e, q)
	}
}
コード例 #2
0
ファイル: sync_test.go プロジェクト: ronoaldo/aetools
func TestKeyRangeForKind(t *testing.T) {
	c, clean, err := aetest.NewContext()
	if err != nil {
		t.Fatal(err)
	}
	defer clean()
	// No entities: empty range
	ranges := bigquerysync.KeyRangesForKind(c, "RangeTest")
	if len(ranges) != 0 {
		t.Errorf("Unexpected ranges returned: %d, expected 0: %#v", len(ranges), ranges)
	}
	// Setup datastore - __scatter__ is replaced with _scatter__ for testing
	aetools.LoadJSON(c, SampleEntities, aetools.LoadSync)
	// No scatter, single range
	ranges = bigquerysync.KeyRangesForKind(c, "Sample")
	if len(ranges) != 1 {
		t.Errorf("Unexpected ranges without scatters: %v, expected length 1", ranges)
	} else {
		if ranges[0].Start == nil {
			t.Errorf("Unexpected nil start for Sample")
		} else {
			if ranges[0].Start.IntID() != 1 || ranges[0].Start.Kind() != "Sample" {
				t.Errorf("Unexpected start key for Sample: %#v", ranges[0].Start)
			}
			if ranges[0].End != nil {
				t.Errorf("Unexpected end key for Sample: %#v, expected nil", ranges[0].End)
			}
		}
	}
	// Scattered entities: sorted key ranges expected
	ranges = bigquerysync.KeyRangesForKind(c, "RangeTest")
	expected := []struct {
		Start int64
		End   int64
	}{
		{1, 30},
		{30, 50},
		{50, 1000},
		{1000, 0},
	}
	if len(ranges) != len(expected) {
		t.Errorf("Unexpected ranges with scatter: %d, expected 3", len(ranges))
	} else {
		for i, e := range expected {
			r := ranges[i]
			if r.Start == nil {
				t.Errorf("Unexpected nil start at range %d", i)
			} else if r.Start.IntID() != e.Start {
				t.Errorf("Unexpected start at range %d: %#v, expected %d", i, r.Start.IntID(), e.Start)
			}
			if e.End == 0 {
				if r.End != nil {
					t.Errorf("Unexpected end at range %d: %#v, expected nil", i, r.End)
				}
			} else if r.End == nil {
				t.Errorf("Unexpected nil end at range %d, expected %d", i, r.End, e.End)
			} else if r.End.IntID() != e.End {
				t.Errorf("Unexpected end at range %d: %#v, expected %d", i, r.End.IntID(), e.End)
			}
		}
		// Check if all entity keys match
		for i, r := range ranges {
			if r.Start != nil {
				if r.Start.Kind() != "RangeTest" {
					t.Errorf("Unexpected kind at range %d: %s", i, r.Start.Kind())
				}
			}
			if r.End != nil {
				if r.End.Kind() != "RangeTest" {
					t.Errorf("Unexpected kind at range %d: %s", i, r.End.Kind())
				}
			}
		}
	}
}