// NewIngestionProcess creates a Process for ingesting data.
func NewIngestionProcess(git *gitinfo.GitInfo, tileDir, datasetName string, ri ingester.ResultIngester, config map[string]string, every time.Duration, nCommits int, minDuration time.Duration, statusDir, metricName string) ProcessStarter {
return func() {
i, err := ingester.NewIngester(git, tileDir, datasetName, ri, nCommits, minDuration, config, statusDir, metricName)
if err != nil {
glog.Fatalf("Failed to create Ingester: %s", err)
}
glog.Infof("Starting %s ingester. Run every %s.", datasetName, every.String())
// oneStep is a single round of ingestion.
oneStep := func() {
glog.Infof("Running ingester: %s", datasetName)
err := i.Update()
if err != nil {
glog.Error(err)
}
glog.Infof("Finished running ingester: %s", datasetName)
}
// Start the ingester.
go func() {
oneStep()
for _ = range time.Tick(every) {
oneStep()
}
}()
}
}
func TestIngestCommits(t *testing.T) {
// Get a known Git repo with 34 commits in it setup.
tr := util.NewTempRepo()
defer tr.Cleanup()
// Create a temporary place for a filetilestore.
tileDir, err := ioutil.TempDir("", "skiaperf")
if err != nil {
t.Fatal("Failed to create testing Tile dir: ", err)
}
defer testutils.RemoveAll(t, tileDir)
git, err := gitinfo.NewGitInfo(filepath.Join(tr.Dir, "testrepo"), false, false)
if err != nil {
glog.Fatalf("Failed loading Git info: %s\n", err)
}
// Construct an Ingestor and have it UpdateCommitInfo.
i, err := ingester.NewIngester(git, tileDir, config.DATASET_NANO, NewNanoBenchIngester(), 1, time.Second, map[string]string{}, "", "")
if err != nil {
t.Fatal("Failed to create ingester:", err)
}
if err := i.UpdateCommitInfo(false); err != nil {
t.Fatal("Failed to ingest commits:", err)
}
// Validate the generated Tiles.
store := filetilestore.NewFileTileStore(tileDir, config.DATASET_NANO, 0)
if !validator.ValidateDataset(store, false, false) {
t.Error("Failed to validate the created Tiles:", err)
}
// Test TileTracker while were here.
tt := ingester.NewTileTracker(store, i.HashToNumber())
err = tt.Move("7a6fe813047d1a84107ef239e81f310f27861473")
if err != nil {
t.Fatal(err)
}
if got, want := tt.LastTileNum(), 2; got != want {
t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want)
}
err = tt.Move("87709bc360f35de52c2f2bc2fc70962fb234db2d")
if err != nil {
t.Fatal(err)
}
if got, want := tt.LastTileNum(), 2; got != want {
t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want)
}
}