// NewIngestionProcess creates a Process for ingesting data. func NewIngestionProcess(git *gitinfo.GitInfo, tileDir, datasetName string, ri ingester.ResultIngester, config map[string]string, every time.Duration, nCommits int, minDuration time.Duration, statusDir, metricName string) ProcessStarter { return func() { i, err := ingester.NewIngester(git, tileDir, datasetName, ri, nCommits, minDuration, config, statusDir, metricName) if err != nil { glog.Fatalf("Failed to create Ingester: %s", err) } glog.Infof("Starting %s ingester. Run every %s.", datasetName, every.String()) // oneStep is a single round of ingestion. oneStep := func() { glog.Infof("Running ingester: %s", datasetName) err := i.Update() if err != nil { glog.Error(err) } glog.Infof("Finished running ingester: %s", datasetName) } // Start the ingester. go func() { oneStep() for _ = range time.Tick(every) { oneStep() } }() } }
func TestIngestCommits(t *testing.T) { // Get a known Git repo with 34 commits in it setup. tr := util.NewTempRepo() defer tr.Cleanup() // Create a temporary place for a filetilestore. tileDir, err := ioutil.TempDir("", "skiaperf") if err != nil { t.Fatal("Failed to create testing Tile dir: ", err) } defer testutils.RemoveAll(t, tileDir) git, err := gitinfo.NewGitInfo(filepath.Join(tr.Dir, "testrepo"), false, false) if err != nil { glog.Fatalf("Failed loading Git info: %s\n", err) } // Construct an Ingestor and have it UpdateCommitInfo. i, err := ingester.NewIngester(git, tileDir, config.DATASET_NANO, NewNanoBenchIngester(), 1, time.Second, map[string]string{}, "", "") if err != nil { t.Fatal("Failed to create ingester:", err) } if err := i.UpdateCommitInfo(false); err != nil { t.Fatal("Failed to ingest commits:", err) } // Validate the generated Tiles. store := filetilestore.NewFileTileStore(tileDir, config.DATASET_NANO, 0) if !validator.ValidateDataset(store, false, false) { t.Error("Failed to validate the created Tiles:", err) } // Test TileTracker while were here. tt := ingester.NewTileTracker(store, i.HashToNumber()) err = tt.Move("7a6fe813047d1a84107ef239e81f310f27861473") if err != nil { t.Fatal(err) } if got, want := tt.LastTileNum(), 2; got != want { t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want) } err = tt.Move("87709bc360f35de52c2f2bc2fc70962fb234db2d") if err != nil { t.Fatal(err) } if got, want := tt.LastTileNum(), 2; got != want { t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want) } }