Ejemplo n.º 1
0
// NewIngestionProcess creates a Process for ingesting data.
func NewIngestionProcess(git *gitinfo.GitInfo, tileDir, datasetName string, ri ingester.ResultIngester, config map[string]string, every time.Duration, nCommits int, minDuration time.Duration, statusDir, metricName string) ProcessStarter {
	return func() {
		i, err := ingester.NewIngester(git, tileDir, datasetName, ri, nCommits, minDuration, config, statusDir, metricName)
		if err != nil {
			glog.Fatalf("Failed to create Ingester: %s", err)
		}

		glog.Infof("Starting %s ingester. Run every %s.", datasetName, every.String())

		// oneStep is a single round of ingestion.
		oneStep := func() {
			glog.Infof("Running ingester: %s", datasetName)
			err := i.Update()
			if err != nil {
				glog.Error(err)
			}
			glog.Infof("Finished running ingester: %s", datasetName)
		}

		// Start the ingester.
		go func() {
			oneStep()
			for _ = range time.Tick(every) {
				oneStep()
			}
		}()
	}
}
Ejemplo n.º 2
0
func TestIngestCommits(t *testing.T) {
	// Get a known Git repo with 34 commits in it setup.
	tr := util.NewTempRepo()
	defer tr.Cleanup()

	// Create a temporary place for a filetilestore.
	tileDir, err := ioutil.TempDir("", "skiaperf")
	if err != nil {
		t.Fatal("Failed to create testing Tile dir: ", err)
	}
	defer testutils.RemoveAll(t, tileDir)

	git, err := gitinfo.NewGitInfo(filepath.Join(tr.Dir, "testrepo"), false, false)
	if err != nil {
		glog.Fatalf("Failed loading Git info: %s\n", err)
	}

	// Construct an Ingestor and have it UpdateCommitInfo.
	i, err := ingester.NewIngester(git, tileDir, config.DATASET_NANO, NewNanoBenchIngester(), 1, time.Second, map[string]string{}, "", "")
	if err != nil {
		t.Fatal("Failed to create ingester:", err)
	}

	if err := i.UpdateCommitInfo(false); err != nil {
		t.Fatal("Failed to ingest commits:", err)
	}

	// Validate the generated Tiles.
	store := filetilestore.NewFileTileStore(tileDir, config.DATASET_NANO, 0)
	if !validator.ValidateDataset(store, false, false) {
		t.Error("Failed to validate the created Tiles:", err)
	}

	// Test TileTracker while were here.
	tt := ingester.NewTileTracker(store, i.HashToNumber())
	err = tt.Move("7a6fe813047d1a84107ef239e81f310f27861473")
	if err != nil {
		t.Fatal(err)
	}
	if got, want := tt.LastTileNum(), 2; got != want {
		t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want)
	}
	err = tt.Move("87709bc360f35de52c2f2bc2fc70962fb234db2d")
	if err != nil {
		t.Fatal(err)
	}
	if got, want := tt.LastTileNum(), 2; got != want {
		t.Errorf("Move failed, wrong tile: Got %d Want %d", got, want)
	}
}