// clearDB initializes the database, upgrading it if needed, and removes all
// data to ensure that the test begins with a clean slate. Returns a testDB
// which must be closed after the test finishes.
func clearDB(t *testing.T) *testDB {
tempDir, err := ioutil.TempDir("", "build_scheduler_test_")
assert.Nil(t, err)
db, err := buildbot.NewLocalDB(path.Join(tempDir, "buildbot.db"))
assert.Nil(t, err)
return &testDB{
db: db,
dir: tempDir,
}
}
func main() {
defer common.LogPanic()
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("datahopper", graphiteServer)
// Shared repo objects.
skiaRepo, err := gitinfo.CloneOrUpdate(SKIA_REPO, path.Join(*workdir, "datahopper_skia"), true)
if err != nil {
glog.Fatal(err)
}
infraRepo, err := gitinfo.CloneOrUpdate(INFRA_REPO, path.Join(*workdir, "datahopper_infra"), true)
if err != nil {
glog.Fatal(err)
}
go func() {
for _ = range time.Tick(5 * time.Minute) {
if err := skiaRepo.Update(true, true); err != nil {
glog.Errorf("Failed to sync Skia repo: %v", err)
}
if err := infraRepo.Update(true, true); err != nil {
glog.Errorf("Failed to sync Infra repo: %v", err)
}
}
}()
// Data generation goroutines.
db, err := buildbot.NewLocalDB(path.Join(*workdir, "buildbot.db"))
if err != nil {
glog.Fatal(err)
}
// Buildbot data ingestion.
if err := buildbot.IngestNewBuildsLoop(db, *workdir); err != nil {
glog.Fatal(err)
}
// Run a server for the buildbot data.
if err := buildbot.RunBuildServer(*grpcPort, db); err != nil {
glog.Fatal(err)
}
// Measure buildbot data ingestion progress.
totalGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.total", go_metrics.DefaultRegistry)
ingestGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.ingested", go_metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(common.SAMPLE_PERIOD) {
totalBuilds, err := buildbot.NumTotalBuilds()
if err != nil {
glog.Error(err)
continue
}
ingestedBuilds, err := db.NumIngestedBuilds()
if err != nil {
glog.Error(err)
continue
}
totalGuage.Update(int64(totalBuilds))
ingestGuage.Update(int64(ingestedBuilds))
}
}()
// Average build and step time.
go func() {
period := 24 * time.Hour
for _ = range time.Tick(10 * time.Minute) {
glog.Info("Loading build and buildstep duration data.")
end := time.Now().UTC()
start := end.Add(-period)
builds, err := db.GetBuildsFromDateRange(start, end)
if err != nil {
glog.Errorf("Failed to obtain build and buildstep duration data: %s", err)
continue
}
for _, b := range builds {
if !b.IsFinished() {
continue
}
// Report build time.
// app.host.measurement.measurement.builder.measurement*
d := b.Finished.Sub(b.Started)
metric := fmt.Sprintf("buildbot.builds.%s.duration", fixName(b.Builder))
metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d))
for _, s := range b.Steps {
if !s.IsFinished() {
continue
}
// app.host.measurement.measurement.builder.step.measurement*
d := s.Finished.Sub(s.Started)
metric := fmt.Sprintf("buildbot.buildstepsbybuilder.%s.%s.duration", fixName(b.Builder), fixName(s.Name))
metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d))
}
}
}
}()
// Number of commits in the repo.
go func() {
skiaGauge := go_metrics.GetOrRegisterGauge("repo.skia.commits", go_metrics.DefaultRegistry)
infraGauge := go_metrics.GetOrRegisterGauge("repo.infra.commits", go_metrics.DefaultRegistry)
for _ = range time.Tick(5 * time.Minute) {
skiaGauge.Update(int64(skiaRepo.NumCommits()))
infraGauge.Update(int64(infraRepo.NumCommits()))
}
}()
// Run a backup server.
go func() {
glog.Fatal(buildbot.RunBackupServer(db, *httpPort))
}()
// Wait while the above goroutines generate data.
select {}
}