func main() {
// Setup flags.
dbConf := buildbot.DBConfigFromFlags()
influxdb.SetupFlags()
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("datahopper", graphiteServer)
// Initialize the buildbot database.
if !*local {
if err := dbConf.GetPasswordFromMetadata(); err != nil {
glog.Fatal(err)
}
}
if err := dbConf.InitDB(); err != nil {
glog.Fatal(err)
}
// Data generation goroutines.
// Buildbot data ingestion.
go buildbot.IngestNewBuildsLoop(*workdir)
// Measure buildbot data ingestion progress.
totalGuage := metrics.GetOrRegisterGauge("buildbot.builds.total", metrics.DefaultRegistry)
ingestGuage := metrics.GetOrRegisterGauge("buildbot.builds.ingested", metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(common.SAMPLE_PERIOD) {
totalBuilds, err := buildbot.NumTotalBuilds()
if err != nil {
glog.Error(err)
continue
}
ingestedBuilds, err := buildbot.NumIngestedBuilds()
if err != nil {
glog.Error(err)
continue
}
totalGuage.Update(int64(totalBuilds))
ingestGuage.Update(int64(ingestedBuilds))
}
}()
// Average duration of buildsteps over a time period.
go func() {
period := 24 * time.Hour
type stepData struct {
Name string `db:"name"`
Duration float64 `db:"duration"`
}
stmt, err := buildbot.DB.Preparex(fmt.Sprintf("SELECT name, AVG(finished-started) AS duration FROM %s WHERE started > ? AND finished > started GROUP BY name ORDER BY duration;", buildbot.TABLE_BUILD_STEPS))
if err != nil {
glog.Fatalf("Failed to prepare buildbot database query: %v", err)
}
defer util.Close(stmt)
for _ = range time.Tick(common.SAMPLE_PERIOD) {
glog.Info("Loading buildstep duration data.")
t := time.Now().UTC().Add(-period).Unix()
steps := []stepData{}
if err := stmt.Select(&steps, t); err != nil {
glog.Error(err)
continue
}
for _, s := range steps {
v := int64(s.Duration * float64(time.Millisecond))
metric := fmt.Sprintf("buildbot.buildsteps.%s.duration", fixName(s.Name))
metrics.GetOrRegisterGauge(metric, metrics.DefaultRegistry).Update(v)
}
}
}()
// Average duration of builds over a time period.
go func() {
period := 24 * time.Hour
type buildData struct {
Builder string `db:"builder"`
Duration float64 `db:"duration"`
}
stmt, err := buildbot.DB.Preparex(fmt.Sprintf("SELECT builder, AVG(finished-started) AS duration FROM %s WHERE started > ? AND finished > started GROUP BY builder ORDER BY duration;", buildbot.TABLE_BUILDS))
if err != nil {
glog.Fatalf("Failed to prepare buildbot database query: %v", err)
}
defer util.Close(stmt)
for _ = range time.Tick(common.SAMPLE_PERIOD) {
glog.Info("Loading build duration data.")
t := time.Now().UTC().Add(-period).Unix()
builds := []buildData{}
if err := stmt.Select(&builds, t); err != nil {
glog.Error(err)
continue
}
for _, s := range builds {
v := int64(s.Duration * float64(time.Millisecond))
metric := fmt.Sprintf("buildbot.builds.%s.duration", fixName(s.Builder))
metrics.GetOrRegisterGauge(metric, metrics.DefaultRegistry).Update(v)
}
}
}()
// Average build step time broken down by builder.
go func() {
period := 24 * time.Hour
type stepData struct {
Builder string `db:"builder"`
StepName string `db:"stepName"`
Duration float64 `db:"duration"`
}
stmt, err := buildbot.DB.Preparex(fmt.Sprintf("SELECT b.builder as builder, s.name as stepName, AVG(s.finished-s.started) AS duration FROM %s s INNER JOIN %s b ON (s.buildId = b.id) WHERE s.started > ? AND s.finished > s.started GROUP BY b.builder, s.name ORDER BY b.builder, duration;", buildbot.TABLE_BUILD_STEPS, buildbot.TABLE_BUILDS))
if err != nil {
glog.Fatalf("Failed to prepare buildbot database query: %v", err)
}
defer util.Close(stmt)
for _ = range time.Tick(common.SAMPLE_PERIOD) {
glog.Info("Loading per-builder buildstep duration data.")
t := time.Now().UTC().Add(-period).Unix()
steps := []stepData{}
if err := stmt.Select(&steps, t); err != nil {
glog.Error(err)
continue
}
for _, s := range steps {
v := int64(s.Duration * float64(time.Millisecond))
metric := fmt.Sprintf("buildbot.buildstepsbybuilder.%s.%s.duration", fixName(s.Builder), fixName(s.StepName))
metrics.GetOrRegisterGauge(metric, metrics.DefaultRegistry).Update(v)
}
}
}()
// Wait while the above goroutines generate data.
select {}
}
func main() {
defer common.LogPanic()
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("datahopper", graphiteServer)
// Shared repo objects.
skiaRepo, err := gitinfo.CloneOrUpdate(SKIA_REPO, path.Join(*workdir, "datahopper_skia"), true)
if err != nil {
glog.Fatal(err)
}
infraRepo, err := gitinfo.CloneOrUpdate(INFRA_REPO, path.Join(*workdir, "datahopper_infra"), true)
if err != nil {
glog.Fatal(err)
}
go func() {
for _ = range time.Tick(5 * time.Minute) {
if err := skiaRepo.Update(true, true); err != nil {
glog.Errorf("Failed to sync Skia repo: %v", err)
}
if err := infraRepo.Update(true, true); err != nil {
glog.Errorf("Failed to sync Infra repo: %v", err)
}
}
}()
// Data generation goroutines.
db, err := buildbot.NewLocalDB(path.Join(*workdir, "buildbot.db"))
if err != nil {
glog.Fatal(err)
}
// Buildbot data ingestion.
if err := buildbot.IngestNewBuildsLoop(db, *workdir); err != nil {
glog.Fatal(err)
}
// Run a server for the buildbot data.
if err := buildbot.RunBuildServer(*grpcPort, db); err != nil {
glog.Fatal(err)
}
// Measure buildbot data ingestion progress.
totalGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.total", go_metrics.DefaultRegistry)
ingestGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.ingested", go_metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(common.SAMPLE_PERIOD) {
totalBuilds, err := buildbot.NumTotalBuilds()
if err != nil {
glog.Error(err)
continue
}
ingestedBuilds, err := db.NumIngestedBuilds()
if err != nil {
glog.Error(err)
continue
}
totalGuage.Update(int64(totalBuilds))
ingestGuage.Update(int64(ingestedBuilds))
}
}()
// Average build and step time.
go func() {
period := 24 * time.Hour
for _ = range time.Tick(10 * time.Minute) {
glog.Info("Loading build and buildstep duration data.")
end := time.Now().UTC()
start := end.Add(-period)
builds, err := db.GetBuildsFromDateRange(start, end)
if err != nil {
glog.Errorf("Failed to obtain build and buildstep duration data: %s", err)
continue
}
for _, b := range builds {
if !b.IsFinished() {
continue
}
// Report build time.
// app.host.measurement.measurement.builder.measurement*
d := b.Finished.Sub(b.Started)
metric := fmt.Sprintf("buildbot.builds.%s.duration", fixName(b.Builder))
metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d))
for _, s := range b.Steps {
if !s.IsFinished() {
continue
}
// app.host.measurement.measurement.builder.step.measurement*
d := s.Finished.Sub(s.Started)
metric := fmt.Sprintf("buildbot.buildstepsbybuilder.%s.%s.duration", fixName(b.Builder), fixName(s.Name))
metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d))
}
}
}
}()
// Number of commits in the repo.
go func() {
skiaGauge := go_metrics.GetOrRegisterGauge("repo.skia.commits", go_metrics.DefaultRegistry)
infraGauge := go_metrics.GetOrRegisterGauge("repo.infra.commits", go_metrics.DefaultRegistry)
for _ = range time.Tick(5 * time.Minute) {
skiaGauge.Update(int64(skiaRepo.NumCommits()))
infraGauge.Update(int64(infraRepo.NumCommits()))
}
}()
// Run a backup server.
go func() {
glog.Fatal(buildbot.RunBackupServer(db, *httpPort))
}()
// Wait while the above goroutines generate data.
select {}
}