// putBuild inserts the build into the database, replacing any previous version. func putBuild(tx *bolt.Tx, b *Build) error { id := b.Id() if tx.Bucket(BUCKET_BUILDS).Get(id) == nil { // Measure the time between build start and first DB insertion. latency := time.Now().Sub(b.Started) if latency > INGEST_LATENCY_ALERT_THRESHOLD { // This is probably going to trigger an alert. Log the build for debugging. glog.Warningf("Build start to ingestion latency is greater than %s (%s): %s %s #%d", INGEST_LATENCY_ALERT_THRESHOLD, latency, b.Master, b.Builder, b.Number) } metrics.GetOrRegisterSlidingWindow("buildbot.startToIngestLatency", metrics.DEFAULT_WINDOW).Update(int64(latency)) } else { if err := deleteBuild(tx, id); err != nil { return err } } return insertBuild(tx, b) }
func (b *Build) replaceIntoDBTx(tx *sqlx.Tx) (rv error) { // Determine whether we've already ingested this build. If so, fix up the ID // so that we update it rather than insert a new copy. existingBuildID, err := GetBuildIDFromDBTx(b.Builder, b.Master, b.Number, tx) if err == nil && existingBuildID >= 0 { b.Id = existingBuildID } buildID := b.Id var stepIDMap map[*BuildStep]int var commentIDMap map[*BuildComment]int // If successful, update the build with new IDs. defer func() { if rv == nil { // Measure the time between build start and first DB insertion. if b.Id == 0 { t := int64(time.Now().Sub(util.UnixFloatToTime(b.Started))) metrics.GetOrRegisterSlidingWindow("buildbot.startToIngestLatency", metrics.DEFAULT_WINDOW).Update(t) } // Update the build with any new IDs. b.Id = buildID for _, s := range b.Steps { s.BuildID = b.Id } for s, id := range stepIDMap { s.Id = id } for c, id := range commentIDMap { c.Id = id } } }() if buildID == 0 { stmt := fmt.Sprintf("INSERT INTO %s (builder,master,number,results,gotRevision,buildslave,started,finished,properties,branch,repository) VALUES (?,?,?,?,?,?,?,?,?,?,?);", TABLE_BUILDS) res, err := tx.Exec(stmt, b.Builder, b.Master, b.Number, b.Results, b.GotRevision, b.BuildSlave, b.Started, b.Finished, b.PropertiesStr, b.Branch, b.Repository) if err != nil { return fmt.Errorf("Failed to push build into database: %v", err) } id, err := res.LastInsertId() if err != nil { return fmt.Errorf("Failed to push build into database; LastInsertId failed: %v", err) } buildID = int(id) } else { stmt := fmt.Sprintf("UPDATE %s set builder=?, master=?, number=?, results=?, gotRevision=?, buildslave=?, started=?, finished=?, properties=?, branch=?, repository=? WHERE id=?;", TABLE_BUILDS) if _, err := tx.Exec(stmt, b.Builder, b.Master, b.Number, b.Results, b.GotRevision, b.BuildSlave, b.Started, b.Finished, b.PropertiesStr, b.Branch, b.Repository, buildID); err != nil { return fmt.Errorf("Failed to push build into database: %v", err) } } // Build steps. stepIDMap, err = replaceStepsIntoDB(tx, b.Steps, buildID) if err != nil { return err } // Commits. if err = replaceCommitsIntoDB(tx, b.Commits, buildID); err != nil { return err } // Comments. commentIDMap, err = replaceCommentsIntoDB(tx, b.Comments, buildID) if err != nil { return err } // The transaction is committed during the deferred function. return nil }
func main() { defer common.LogPanic() // Global init to initialize glog and parse arguments. common.InitWithMetrics("datahopper", graphiteServer) // Shared repo objects. skiaRepo, err := gitinfo.CloneOrUpdate(SKIA_REPO, path.Join(*workdir, "datahopper_skia"), true) if err != nil { glog.Fatal(err) } infraRepo, err := gitinfo.CloneOrUpdate(INFRA_REPO, path.Join(*workdir, "datahopper_infra"), true) if err != nil { glog.Fatal(err) } go func() { for _ = range time.Tick(5 * time.Minute) { if err := skiaRepo.Update(true, true); err != nil { glog.Errorf("Failed to sync Skia repo: %v", err) } if err := infraRepo.Update(true, true); err != nil { glog.Errorf("Failed to sync Infra repo: %v", err) } } }() // Data generation goroutines. db, err := buildbot.NewLocalDB(path.Join(*workdir, "buildbot.db")) if err != nil { glog.Fatal(err) } // Buildbot data ingestion. if err := buildbot.IngestNewBuildsLoop(db, *workdir); err != nil { glog.Fatal(err) } // Run a server for the buildbot data. if err := buildbot.RunBuildServer(*grpcPort, db); err != nil { glog.Fatal(err) } // Measure buildbot data ingestion progress. totalGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.total", go_metrics.DefaultRegistry) ingestGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.ingested", go_metrics.DefaultRegistry) go func() { for _ = range time.Tick(common.SAMPLE_PERIOD) { totalBuilds, err := buildbot.NumTotalBuilds() if err != nil { glog.Error(err) continue } ingestedBuilds, err := db.NumIngestedBuilds() if err != nil { glog.Error(err) continue } totalGuage.Update(int64(totalBuilds)) ingestGuage.Update(int64(ingestedBuilds)) } }() // Average build and step time. go func() { period := 24 * time.Hour for _ = range time.Tick(10 * time.Minute) { glog.Info("Loading build and buildstep duration data.") end := time.Now().UTC() start := end.Add(-period) builds, err := db.GetBuildsFromDateRange(start, end) if err != nil { glog.Errorf("Failed to obtain build and buildstep duration data: %s", err) continue } for _, b := range builds { if !b.IsFinished() { continue } // Report build time. // app.host.measurement.measurement.builder.measurement* d := b.Finished.Sub(b.Started) metric := fmt.Sprintf("buildbot.builds.%s.duration", fixName(b.Builder)) metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d)) for _, s := range b.Steps { if !s.IsFinished() { continue } // app.host.measurement.measurement.builder.step.measurement* d := s.Finished.Sub(s.Started) metric := fmt.Sprintf("buildbot.buildstepsbybuilder.%s.%s.duration", fixName(b.Builder), fixName(s.Name)) metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d)) } } } }() // Number of commits in the repo. go func() { skiaGauge := go_metrics.GetOrRegisterGauge("repo.skia.commits", go_metrics.DefaultRegistry) infraGauge := go_metrics.GetOrRegisterGauge("repo.infra.commits", go_metrics.DefaultRegistry) for _ = range time.Tick(5 * time.Minute) { skiaGauge.Update(int64(skiaRepo.NumCommits())) infraGauge.Update(int64(infraRepo.NumCommits())) } }() // Run a backup server. go func() { glog.Fatal(buildbot.RunBackupServer(db, *httpPort)) }() // Wait while the above goroutines generate data. select {} }