// putBuild inserts the build into the database, replacing any previous version.
func putBuild(tx *bolt.Tx, b *Build) error {
id := b.Id()
if tx.Bucket(BUCKET_BUILDS).Get(id) == nil {
// Measure the time between build start and first DB insertion.
latency := time.Now().Sub(b.Started)
if latency > INGEST_LATENCY_ALERT_THRESHOLD {
// This is probably going to trigger an alert. Log the build for debugging.
glog.Warningf("Build start to ingestion latency is greater than %s (%s): %s %s #%d", INGEST_LATENCY_ALERT_THRESHOLD, latency, b.Master, b.Builder, b.Number)
}
metrics.GetOrRegisterSlidingWindow("buildbot.startToIngestLatency", metrics.DEFAULT_WINDOW).Update(int64(latency))
} else {
if err := deleteBuild(tx, id); err != nil {
return err
}
}
return insertBuild(tx, b)
}
func (b *Build) replaceIntoDBTx(tx *sqlx.Tx) (rv error) {
// Determine whether we've already ingested this build. If so, fix up the ID
// so that we update it rather than insert a new copy.
existingBuildID, err := GetBuildIDFromDBTx(b.Builder, b.Master, b.Number, tx)
if err == nil && existingBuildID >= 0 {
b.Id = existingBuildID
}
buildID := b.Id
var stepIDMap map[*BuildStep]int
var commentIDMap map[*BuildComment]int
// If successful, update the build with new IDs.
defer func() {
if rv == nil {
// Measure the time between build start and first DB insertion.
if b.Id == 0 {
t := int64(time.Now().Sub(util.UnixFloatToTime(b.Started)))
metrics.GetOrRegisterSlidingWindow("buildbot.startToIngestLatency", metrics.DEFAULT_WINDOW).Update(t)
}
// Update the build with any new IDs.
b.Id = buildID
for _, s := range b.Steps {
s.BuildID = b.Id
}
for s, id := range stepIDMap {
s.Id = id
}
for c, id := range commentIDMap {
c.Id = id
}
}
}()
if buildID == 0 {
stmt := fmt.Sprintf("INSERT INTO %s (builder,master,number,results,gotRevision,buildslave,started,finished,properties,branch,repository) VALUES (?,?,?,?,?,?,?,?,?,?,?);", TABLE_BUILDS)
res, err := tx.Exec(stmt, b.Builder, b.Master, b.Number, b.Results, b.GotRevision, b.BuildSlave, b.Started, b.Finished, b.PropertiesStr, b.Branch, b.Repository)
if err != nil {
return fmt.Errorf("Failed to push build into database: %v", err)
}
id, err := res.LastInsertId()
if err != nil {
return fmt.Errorf("Failed to push build into database; LastInsertId failed: %v", err)
}
buildID = int(id)
} else {
stmt := fmt.Sprintf("UPDATE %s set builder=?, master=?, number=?, results=?, gotRevision=?, buildslave=?, started=?, finished=?, properties=?, branch=?, repository=? WHERE id=?;", TABLE_BUILDS)
if _, err := tx.Exec(stmt, b.Builder, b.Master, b.Number, b.Results, b.GotRevision, b.BuildSlave, b.Started, b.Finished, b.PropertiesStr, b.Branch, b.Repository, buildID); err != nil {
return fmt.Errorf("Failed to push build into database: %v", err)
}
}
// Build steps.
stepIDMap, err = replaceStepsIntoDB(tx, b.Steps, buildID)
if err != nil {
return err
}
// Commits.
if err = replaceCommitsIntoDB(tx, b.Commits, buildID); err != nil {
return err
}
// Comments.
commentIDMap, err = replaceCommentsIntoDB(tx, b.Comments, buildID)
if err != nil {
return err
}
// The transaction is committed during the deferred function.
return nil
}
func main() {
defer common.LogPanic()
// Global init to initialize glog and parse arguments.
common.InitWithMetrics("datahopper", graphiteServer)
// Shared repo objects.
skiaRepo, err := gitinfo.CloneOrUpdate(SKIA_REPO, path.Join(*workdir, "datahopper_skia"), true)
if err != nil {
glog.Fatal(err)
}
infraRepo, err := gitinfo.CloneOrUpdate(INFRA_REPO, path.Join(*workdir, "datahopper_infra"), true)
if err != nil {
glog.Fatal(err)
}
go func() {
for _ = range time.Tick(5 * time.Minute) {
if err := skiaRepo.Update(true, true); err != nil {
glog.Errorf("Failed to sync Skia repo: %v", err)
}
if err := infraRepo.Update(true, true); err != nil {
glog.Errorf("Failed to sync Infra repo: %v", err)
}
}
}()
// Data generation goroutines.
db, err := buildbot.NewLocalDB(path.Join(*workdir, "buildbot.db"))
if err != nil {
glog.Fatal(err)
}
// Buildbot data ingestion.
if err := buildbot.IngestNewBuildsLoop(db, *workdir); err != nil {
glog.Fatal(err)
}
// Run a server for the buildbot data.
if err := buildbot.RunBuildServer(*grpcPort, db); err != nil {
glog.Fatal(err)
}
// Measure buildbot data ingestion progress.
totalGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.total", go_metrics.DefaultRegistry)
ingestGuage := go_metrics.GetOrRegisterGauge("buildbot.builds.ingested", go_metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(common.SAMPLE_PERIOD) {
totalBuilds, err := buildbot.NumTotalBuilds()
if err != nil {
glog.Error(err)
continue
}
ingestedBuilds, err := db.NumIngestedBuilds()
if err != nil {
glog.Error(err)
continue
}
totalGuage.Update(int64(totalBuilds))
ingestGuage.Update(int64(ingestedBuilds))
}
}()
// Average build and step time.
go func() {
period := 24 * time.Hour
for _ = range time.Tick(10 * time.Minute) {
glog.Info("Loading build and buildstep duration data.")
end := time.Now().UTC()
start := end.Add(-period)
builds, err := db.GetBuildsFromDateRange(start, end)
if err != nil {
glog.Errorf("Failed to obtain build and buildstep duration data: %s", err)
continue
}
for _, b := range builds {
if !b.IsFinished() {
continue
}
// Report build time.
// app.host.measurement.measurement.builder.measurement*
d := b.Finished.Sub(b.Started)
metric := fmt.Sprintf("buildbot.builds.%s.duration", fixName(b.Builder))
metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d))
for _, s := range b.Steps {
if !s.IsFinished() {
continue
}
// app.host.measurement.measurement.builder.step.measurement*
d := s.Finished.Sub(s.Started)
metric := fmt.Sprintf("buildbot.buildstepsbybuilder.%s.%s.duration", fixName(b.Builder), fixName(s.Name))
metrics.GetOrRegisterSlidingWindow(metric, metrics.DEFAULT_WINDOW).Update(int64(d))
}
}
}
}()
// Number of commits in the repo.
go func() {
skiaGauge := go_metrics.GetOrRegisterGauge("repo.skia.commits", go_metrics.DefaultRegistry)
infraGauge := go_metrics.GetOrRegisterGauge("repo.infra.commits", go_metrics.DefaultRegistry)
for _ = range time.Tick(5 * time.Minute) {
skiaGauge.Update(int64(skiaRepo.NumCommits()))
infraGauge.Update(int64(infraRepo.NumCommits()))
}
}()
// Run a backup server.
go func() {
glog.Fatal(buildbot.RunBackupServer(db, *httpPort))
}()
// Wait while the above goroutines generate data.
select {}
}