// FindCommitsForBuild determines which commits were first included in the // given build. Assumes that all previous builds for the given builder/master // are already in the database. func FindCommitsForBuild(bf BuildFinder, b *Build, repos *gitinfo.RepoMap) ([]string, int, []string, error) { defer metrics.NewTimer("buildbot.FindCommitsForBuild").Stop() // Shortcut: Don't bother computing commit blamelists for trybots. if IsTrybot(b.Builder) { return []string{}, -1, []string{}, nil } if b.Repository == "" { return []string{}, -1, []string{}, nil } repo, err := repos.Repo(b.Repository) if err != nil { return nil, -1, nil, fmt.Errorf("Could not find commits for build: %v", err) } // Shortcut for the first build for a given builder: this build must be // the first inclusion for all revisions prior to b.GotRevision. if b.Number == 0 && b.GotRevision != "" { revlist, err := repo.RevList(b.GotRevision) return revlist, -1, []string{}, err } // Start tracing commits back in time until we hit a previous build. commitMap, stealFrom, stolen, err := findCommitsRecursive(bf, map[string]bool{}, b, b.GotRevision, repo, -1, []string{}) if err != nil { return nil, -1, nil, err } commits := make([]string, 0, len(commitMap)) for c, _ := range commitMap { commits = append(commits, c) } return commits, stealFrom, stolen, nil }
// ingestNewBuilds finds the set of uningested builds and ingests them. func ingestNewBuilds(m string, repos *gitinfo.RepoMap) error { defer metrics.NewTimer("buildbot.ingestNewBuilds").Stop() glog.Infof("Ingesting builds for %s", m) // TODO(borenet): Investigate the use of channels here. We should be // able to start ingesting builds as the data becomes available rather // than waiting until the end. buildsToProcess, err := getUningestedBuilds(m) if err != nil { return fmt.Errorf("Failed to obtain the set of uningested builds: %v", err) } unfinished, err := getUnfinishedBuilds(m) if err != nil { return fmt.Errorf("Failed to obtain the set of unfinished builds: %v", err) } for _, b := range unfinished { if _, ok := buildsToProcess[b.Builder]; !ok { buildsToProcess[b.Builder] = []int{} } buildsToProcess[b.Builder] = append(buildsToProcess[b.Builder], b.Number) } if err := repos.Update(); err != nil { return err } // TODO(borenet): Can we ingest builders in parallel? errs := map[string]error{} for b, w := range buildsToProcess { for _, n := range w { if BUILD_BLACKLIST[b][n] { glog.Warningf("Skipping blacklisted build: %s # %d", b, n) continue } if IsTrybot(b) { continue } glog.Infof("Ingesting build: %s, %s, %d", m, b, n) build, err := retryGetBuildFromMaster(m, b, n, repos) if err != nil { errs[b] = fmt.Errorf("Failed to ingest build: %v", err) break } if err := IngestBuild(build, repos); err != nil { errs[b] = fmt.Errorf("Failed to ingest build: %v", err) break } } } if len(errs) > 0 { msg := fmt.Sprintf("Encountered errors ingesting builds for %s:", m) for b, err := range errs { msg += fmt.Sprintf("\n%s: %v", b, err) } return fmt.Errorf(msg) } glog.Infof("Done ingesting builds for %s", m) return nil }
// tx is a wrapper for a BoltDB transaction which tracks statistics. func (d *localDB) tx(name string, fn func(*bolt.Tx) error, update bool) error { txId := d.startTx(name) defer d.endTx(txId) defer metrics.NewTimer(fmt.Sprintf("buildbot.tx.%s", name)).Stop() if update { return d.db.Update(fn) } else { return d.db.View(fn) } }
// ReplaceIntoDB inserts or updates the Build in the database. func (b *Build) ReplaceIntoDB() error { defer metrics.NewTimer("buildbot.ReplaceIntoDB").Stop() var err error for attempt := 0; attempt < 5; attempt++ { if err = b.replaceIntoDB(); err == nil { return nil } time.Sleep(500 * time.Millisecond) } return err }
// getUningestedBuilds returns a map whose keys are master names and values are // sub-maps whose keys are builder names and values are slices of ints // representing the numbers of builds which have not yet been ingested. func getUningestedBuilds(db DB, m string) (map[string][]int, error) { defer metrics.NewTimer("buildbot.getUningestedBuilds").Stop() // Get the latest and last-processed builds for all builders. latest, err := getLatestBuilds(m) if err != nil { return nil, fmt.Errorf("Failed to get latest builds: %s", err) } lastProcessed, err := db.GetLastProcessedBuilds(m) if err != nil { return nil, fmt.Errorf("Failed to get last-processed builds: %s", err) } // Find the range of uningested builds for each builder. type numRange struct { Start int // The last-ingested build number. End int // The latest build number. } ranges := map[string]*numRange{} for _, id := range lastProcessed { b, err := db.GetBuild(id) if err != nil { return nil, err } ranges[b.Builder] = &numRange{ Start: b.Number, End: b.Number, } } for b, n := range latest { if _, ok := ranges[b]; !ok { ranges[b] = &numRange{ Start: -1, End: n, } } else { ranges[b].End = n } } // Create a slice of build numbers for the uningested builds. unprocessed := map[string][]int{} for b, r := range ranges { if r.End < r.Start { glog.Warningf("Cannot create slice of builds to ingest for %q; invalid range (%d, %d)", b, r.Start, r.End) continue } builds := make([]int, r.End-r.Start) for i := r.Start + 1; i <= r.End; i++ { builds[i-r.Start-1] = i } if len(builds) > 0 { unprocessed[b] = builds } } return unprocessed, nil }
// retryGetBuildFromMaster retrieves the given build from the build master's JSON // interface as specified by the master, builder, and build number. Makes // multiple attempts in case the master fails to respond. func retryGetBuildFromMaster(master, builder string, buildNumber int, repos *gitinfo.RepoMap) (*Build, error) { defer metrics.NewTimer("buildbot.retryGetBuildFromMaster").Stop() var b *Build var err error for attempt := 0; attempt < 3; attempt++ { b, err = getBuildFromMaster(master, builder, buildNumber, repos) if err == nil { break } time.Sleep(500 * time.Millisecond) } return b, err }
// IngestBuild retrieves the given build from the build master's JSON interface // and pushes it into the database. func IngestBuild(b *Build, repos *gitinfo.RepoMap) error { defer metrics.NewTimer("buildbot.IngestBuild").Stop() // Find the commits for this build. commits, stoleFrom, stolen, err := FindCommitsForBuild(&bf, b, repos) if err != nil { return err } b.Commits = commits // Log the case where we found no revisions for the build. if !(IsTrybot(b.Builder) || strings.Contains(b.Builder, "Housekeeper")) && len(b.Commits) == 0 { glog.Infof("Got build with 0 revs: %s #%d GotRev=%s", b.Builder, b.Number, b.GotRevision) } // Determine whether we've already ingested this build. If so, fix up the ID // so that we update it rather than insert a new copy. existingBuildID, err := GetBuildIDFromDB(b.Builder, b.Master, b.Number) if err == nil { b.Id = existingBuildID } // Insert the build. if stoleFrom >= 0 && stolen != nil && len(stolen) > 0 { // Remove the commits we stole from the previous owner. oldBuild, err := GetBuildFromDB(b.Builder, b.Master, stoleFrom) if err != nil { return err } if oldBuild == nil { return fmt.Errorf("Attempted to retrieve %s #%d, but got a nil build from the DB.", b.Builder, stoleFrom) } newCommits := make([]string, 0, len(oldBuild.Commits)) for _, c := range oldBuild.Commits { keep := true for _, s := range stolen { if c == s { keep = false break } } if keep { newCommits = append(newCommits, c) } } oldBuild.Commits = newCommits return ReplaceMultipleBuildsIntoDB([]*Build{b, oldBuild}) } else { return b.ReplaceIntoDB() } }
// insertBuilds inserts a batch of builds into the database. In the case of // failure, it continually retries until it succeeds. func insertBuilds(builds []*Build) { defer metrics.NewTimer("buildbot.insertBuilds").Stop() for { // Insert the builds. glog.Infof("Inserting %d builds.", len(builds)) if err := ReplaceMultipleBuildsIntoDB(builds); err != nil { glog.Errorf("Failed to insert builds, retrying: %s", err) time.Sleep(100 * time.Millisecond) } else { break } } glog.Infof("Finished inserting %d builds.", len(builds)) go_metrics.GetOrRegisterCounter("buildbot.NumInsertedBuilds", go_metrics.DefaultRegistry).Inc(int64(len(builds))) }
// IngestBuild retrieves the given build from the build master's JSON interface // and pushes it into the database. func IngestBuild(db DB, b *Build, repos *gitinfo.RepoMap) error { defer metrics.NewTimer("buildbot.IngestBuild").Stop() defer go_metrics.GetOrRegisterCounter("buildbot.NumIngestedBuilds", go_metrics.DefaultRegistry).Inc(1) // Find the commits for this build. commits, stoleFrom, stolen, err := FindCommitsForBuild(db, b, repos) if err != nil { return err } b.Commits = commits // Log the case where we found no revisions for the build. if !(IsTrybot(b.Builder) || strings.Contains(b.Builder, "Housekeeper")) && len(b.Commits) == 0 { glog.Infof("Got build with 0 revs: %s #%d GotRev=%s", b.Builder, b.Number, b.GotRevision) } // Insert the build. if stoleFrom >= 0 && stolen != nil && len(stolen) > 0 { // Remove the commits we stole from the previous owner. oldBuild, err := db.GetBuildFromDB(b.Master, b.Builder, stoleFrom) if err != nil { return err } if oldBuild == nil { return fmt.Errorf("Attempted to retrieve %s #%d, but got a nil build from the DB.", b.Builder, stoleFrom) } newCommits := make([]string, 0, len(oldBuild.Commits)) for _, c := range oldBuild.Commits { keep := true for _, s := range stolen { if c == s { keep = false break } } if keep { newCommits = append(newCommits, c) } } oldBuild.Commits = newCommits return db.PutBuilds([]*Build{b, oldBuild}) } else { return db.PutBuild(b) } }
func ReplaceMultipleBuildsIntoDB(builds []*Build) (rv error) { defer metrics.NewTimer("buildbot.ReplaceMultipleBuildsIntoDB").Stop() tx, err := DB.Beginx() if err != nil { return fmt.Errorf("Unable to push builds into database - Could not begin transaction: %v", err) } // Defer committing/rolling back the transaction. defer func() { rv = database.CommitOrRollback(tx, rv) }() // Insert the builds. // TODO(borenet): Insert/update all of the builds at once. for _, b := range builds { if err := b.replaceIntoDBTx(tx); err != nil { return err } } return nil }
// See documentation for DB interface. func (d *localDB) PutBuilds(builds []*Build) error { defer metrics.NewTimer("buildbot.PutBuilds").Stop() return d.update("PutBuilds", func(tx *bolt.Tx) error { return putBuilds(tx, builds) }) }