// FindCommitsForBuild determines which commits were first included in the
// given build. Assumes that all previous builds for the given builder/master
// are already in the database.
func FindCommitsForBuild(bf BuildFinder, b *Build, repos *gitinfo.RepoMap) ([]string, int, []string, error) {
defer metrics.NewTimer("buildbot.FindCommitsForBuild").Stop()
// Shortcut: Don't bother computing commit blamelists for trybots.
if IsTrybot(b.Builder) {
return []string{}, -1, []string{}, nil
}
if b.Repository == "" {
return []string{}, -1, []string{}, nil
}
repo, err := repos.Repo(b.Repository)
if err != nil {
return nil, -1, nil, fmt.Errorf("Could not find commits for build: %v", err)
}
// Shortcut for the first build for a given builder: this build must be
// the first inclusion for all revisions prior to b.GotRevision.
if b.Number == 0 && b.GotRevision != "" {
revlist, err := repo.RevList(b.GotRevision)
return revlist, -1, []string{}, err
}
// Start tracing commits back in time until we hit a previous build.
commitMap, stealFrom, stolen, err := findCommitsRecursive(bf, map[string]bool{}, b, b.GotRevision, repo, -1, []string{})
if err != nil {
return nil, -1, nil, err
}
commits := make([]string, 0, len(commitMap))
for c, _ := range commitMap {
commits = append(commits, c)
}
return commits, stealFrom, stolen, nil
}
// ingestNewBuilds finds the set of uningested builds and ingests them.
func ingestNewBuilds(m string, repos *gitinfo.RepoMap) error {
defer metrics.NewTimer("buildbot.ingestNewBuilds").Stop()
glog.Infof("Ingesting builds for %s", m)
// TODO(borenet): Investigate the use of channels here. We should be
// able to start ingesting builds as the data becomes available rather
// than waiting until the end.
buildsToProcess, err := getUningestedBuilds(m)
if err != nil {
return fmt.Errorf("Failed to obtain the set of uningested builds: %v", err)
}
unfinished, err := getUnfinishedBuilds(m)
if err != nil {
return fmt.Errorf("Failed to obtain the set of unfinished builds: %v", err)
}
for _, b := range unfinished {
if _, ok := buildsToProcess[b.Builder]; !ok {
buildsToProcess[b.Builder] = []int{}
}
buildsToProcess[b.Builder] = append(buildsToProcess[b.Builder], b.Number)
}
if err := repos.Update(); err != nil {
return err
}
// TODO(borenet): Can we ingest builders in parallel?
errs := map[string]error{}
for b, w := range buildsToProcess {
for _, n := range w {
if BUILD_BLACKLIST[b][n] {
glog.Warningf("Skipping blacklisted build: %s # %d", b, n)
continue
}
if IsTrybot(b) {
continue
}
glog.Infof("Ingesting build: %s, %s, %d", m, b, n)
build, err := retryGetBuildFromMaster(m, b, n, repos)
if err != nil {
errs[b] = fmt.Errorf("Failed to ingest build: %v", err)
break
}
if err := IngestBuild(build, repos); err != nil {
errs[b] = fmt.Errorf("Failed to ingest build: %v", err)
break
}
}
}
if len(errs) > 0 {
msg := fmt.Sprintf("Encountered errors ingesting builds for %s:", m)
for b, err := range errs {
msg += fmt.Sprintf("\n%s: %v", b, err)
}
return fmt.Errorf(msg)
}
glog.Infof("Done ingesting builds for %s", m)
return nil
}
// tx is a wrapper for a BoltDB transaction which tracks statistics.
func (d *localDB) tx(name string, fn func(*bolt.Tx) error, update bool) error {
txId := d.startTx(name)
defer d.endTx(txId)
defer metrics.NewTimer(fmt.Sprintf("buildbot.tx.%s", name)).Stop()
if update {
return d.db.Update(fn)
} else {
return d.db.View(fn)
}
}
// ReplaceIntoDB inserts or updates the Build in the database.
func (b *Build) ReplaceIntoDB() error {
defer metrics.NewTimer("buildbot.ReplaceIntoDB").Stop()
var err error
for attempt := 0; attempt < 5; attempt++ {
if err = b.replaceIntoDB(); err == nil {
return nil
}
time.Sleep(500 * time.Millisecond)
}
return err
}
// getUningestedBuilds returns a map whose keys are master names and values are
// sub-maps whose keys are builder names and values are slices of ints
// representing the numbers of builds which have not yet been ingested.
func getUningestedBuilds(db DB, m string) (map[string][]int, error) {
defer metrics.NewTimer("buildbot.getUningestedBuilds").Stop()
// Get the latest and last-processed builds for all builders.
latest, err := getLatestBuilds(m)
if err != nil {
return nil, fmt.Errorf("Failed to get latest builds: %s", err)
}
lastProcessed, err := db.GetLastProcessedBuilds(m)
if err != nil {
return nil, fmt.Errorf("Failed to get last-processed builds: %s", err)
}
// Find the range of uningested builds for each builder.
type numRange struct {
Start int // The last-ingested build number.
End int // The latest build number.
}
ranges := map[string]*numRange{}
for _, id := range lastProcessed {
b, err := db.GetBuild(id)
if err != nil {
return nil, err
}
ranges[b.Builder] = &numRange{
Start: b.Number,
End: b.Number,
}
}
for b, n := range latest {
if _, ok := ranges[b]; !ok {
ranges[b] = &numRange{
Start: -1,
End: n,
}
} else {
ranges[b].End = n
}
}
// Create a slice of build numbers for the uningested builds.
unprocessed := map[string][]int{}
for b, r := range ranges {
if r.End < r.Start {
glog.Warningf("Cannot create slice of builds to ingest for %q; invalid range (%d, %d)", b, r.Start, r.End)
continue
}
builds := make([]int, r.End-r.Start)
for i := r.Start + 1; i <= r.End; i++ {
builds[i-r.Start-1] = i
}
if len(builds) > 0 {
unprocessed[b] = builds
}
}
return unprocessed, nil
}
// retryGetBuildFromMaster retrieves the given build from the build master's JSON
// interface as specified by the master, builder, and build number. Makes
// multiple attempts in case the master fails to respond.
func retryGetBuildFromMaster(master, builder string, buildNumber int, repos *gitinfo.RepoMap) (*Build, error) {
defer metrics.NewTimer("buildbot.retryGetBuildFromMaster").Stop()
var b *Build
var err error
for attempt := 0; attempt < 3; attempt++ {
b, err = getBuildFromMaster(master, builder, buildNumber, repos)
if err == nil {
break
}
time.Sleep(500 * time.Millisecond)
}
return b, err
}
// IngestBuild retrieves the given build from the build master's JSON interface
// and pushes it into the database.
func IngestBuild(b *Build, repos *gitinfo.RepoMap) error {
defer metrics.NewTimer("buildbot.IngestBuild").Stop()
// Find the commits for this build.
commits, stoleFrom, stolen, err := FindCommitsForBuild(&bf, b, repos)
if err != nil {
return err
}
b.Commits = commits
// Log the case where we found no revisions for the build.
if !(IsTrybot(b.Builder) || strings.Contains(b.Builder, "Housekeeper")) && len(b.Commits) == 0 {
glog.Infof("Got build with 0 revs: %s #%d GotRev=%s", b.Builder, b.Number, b.GotRevision)
}
// Determine whether we've already ingested this build. If so, fix up the ID
// so that we update it rather than insert a new copy.
existingBuildID, err := GetBuildIDFromDB(b.Builder, b.Master, b.Number)
if err == nil {
b.Id = existingBuildID
}
// Insert the build.
if stoleFrom >= 0 && stolen != nil && len(stolen) > 0 {
// Remove the commits we stole from the previous owner.
oldBuild, err := GetBuildFromDB(b.Builder, b.Master, stoleFrom)
if err != nil {
return err
}
if oldBuild == nil {
return fmt.Errorf("Attempted to retrieve %s #%d, but got a nil build from the DB.", b.Builder, stoleFrom)
}
newCommits := make([]string, 0, len(oldBuild.Commits))
for _, c := range oldBuild.Commits {
keep := true
for _, s := range stolen {
if c == s {
keep = false
break
}
}
if keep {
newCommits = append(newCommits, c)
}
}
oldBuild.Commits = newCommits
return ReplaceMultipleBuildsIntoDB([]*Build{b, oldBuild})
} else {
return b.ReplaceIntoDB()
}
}
// insertBuilds inserts a batch of builds into the database. In the case of
// failure, it continually retries until it succeeds.
func insertBuilds(builds []*Build) {
defer metrics.NewTimer("buildbot.insertBuilds").Stop()
for {
// Insert the builds.
glog.Infof("Inserting %d builds.", len(builds))
if err := ReplaceMultipleBuildsIntoDB(builds); err != nil {
glog.Errorf("Failed to insert builds, retrying: %s", err)
time.Sleep(100 * time.Millisecond)
} else {
break
}
}
glog.Infof("Finished inserting %d builds.", len(builds))
go_metrics.GetOrRegisterCounter("buildbot.NumInsertedBuilds", go_metrics.DefaultRegistry).Inc(int64(len(builds)))
}
// IngestBuild retrieves the given build from the build master's JSON interface
// and pushes it into the database.
func IngestBuild(db DB, b *Build, repos *gitinfo.RepoMap) error {
defer metrics.NewTimer("buildbot.IngestBuild").Stop()
defer go_metrics.GetOrRegisterCounter("buildbot.NumIngestedBuilds", go_metrics.DefaultRegistry).Inc(1)
// Find the commits for this build.
commits, stoleFrom, stolen, err := FindCommitsForBuild(db, b, repos)
if err != nil {
return err
}
b.Commits = commits
// Log the case where we found no revisions for the build.
if !(IsTrybot(b.Builder) || strings.Contains(b.Builder, "Housekeeper")) && len(b.Commits) == 0 {
glog.Infof("Got build with 0 revs: %s #%d GotRev=%s", b.Builder, b.Number, b.GotRevision)
}
// Insert the build.
if stoleFrom >= 0 && stolen != nil && len(stolen) > 0 {
// Remove the commits we stole from the previous owner.
oldBuild, err := db.GetBuildFromDB(b.Master, b.Builder, stoleFrom)
if err != nil {
return err
}
if oldBuild == nil {
return fmt.Errorf("Attempted to retrieve %s #%d, but got a nil build from the DB.", b.Builder, stoleFrom)
}
newCommits := make([]string, 0, len(oldBuild.Commits))
for _, c := range oldBuild.Commits {
keep := true
for _, s := range stolen {
if c == s {
keep = false
break
}
}
if keep {
newCommits = append(newCommits, c)
}
}
oldBuild.Commits = newCommits
return db.PutBuilds([]*Build{b, oldBuild})
} else {
return db.PutBuild(b)
}
}
func ReplaceMultipleBuildsIntoDB(builds []*Build) (rv error) {
defer metrics.NewTimer("buildbot.ReplaceMultipleBuildsIntoDB").Stop()
tx, err := DB.Beginx()
if err != nil {
return fmt.Errorf("Unable to push builds into database - Could not begin transaction: %v", err)
}
// Defer committing/rolling back the transaction.
defer func() {
rv = database.CommitOrRollback(tx, rv)
}()
// Insert the builds.
// TODO(borenet): Insert/update all of the builds at once.
for _, b := range builds {
if err := b.replaceIntoDBTx(tx); err != nil {
return err
}
}
return nil
}
// See documentation for DB interface.
func (d *localDB) PutBuilds(builds []*Build) error {
defer metrics.NewTimer("buildbot.PutBuilds").Stop()
return d.update("PutBuilds", func(tx *bolt.Tx) error {
return putBuilds(tx, builds)
})
}
