// Start calculating and reporting statistics on the repo and tiles.
//
// We presume the git.Update(true) is called somewhere else, usually this is done
// in the trace/db.Builder, so the repo is always as good as the loaded tiles.
func Start(nanoTileStore *db.Builder, git *gitinfo.GitInfo) {
coverage := metrics.NewRegisteredGaugeFloat64("stats.tests.bench_runs_per_changelist", metrics.DefaultRegistry)
skpLatency := metrics.NewRegisteredTimer("stats.skp.update_latency", metrics.DefaultRegistry)
commits := metrics.NewRegisteredGauge("stats.commits.total", metrics.DefaultRegistry)
go func() {
for _ = range time.Tick(2 * time.Minute) {
tile := nanoTileStore.GetTile()
numCommits := tile.LastCommitIndex() + 1
numTraces := len(tile.Traces)
total := 0
for _, tr := range tile.Traces {
for i := 0; i < numCommits; i++ {
if !tr.IsMissing(i) {
total += 1
}
}
}
cov := float64(total) / float64(numCommits*numTraces)
glog.Info("Coverage: ", cov)
coverage.Update(cov)
last, err := git.LastSkpCommit()
if err != nil {
glog.Warning("Failed to read last SKP commit: %s", err)
continue
}
skpLatency.Update(time.Since(last))
commits.Update(int64(git.NumCommits()))
}
}()
}
// findCommitsRecursive is a recursive function called by FindCommitsForBuild.
// It traces the history to find builds which were first included in the given
// build.
func findCommitsRecursive(commits map[string]bool, b *Build, hash string, repo *gitinfo.GitInfo, stealFrom int, stolen []string) (map[string]bool, int, []string, error) {
// Shortcut for empty hashes. This can happen when a commit has no
// parents (initial commit) or when a Build has no GotRevision.
if hash == "" {
return commits, stealFrom, stolen, nil
}
// Determine whether any build already includes this commit.
n, err := GetBuildForCommit(b.Builder, b.Master, hash)
if err != nil {
return commits, stealFrom, stolen, fmt.Errorf("Could not find build for commit %s: %v", hash, err)
}
// If so, we have to make a decision.
if n >= 0 {
// If the build we found is the current build, keep going,
// since we may have already ingested data for this build but still
// need to find accurate revision data.
if n != b.Number {
// If this Build's GotRevision is already included in a different
// Build, then we're "inserting" this one in between two already-ingested
// Builds. In that case, this build is providing "better" information
// on the already-claimed commits, so we steal them from the other Build.
if hash == b.GotRevision {
stealFrom = n
}
if stealFrom == n {
stolen = append(stolen, hash)
} else {
return commits, stealFrom, stolen, nil
}
}
}
// Add the commit.
commits[hash] = true
// Recurse on the commit's parents.
c, err := repo.Details(hash)
if err != nil {
// Special case. Commits can disappear from the repository
// after they're picked up by the buildbots but before they're
// ingested here. If we can't find a commit, log an error and
// skip the commit.
glog.Errorf("Failed to obtain details for %s: %v", hash, err)
delete(commits, hash)
return commits, stealFrom, stolen, nil
}
for _, p := range c.Parents {
// If we've already seen this parent commit, don't revisit it.
if _, ok := commits[p]; ok {
continue
}
commits, stealFrom, stolen, err = findCommitsRecursive(commits, b, p, repo, stealFrom, stolen)
if err != nil {
return commits, stealFrom, stolen, err
}
}
return commits, stealFrom, stolen, nil
}
// updateRepo syncs the given repo and returns a set of BuildCandidates for it.
func (q *BuildQueue) updateRepo(repoUrl string, repo *gitinfo.GitInfo, now time.Time) (map[string][]*BuildCandidate, error) {
from := now.Add(-q.period)
if q.period == PERIOD_FOREVER {
from = time.Unix(0, 0)
}
recentCommits := repo.From(from)
commitDetails := map[string]*gitinfo.LongCommit{}
for _, c := range recentCommits {
details, err := repo.Details(c)
if err != nil {
return nil, err
}
commitDetails[c] = details
}
// Get all builds associated with the recent commits.
buildsByCommit, err := buildbot.GetBuildsForCommits(recentCommits, map[int]bool{})
if err != nil {
return nil, err
}
// Find the sets of all bots and masters, organize builds by
// commit/builder and builder/number.
masters := map[string]string{}
builds := map[string]map[string]*buildbot.Build{}
buildsByBuilderAndNum := map[string]map[int]*buildbot.Build{}
for commit, buildsForCommit := range buildsByCommit {
builds[commit] = map[string]*buildbot.Build{}
for _, build := range buildsForCommit {
if !util.In(build.Builder, q.botWhitelist) {
continue
}
masters[build.Builder] = build.Master
builds[commit][build.Builder] = build
if _, ok := buildsByBuilderAndNum[build.Builder]; !ok {
buildsByBuilderAndNum[build.Builder] = map[int]*buildbot.Build{}
}
buildsByBuilderAndNum[build.Builder][build.Number] = build
}
}
allBots := make([]string, 0, len(masters))
for builder, _ := range masters {
allBots = append(allBots, builder)
}
// Find the current scores for each commit/builder pair.
currentScores := map[string]map[string]float64{}
for _, commit := range recentCommits {
myBuilds, ok := builds[commit]
if !ok {
myBuilds = map[string]*buildbot.Build{}
}
currentScores[commit] = map[string]float64{}
for _, builder := range allBots {
currentScores[commit][builder] = scoreBuild(commitDetails[commit], myBuilds[builder], now, q.timeLambda)
}
}
// For each commit/builder pair, determine the score increase obtained
// by running a build at that commit.
scoreIncrease := map[string]map[string]float64{}
for _, commit := range recentCommits {
scoreIncrease[commit] = map[string]float64{}
for _, builder := range allBots {
// Shortcut: Don't bisect builds with a huge number
// of commits. This saves lots of time and only affects
// the first successful build for a bot.
if _, ok := builds[commit][builder]; ok {
if len(builds[commit][builder].Commits) > NO_BISECT_COMMIT_LIMIT {
glog.Warningf("Skipping %s on %s; previous build has too many commits.", commit[0:7], builder)
scoreIncrease[commit][builder] = 0.0
continue
}
}
newScores := map[string]float64{}
// Pretend to create a new Build at the given commit.
newBuild := buildbot.Build{
Builder: builder,
Master: masters[builder],
Number: math.MaxInt32,
GotRevision: commit,
Repository: repoUrl,
}
commits, stealFrom, stolen, err := buildbot.FindCommitsForBuild(&newBuild, q.repos)
if err != nil {
return nil, err
}
// Re-score all commits in the new build.
newBuild.Commits = commits
for _, c := range commits {
if _, ok := currentScores[c]; !ok {
// If this build has commits which are outside of our window,
// insert them into currentScores to account for them.
score := scoreBuild(commitDetails[commit], builds[commit][builder], now, q.timeLambda)
currentScores[c] = map[string]float64{
builder: score,
}
}
if _, ok := commitDetails[c]; !ok {
d, err := repo.Details(c)
if err != nil {
return nil, err
}
commitDetails[c] = d
}
newScores[c] = scoreBuild(commitDetails[c], &newBuild, now, q.timeLambda)
}
// If the new build includes commits previously included in
// another build, update scores for commits in the build we stole
// them from.
if stealFrom != -1 {
stoleFromOrig, ok := buildsByBuilderAndNum[builder][stealFrom]
if !ok {
// The build may not be cached. Fall back on getting it from the DB.
stoleFromOrig, err = buildbot.GetBuildFromDB(builder, masters[builder], stealFrom)
if err != nil {
return nil, err
}
buildsByBuilderAndNum[builder][stealFrom] = stoleFromOrig
}
// "copy" the build so that we can assign new commits to it
// without modifying the cached build.
stoleFromBuild := *stoleFromOrig
newCommits := []string{}
for _, c := range stoleFromBuild.Commits {
if !util.In(c, stolen) {
newCommits = append(newCommits, c)
}
}
stoleFromBuild.Commits = newCommits
for _, c := range stoleFromBuild.Commits {
newScores[c] = scoreBuild(commitDetails[c], &stoleFromBuild, now, q.timeLambda)
}
}
// Sum the old and new scores.
// First, sort the old and new scores to help with numerical stability.
oldScoresList := make([]float64, 0, len(newScores))
newScoresList := make([]float64, 0, len(newScores))
for c, score := range newScores {
oldScoresList = append(oldScoresList, currentScores[c][builder])
newScoresList = append(newScoresList, score)
}
sort.Sort(sort.Float64Slice(oldScoresList))
sort.Sort(sort.Float64Slice(newScoresList))
oldTotal := 0.0
newTotal := 0.0
for i, _ := range oldScoresList {
oldTotal += oldScoresList[i]
newTotal += newScoresList[i]
}
scoreIncrease[commit][builder] = newTotal - oldTotal
}
}
// Arrange the score increases by builder.
candidates := map[string][]*BuildCandidate{}
for commit, builders := range scoreIncrease {
for builder, scoreIncrease := range builders {
if _, ok := candidates[builder]; !ok {
candidates[builder] = []*BuildCandidate{}
}
// Don't schedule builds below the given threshold.
if scoreIncrease > q.scoreThreshold {
candidates[builder] = append(candidates[builder], &BuildCandidate{
Author: commitDetails[commit].Author,
Builder: builder,
Commit: commit,
Repo: repoUrl,
Score: scoreIncrease,
})
}
}
}
return candidates, nil
}