Exemplo n.º 1
0
// LoadData loads the build data for the given commits.
func LoadData(commits []string) (map[int]*buildbot.Build, map[string]map[string]*buildbot.BuildSummary, map[string][]*buildbot.BuilderComment, error) {
	defer timer.New("build_cache.LoadData()").Stop()
	builds, err := buildbot.GetBuildsForCommits(commits, nil)
	if err != nil {
		return nil, nil, nil, err
	}
	byId := map[int]*buildbot.Build{}
	byCommit := map[string]map[string]*buildbot.BuildSummary{}
	builders := map[string]bool{}
	for hash, buildList := range builds {
		byBuilder := map[string]*buildbot.BuildSummary{}
		for _, b := range buildList {
			byId[b.Id] = b
			if !util.AnyMatch(BOT_BLACKLIST, b.Builder) {
				byBuilder[b.Builder] = b.GetSummary()
				builders[b.Builder] = true
			}
		}
		byCommit[hash] = byBuilder
	}
	builderList := make([]string, 0, len(builders))
	for b, _ := range builders {
		builderList = append(builderList, b)
	}
	builderComments, err := buildbot.GetBuildersComments(builderList)
	if err != nil {
		return nil, nil, nil, err
	}
	return byId, byCommit, builderComments, nil
}
Exemplo n.º 2
0
// updateRepo syncs the given repo and returns a set of BuildCandidates for
// each builder which uses it.
func (q *BuildQueue) updateRepo(repoUrl string, now time.Time) (map[string][]*BuildCandidate, error) {
	defer timer.New("BuildQueue.updateRepo()").Stop()
	errMsg := "Failed to update the repo: %v"

	// Sync/update the code.
	repo, err := q.repos.Repo(repoUrl)
	if err != nil {
		return nil, fmt.Errorf(errMsg, err)
	}
	if err := repo.Update(true, true); err != nil {
		return nil, fmt.Errorf(errMsg, err)
	}

	// Get the details for all recent commits.
	from := now.Add(-q.period)
	if q.period == PERIOD_FOREVER {
		from = time.Unix(0, 0)
	}
	recentCommits := util.Reverse(repo.From(from))

	// Pre-load commit details, from a larger window than we actually care about.
	fromPreload := now.Add(time.Duration(int64(-1.5 * float64(q.period))))
	if q.period == PERIOD_FOREVER {
		fromPreload = time.Unix(0, 0)
	}
	recentCommitsPreload := repo.From(fromPreload)
	for _, c := range recentCommitsPreload {
		if _, err := repo.Details(c); err != nil {
			return nil, fmt.Errorf(errMsg, err)
		}
	}

	// Get all builds associated with the recent commits.
	buildsByCommit, err := buildbot.GetBuildsForCommits(recentCommitsPreload, map[int]bool{})
	if err != nil {
		return nil, fmt.Errorf(errMsg, err)
	}

	// Create buildCaches for each builder.
	buildCaches := map[string]*buildCache{}
	for _, buildsForCommit := range buildsByCommit {
		for _, build := range buildsForCommit {
			if util.AnyMatch(q.botBlacklist, build.Builder) {
				glog.Infof("Skipping blacklisted builder %s", build.Builder)
				continue
			}
			if _, ok := buildCaches[build.Builder]; !ok {
				bc, err := newBuildCache(build.Builder, build.Master, build.Repository)
				if err != nil {
					return nil, fmt.Errorf(errMsg, err)
				}
				buildCaches[build.Builder] = bc
			}
			if err := buildCaches[build.Builder].Put(build); err != nil {
				return nil, err
			}
		}
	}

	// Find candidates for each builder.
	candidates := map[string][]*BuildCandidate{}
	errs := map[string]error{}
	mutex := sync.Mutex{}
	var wg sync.WaitGroup
	for builder, finder := range buildCaches {
		wg.Add(1)
		go func(b string, bc *buildCache) {
			defer wg.Done()
			c, err := q.getCandidatesForBuilder(bc, recentCommits, now)
			mutex.Lock()
			defer mutex.Unlock()
			if err != nil {
				errs[b] = err
				return
			}
			candidates[b] = c
		}(builder, finder)
	}
	wg.Wait()
	if len(errs) > 0 {
		msg := "Failed to update the repo:"
		for _, err := range errs {
			msg += fmt.Sprintf("\n%v", err)
		}
		return nil, fmt.Errorf(msg)
	}
	return candidates, nil
}
Exemplo n.º 3
0
// updateRepo syncs the given repo and returns a set of BuildCandidates for it.
func (q *BuildQueue) updateRepo(repoUrl string, repo *gitinfo.GitInfo, now time.Time) (map[string][]*BuildCandidate, error) {
	from := now.Add(-q.period)
	if q.period == PERIOD_FOREVER {
		from = time.Unix(0, 0)
	}
	recentCommits := repo.From(from)
	commitDetails := map[string]*gitinfo.LongCommit{}
	for _, c := range recentCommits {
		details, err := repo.Details(c)
		if err != nil {
			return nil, err
		}
		commitDetails[c] = details
	}

	// Get all builds associated with the recent commits.
	buildsByCommit, err := buildbot.GetBuildsForCommits(recentCommits, map[int]bool{})
	if err != nil {
		return nil, err
	}

	// Find the sets of all bots and masters, organize builds by
	// commit/builder and builder/number.
	masters := map[string]string{}
	builds := map[string]map[string]*buildbot.Build{}
	buildsByBuilderAndNum := map[string]map[int]*buildbot.Build{}
	for commit, buildsForCommit := range buildsByCommit {
		builds[commit] = map[string]*buildbot.Build{}
		for _, build := range buildsForCommit {
			if !util.In(build.Builder, q.botWhitelist) {
				continue
			}
			masters[build.Builder] = build.Master
			builds[commit][build.Builder] = build
			if _, ok := buildsByBuilderAndNum[build.Builder]; !ok {
				buildsByBuilderAndNum[build.Builder] = map[int]*buildbot.Build{}
			}
			buildsByBuilderAndNum[build.Builder][build.Number] = build
		}
	}
	allBots := make([]string, 0, len(masters))
	for builder, _ := range masters {
		allBots = append(allBots, builder)
	}

	// Find the current scores for each commit/builder pair.
	currentScores := map[string]map[string]float64{}
	for _, commit := range recentCommits {
		myBuilds, ok := builds[commit]
		if !ok {
			myBuilds = map[string]*buildbot.Build{}
		}
		currentScores[commit] = map[string]float64{}
		for _, builder := range allBots {
			currentScores[commit][builder] = scoreBuild(commitDetails[commit], myBuilds[builder], now, q.timeLambda)
		}
	}

	// For each commit/builder pair, determine the score increase obtained
	// by running a build at that commit.
	scoreIncrease := map[string]map[string]float64{}
	for _, commit := range recentCommits {
		scoreIncrease[commit] = map[string]float64{}
		for _, builder := range allBots {
			// Shortcut: Don't bisect builds with a huge number
			// of commits.  This saves lots of time and only affects
			// the first successful build for a bot.
			if _, ok := builds[commit][builder]; ok {
				if len(builds[commit][builder].Commits) > NO_BISECT_COMMIT_LIMIT {
					glog.Warningf("Skipping %s on %s; previous build has too many commits.", commit[0:7], builder)
					scoreIncrease[commit][builder] = 0.0
					continue
				}
			}

			newScores := map[string]float64{}
			// Pretend to create a new Build at the given commit.
			newBuild := buildbot.Build{
				Builder:     builder,
				Master:      masters[builder],
				Number:      math.MaxInt32,
				GotRevision: commit,
				Repository:  repoUrl,
			}
			commits, stealFrom, stolen, err := buildbot.FindCommitsForBuild(&newBuild, q.repos)
			if err != nil {
				return nil, err
			}
			// Re-score all commits in the new build.
			newBuild.Commits = commits
			for _, c := range commits {
				if _, ok := currentScores[c]; !ok {
					// If this build has commits which are outside of our window,
					// insert them into currentScores to account for them.
					score := scoreBuild(commitDetails[commit], builds[commit][builder], now, q.timeLambda)
					currentScores[c] = map[string]float64{
						builder: score,
					}
				}
				if _, ok := commitDetails[c]; !ok {
					d, err := repo.Details(c)
					if err != nil {
						return nil, err
					}
					commitDetails[c] = d
				}
				newScores[c] = scoreBuild(commitDetails[c], &newBuild, now, q.timeLambda)
			}
			// If the new build includes commits previously included in
			// another build, update scores for commits in the build we stole
			// them from.
			if stealFrom != -1 {
				stoleFromOrig, ok := buildsByBuilderAndNum[builder][stealFrom]
				if !ok {
					// The build may not be cached. Fall back on getting it from the DB.
					stoleFromOrig, err = buildbot.GetBuildFromDB(builder, masters[builder], stealFrom)
					if err != nil {
						return nil, err
					}
					buildsByBuilderAndNum[builder][stealFrom] = stoleFromOrig
				}
				// "copy" the build so that we can assign new commits to it
				// without modifying the cached build.
				stoleFromBuild := *stoleFromOrig
				newCommits := []string{}
				for _, c := range stoleFromBuild.Commits {
					if !util.In(c, stolen) {
						newCommits = append(newCommits, c)
					}
				}
				stoleFromBuild.Commits = newCommits
				for _, c := range stoleFromBuild.Commits {
					newScores[c] = scoreBuild(commitDetails[c], &stoleFromBuild, now, q.timeLambda)
				}
			}
			// Sum the old and new scores.
			// First, sort the old and new scores to help with numerical stability.
			oldScoresList := make([]float64, 0, len(newScores))
			newScoresList := make([]float64, 0, len(newScores))
			for c, score := range newScores {
				oldScoresList = append(oldScoresList, currentScores[c][builder])
				newScoresList = append(newScoresList, score)
			}
			sort.Sort(sort.Float64Slice(oldScoresList))
			sort.Sort(sort.Float64Slice(newScoresList))
			oldTotal := 0.0
			newTotal := 0.0
			for i, _ := range oldScoresList {
				oldTotal += oldScoresList[i]
				newTotal += newScoresList[i]
			}
			scoreIncrease[commit][builder] = newTotal - oldTotal
		}
	}

	// Arrange the score increases by builder.
	candidates := map[string][]*BuildCandidate{}
	for commit, builders := range scoreIncrease {
		for builder, scoreIncrease := range builders {
			if _, ok := candidates[builder]; !ok {
				candidates[builder] = []*BuildCandidate{}
			}
			// Don't schedule builds below the given threshold.
			if scoreIncrease > q.scoreThreshold {
				candidates[builder] = append(candidates[builder], &BuildCandidate{
					Author:  commitDetails[commit].Author,
					Builder: builder,
					Commit:  commit,
					Repo:    repoUrl,
					Score:   scoreIncrease,
				})
			}
		}
	}

	return candidates, nil
}