// LoadData loads the build data for the given commits.
func LoadData(commits []string) (map[int]*buildbot.Build, map[string]map[string]*buildbot.BuildSummary, map[string][]*buildbot.BuilderComment, error) {
defer timer.New("build_cache.LoadData()").Stop()
builds, err := buildbot.GetBuildsForCommits(commits, nil)
if err != nil {
return nil, nil, nil, err
}
byId := map[int]*buildbot.Build{}
byCommit := map[string]map[string]*buildbot.BuildSummary{}
builders := map[string]bool{}
for hash, buildList := range builds {
byBuilder := map[string]*buildbot.BuildSummary{}
for _, b := range buildList {
byId[b.Id] = b
if !util.AnyMatch(BOT_BLACKLIST, b.Builder) {
byBuilder[b.Builder] = b.GetSummary()
builders[b.Builder] = true
}
}
byCommit[hash] = byBuilder
}
builderList := make([]string, 0, len(builders))
for b, _ := range builders {
builderList = append(builderList, b)
}
builderComments, err := buildbot.GetBuildersComments(builderList)
if err != nil {
return nil, nil, nil, err
}
return byId, byCommit, builderComments, nil
}
// getFreeBuildslaves returns a slice of names of buildslaves which are free.
func getFreeBuildslaves() ([]*buildslave, error) {
errMsg := "Failed to get free buildslaves: %v"
// Get the set of builders for each master.
builders, err := getBuilders()
if err != nil {
return nil, fmt.Errorf(errMsg, err)
}
// Get the set of buildslaves for each master.
buildslaves, err := getBuildslaves()
if err != nil {
return nil, fmt.Errorf(errMsg, err)
}
// Map the builders to buildslaves.
for _, b := range builders {
// Only include builders in the whitelist, and those only if
// there are no already-pending builds.
if !util.AnyMatch(BOT_BLACKLIST, b.Name) && b.PendingBuilds == 0 {
for _, slave := range b.Slaves {
buildslaves[slave].Builders = append(buildslaves[slave].Builders, b.Name)
}
}
}
// Return the builders which are connected and idle.
rv := []*buildslave{}
for _, s := range buildslaves {
if len(s.RunningBuilds) == 0 && s.Connected {
rv = append(rv, s)
}
}
return rv, nil
}
// updateRepo syncs the given repo and returns a set of BuildCandidates for
// each builder which uses it.
func (q *BuildQueue) updateRepo(repoUrl string, now time.Time) (map[string][]*BuildCandidate, error) {
defer timer.New("BuildQueue.updateRepo()").Stop()
errMsg := "Failed to update the repo: %v"
// Sync/update the code.
repo, err := q.repos.Repo(repoUrl)
if err != nil {
return nil, fmt.Errorf(errMsg, err)
}
if err := repo.Update(true, true); err != nil {
return nil, fmt.Errorf(errMsg, err)
}
// Get the details for all recent commits.
from := now.Add(-q.period)
if q.period == PERIOD_FOREVER {
from = time.Unix(0, 0)
}
recentCommits := util.Reverse(repo.From(from))
// Pre-load commit details, from a larger window than we actually care about.
fromPreload := now.Add(time.Duration(int64(-1.5 * float64(q.period))))
if q.period == PERIOD_FOREVER {
fromPreload = time.Unix(0, 0)
}
recentCommitsPreload := repo.From(fromPreload)
for _, c := range recentCommitsPreload {
if _, err := repo.Details(c); err != nil {
return nil, fmt.Errorf(errMsg, err)
}
}
// Get all builds associated with the recent commits.
buildsByCommit, err := buildbot.GetBuildsForCommits(recentCommitsPreload, map[int]bool{})
if err != nil {
return nil, fmt.Errorf(errMsg, err)
}
// Create buildCaches for each builder.
buildCaches := map[string]*buildCache{}
for _, buildsForCommit := range buildsByCommit {
for _, build := range buildsForCommit {
if util.AnyMatch(q.botBlacklist, build.Builder) {
glog.Infof("Skipping blacklisted builder %s", build.Builder)
continue
}
if _, ok := buildCaches[build.Builder]; !ok {
bc, err := newBuildCache(build.Builder, build.Master, build.Repository)
if err != nil {
return nil, fmt.Errorf(errMsg, err)
}
buildCaches[build.Builder] = bc
}
if err := buildCaches[build.Builder].Put(build); err != nil {
return nil, err
}
}
}
// Find candidates for each builder.
candidates := map[string][]*BuildCandidate{}
errs := map[string]error{}
mutex := sync.Mutex{}
var wg sync.WaitGroup
for builder, finder := range buildCaches {
wg.Add(1)
go func(b string, bc *buildCache) {
defer wg.Done()
c, err := q.getCandidatesForBuilder(bc, recentCommits, now)
mutex.Lock()
defer mutex.Unlock()
if err != nil {
errs[b] = err
return
}
candidates[b] = c
}(builder, finder)
}
wg.Wait()
if len(errs) > 0 {
msg := "Failed to update the repo:"
for _, err := range errs {
msg += fmt.Sprintf("\n%v", err)
}
return nil, fmt.Errorf(msg)
}
return candidates, nil
}
