// Makes sure that the dependencies for the tasks in the project form a
// valid dependency graph (no cycles).
func checkDependencyGraph(project *model.Project) []ValidationError {
errs := []ValidationError{}
// map of task name and variant -> BuildVariantTask
tasksByNameAndVariant := map[model.TVPair]model.BuildVariantTask{}
// generate task nodes for every task and variant combination
visited := map[model.TVPair]bool{}
allNodes := []model.TVPair{}
for _, bv := range project.BuildVariants {
for _, t := range bv.Tasks {
t.Populate(project.GetSpecForTask(t.Name))
node := model.TVPair{bv.Name, t.Name}
tasksByNameAndVariant[node] = t
visited[node] = false
allNodes = append(allNodes, node)
}
}
// run through the task nodes, checking their dependency graphs for cycles
for _, node := range allNodes {
// the visited nodes
if err := dependencyCycleExists(node, visited, tasksByNameAndVariant); err != nil {
errs = append(errs,
ValidationError{
Message: fmt.Sprintf(
"dependency error for '%v' task: %v", node.TaskName, err),
},
)
}
}
return errs
}
func applyPatch(patch *service.RestPatch, rootCloneDir string, conf *model.Project, variant *model.BuildVariant) error {
// patch sets and contain multiple patches, some of them for modules
for _, patchPart := range patch.Patches {
var dir string
if patchPart.ModuleName == "" {
// if patch is not part of a module, just apply patch against src root
dir = rootCloneDir
} else {
fmt.Println("Applying patches for module", patchPart.ModuleName)
// if patch is part of a module, apply patch in module root
module, err := conf.GetModuleByName(patchPart.ModuleName)
if err != nil || module == nil {
return fmt.Errorf("can't find module %v: %v", patchPart.ModuleName, err)
}
// skip the module if this build variant does not use it
if !util.SliceContains(variant.Modules, module.Name) {
continue
}
dir = filepath.Join(rootCloneDir, module.Prefix, module.Name)
}
args := []string{"apply", "--whitespace=fix"}
applyCmd := exec.Command("git", args...)
applyCmd.Stdout, applyCmd.Stderr, applyCmd.Dir = os.Stdout, os.Stderr, dir
applyCmd.Stdin = bytes.NewReader([]byte(patchPart.PatchSet.Patch))
err := applyCmd.Run()
if err != nil {
return err
}
}
return nil
}
func cloneSource(task *service.RestTask, project *model.ProjectRef, config *model.Project, cloneDir string) error {
// Fetch the outermost repo for the task
err := clone(
cloneOptions{
repo: fmt.Sprintf("[email protected]:%v/%v.git", project.Owner, project.Repo),
revision: task.Revision,
rootDir: cloneDir,
depth: defaultCloneDepth,
},
false,
)
if err != nil {
return err
}
// Then fetch each of the modules
variant := config.FindBuildVariant(task.BuildVariant)
if variant == nil {
return fmt.Errorf("couldn't find build variant '%v' in config", task.BuildVariant)
}
for _, moduleName := range variant.Modules {
module, err := config.GetModuleByName(moduleName)
if err != nil || module == nil {
return fmt.Errorf("variant refers to a module '%v' that doesn't exist.", moduleName)
}
moduleBase := filepath.Join(cloneDir, module.Prefix, module.Name)
fmt.Printf("Fetching module %v at %v\n", moduleName, module.Branch)
err = clone(cloneOptions{
repo: module.Repo,
revision: module.Branch,
rootDir: filepath.ToSlash(moduleBase),
}, false)
if err != nil {
return err
}
}
return nil
}
// Makes sure that the dependencies for the tasks have the correct fields,
// and that the fields reference valid tasks.
func verifyTaskRequirements(project *model.Project) []ValidationError {
errs := []ValidationError{}
for _, bvt := range project.FindAllBuildVariantTasks() {
for _, r := range bvt.Requires {
if project.FindProjectTask(r.Name) == nil {
if r.Name == model.AllDependencies {
errs = append(errs, ValidationError{Message: fmt.Sprintf(
"task '%v': * is not supported for requirement selectors", bvt.Name)})
} else {
errs = append(errs,
ValidationError{Message: fmt.Sprintf(
"task '%v' requires non-existent task '%v'", bvt.Name, r.Name)})
}
}
if r.Variant != "" && r.Variant != model.AllVariants && project.FindBuildVariant(r.Variant) == nil {
errs = append(errs, ValidationError{Message: fmt.Sprintf(
"task '%v' requires non-existent variant '%v'", bvt.Name, r.Variant)})
}
}
}
return errs
}
// Fetch versions until 'numVersionElements' elements are created, including
// elements consisting of multiple versions rolled-up into one.
// The skip value indicates how many versions back in time should be skipped
// before starting to fetch versions, the project indicates which project the
// returned versions should be a part of.
func getVersionsAndVariants(skip, numVersionElements int, project *model.Project) (versionVariantData, error) {
// the final array of versions to return
finalVersions := []waterfallVersion{}
// keep track of the build variants we see
bvSet := map[string]bool{}
waterfallRows := map[string]waterfallRow{}
// build variant mappings - used so we can store the display name as
// the build variant field of a build
buildVariantMappings := project.GetVariantMappings()
// keep track of the last rolled-up version, so inactive versions can
// be added
var lastRolledUpVersion *waterfallVersion = nil
// loop until we have enough from the db
for len(finalVersions) < numVersionElements {
// fetch the versions and associated builds
versionsFromDB, buildsByVersion, err :=
fetchVersionsAndAssociatedBuilds(project, skip, numVersionElements)
if err != nil {
return versionVariantData{}, fmt.Errorf("error fetching versions and builds:"+
" %v", err)
}
// if we've reached the beginning of all versions
if len(versionsFromDB) == 0 {
break
}
// to fetch started tasks and failed tests for providing additional context
// in a tooltip
failedAndStartedTaskIds := []string{}
// update the amount skipped
skip += len(versionsFromDB)
// create the necessary versions, rolling up inactive ones
for _, versionFromDB := range versionsFromDB {
// if we have hit enough versions, break out
if len(finalVersions) == numVersionElements {
break
}
// the builds for the version
buildsInVersion := buildsByVersion[versionFromDB.Id]
// see if there are any active tasks in the version
versionActive := anyActiveTasks(buildsInVersion)
// add any represented build variants to the set and initialize rows
for _, b := range buildsInVersion {
bvSet[b.BuildVariant] = true
buildVariant := waterfallBuildVariant{
Id: b.BuildVariant,
DisplayName: buildVariantMappings[b.BuildVariant],
}
if buildVariant.DisplayName == "" {
buildVariant.DisplayName = b.BuildVariant +
" (removed)"
}
if _, ok := waterfallRows[b.BuildVariant]; !ok {
waterfallRows[b.BuildVariant] = waterfallRow{
Builds: map[string]waterfallBuild{},
BuildVariant: buildVariant,
}
}
}
// if it is inactive, roll up the version and don't create any
// builds for it
if !versionActive {
if lastRolledUpVersion == nil {
lastRolledUpVersion = &waterfallVersion{RolledUp: true, RevisionOrderNumber: versionFromDB.RevisionOrderNumber}
}
// add the version metadata into the last rolled-up version
lastRolledUpVersion.Ids = append(lastRolledUpVersion.Ids,
versionFromDB.Id)
lastRolledUpVersion.Authors = append(lastRolledUpVersion.Authors,
versionFromDB.Author)
lastRolledUpVersion.Errors = append(
lastRolledUpVersion.Errors, waterfallVersionError{versionFromDB.Errors})
lastRolledUpVersion.Warnings = append(
lastRolledUpVersion.Warnings, waterfallVersionError{versionFromDB.Warnings})
lastRolledUpVersion.Messages = append(
lastRolledUpVersion.Messages, versionFromDB.Message)
lastRolledUpVersion.Ignoreds = append(
lastRolledUpVersion.Ignoreds, versionFromDB.Ignored)
lastRolledUpVersion.CreateTimes = append(
lastRolledUpVersion.CreateTimes, versionFromDB.CreateTime)
lastRolledUpVersion.Revisions = append(
lastRolledUpVersion.Revisions, versionFromDB.Revision)
// move on to the next version
continue
}
// add a pending rolled-up version, if it exists
if lastRolledUpVersion != nil {
finalVersions = append(finalVersions, *lastRolledUpVersion)
lastRolledUpVersion = nil
}
// if we have hit enough versions, break out
if len(finalVersions) == numVersionElements {
break
}
// if the version can not be rolled up, create a fully fledged
// version for it
activeVersion := waterfallVersion{
Ids: []string{versionFromDB.Id},
Messages: []string{versionFromDB.Message},
Authors: []string{versionFromDB.Author},
CreateTimes: []time.Time{versionFromDB.CreateTime},
Revisions: []string{versionFromDB.Revision},
Errors: []waterfallVersionError{{versionFromDB.Errors}},
Warnings: []waterfallVersionError{{versionFromDB.Warnings}},
Ignoreds: []bool{versionFromDB.Ignored},
RevisionOrderNumber: versionFromDB.RevisionOrderNumber,
}
// add the builds to the waterfall row
for _, b := range buildsInVersion {
currentRow := waterfallRows[b.BuildVariant]
buildForWaterfall := waterfallBuild{
Id: b.Id,
Version: versionFromDB.Id,
}
tasks, statusCount := createWaterfallTasks(b.Tasks)
buildForWaterfall.Tasks = tasks
buildForWaterfall.TaskStatusCount = statusCount
currentRow.Builds[versionFromDB.Id] = buildForWaterfall
waterfallRows[b.BuildVariant] = currentRow
for _, task := range buildForWaterfall.Tasks {
if task.Status == evergreen.TaskFailed || task.Status == evergreen.TaskStarted {
failedAndStartedTaskIds = append(failedAndStartedTaskIds, task.Id)
}
}
}
// add the version
finalVersions = append(finalVersions, activeVersion)
}
failedAndStartedTasks, err := task.Find(task.ByIds(failedAndStartedTaskIds))
if err != nil {
return versionVariantData{}, fmt.Errorf("error fetching failed tasks:"+
" %v", err)
}
addFailedAndStartedTests(waterfallRows, failedAndStartedTasks)
}
// if the last version was rolled-up, add it
if lastRolledUpVersion != nil {
finalVersions = append(finalVersions, *lastRolledUpVersion)
}
// create the list of display names for the build variants represented
buildVariants := waterfallBuildVariants{}
for name := range bvSet {
displayName := buildVariantMappings[name]
if displayName == "" {
displayName = name + " (removed)"
}
buildVariants = append(buildVariants, waterfallBuildVariant{Id: name, DisplayName: displayName})
}
return versionVariantData{
Rows: waterfallRows,
Versions: finalVersions,
BuildVariants: buildVariants,
}, nil
}
// Fetch versions until 'numVersionElements' elements are created, including
// elements consisting of multiple versions rolled-up into one.
// The skip value indicates how many versions back in time should be skipped
// before starting to fetch versions, the project indicates which project the
// returned versions should be a part of.
func getVersionsAndVariants(skip int, numVersionElements int, project *model.Project) ([]waterfallVersion, []string, error) {
// the final array of versions to return
finalVersions := []waterfallVersion{}
// keep track of the build variants we see
bvSet := map[string]bool{}
// build variant mappings - used so we can store the display name as
// the build variant field of a build
buildVariantMappings := project.GetVariantMappings()
// keep track of the last rolled-up version, so inactive versions can
// be added
var lastRolledUpVersion *waterfallVersion = nil
// loop until we have enough from the db
for len(finalVersions) < numVersionElements {
// fetch the versions and associated builds
versionsFromDB, buildsByVersion, err :=
fetchVersionsAndAssociatedBuilds(project, skip, numVersionElements)
if err != nil {
return nil, nil, fmt.Errorf("error fetching versions and builds:"+
" %v", err)
}
// if we've reached the beginning of all versions
if len(versionsFromDB) == 0 {
break
}
// update the amount skipped
skip += len(versionsFromDB)
// create the necessary versions, rolling up inactive ones
for _, version := range versionsFromDB {
// if we have hit enough versions, break out
if len(finalVersions) == numVersionElements {
break
}
// the builds for the version
buildsInVersion := buildsByVersion[version.Id]
// see if there are any active tasks in the version
versionActive := anyActiveTasks(buildsInVersion)
// add any represented build variants to the set
for _, build := range buildsInVersion {
bvSet[build.BuildVariant] = true
}
// if it is inactive, roll up the version and don't create any
// builds for it
if !versionActive {
if lastRolledUpVersion == nil {
lastRolledUpVersion = &waterfallVersion{RolledUp: true}
}
// add the version metadata into the last rolled-up version
lastRolledUpVersion.Ids = append(lastRolledUpVersion.Ids,
version.Id)
lastRolledUpVersion.Authors = append(lastRolledUpVersion.Authors,
version.Author)
lastRolledUpVersion.Errors = append(
lastRolledUpVersion.Errors, waterfallVersionError{version.Errors})
lastRolledUpVersion.Warnings = append(
lastRolledUpVersion.Warnings, waterfallVersionError{version.Warnings})
lastRolledUpVersion.Messages = append(
lastRolledUpVersion.Messages, version.Message)
lastRolledUpVersion.CreateTimes = append(
lastRolledUpVersion.CreateTimes, version.CreateTime)
lastRolledUpVersion.Revisions = append(
lastRolledUpVersion.Revisions, version.Revision)
// move on to the next version
continue
}
// add a pending rolled-up version, if it exists
if lastRolledUpVersion != nil {
finalVersions = append(finalVersions, *lastRolledUpVersion)
lastRolledUpVersion = nil
}
// if we have hit enough versions, break out
if len(finalVersions) == numVersionElements {
break
}
// if the version can not be rolled up, create a fully fledged
// version for it
activeVersion := waterfallVersion{
Ids: []string{version.Id},
Messages: []string{version.Message},
Authors: []string{version.Author},
CreateTimes: []time.Time{version.CreateTime},
Revisions: []string{version.Revision},
Errors: []waterfallVersionError{
waterfallVersionError{version.Errors}},
Warnings: []waterfallVersionError{
waterfallVersionError{version.Warnings}},
}
// add the builds to the version
for _, build := range buildsInVersion {
buildForWaterfall := waterfallBuild{
Id: build.Id,
BuildVariant: buildVariantMappings[build.BuildVariant],
}
if buildForWaterfall.BuildVariant == "" {
buildForWaterfall.BuildVariant = build.BuildVariant +
" (removed)"
}
// add the tasks to the build
for _, task := range build.Tasks {
taskForWaterfall := waterfallTask{
Id: task.Id,
Status: task.Status,
StatusDetails: task.StatusDetails,
DisplayName: task.DisplayName,
Activated: task.Activated,
TimeTaken: task.TimeTaken,
}
// if the task is inactive, set its status to inactive
if !task.Activated {
taskForWaterfall.Status = InactiveStatus
}
buildForWaterfall.Tasks = append(buildForWaterfall.Tasks, taskForWaterfall)
}
activeVersion.Builds =
append(activeVersion.Builds, buildForWaterfall)
}
// add the version
finalVersions = append(finalVersions, activeVersion)
}
}
// if the last version was rolled-up, add it
if lastRolledUpVersion != nil {
finalVersions = append(finalVersions, *lastRolledUpVersion)
}
// create the list of display names for the build variants represented
buildVariants := []string{}
for name, _ := range bvSet {
displayName := buildVariantMappings[name]
if displayName == "" {
displayName = name + " (removed)"
}
buildVariants = append(buildVariants, displayName)
}
return finalVersions, buildVariants, nil
}
// getDeps returns all recursive dependencies of task and their variants.
// If task has a non-patchable dependency, getDeps will return TaskNotPatchableError
// The returned slices may contain duplicates.
func getDeps(task string, variant string, p *model.Project) ([]string, []string, error) {
projectTask := p.FindTaskForVariant(task, variant)
if projectTask == nil {
return nil, nil, fmt.Errorf("Task not found in project: %v", task)
}
if patchable := projectTask.Patchable; (patchable != nil && !*patchable) || task == evergreen.PushStage { //TODO remove PushStage
return nil, nil, TaskNotPatchableError
}
deps := make([]string, 0)
variants := make([]string, 0)
for _, dep := range projectTask.DependsOn {
if dep.Variant == model.AllVariants {
if dep.Name == model.AllDependencies {
// Case: name = *, variant = *
for _, v := range p.FindAllVariants() {
variants = append(variants, v)
for _, t := range p.FindTasksForVariant(v) {
if t == task && v == variant {
continue
}
if depTask := p.FindTaskForVariant(t, v); t == evergreen.PushStage ||
(depTask.Patchable != nil && !*depTask.Patchable) {
return nil, nil, TaskNotPatchableError // TODO remove PushStage
}
deps = append(deps, t)
}
}
} else {
// Case: variant = *, specific name
deps = append(deps, dep.Name)
for _, v := range p.FindAllVariants() {
for _, t := range p.FindTasksForVariant(v) {
if t == dep.Name {
if t == task && v == variant {
continue
}
recDeps, recVariants, err := getDeps(t, v, p)
if err != nil {
return nil, nil, err
}
deps = append(deps, recDeps...)
variants = append(variants, v)
variants = append(variants, recVariants...)
}
}
}
}
} else {
if dep.Name == model.AllDependencies {
// Case: name = *, specific variant
v := dep.Variant
if v == "" {
v = variant
}
variants = append(variants, v)
for _, t := range p.FindTasksForVariant(v) {
if t == task && v == variant {
continue
}
recDeps, recVariants, err := getDeps(t, v, p)
if err != nil {
return nil, nil, err
}
deps = append(deps, t)
deps = append(deps, recDeps...)
variants = append(variants, recVariants...)
}
} else {
// Case: specific name, specific variant
v := dep.Variant
if v == "" {
v = variant
}
recDeps, recVariants, err := getDeps(dep.Name, v, p)
if err != nil {
return nil, nil, err
}
deps = append(deps, dep.Name)
deps = append(deps, recDeps...)
variants = append(variants, v)
variants = append(variants, recVariants...)
}
}
}
return deps, variants, nil
}