Esempio n. 1
0
// Makes sure that the dependencies for the tasks in the project form a
// valid dependency graph (no cycles).
func checkDependencyGraph(project *model.Project) []ValidationError {
	errs := []ValidationError{}

	// map of task name and variant -> BuildVariantTask
	tasksByNameAndVariant := map[model.TVPair]model.BuildVariantTask{}

	// generate task nodes for every task and variant combination
	visited := map[model.TVPair]bool{}
	allNodes := []model.TVPair{}
	for _, bv := range project.BuildVariants {
		for _, t := range bv.Tasks {
			t.Populate(project.GetSpecForTask(t.Name))
			node := model.TVPair{bv.Name, t.Name}

			tasksByNameAndVariant[node] = t
			visited[node] = false
			allNodes = append(allNodes, node)
		}
	}

	// run through the task nodes, checking their dependency graphs for cycles
	for _, node := range allNodes {
		// the visited nodes
		if err := dependencyCycleExists(node, visited, tasksByNameAndVariant); err != nil {
			errs = append(errs,
				ValidationError{
					Message: fmt.Sprintf(
						"dependency error for '%v' task: %v", node.TaskName, err),
				},
			)
		}
	}

	return errs
}
Esempio n. 2
0
func applyPatch(patch *service.RestPatch, rootCloneDir string, conf *model.Project, variant *model.BuildVariant) error {
	// patch sets and contain multiple patches, some of them for modules
	for _, patchPart := range patch.Patches {
		var dir string
		if patchPart.ModuleName == "" {
			// if patch is not part of a module, just apply patch against src root
			dir = rootCloneDir
		} else {
			fmt.Println("Applying patches for module", patchPart.ModuleName)
			// if patch is part of a module, apply patch in module root
			module, err := conf.GetModuleByName(patchPart.ModuleName)
			if err != nil || module == nil {
				return fmt.Errorf("can't find module %v: %v", patchPart.ModuleName, err)
			}

			// skip the module if this build variant does not use it
			if !util.SliceContains(variant.Modules, module.Name) {
				continue
			}

			dir = filepath.Join(rootCloneDir, module.Prefix, module.Name)
		}

		args := []string{"apply", "--whitespace=fix"}
		applyCmd := exec.Command("git", args...)
		applyCmd.Stdout, applyCmd.Stderr, applyCmd.Dir = os.Stdout, os.Stderr, dir
		applyCmd.Stdin = bytes.NewReader([]byte(patchPart.PatchSet.Patch))
		err := applyCmd.Run()
		if err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 3
0
func cloneSource(task *service.RestTask, project *model.ProjectRef, config *model.Project, cloneDir string) error {
	// Fetch the outermost repo for the task
	err := clone(
		cloneOptions{
			repo:     fmt.Sprintf("[email protected]:%v/%v.git", project.Owner, project.Repo),
			revision: task.Revision,
			rootDir:  cloneDir,
			depth:    defaultCloneDepth,
		},
		false,
	)

	if err != nil {
		return err
	}

	// Then fetch each of the modules
	variant := config.FindBuildVariant(task.BuildVariant)
	if variant == nil {
		return fmt.Errorf("couldn't find build variant '%v' in config", task.BuildVariant)
	}
	for _, moduleName := range variant.Modules {
		module, err := config.GetModuleByName(moduleName)
		if err != nil || module == nil {
			return fmt.Errorf("variant refers to a module '%v' that doesn't exist.", moduleName)
		}
		moduleBase := filepath.Join(cloneDir, module.Prefix, module.Name)
		fmt.Printf("Fetching module %v at %v\n", moduleName, module.Branch)
		err = clone(cloneOptions{
			repo:     module.Repo,
			revision: module.Branch,
			rootDir:  filepath.ToSlash(moduleBase),
		}, false)
		if err != nil {
			return err
		}
	}
	return nil
}
Esempio n. 4
0
// Makes sure that the dependencies for the tasks have the correct fields,
// and that the fields reference valid tasks.
func verifyTaskRequirements(project *model.Project) []ValidationError {
	errs := []ValidationError{}
	for _, bvt := range project.FindAllBuildVariantTasks() {
		for _, r := range bvt.Requires {
			if project.FindProjectTask(r.Name) == nil {
				if r.Name == model.AllDependencies {
					errs = append(errs, ValidationError{Message: fmt.Sprintf(
						"task '%v': * is not supported for requirement selectors", bvt.Name)})
				} else {
					errs = append(errs,
						ValidationError{Message: fmt.Sprintf(
							"task '%v' requires non-existent task '%v'", bvt.Name, r.Name)})
				}
			}
			if r.Variant != "" && r.Variant != model.AllVariants && project.FindBuildVariant(r.Variant) == nil {
				errs = append(errs, ValidationError{Message: fmt.Sprintf(
					"task '%v' requires non-existent variant '%v'", bvt.Name, r.Variant)})
			}
		}
	}
	return errs
}
Esempio n. 5
0
// Fetch versions until 'numVersionElements' elements are created, including
// elements consisting of multiple versions rolled-up into one.
// The skip value indicates how many versions back in time should be skipped
// before starting to fetch versions, the project indicates which project the
// returned versions should be a part of.
func getVersionsAndVariants(skip, numVersionElements int, project *model.Project) (versionVariantData, error) {
	// the final array of versions to return
	finalVersions := []waterfallVersion{}

	// keep track of the build variants we see
	bvSet := map[string]bool{}

	waterfallRows := map[string]waterfallRow{}

	// build variant mappings - used so we can store the display name as
	// the build variant field of a build
	buildVariantMappings := project.GetVariantMappings()

	// keep track of the last rolled-up version, so inactive versions can
	// be added
	var lastRolledUpVersion *waterfallVersion = nil

	// loop until we have enough from the db
	for len(finalVersions) < numVersionElements {

		// fetch the versions and associated builds
		versionsFromDB, buildsByVersion, err :=
			fetchVersionsAndAssociatedBuilds(project, skip, numVersionElements)

		if err != nil {
			return versionVariantData{}, fmt.Errorf("error fetching versions and builds:"+
				" %v", err)
		}

		// if we've reached the beginning of all versions
		if len(versionsFromDB) == 0 {
			break
		}

		// to fetch started tasks and failed tests for providing additional context
		// in a tooltip
		failedAndStartedTaskIds := []string{}

		// update the amount skipped
		skip += len(versionsFromDB)

		// create the necessary versions, rolling up inactive ones
		for _, versionFromDB := range versionsFromDB {

			// if we have hit enough versions, break out
			if len(finalVersions) == numVersionElements {
				break
			}

			// the builds for the version
			buildsInVersion := buildsByVersion[versionFromDB.Id]

			// see if there are any active tasks in the version
			versionActive := anyActiveTasks(buildsInVersion)

			// add any represented build variants to the set and initialize rows
			for _, b := range buildsInVersion {
				bvSet[b.BuildVariant] = true

				buildVariant := waterfallBuildVariant{
					Id:          b.BuildVariant,
					DisplayName: buildVariantMappings[b.BuildVariant],
				}

				if buildVariant.DisplayName == "" {
					buildVariant.DisplayName = b.BuildVariant +
						" (removed)"
				}

				if _, ok := waterfallRows[b.BuildVariant]; !ok {
					waterfallRows[b.BuildVariant] = waterfallRow{
						Builds:       map[string]waterfallBuild{},
						BuildVariant: buildVariant,
					}
				}

			}

			// if it is inactive, roll up the version and don't create any
			// builds for it
			if !versionActive {
				if lastRolledUpVersion == nil {
					lastRolledUpVersion = &waterfallVersion{RolledUp: true, RevisionOrderNumber: versionFromDB.RevisionOrderNumber}
				}

				// add the version metadata into the last rolled-up version
				lastRolledUpVersion.Ids = append(lastRolledUpVersion.Ids,
					versionFromDB.Id)
				lastRolledUpVersion.Authors = append(lastRolledUpVersion.Authors,
					versionFromDB.Author)
				lastRolledUpVersion.Errors = append(
					lastRolledUpVersion.Errors, waterfallVersionError{versionFromDB.Errors})
				lastRolledUpVersion.Warnings = append(
					lastRolledUpVersion.Warnings, waterfallVersionError{versionFromDB.Warnings})
				lastRolledUpVersion.Messages = append(
					lastRolledUpVersion.Messages, versionFromDB.Message)
				lastRolledUpVersion.Ignoreds = append(
					lastRolledUpVersion.Ignoreds, versionFromDB.Ignored)
				lastRolledUpVersion.CreateTimes = append(
					lastRolledUpVersion.CreateTimes, versionFromDB.CreateTime)
				lastRolledUpVersion.Revisions = append(
					lastRolledUpVersion.Revisions, versionFromDB.Revision)

				// move on to the next version
				continue
			}

			// add a pending rolled-up version, if it exists
			if lastRolledUpVersion != nil {
				finalVersions = append(finalVersions, *lastRolledUpVersion)
				lastRolledUpVersion = nil
			}

			// if we have hit enough versions, break out
			if len(finalVersions) == numVersionElements {
				break
			}

			// if the version can not be rolled up, create a fully fledged
			// version for it
			activeVersion := waterfallVersion{
				Ids:                 []string{versionFromDB.Id},
				Messages:            []string{versionFromDB.Message},
				Authors:             []string{versionFromDB.Author},
				CreateTimes:         []time.Time{versionFromDB.CreateTime},
				Revisions:           []string{versionFromDB.Revision},
				Errors:              []waterfallVersionError{{versionFromDB.Errors}},
				Warnings:            []waterfallVersionError{{versionFromDB.Warnings}},
				Ignoreds:            []bool{versionFromDB.Ignored},
				RevisionOrderNumber: versionFromDB.RevisionOrderNumber,
			}

			// add the builds to the waterfall row
			for _, b := range buildsInVersion {
				currentRow := waterfallRows[b.BuildVariant]
				buildForWaterfall := waterfallBuild{
					Id:      b.Id,
					Version: versionFromDB.Id,
				}

				tasks, statusCount := createWaterfallTasks(b.Tasks)
				buildForWaterfall.Tasks = tasks
				buildForWaterfall.TaskStatusCount = statusCount
				currentRow.Builds[versionFromDB.Id] = buildForWaterfall
				waterfallRows[b.BuildVariant] = currentRow
				for _, task := range buildForWaterfall.Tasks {
					if task.Status == evergreen.TaskFailed || task.Status == evergreen.TaskStarted {
						failedAndStartedTaskIds = append(failedAndStartedTaskIds, task.Id)
					}
				}
			}

			// add the version
			finalVersions = append(finalVersions, activeVersion)

		}

		failedAndStartedTasks, err := task.Find(task.ByIds(failedAndStartedTaskIds))
		if err != nil {
			return versionVariantData{}, fmt.Errorf("error fetching failed tasks:"+
				" %v", err)

		}
		addFailedAndStartedTests(waterfallRows, failedAndStartedTasks)
	}

	// if the last version was rolled-up, add it
	if lastRolledUpVersion != nil {
		finalVersions = append(finalVersions, *lastRolledUpVersion)
	}

	// create the list of display names for the build variants represented
	buildVariants := waterfallBuildVariants{}
	for name := range bvSet {
		displayName := buildVariantMappings[name]
		if displayName == "" {
			displayName = name + " (removed)"
		}
		buildVariants = append(buildVariants, waterfallBuildVariant{Id: name, DisplayName: displayName})
	}

	return versionVariantData{
		Rows:          waterfallRows,
		Versions:      finalVersions,
		BuildVariants: buildVariants,
	}, nil

}
Esempio n. 6
0
// Fetch versions until 'numVersionElements' elements are created, including
// elements consisting of multiple versions rolled-up into one.
// The skip value indicates how many versions back in time should be skipped
// before starting to fetch versions, the project indicates which project the
// returned versions should be a part of.
func getVersionsAndVariants(skip int, numVersionElements int, project *model.Project) ([]waterfallVersion, []string, error) {
	// the final array of versions to return
	finalVersions := []waterfallVersion{}

	// keep track of the build variants we see
	bvSet := map[string]bool{}

	// build variant mappings - used so we can store the display name as
	// the build variant field of a build
	buildVariantMappings := project.GetVariantMappings()

	// keep track of the last rolled-up version, so inactive versions can
	// be added
	var lastRolledUpVersion *waterfallVersion = nil

	// loop until we have enough from the db
	for len(finalVersions) < numVersionElements {

		// fetch the versions and associated builds
		versionsFromDB, buildsByVersion, err :=
			fetchVersionsAndAssociatedBuilds(project, skip, numVersionElements)

		if err != nil {
			return nil, nil, fmt.Errorf("error fetching versions and builds:"+
				" %v", err)
		}

		// if we've reached the beginning of all versions
		if len(versionsFromDB) == 0 {
			break
		}

		// update the amount skipped
		skip += len(versionsFromDB)

		// create the necessary versions, rolling up inactive ones
		for _, version := range versionsFromDB {

			// if we have hit enough versions, break out
			if len(finalVersions) == numVersionElements {
				break
			}

			// the builds for the version
			buildsInVersion := buildsByVersion[version.Id]

			// see if there are any active tasks in the version
			versionActive := anyActiveTasks(buildsInVersion)

			// add any represented build variants to the set
			for _, build := range buildsInVersion {
				bvSet[build.BuildVariant] = true
			}

			// if it is inactive, roll up the version and don't create any
			// builds for it
			if !versionActive {
				if lastRolledUpVersion == nil {
					lastRolledUpVersion = &waterfallVersion{RolledUp: true}
				}

				// add the version metadata into the last rolled-up version
				lastRolledUpVersion.Ids = append(lastRolledUpVersion.Ids,
					version.Id)
				lastRolledUpVersion.Authors = append(lastRolledUpVersion.Authors,
					version.Author)
				lastRolledUpVersion.Errors = append(
					lastRolledUpVersion.Errors, waterfallVersionError{version.Errors})
				lastRolledUpVersion.Warnings = append(
					lastRolledUpVersion.Warnings, waterfallVersionError{version.Warnings})
				lastRolledUpVersion.Messages = append(
					lastRolledUpVersion.Messages, version.Message)
				lastRolledUpVersion.CreateTimes = append(
					lastRolledUpVersion.CreateTimes, version.CreateTime)
				lastRolledUpVersion.Revisions = append(
					lastRolledUpVersion.Revisions, version.Revision)

				// move on to the next version
				continue
			}

			// add a pending rolled-up version, if it exists
			if lastRolledUpVersion != nil {
				finalVersions = append(finalVersions, *lastRolledUpVersion)
				lastRolledUpVersion = nil
			}

			// if we have hit enough versions, break out
			if len(finalVersions) == numVersionElements {
				break
			}

			// if the version can not be rolled up, create a fully fledged
			// version for it
			activeVersion := waterfallVersion{
				Ids:         []string{version.Id},
				Messages:    []string{version.Message},
				Authors:     []string{version.Author},
				CreateTimes: []time.Time{version.CreateTime},
				Revisions:   []string{version.Revision},
				Errors: []waterfallVersionError{
					waterfallVersionError{version.Errors}},
				Warnings: []waterfallVersionError{
					waterfallVersionError{version.Warnings}},
			}

			// add the builds to the version
			for _, build := range buildsInVersion {

				buildForWaterfall := waterfallBuild{
					Id:           build.Id,
					BuildVariant: buildVariantMappings[build.BuildVariant],
				}

				if buildForWaterfall.BuildVariant == "" {
					buildForWaterfall.BuildVariant = build.BuildVariant +
						" (removed)"
				}

				// add the tasks to the build
				for _, task := range build.Tasks {
					taskForWaterfall := waterfallTask{
						Id:            task.Id,
						Status:        task.Status,
						StatusDetails: task.StatusDetails,
						DisplayName:   task.DisplayName,
						Activated:     task.Activated,
						TimeTaken:     task.TimeTaken,
					}

					// if the task is inactive, set its status to inactive
					if !task.Activated {
						taskForWaterfall.Status = InactiveStatus
					}

					buildForWaterfall.Tasks = append(buildForWaterfall.Tasks, taskForWaterfall)
				}

				activeVersion.Builds =
					append(activeVersion.Builds, buildForWaterfall)
			}

			// add the version
			finalVersions = append(finalVersions, activeVersion)

		}

	}

	// if the last version was rolled-up, add it
	if lastRolledUpVersion != nil {
		finalVersions = append(finalVersions, *lastRolledUpVersion)
	}

	// create the list of display names for the build variants represented
	buildVariants := []string{}
	for name, _ := range bvSet {
		displayName := buildVariantMappings[name]
		if displayName == "" {
			displayName = name + " (removed)"
		}
		buildVariants = append(buildVariants, displayName)
	}

	return finalVersions, buildVariants, nil
}
Esempio n. 7
0
// getDeps returns all recursive dependencies of task and their variants.
// If task has a non-patchable dependency, getDeps will return TaskNotPatchableError
// The returned slices may contain duplicates.
func getDeps(task string, variant string, p *model.Project) ([]string, []string, error) {
	projectTask := p.FindTaskForVariant(task, variant)
	if projectTask == nil {
		return nil, nil, fmt.Errorf("Task not found in project: %v", task)
	}
	if patchable := projectTask.Patchable; (patchable != nil && !*patchable) || task == evergreen.PushStage { //TODO remove PushStage
		return nil, nil, TaskNotPatchableError
	}
	deps := make([]string, 0)
	variants := make([]string, 0)
	for _, dep := range projectTask.DependsOn {
		if dep.Variant == model.AllVariants {
			if dep.Name == model.AllDependencies {
				// Case: name = *, variant = *
				for _, v := range p.FindAllVariants() {
					variants = append(variants, v)
					for _, t := range p.FindTasksForVariant(v) {
						if t == task && v == variant {
							continue
						}
						if depTask := p.FindTaskForVariant(t, v); t == evergreen.PushStage ||
							(depTask.Patchable != nil && !*depTask.Patchable) {
							return nil, nil, TaskNotPatchableError // TODO remove PushStage
						}
						deps = append(deps, t)
					}
				}
			} else {
				// Case: variant = *, specific name
				deps = append(deps, dep.Name)
				for _, v := range p.FindAllVariants() {
					for _, t := range p.FindTasksForVariant(v) {
						if t == dep.Name {
							if t == task && v == variant {
								continue
							}
							recDeps, recVariants, err := getDeps(t, v, p)
							if err != nil {
								return nil, nil, err
							}
							deps = append(deps, recDeps...)
							variants = append(variants, v)
							variants = append(variants, recVariants...)
						}
					}
				}
			}
		} else {
			if dep.Name == model.AllDependencies {
				// Case: name = *, specific variant
				v := dep.Variant
				if v == "" {
					v = variant
				}
				variants = append(variants, v)
				for _, t := range p.FindTasksForVariant(v) {
					if t == task && v == variant {
						continue
					}
					recDeps, recVariants, err := getDeps(t, v, p)
					if err != nil {
						return nil, nil, err
					}
					deps = append(deps, t)
					deps = append(deps, recDeps...)
					variants = append(variants, recVariants...)
				}
			} else {
				// Case: specific name, specific variant
				v := dep.Variant
				if v == "" {
					v = variant
				}
				recDeps, recVariants, err := getDeps(dep.Name, v, p)
				if err != nil {
					return nil, nil, err
				}
				deps = append(deps, dep.Name)
				deps = append(deps, recDeps...)
				variants = append(variants, v)
				variants = append(variants, recVariants...)
			}
		}
	}
	return deps, variants, nil
}