func TestGetVariantsWithTask(t *testing.T) {
Convey("With a project", t, func() {
project := &Project{
BuildVariants: []BuildVariant{
BuildVariant{
Name: "bv1",
Tasks: []BuildVariantTask{
BuildVariantTask{
Name: "suite1",
},
},
},
BuildVariant{
Name: "bv2",
Tasks: []BuildVariantTask{
BuildVariantTask{
Name: "suite1",
},
BuildVariantTask{
Name: "suite2",
},
},
},
BuildVariant{
Name: "bv3",
Tasks: []BuildVariantTask{
BuildVariantTask{
Name: "suite2",
},
},
},
},
}
Convey("when getting the build variants where a task applies", func() {
Convey("it should be run on any build variants where the test is"+
" specified to run", func() {
variants := project.GetVariantsWithTask("suite1")
So(len(variants), ShouldEqual, 2)
So(util.SliceContains(variants, "bv1"), ShouldBeTrue)
So(util.SliceContains(variants, "bv2"), ShouldBeTrue)
variants = project.GetVariantsWithTask("suite2")
So(len(variants), ShouldEqual, 2)
So(util.SliceContains(variants, "bv2"), ShouldBeTrue)
So(util.SliceContains(variants, "bv3"), ShouldBeTrue)
})
})
})
}
// creates/returns slice of 'relevant' NotificationKeys a
// notification is relevant if it has at least one recipient
func notificationsToStruct(mciNotification *MCINotification) (notifyOn []NotificationKey) {
// Get default notifications
for _, notification := range mciNotification.Notifications {
if len(notification.Recipients) != 0 {
// flag the notification as needed
key := NotificationKey{
Project: notification.Project,
NotificationName: notification.Name,
NotificationType: getType(notification.Name),
NotificationRequester: evergreen.RepotrackerVersionRequester,
}
// prevent duplicate notifications from being sent
if !util.SliceContains(notifyOn, key) {
notifyOn = append(notifyOn, key)
}
}
}
// Get team notifications
for _, team := range mciNotification.Teams {
for _, subscription := range team.Subscriptions {
for _, name := range subscription.NotifyOn {
key := NotificationKey{
Project: subscription.Project,
NotificationName: name,
NotificationType: getType(name),
NotificationRequester: evergreen.RepotrackerVersionRequester,
}
// prevent duplicate notifications from being sent
if !util.SliceContains(notifyOn, key) {
notifyOn = append(notifyOn, key)
}
}
}
}
// Get patch notifications
for _, subscription := range mciNotification.PatchNotifications {
for _, notification := range subscription.NotifyOn {
key := NotificationKey{
Project: subscription.Project,
NotificationName: notification,
NotificationType: getType(notification),
NotificationRequester: evergreen.PatchVersionRequester,
}
// prevent duplicate notifications from being sent
if !util.SliceContains(notifyOn, key) {
notifyOn = append(notifyOn, key)
}
}
}
return
}
// NewPatchTaskIdTable constructs a new TaskIdTable (map of [variant, task display name]->[task id])
func NewPatchTaskIdTable(proj *Project, v *version.Version, patchConfig TVPairSet) TaskIdTable {
table := TaskIdTable{}
processedVariants := map[string]bool{}
for _, vt := range patchConfig {
// don't hit the same variant more than once
if _, ok := processedVariants[vt.Variant]; ok {
continue
}
processedVariants[vt.Variant] = true
// we must track the project's variants definitions as well,
// so that we don't create Ids for variants that don't exist.
projBV := proj.FindBuildVariant(vt.Variant)
taskNamesForVariant := patchConfig.TaskNames(vt.Variant)
for _, t := range projBV.Tasks {
// create Ids for each task that can run on the variant and is requested by the patch.
if util.SliceContains(taskNamesForVariant, t.Name) {
taskId := util.CleanName(
fmt.Sprintf("%v_%v_%v_%v_%v",
proj.Identifier, projBV.Name, t.Name, v.Revision,
v.CreateTime.Format(build.IdTimeLayout)))
table[TVPair{vt.Variant, t.Name}] = taskId
}
}
}
return table
}
// Validate that all necessary params are set and valid.
func (s3pc *S3PutCommand) validateParams() error {
if s3pc.AwsKey == "" {
return fmt.Errorf("aws_key cannot be blank")
}
if s3pc.AwsSecret == "" {
return fmt.Errorf("aws_secret cannot be blank")
}
if s3pc.LocalFile == "" {
return fmt.Errorf("local_file cannot be blank")
}
if s3pc.RemoteFile == "" {
return fmt.Errorf("remote_file cannot be blank")
}
if s3pc.ContentType == "" {
return fmt.Errorf("content_type cannot be blank")
}
if !util.SliceContains(artifact.ValidVisibilities, s3pc.Visibility) {
return fmt.Errorf("invalid visibility setting: %v", s3pc.Visibility)
}
// make sure the bucket is valid
if err := validateS3BucketName(s3pc.Bucket); err != nil {
return fmt.Errorf("%v is an invalid bucket name: %v", s3pc.Bucket, err)
}
// make sure the s3 permissions are valid
if !validS3Permissions(s3pc.Permissions) {
return fmt.Errorf("permissions '%v' are not valid", s3pc.Permissions)
}
return nil
}
// computeScheduledTasksDuration returns the total estimated duration of all
// tasks scheduled to be run in a given task queue
func computeScheduledTasksDuration(
scheduledDistroTasksData *ScheduledDistroTasksData) (
scheduledTasksDuration float64, sharedTasksDuration map[string]float64) {
taskQueueItems := scheduledDistroTasksData.taskQueueItems
taskRunDistros := scheduledDistroTasksData.taskRunDistros
tasksAccountedFor := scheduledDistroTasksData.tasksAccountedFor
currentDistroId := scheduledDistroTasksData.currentDistroId
sharedTasksDuration = make(map[string]float64)
// compute the total expected duration for tasks in this queue
for _, taskQueueItem := range taskQueueItems {
if !tasksAccountedFor[taskQueueItem.Id] {
scheduledTasksDuration += taskQueueItem.ExpectedDuration.Seconds()
tasksAccountedFor[taskQueueItem.Id] = true
}
// if the task can be run on multiple distros - including this one - add
// it to the total duration of 'shared tasks' for the distro and all
// other distros it can be run on
distroIds, ok := taskRunDistros[taskQueueItem.Id]
if ok && util.SliceContains(distroIds, currentDistroId) {
for _, distroId := range distroIds {
sharedTasksDuration[distroId] +=
taskQueueItem.ExpectedDuration.Seconds()
}
}
}
return
}
func (pp *PerfPlugin) GetPanelConfig() (*plugin.PanelConfig, error) {
root := plugin.StaticWebRootFromSourceFile()
panelHTML, err := ioutil.ReadFile(root + "/task_perf_data.html")
if err != nil {
return nil, fmt.Errorf("Can't load panel html file: %v", err)
}
return &plugin.PanelConfig{
StaticRoot: plugin.StaticWebRootFromSourceFile(),
Panels: []plugin.UIPanel{
{
Includes: includes,
Page: plugin.TaskPage,
Position: plugin.PageCenter,
PanelHTML: template.HTML(panelHTML),
DataFunc: func(context plugin.UIContext) (interface{}, error) {
return struct {
Enabled bool `json:"enabled"`
}{util.SliceContains(pp.Projects, context.ProjectRef.Identifier)}, nil
},
},
},
}, nil
return nil, nil
}
func applyPatch(patch *service.RestPatch, rootCloneDir string, conf *model.Project, variant *model.BuildVariant) error {
// patch sets and contain multiple patches, some of them for modules
for _, patchPart := range patch.Patches {
var dir string
if patchPart.ModuleName == "" {
// if patch is not part of a module, just apply patch against src root
dir = rootCloneDir
} else {
fmt.Println("Applying patches for module", patchPart.ModuleName)
// if patch is part of a module, apply patch in module root
module, err := conf.GetModuleByName(patchPart.ModuleName)
if err != nil || module == nil {
return fmt.Errorf("can't find module %v: %v", patchPart.ModuleName, err)
}
// skip the module if this build variant does not use it
if !util.SliceContains(variant.Modules, module.Name) {
continue
}
dir = filepath.Join(rootCloneDir, module.Prefix, module.Name)
}
args := []string{"apply", "--whitespace=fix"}
applyCmd := exec.Command("git", args...)
applyCmd.Stdout, applyCmd.Stderr, applyCmd.Dir = os.Stdout, os.Stderr, dir
applyCmd.Stdin = bytes.NewReader([]byte(patchPart.PatchSet.Patch))
err := applyCmd.Run()
if err != nil {
return err
}
}
return nil
}
// Given a patch version and a list of task names, creates a new task with
// the given name for each variant, if applicable.
func AddNewTasksForPatch(p *patch.Patch, patchVersion *version.Version, project *Project,
taskNames []string) error {
// create new tasks for all of the added patch tasks
var newTasks []string
for _, taskName := range taskNames {
if !util.SliceContains(p.Tasks, taskName) {
newTasks = append(newTasks, taskName)
}
}
// add tasks to the patch in the db
if err := p.AddTasks(taskNames); err != nil {
return err
}
// add new tasks to the build, if they exist
if len(newTasks) > 0 {
builds, err := build.Find(build.ByIds(patchVersion.BuildIds))
if err != nil {
return err
}
for _, b := range builds {
if _, err = AddTasksToBuild(&b, project, patchVersion, newTasks); err != nil {
return err
}
}
}
return nil
}
// confirm asks the user a yes/no question and returns true/false if they reply with y/yes/n/no.
// if defaultYes is true, allows user to just hit enter without typing an explicit yes.
func confirm(message string, defaultYes bool) bool {
reply := ""
yes := []string{"y", "yes"}
no := []string{"n", "no"}
if defaultYes {
yes = append(yes, "")
}
for {
reply = prompt(message)
if util.SliceContains(yes, strings.ToLower(reply)) {
return true
}
if util.SliceContains(no, strings.ToLower(reply)) {
return false
}
}
}
// ShouldContainResembling tests whether a slice contains an element that DeepEquals
// the expected input.
func ShouldContainResembling(actual interface{}, expected ...interface{}) string {
if len(expected) != 1 {
return "ShouldContainResembling takes 1 argument"
}
if !util.SliceContains(actual, expected[0]) {
return fmt.Sprintf("%#v does not contain %#v", actual, expected[0])
}
return ""
}
func (self *S3GetCommand) shouldRunForVariant(buildVariantName string) bool {
//No buildvariant filter, so run always
if len(self.BuildVariants) == 0 {
return true
}
//Only run if the buildvariant specified appears in our list.
return util.SliceContains(self.BuildVariants, buildVariantName)
}
func (self *BuildEmail) ShouldSkip(skipVariants []string) bool {
buildVariant := self.Trigger.Current.BuildVariant
if util.SliceContains(skipVariants, buildVariant) {
evergreen.Logger.Logf(slogger.DEBUG, "Skipping buildvariant %v “%v” notification: “%v”",
buildVariant, self.Trigger.Key.NotificationName, self.Subject)
return true
}
return false
}
// get the failed task(s) for a given build
func getFailedTasks(current *build.Build, notificationName string) (failedTasks []build.TaskCache) {
if util.SliceContains(buildFailureKeys, notificationName) {
for _, task := range current.Tasks {
if task.Status == evergreen.TaskFailed {
failedTasks = append(failedTasks, task)
}
}
}
return
}
// Given the patch version and a list of build variants, creates new builds
// with the patch's tasks.
func AddNewBuildsForPatch(p *patch.Patch, patchVersion *version.Version, project *Project,
buildVariants []string) (*version.Version, error) {
// compute a list of the newly added build variants
var newVariants []string
for _, variant := range buildVariants {
if !util.SliceContains(p.BuildVariants, variant) {
newVariants = append(newVariants, variant)
}
}
// update the patch
if err := p.AddBuildVariants(buildVariants); err != nil {
return nil, err
}
newBuildIds := make([]string, 0)
newBuildStatuses := make([]version.BuildStatus, 0)
tt := BuildTaskIdTable(project, patchVersion)
for _, buildVariant := range newVariants {
evergreen.Logger.Logf(slogger.INFO,
"Creating build for version %v, buildVariant %v, activated = %v",
patchVersion.Id, buildVariant, p.Activated)
buildId, err := CreateBuildFromVersion(
project, patchVersion, tt, buildVariant, p.Activated, p.Tasks)
if err != nil {
return nil, err
}
newBuildIds = append(newBuildIds, buildId)
newBuildStatuses = append(newBuildStatuses,
version.BuildStatus{
BuildVariant: buildVariant,
BuildId: buildId,
Activated: p.Activated,
},
)
patchVersion.BuildIds = append(patchVersion.BuildIds, buildId)
}
err := version.UpdateOne(
bson.M{version.IdKey: patchVersion.Id},
bson.M{
"$push": bson.M{
version.BuildIdsKey: bson.M{"$each": newBuildIds},
version.BuildVariantsKey: bson.M{"$each": newBuildStatuses},
},
},
)
if err != nil {
return nil, err
}
return patchVersion, nil
}
func (uis *UIServer) modifyHosts(w http.ResponseWriter, r *http.Request) {
_ = MustHaveUser(r)
opts := &uiParams{}
err := util.ReadJSONInto(r.Body, opts)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
hostIds := opts.HostIds
if len(hostIds) == 1 && strings.TrimSpace(hostIds[0]) == "" {
http.Error(w, "No host ID's found in request", http.StatusBadRequest)
return
}
// fetch all relevant hosts
hosts, err := host.Find(host.ByIds(hostIds))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error finding hosts: %v", err))
return
}
if len(hosts) == 0 {
http.Error(w, "No matching hosts found.", http.StatusBadRequest)
return
}
// determine what action needs to be taken
switch opts.Action {
case "updateStatus":
newStatus := opts.Status
if !util.SliceContains(validUpdateToStatuses, newStatus) {
http.Error(w, fmt.Sprintf("Invalid status: %v", opts.Status), http.StatusBadRequest)
return
}
numHostsUpdated := 0
for _, host := range hosts {
err := host.SetStatus(newStatus)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error updating host %v", err))
return
}
numHostsUpdated += 1
}
msg := NewSuccessFlash(fmt.Sprintf("%v host(s) status successfully updated to '%v'",
numHostsUpdated, newStatus))
PushFlash(uis.CookieStore, r, w, msg)
return
default:
http.Error(w, fmt.Sprintf("Unrecognized action: %v", opts.Action), http.StatusBadRequest)
return
}
}
// IsSuperUser verifies that a given user has super user permissions.
// A user has these permission if they are in the super users list or if the list is empty,
// in which case all users are super users.
func IsSuperUser(settings evergreen.Settings, u User) bool {
if u == nil {
return false
}
if util.SliceContains(settings.SuperUsers, u.Username()) ||
len(settings.SuperUsers) == 0 {
return true
}
return false
}
// contains returns whether a value is contained by a definition.
// Note that a value that doesn't contain every matrix axis will still
// be evaluated based on the axes that exist.
func (mdef matrixDefinition) contains(mv matrixValue) bool {
for k, v := range mv {
axis, ok := mdef[k]
if !ok {
return false
}
if !util.SliceContains(axis, v) {
return false
}
}
return true
}
// isSuperUser verifies that a given user has super user permissions.
// A user has these permission if they are in the super users list or if the list is empty,
// in which case all users are super users.
func (uis *UIServer) isSuperUser(u *user.DBUser) bool {
if u == nil {
return false
}
if util.SliceContains(uis.Settings.SuperUsers, u.Id) ||
len(uis.Settings.SuperUsers) == 0 {
return true
}
return false
}
func validS3Permissions(perm string) bool {
return util.SliceContains(
[]s3.ACL{
s3.Private,
s3.PublicRead,
s3.PublicReadWrite,
s3.AuthenticatedRead,
s3.BucketOwnerRead,
s3.BucketOwnerFull,
},
s3.ACL(perm),
)
}
// get the specific failed test(s) for this task
func getFailedTests(current *model.Task, notificationName string) (failedTests []model.TestResult) {
if util.SliceContains(taskFailureKeys, notificationName) {
for _, test := range current.TestResults {
if test.Status == "fail" {
// get the base name for windows/non-windows paths
test.TestFile = path.Base(strings.Replace(test.TestFile, "\\", "/", -1))
failedTests = append(failedTests, test)
}
}
}
return
}
// create a slice of all valid distro names
func getDistroIds() ([]string, error) {
// create a slice of all known distros
distros, err := distro.Find(distro.All)
if err != nil {
return nil, err
}
distroIds := []string{}
for _, d := range distros {
if !util.SliceContains(distroIds, d.Id) {
distroIds = append(distroIds, d.Id)
}
}
return distroIds, nil
}
// S3Copy is responsible for carrying out the core of the S3CopyPlugin's
// function - it makes an API calls to copy a given staged file to it's final
// production destination
func (scc *S3CopyCommand) S3Copy(taskConfig *model.TaskConfig,
pluginLogger plugin.Logger, pluginCom plugin.PluginCommunicator) error {
for _, s3CopyFile := range scc.S3CopyFiles {
if len(s3CopyFile.BuildVariants) > 0 && !util.SliceContains(
s3CopyFile.BuildVariants, taskConfig.BuildVariant.Name) {
continue
}
pluginLogger.LogExecution(slogger.INFO, "Making API push copy call to "+
"transfer %v/%v => %v/%v", s3CopyFile.Source.Bucket,
s3CopyFile.Source.Path, s3CopyFile.Destination.Bucket,
s3CopyFile.Destination.Path)
s3CopyReq := S3CopyRequest{
AwsKey: scc.AwsKey,
AwsSecret: scc.AwsSecret,
S3SourceBucket: s3CopyFile.Source.Bucket,
S3SourcePath: s3CopyFile.Source.Path,
S3DestinationBucket: s3CopyFile.Destination.Bucket,
S3DestinationPath: s3CopyFile.Destination.Path,
S3DisplayName: s3CopyFile.DisplayName,
}
resp, err := pluginCom.TaskPostJSON(s3CopyAPIEndpoint, s3CopyReq)
if resp != nil {
defer resp.Body.Close()
}
if resp != nil && resp.StatusCode != http.StatusOK {
body, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("S3 push copy failed (%v): %v",
resp.StatusCode, string(body))
}
if err != nil {
body, _ := ioutil.ReadAll(resp.Body)
return fmt.Errorf("S3 push copy failed (%v): %v",
resp.StatusCode, string(body))
}
pluginLogger.LogExecution(slogger.INFO, "API push copy call succeeded")
err = scc.AttachTaskFiles(pluginLogger, pluginCom, s3CopyReq)
if err != nil {
body, readAllErr := ioutil.ReadAll(resp.Body)
if readAllErr != nil {
return fmt.Errorf("Error: %v", err)
}
return fmt.Errorf("Error: %v, (%v): %v",
resp.StatusCode, err, string(body))
}
}
return nil
}
// create a slice of all valid distro names
func populateDistroIds() *ValidationError {
// create a slice of all known distros
distros, err := distro.Find(distro.All)
if err != nil {
return &ValidationError{
Message: fmt.Sprintf("error finding distros: %v", err),
Level: Error,
}
}
distroIds = []string{}
for _, d := range distros {
if !util.SliceContains(distroIds, d.Id) {
distroIds = append(distroIds, d.Id)
}
}
return nil
}
func (uis *UIServer) modifyHost(w http.ResponseWriter, r *http.Request) {
_ = MustHaveUser(r)
vars := mux.Vars(r)
id := vars["host_id"]
host, err := host.FindOne(host.ById(id))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if host == nil {
http.Error(w, "Host not found", http.StatusNotFound)
return
}
opts := &uiParams{}
err = util.ReadJSONInto(r.Body, opts)
if err != nil {
uis.LoggedError(w, r, http.StatusBadRequest, err)
return
}
// determine what action needs to be taken
switch opts.Action {
case "updateStatus":
currentStatus := host.Status
newStatus := opts.Status
if !util.SliceContains(validUpdateToStatuses, newStatus) {
http.Error(w, fmt.Sprintf("'%v' is not a valid status", newStatus), http.StatusBadRequest)
return
}
err := host.SetStatus(newStatus)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error updating host: %v", err))
return
}
msg := NewSuccessFlash(fmt.Sprintf("Host status successfully updated from '%v' to '%v'", currentStatus, host.Status))
PushFlash(uis.CookieStore, r, w, msg)
uis.WriteJSON(w, http.StatusOK, "Successfully updated host status")
default:
uis.WriteJSON(w, http.StatusBadRequest, fmt.Sprintf("Unrecognized action: %v", opts.Action))
}
}
// createTasksForBuild creates all of the necessary tasks for the build. Returns a
// slice of all of the tasks created, as well as an error if any occurs.
// The slice of tasks will be in the same order as the project's specified tasks
// appear in the specified build variant.
func createTasksForBuild(project *Project, buildVariant *BuildVariant,
b *build.Build, v *version.Version, tt TaskIdTable, taskNames []string) ([]*Task, error) {
// the list of tasks we should create. if tasks are passed in, then
// use those, else use the default set
tasksToCreate := []BuildVariantTask{}
createAll := len(taskNames) == 0
for _, task := range buildVariant.Tasks {
// get the task spec out of the project
taskSpec := project.GetSpecForTask(task.Name)
// sanity check that the config isn't malformed
if taskSpec.Name == "" {
return nil, fmt.Errorf("config is malformed: variant '%v' runs "+
"task called '%v' but no such task exists for repo %v for "+
"version %v", buildVariant.Name, task.Name, project.Identifier, v.Id)
}
// update task document with spec fields
task.Populate(taskSpec)
if ((task.Patchable != nil && *task.Patchable == false) || task.Name == evergreen.PushStage) && //TODO remove PushStage
b.Requester == evergreen.PatchVersionRequester {
continue
}
if createAll || util.SliceContains(taskNames, task.Name) {
tasksToCreate = append(tasksToCreate, task)
}
}
// if any tasks already exist in the build, add them to the id table
// so they can be used as dependencies
for _, task := range b.Tasks {
tt.AddId(b.BuildVariant, task.DisplayName, task.Id)
}
// create and insert all of the actual tasks
tasks := make([]*Task, 0, len(tasksToCreate))
for _, task := range tasksToCreate {
newTask := createOneTask(tt.GetId(b.BuildVariant, task.Name), task, project, buildVariant, b, v)
// set Tags based on the spec
newTask.Tags = project.GetSpecForTask(task.Name).Tags
// set the new task's dependencies
// TODO encapsulate better
if len(task.DependsOn) == 1 &&
task.DependsOn[0].Name == AllDependencies &&
task.DependsOn[0].Variant != AllVariants {
// the task depends on all of the other tasks in the build
newTask.DependsOn = make([]Dependency, 0, len(tasksToCreate)-1)
for _, dep := range tasksToCreate {
status := evergreen.TaskSucceeded
if task.DependsOn[0].Status != "" {
status = task.DependsOn[0].Status
}
newDep := Dependency{
TaskId: tt.GetId(b.BuildVariant, dep.Name),
Status: status,
}
if dep.Name != newTask.DisplayName {
newTask.DependsOn = append(newTask.DependsOn, newDep)
}
}
} else {
// the task has specific dependencies
newTask.DependsOn = make([]Dependency, 0, len(task.DependsOn))
for _, dep := range task.DependsOn {
// only add as a dependency if the dependency is valid/exists
status := evergreen.TaskSucceeded
if dep.Status != "" {
status = dep.Status
}
bv := b.BuildVariant
if dep.Variant != "" {
bv = dep.Variant
}
newDeps := []Dependency{}
if dep.Variant == AllVariants {
// for * case, we need to add all variants of the task
var ids []string
if dep.Name != AllDependencies {
ids = tt.GetIdsForAllVariants(b.BuildVariant, dep.Name)
} else {
// edge case where variant and task are both *
ids = tt.GetIdsForAllTasks(b.BuildVariant, newTask.DisplayName)
}
for _, id := range ids {
newDeps = append(newDeps, Dependency{TaskId: id, Status: status})
}
} else {
// general case
newDep := Dependency{
TaskId: tt.GetId(bv, dep.Name),
Status: status,
}
if newDep.TaskId != "" {
newDeps = []Dependency{newDep}
}
}
newTask.DependsOn = append(newTask.DependsOn, newDeps...)
}
}
// append the task to the list of the created tasks
tasks = append(tasks, newTask)
}
// Set the NumDependents field
// Existing tasks in the db and tasks in other builds are not updated
setNumDeps(tasks)
// return all of the tasks created
return tasks, nil
}
// Ensures that:
// 1. a referenced task within a buildvariant task object exists in
// the set of project tasks
// 2. any referenced distro exists within the current setting's distro directory
func ensureReferentialIntegrity(project *model.Project) []ValidationError {
errs := []ValidationError{}
// create a set of all the task names
allTaskNames := map[string]bool{}
for _, task := range project.Tasks {
allTaskNames[task.Name] = true
}
for _, buildVariant := range project.BuildVariants {
buildVariantTasks := map[string]bool{}
for _, task := range buildVariant.Tasks {
if _, ok := allTaskNames[task.Name]; !ok {
if task.Name == "" {
errs = append(errs,
ValidationError{
Message: fmt.Sprintf("tasks for buildvariant '%v' "+
"in project '%v' must each have a name field",
project.Identifier, buildVariant.Name),
},
)
} else {
errs = append(errs,
ValidationError{
Message: fmt.Sprintf("buildvariant '%v' in "+
"project '%v' references a non-existent "+
"task '%v'", buildVariant.Name,
project.Identifier, task.Name),
},
)
}
}
buildVariantTasks[task.Name] = true
for _, distroId := range task.Distros {
if !util.SliceContains(distroIds, distroId) {
errs = append(errs,
ValidationError{
Message: fmt.Sprintf("task '%v' in buildvariant "+
"'%v' in project '%v' references a "+
"non-existent distro '%v'.\nValid distros "+
"include: \n\t- %v", task.Name,
buildVariant.Name, project.Identifier,
distroId, strings.Join(distroIds, "\n\t- ")),
},
)
}
}
}
for _, distroId := range buildVariant.RunOn {
if !util.SliceContains(distroIds, distroId) {
errs = append(errs,
ValidationError{
Message: fmt.Sprintf("buildvariant '%v' in project "+
"'%v' references a non-existent distro '%v'.\n"+
"Valid distros include: \n\t- %v",
buildVariant.Name, project.Identifier, distroId,
strings.Join(distroIds, "\n\t- ")),
},
)
}
}
}
return errs
}
// This function is responsible for validating the notifications file
func ValidateNotifications(configName string, mciNotification *MCINotification) error {
evergreen.Logger.Logf(slogger.INFO, "Validating notifications...")
allNotifications := []string{}
projectNameToBuildVariants, err := findProjectBuildVariants(configName)
if err != nil {
return fmt.Errorf("Error loading project build variants: %v", err)
}
// Validate default notification recipients
for _, notification := range mciNotification.Notifications {
if notification.Project == "" {
return fmt.Errorf("Must specify a project for each notification - see %v", notification.Name)
}
buildVariants, ok := projectNameToBuildVariants[notification.Project]
if !ok {
return fmt.Errorf("Notifications validation failed: "+
"project `%v` not found", notification.Project)
}
// ensure all supplied build variants are valid
for _, buildVariant := range notification.SkipVariants {
if !util.SliceContains(buildVariants, buildVariant) {
return fmt.Errorf("Nonexistent buildvariant - ”%v” - specified for ”%v” notification", buildVariant, notification.Name)
}
}
allNotifications = append(allNotifications, notification.Name)
}
// Validate team name and addresses
for _, team := range mciNotification.Teams {
if team.Name == "" {
return fmt.Errorf("Each notification team must have a name")
}
for _, subscription := range team.Subscriptions {
for _, notification := range subscription.NotifyOn {
if !util.SliceContains(allNotifications, notification) {
return fmt.Errorf("Team ”%v” contains a non-existent subscription - %v", team.Name, notification)
}
}
for _, buildVariant := range subscription.SkipVariants {
buildVariants, ok := projectNameToBuildVariants[subscription.Project]
if !ok {
return fmt.Errorf("Teams validation failed: project `%v` not found", subscription.Project)
}
if !util.SliceContains(buildVariants, buildVariant) {
return fmt.Errorf("Nonexistent buildvariant - ”%v” - specified for team ”%v” ", buildVariant, team.Name)
}
}
}
}
// Validate patch notifications
for _, subscription := range mciNotification.PatchNotifications {
for _, notification := range subscription.NotifyOn {
if !util.SliceContains(allNotifications, notification) {
return fmt.Errorf("Nonexistent patch notification - ”%v” - specified", notification)
}
}
}
// validate the patch notification buildvariatns
for _, subscription := range mciNotification.PatchNotifications {
buildVariants, ok := projectNameToBuildVariants[subscription.Project]
if !ok {
return fmt.Errorf("Patch notification build variants validation failed: "+
"project `%v` not found", subscription.Project)
}
for _, buildVariant := range subscription.SkipVariants {
if !util.SliceContains(buildVariants, buildVariant) {
return fmt.Errorf("Nonexistent buildvariant - ”%v” - specified for patch notifications", buildVariant)
}
}
}
// all good!
return nil
}
// RunOnVariant returns true if the plugin command should run on variant; returns false otherwise
func (p PluginCommandConf) RunOnVariant(variant string) bool {
return len(p.Variants) == 0 || util.SliceContains(p.Variants, variant)
}
// applyPatch is used by the agent to copy patch data onto disk
// and then call the necessary git commands to apply the patch file
func (gapc *GitApplyPatchCommand) applyPatch(conf *model.TaskConfig,
patch *patch.Patch, pluginLogger plugin.Logger) error {
// patch sets and contain multiple patches, some of them for modules
for _, patchPart := range patch.Patches {
var dir string
if patchPart.ModuleName == "" {
// if patch is not part of a module, just apply patch against src root
dir = gapc.Directory
pluginLogger.LogExecution(slogger.INFO, "Applying patch with git...")
} else {
// if patch is part of a module, apply patch in module root
module, err := conf.Project.GetModuleByName(patchPart.ModuleName)
if err != nil {
return fmt.Errorf("Error getting module: %v", err)
}
if module == nil {
return fmt.Errorf("Module not found: %v", patchPart.ModuleName)
}
// skip the module if this build variant does not use it
if !util.SliceContains(conf.BuildVariant.Modules, module.Name) {
pluginLogger.LogExecution(slogger.INFO, "Skipping patch for"+
" module %v, since the current build variant does not"+
" use it", module.Name)
continue
}
dir = filepath.Join(gapc.Directory, module.Prefix, module.Name)
pluginLogger.LogExecution(slogger.INFO, "Applying module patch with git...")
}
// create a temporary folder and store patch files on disk,
// for later use in shell script
tempFile, err := ioutil.TempFile("", "mcipatch_")
if err != nil {
return err
}
defer tempFile.Close()
_, err = io.WriteString(tempFile, patchPart.PatchSet.Patch)
if err != nil {
return err
}
tempAbsPath := tempFile.Name()
// this applies the patch using the patch files in the temp directory
patchCommandStrings := []string{
fmt.Sprintf("set -o verbose"),
fmt.Sprintf("set -o errexit"),
fmt.Sprintf("ls"),
fmt.Sprintf("cd '%v'", dir),
fmt.Sprintf("git checkout '%v'", patchPart.Githash),
fmt.Sprintf("git apply --check --whitespace=fix '%v'", tempAbsPath),
fmt.Sprintf("git apply --stat '%v'", tempAbsPath),
fmt.Sprintf("git apply --whitespace=fix < '%v'", tempAbsPath),
}
cmdsJoined := strings.Join(patchCommandStrings, "\n")
patchCmd := &command.LocalCommand{
CmdString: cmdsJoined,
WorkingDirectory: conf.WorkDir,
Stdout: pluginLogger.GetTaskLogWriter(slogger.INFO),
Stderr: pluginLogger.GetTaskLogWriter(slogger.ERROR),
ScriptMode: true,
}
err = patchCmd.Run()
if err != nil {
return err
}
pluginLogger.Flush()
}
return nil
}
// isAdmin returns false if the user is nil or if its id is not
// located in ProjectRef's Admins field.
func isAdmin(u *user.DBUser, project *model.ProjectRef) bool {
if u == nil {
return false
}
return util.SliceContains(project.Admins, u.Id)
}