Пример #1
0
// Given a patch version and a list of task names, creates a new task with
// the given name for each variant, if applicable.
func AddNewTasksForPatch(p *patch.Patch, patchVersion *version.Version, project *Project,
	taskNames []string) error {
	// create new tasks for all of the added patch tasks
	var newTasks []string
	for _, taskName := range taskNames {
		if !util.SliceContains(p.Tasks, taskName) {
			newTasks = append(newTasks, taskName)
		}
	}

	// add tasks to the patch in the db
	if err := p.AddTasks(taskNames); err != nil {
		return err
	}

	// add new tasks to the build, if they exist
	if len(newTasks) > 0 {
		builds, err := build.Find(build.ByIds(patchVersion.BuildIds))
		if err != nil {
			return err
		}

		for _, b := range builds {
			if _, err = AddTasksToBuild(&b, project, patchVersion, newTasks); err != nil {
				return err
			}
		}
	}
	return nil
}
Пример #2
0
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
	if p.Version != "" {
		return nil, fmt.Errorf("Patch %v already finalized", p.Version)
	}
	projectRef, err := model.FindOneProjectRef(p.Project)
	if err != nil {
		return nil, err
	}

	// try to get the remote project file data at the requested revision
	var projectFileBytes []byte
	projectFileURL := thirdparty.GetGithubFileURL(
		projectRef.Owner,
		projectRef.Repo,
		projectRef.RemotePath,
		p.Githash,
	)
	githubFile, err := thirdparty.GetGithubFile(settings.Credentials["github"], projectFileURL)
	if err != nil {
		// if the project file doesn't exist, but our patch includes a project file,
		// we try to apply the diff and proceed.
		if !(p.ConfigChanged(projectRef.RemotePath) && thirdparty.IsFileNotFound(err)) {
			// return an error if the github error is network/auth-related or we aren't patching the config
			return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
		}
	} else {
		// we successfully got the project file in base64, so we decode it
		projectFileBytes, err = base64.StdEncoding.DecodeString(githubFile.Content)
		if err != nil {
			return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
		}
	}

	project := &model.Project{}
	if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
		return nil, err
	}
	// apply remote configuration patch if needed
	if p.ConfigChanged(projectRef.RemotePath) {
		project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
		if err != nil {
			return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
		}
		// overwrite project fields with the project ref to disallow tracking a
		// different project or doing other crazy things via config patches
		errs := CheckProjectSyntax(project)
		if len(errs) != 0 {
			var message string
			for _, err := range errs {
				message += fmt.Sprintf("\n\t=> %v", err)
			}
			return nil, fmt.Errorf(message)
		}
	}
	return project, nil
}
Пример #3
0
// Given the patch version and a list of build variants, creates new builds
// with the patch's tasks.
func AddNewBuildsForPatch(p *patch.Patch, patchVersion *version.Version, project *Project,
	buildVariants []string) (*version.Version, error) {

	// compute a list of the newly added build variants
	var newVariants []string
	for _, variant := range buildVariants {
		if !util.SliceContains(p.BuildVariants, variant) {
			newVariants = append(newVariants, variant)
		}
	}

	// update the patch
	if err := p.AddBuildVariants(buildVariants); err != nil {
		return nil, err
	}

	newBuildIds := make([]string, 0)
	newBuildStatuses := make([]version.BuildStatus, 0)
	tt := BuildTaskIdTable(project, patchVersion)
	for _, buildVariant := range newVariants {
		evergreen.Logger.Logf(slogger.INFO,
			"Creating build for version %v, buildVariant %v, activated = %v",
			patchVersion.Id, buildVariant, p.Activated)
		buildId, err := CreateBuildFromVersion(
			project, patchVersion, tt, buildVariant, p.Activated, p.Tasks)
		if err != nil {
			return nil, err
		}
		newBuildIds = append(newBuildIds, buildId)

		newBuildStatuses = append(newBuildStatuses,
			version.BuildStatus{
				BuildVariant: buildVariant,
				BuildId:      buildId,
				Activated:    p.Activated,
			},
		)
		patchVersion.BuildIds = append(patchVersion.BuildIds, buildId)
	}

	err := version.UpdateOne(
		bson.M{version.IdKey: patchVersion.Id},
		bson.M{
			"$push": bson.M{
				version.BuildIdsKey:      bson.M{"$each": newBuildIds},
				version.BuildVariantsKey: bson.M{"$each": newBuildStatuses},
			},
		},
	)
	if err != nil {
		return nil, err
	}

	return patchVersion, nil
}
Пример #4
0
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
	if p.Version != "" {
		return nil, fmt.Errorf("Patch %v already finalized", p.Version)
	}
	projectRef, err := model.FindOneProjectRef(p.Project)
	if err != nil {
		return nil, err
	}

	// get the remote file at the requested revision
	projectFileURL := thirdparty.GetGithubFileURL(
		projectRef.Owner,
		projectRef.Repo,
		projectRef.RemotePath,
		p.Githash,
	)

	githubFile, err := thirdparty.GetGithubFile(
		settings.Credentials["github"],
		projectFileURL,
	)
	if err != nil {
		return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
	}

	projectFileBytes, err := base64.StdEncoding.DecodeString(githubFile.Content)
	if err != nil {
		return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
	}

	project := &model.Project{}

	if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
		return nil, err
	}
	// apply remote configuration patch if needed
	if p.ConfigChanged(projectRef.RemotePath) {
		project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
		if err != nil {
			return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
		}
		// overwrite project fields with the project ref to disallow tracking a
		// different project or doing other crazy things via config patches
		errs := CheckProjectSyntax(project)
		if len(errs) != 0 {
			var message string
			for _, err := range errs {
				message += fmt.Sprintf("\n\t=> %v", err)
			}
			return nil, fmt.Errorf(message)
		}
	}
	return project, nil
}
Пример #5
0
// Finalizes a patch:
// Patches a remote project's configuration file if needed.
// Creates a version for this patch and links it.
// Creates builds based on the version.
func FinalizePatch(p *patch.Patch, settings *evergreen.Settings) (*version.Version, error) {
	// unmarshal the project YAML for storage
	project := &Project{}
	err := yaml.Unmarshal([]byte(p.PatchedConfig), project)
	if err != nil {
		return nil, fmt.Errorf(
			"Error marshalling patched project config from repository revision “%v”: %v",
			p.Githash, err)
	}

	projectRef, err := FindOneProjectRef(p.Project)
	if err != nil {
		return nil, err
	}

	gitCommit, err := thirdparty.GetCommitEvent(
		settings.Credentials["github"],
		projectRef.Owner, projectRef.Repo, p.Githash,
	)
	if err != nil {
		return nil, fmt.Errorf("Couldn't fetch commit information: %v", err)
	}
	if gitCommit == nil {
		return nil, fmt.Errorf("Couldn't fetch commit information: git commit doesn't exist?")
	}

	patchVersion := &version.Version{
		Id:            p.Id.Hex(),
		CreateTime:    time.Now(),
		Identifier:    p.Project,
		Revision:      p.Githash,
		Author:        p.Author,
		Message:       p.Description,
		BuildIds:      []string{},
		BuildVariants: []version.BuildStatus{},
		Config:        string(p.PatchedConfig),
		Status:        evergreen.PatchCreated,
		Requester:     evergreen.PatchVersionRequester,
		Branch:        project.Branch,
	}

	var pairs []TVPair
	if len(p.VariantsTasks) > 0 {
		pairs = VariantTasksToTVPairs(p.VariantsTasks)
	} else {
		// handle case where the patch is being finalized but only has the old schema tasks/variants
		// instead of the new one.
		for _, v := range p.BuildVariants {
			for _, t := range p.Tasks {
				if project.FindTaskForVariant(t, v) != nil {
					pairs = append(pairs, TVPair{v, t})
				}
			}
		}
		p.VariantsTasks = TVPairsToVariantTasks(pairs)
	}

	tt := NewPatchTaskIdTable(project, patchVersion, pairs)
	variantsProcessed := map[string]bool{}
	for _, vt := range p.VariantsTasks {
		if _, ok := variantsProcessed[vt.Variant]; ok {
			continue
		}
		buildId, err := CreateBuildFromVersion(project, patchVersion, tt, vt.Variant, true, vt.Tasks)
		if err != nil {
			return nil, err
		}
		patchVersion.BuildIds = append(patchVersion.BuildIds, buildId)
		patchVersion.BuildVariants = append(patchVersion.BuildVariants,
			version.BuildStatus{
				BuildVariant: vt.Variant,
				Activated:    true,
				BuildId:      buildId,
			},
		)
	}

	if err = patchVersion.Insert(); err != nil {
		return nil, err
	}
	if err = p.SetActivated(patchVersion.Id); err != nil {
		return nil, err
	}
	return patchVersion, nil
}
Пример #6
0
// Finalizes a patch:
// Patches a remote project's configuration file if needed.
// Creates a version for this patch and links it.
// Creates builds based on the version.
func FinalizePatch(p *patch.Patch, settings *evergreen.Settings) (
	patchVersion *version.Version, err error) {
	// unmarshal the project YAML for storage
	project := &Project{}
	err = yaml.Unmarshal([]byte(p.PatchedConfig), project)
	if err != nil {
		return nil, fmt.Errorf(
			"Error marshalling patched project config from repository revision “%v”: %v",
			p.Githash, err)
	}

	projectRef, err := FindOneProjectRef(p.Project)
	if err != nil {
		return
	}

	gitCommit, err := thirdparty.GetCommitEvent(
		settings.Credentials["github"],
		projectRef.Owner, projectRef.Repo, p.Githash,
	)
	if err != nil {
		return nil, fmt.Errorf("Couldn't fetch commit information: %v", err)
	}
	if gitCommit == nil {
		return nil, fmt.Errorf("Couldn't fetch commit information: git commit" +
			" doesn't exist?")
	}

	patchVersion = &version.Version{
		Id:            fmt.Sprintf("%v_%v", p.Id.Hex(), 0),
		CreateTime:    time.Now(),
		Identifier:    p.Project,
		Revision:      p.Githash,
		Author:        gitCommit.Commit.Committer.Name,
		AuthorEmail:   gitCommit.Commit.Committer.Email,
		Message:       gitCommit.Commit.Message,
		BuildIds:      []string{},
		BuildVariants: []version.BuildStatus{},
		Config:        string(p.PatchedConfig),
		Status:        evergreen.PatchCreated,
		Requester:     evergreen.PatchVersionRequester,
	}

	//expand tasks and build variants
	buildVariants := p.BuildVariants
	if len(p.BuildVariants) == 1 && p.BuildVariants[0] == "all" {
		buildVariants = make([]string, 0)
		for _, buildVariant := range project.BuildVariants {
			if buildVariant.Disabled {
				continue
			}
			buildVariants = append(buildVariants, buildVariant.Name)
		}
	}
	tasks := p.Tasks
	if len(p.Tasks) == 1 && p.Tasks[0] == "all" {
		tasks = make([]string, 0)
		for _, t := range project.Tasks {
			tasks = append(tasks, t.Name)
		}
	}
	tt := BuildTaskIdTable(project, patchVersion)
	for _, buildvariant := range buildVariants {
		buildId, err := CreateBuildFromVersion(project, patchVersion, tt, buildvariant, true, tasks)
		if err != nil {
			return nil, err
		}
		patchVersion.BuildIds = append(patchVersion.BuildIds, buildId)
		patchVersion.BuildVariants = append(patchVersion.BuildVariants,
			version.BuildStatus{
				BuildVariant: buildvariant,
				Activated:    true,
				BuildId:      buildId,
			},
		)
	}

	if err = patchVersion.Insert(); err != nil {
		return nil, err
	}
	if err = p.SetActivated(patchVersion.Id); err != nil {
		return nil, err
	}
	return patchVersion, nil
}
Пример #7
0
// MakePatchedConfig takes in the path to a remote configuration a stringified version
// of the current project and returns an unmarshalled version of the project
// with the patch applied
func MakePatchedConfig(p *patch.Patch, remoteConfigPath, projectConfig string) (
	*Project, error) {
	// Dereference all the patch data so that we can use it to write temp files
	err := p.FetchPatchFiles()
	if err != nil {
		return nil, err
	}
	for _, patchPart := range p.Patches {
		// we only need to patch the main project and not any other modules
		if patchPart.ModuleName != "" {
			continue
		}
		// write patch file
		patchFilePath, err := util.WriteToTempFile(patchPart.PatchSet.Patch)
		if err != nil {
			return nil, fmt.Errorf("could not write patch file: %v", err)
		}
		defer os.Remove(patchFilePath)
		// write project configuration
		configFilePath, err := util.WriteToTempFile(projectConfig)
		if err != nil {
			return nil, fmt.Errorf("could not write config file: %v", err)
		}
		defer os.Remove(configFilePath)

		// clean the working directory
		workingDirectory := filepath.Dir(patchFilePath)
		localConfigPath := filepath.Join(
			workingDirectory,
			remoteConfigPath,
		)
		parentDir := strings.Split(
			remoteConfigPath,
			string(os.PathSeparator),
		)[0]
		err = os.RemoveAll(filepath.Join(workingDirectory, parentDir))
		if err != nil {
			return nil, err
		}
		if err = os.MkdirAll(filepath.Dir(localConfigPath), 0755); err != nil {
			return nil, err
		}
		// rename the temporary config file name to the remote config
		// file path if we are patching an existing remote config
		if len(projectConfig) > 0 {
			if err = os.Rename(configFilePath, localConfigPath); err != nil {
				return nil, fmt.Errorf("could not rename file '%v' to '%v': %v",
					configFilePath, localConfigPath, err)
			}
			defer os.Remove(localConfigPath)
		}

		// selectively apply the patch to the config file
		patchCommandStrings := []string{
			fmt.Sprintf("set -o verbose"),
			fmt.Sprintf("set -o errexit"),
			fmt.Sprintf("git apply --whitespace=fix --include=%v < '%v'",
				remoteConfigPath, patchFilePath),
		}

		patchCmd := &command.LocalCommand{
			CmdString:        strings.Join(patchCommandStrings, "\n"),
			WorkingDirectory: workingDirectory,
			Stdout:           evergreen.NewInfoLoggingWriter(&evergreen.Logger),
			Stderr:           evergreen.NewErrorLoggingWriter(&evergreen.Logger),
			ScriptMode:       true,
		}

		if err = patchCmd.Run(); err != nil {
			return nil, fmt.Errorf("could not run patch command: %v", err)
		}
		// read in the patched config file
		data, err := ioutil.ReadFile(localConfigPath)
		if err != nil {
			return nil, fmt.Errorf("could not read patched config file: %v",
				err)
		}
		project := &Project{}
		if err = LoadProjectInto(data, p.Project, project); err != nil {
			return nil, err
		}
		return project, nil
	}
	return nil, fmt.Errorf("no patch on project")
}
Пример #8
0
// Finalizes a patch:
// Patches a remote project's configuration file if needed.
// Creates a version for this patch and links it.
// Creates builds based on the version.
func FinalizePatch(p *patch.Patch, gitCommit *thirdparty.CommitEvent,
	settings *evergreen.Settings, project *Project) (
	patchVersion *version.Version, err error) {
	// marshal the project YAML for storage
	projectYamlBytes, err := yaml.Marshal(project)

	if err != nil {
		return nil, fmt.Errorf(
			"Error marshalling patched project config from repository revision “%v”: %v",
			p.Githash, err)
	}

	patchVersion = &version.Version{
		Id:            fmt.Sprintf("%v_%v", p.Id.Hex(), 0),
		CreateTime:    time.Now(),
		Project:       p.Project,
		Revision:      p.Githash,
		Author:        gitCommit.Commit.Committer.Name,
		AuthorEmail:   gitCommit.Commit.Committer.Email,
		Message:       gitCommit.Commit.Message,
		BuildIds:      []string{},
		BuildVariants: []version.BuildStatus{},
		Config:        string(projectYamlBytes),
		Status:        evergreen.PatchCreated,
		Requester:     evergreen.PatchVersionRequester,
	}

	buildVariants := p.BuildVariants
	if len(p.BuildVariants) == 1 && p.BuildVariants[0] == "all" {
		buildVariants = make([]string, 0)
		for _, buildVariant := range project.BuildVariants {
			if buildVariant.Disabled {
				continue
			}
			buildVariants = append(buildVariants, buildVariant.Name)
		}
	}

	tt := BuildTaskIdTable(project, patchVersion)
	for _, buildvariant := range buildVariants {
		buildId, err := CreateBuildFromVersion(project, patchVersion, tt, buildvariant, true, p.Tasks)
		if err != nil {
			return nil, err
		}
		patchVersion.BuildIds = append(patchVersion.BuildIds, buildId)
		patchVersion.BuildVariants = append(patchVersion.BuildVariants,
			version.BuildStatus{
				BuildVariant: buildvariant,
				Activated:    true,
				BuildId:      buildId,
			},
		)
	}

	if err = patchVersion.Insert(); err != nil {
		return nil, err
	}
	if err = p.SetActivated(patchVersion.Id); err != nil {
		return nil, err
	}
	return patchVersion, nil
}