func setupPatches(patchMode patchTestMode, b *build.Build, t *testing.T, patches ...patchRequest) {
if patchMode == NoPatch {
return
}
ptch := &patch.Patch{
Status: evergreen.PatchCreated,
Version: b.Version,
Patches: []patch.ModulePatch{},
}
for _, p := range patches {
patchContent, err := ioutil.ReadFile(p.filePath)
testutil.HandleTestingErr(err, t, "failed to read test patch file")
if patchMode == InlinePatch {
ptch.Patches = append(ptch.Patches, patch.ModulePatch{
ModuleName: p.moduleName,
Githash: p.githash,
PatchSet: patch.PatchSet{Patch: string(patchContent)},
})
} else {
pId := bson.NewObjectId().Hex()
So(dbutil.WriteGridFile(patch.GridFSPrefix, pId, strings.NewReader(string(patchContent))), ShouldBeNil)
ptch.Patches = append(ptch.Patches, patch.ModulePatch{
ModuleName: p.moduleName,
Githash: p.githash,
PatchSet: patch.PatchSet{PatchFileId: pId},
})
}
}
testutil.HandleTestingErr(ptch.Insert(), t, "failed to insert patch")
}
func setupAPITestData(testConfig *evergreen.Settings, taskDisplayName string,
variant string, patchMode patchTestMode, t *testing.T) (*model.Task, *build.Build, error) {
//ignore errs here because the ns might just not exist.
clearDataMsg := "Failed to clear test data collection"
testCollections := []string{
model.TasksCollection, build.Collection, host.Collection,
distro.Collection, version.Collection, patch.Collection,
model.PushlogCollection, model.ProjectVarsCollection, model.TaskQueuesCollection,
manifest.Collection, model.ProjectRefCollection}
testutil.HandleTestingErr(dbutil.ClearCollections(testCollections...), t, clearDataMsg)
projectVars := &model.ProjectVars{
Id: "evergreen-ci-render",
Vars: map[string]string{
"aws_key": testConfig.Providers.AWS.Id,
"aws_secret": testConfig.Providers.AWS.Secret,
"fetch_key": "fetch_expansion_value",
},
}
_, err := projectVars.Upsert()
testutil.HandleTestingErr(err, t, clearDataMsg)
taskOne := &model.Task{
Id: "testTaskId",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: "evergreen-ci-render",
DisplayName: taskDisplayName,
HostId: "testHost",
Secret: "testTaskSecret",
Version: "testVersionId",
Status: evergreen.TaskDispatched,
Requester: evergreen.RepotrackerVersionRequester,
}
taskTwo := &model.Task{
Id: "testTaskIdTwo",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: "evergreen-ci-render",
DisplayName: taskDisplayName,
HostId: "",
Secret: "testTaskSecret",
Activated: true,
Version: "testVersionId",
Status: evergreen.TaskUndispatched,
Requester: evergreen.RepotrackerVersionRequester,
}
if patchMode != NoPatch {
taskOne.Requester = evergreen.PatchVersionRequester
}
testutil.HandleTestingErr(taskOne.Insert(), t, "failed to insert taskOne")
testutil.HandleTestingErr(taskTwo.Insert(), t, "failed to insert taskTwo")
// set up task queue for task end tests
taskQueue := &model.TaskQueue{
Distro: "test-distro-one",
Queue: []model.TaskQueueItem{
model.TaskQueueItem{
Id: "testTaskIdTwo",
DisplayName: taskDisplayName,
},
},
}
testutil.HandleTestingErr(taskQueue.Save(), t, "failed to insert taskqueue")
workDir, err := ioutil.TempDir("", "agent_test_")
testutil.HandleTestingErr(err, t, "failed to create working directory")
host := &host.Host{
Id: "testHost",
Host: "testHost",
Distro: distro.Distro{
Id: "test-distro-one",
WorkDir: workDir,
Expansions: []distro.Expansion{{"distro_exp", "DISTRO_EXP"}},
},
RunningTask: "testTaskId",
StartedBy: evergreen.User,
AgentRevision: agentRevision,
}
testutil.HandleTestingErr(host.Insert(), t, "failed to insert host")
// read in the project configuration
projectFile := "testdata/config_test_plugin/project/evergreen-ci-render.yml"
projectConfig, err := ioutil.ReadFile(projectFile)
testutil.HandleTestingErr(err, t, "failed to read project config")
projectRef := &model.ProjectRef{
Identifier: "evergreen-ci-render",
Owner: "evergreen-ci",
Repo: "render",
RepoKind: "github",
Branch: "master",
Enabled: true,
BatchTime: 180,
}
testutil.HandleTestingErr(projectRef.Insert(), t, "failed to insert projectRef")
err = testutil.CreateTestLocalConfig(testConfig, "evergreen-ci-render", "testdata/config_test_plugin/project/evergreen-ci-render.yml")
testutil.HandleTestingErr(err, t, "failed to marshall project config")
// unmarshall the project configuration into a struct
project := &model.Project{}
testutil.HandleTestingErr(yaml.Unmarshal(projectConfig, project), t, "failed to unmarshal project config")
// now then marshall the project YAML for storage
projectYamlBytes, err := yaml.Marshal(project)
testutil.HandleTestingErr(err, t, "failed to marshall project config")
// insert the version document
v := &version.Version{
Id: "testVersionId",
BuildIds: []string{taskOne.BuildId},
Config: string(projectYamlBytes),
}
testutil.HandleTestingErr(v.Insert(), t, "failed to insert version")
if patchMode != NoPatch {
mainPatchContent, err := ioutil.ReadFile("testdata/test.patch")
testutil.HandleTestingErr(err, t, "failed to read test patch file")
modulePatchContent, err := ioutil.ReadFile("testdata/testmodule.patch")
testutil.HandleTestingErr(err, t, "failed to read test module patch file")
ptch := &patch.Patch{
Status: evergreen.PatchCreated,
Version: v.Id,
}
if patchMode == InlinePatch {
ptch.Patches = []patch.ModulePatch{
{
ModuleName: "",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{Patch: string(mainPatchContent)},
},
{
ModuleName: "recursive",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{Patch: string(modulePatchContent)},
},
}
} else {
p1Id, p2Id := bson.NewObjectId().Hex(), bson.NewObjectId().Hex()
So(dbutil.WriteGridFile(patch.GridFSPrefix, p1Id, strings.NewReader(string(mainPatchContent))), ShouldBeNil)
So(dbutil.WriteGridFile(patch.GridFSPrefix, p2Id, strings.NewReader(string(modulePatchContent))), ShouldBeNil)
ptch.Patches = []patch.ModulePatch{
{
ModuleName: "",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{PatchFileId: p1Id},
},
{
ModuleName: "recursive",
Githash: "1e5232709595db427893826ce19289461cba3f75",
PatchSet: patch.PatchSet{PatchFileId: p2Id},
},
}
}
testutil.HandleTestingErr(ptch.Insert(), t, "failed to insert patch")
}
session, _, err := dbutil.GetGlobalSessionFactory().GetSession()
testutil.HandleTestingErr(err, t, "couldn't get db session!")
// Remove any logs for our test task from previous runs.
_, err = session.DB(model.TaskLogDB).C(model.TaskLogCollection).
RemoveAll(bson.M{"t_id": bson.M{"$in": []string{taskOne.Id, taskTwo.Id}}})
testutil.HandleTestingErr(err, t, "failed to remove logs")
build := &build.Build{
Id: "testBuildId",
Tasks: []build.TaskCache{
build.NewTaskCache(taskOne.Id, taskOne.DisplayName, true),
build.NewTaskCache(taskTwo.Id, taskTwo.DisplayName, true),
},
Version: "testVersionId",
}
testutil.HandleTestingErr(build.Insert(), t, "failed to insert build")
return taskOne, build, nil
}
func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) {
p, err := getPatchFromRequest(r)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, err.Error())
return
}
var moduleName, patchContent, githash string
if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
moduleName, patchContent, githash = r.FormValue("module"), r.FormValue("patch"), r.FormValue("githash")
} else {
data := struct {
Module string `json:"module"`
Patch string `json:"patch"`
Githash string `json:"githash"`
}{}
if err := util.ReadJSONInto(r.Body, &data); err != nil {
as.LoggedError(w, r, http.StatusBadRequest, err)
return
}
moduleName, patchContent, githash = data.Module, data.Patch, data.Githash
}
if len(patchContent) == 0 {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
return
}
projectRef, err := model.FindOneProjectRef(p.Project)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting project ref with id %v: %v", p.Project, err))
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting patch: %v", err))
return
}
if project == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("can't find project: %v", p.Project))
return
}
module, err := project.GetModuleByName(moduleName)
if err != nil || module == nil {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("No such module", moduleName))
return
}
gitOutput, err := thirdparty.GitApplyNumstat(patchContent)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Invalid patch: %v", err))
return
}
if gitOutput == nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Empty diff"))
return
}
summaries, err := thirdparty.ParseGitSummary(gitOutput)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Can't validate patch: %v", err))
return
}
repoOwner, repo := module.GetRepoOwnerAndName()
commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"], repoOwner, repo, githash)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if commitInfo == nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("commit hash doesn't seem to exist"))
return
}
// write the patch content into a GridFS file under a new ObjectId.
patchFileId := bson.NewObjectId().Hex()
err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(patchContent))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err))
return
}
modulePatch := patch.ModulePatch{
ModuleName: moduleName,
Githash: githash,
PatchSet: patch.PatchSet{
PatchFileId: patchFileId,
Summary: summaries,
},
}
if err = p.UpdateModulePatch(modulePatch); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
as.WriteJSON(w, http.StatusOK, "Patch module updated")
return
}
func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) {
user := MustHaveUser(r)
var apiRequest PatchAPIRequest
var projId, description string
var finalize bool
if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
patchContent := r.FormValue("patch")
if patchContent == "" {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
return
}
apiRequest = PatchAPIRequest{
ProjectFileName: r.FormValue("project"),
ModuleName: r.FormValue("module"),
Githash: r.FormValue("githash"),
PatchContent: r.FormValue("patch"),
BuildVariants: strings.Split(r.FormValue("buildvariants"), ","),
}
projId = r.FormValue("project")
description = r.FormValue("desc")
finalize = strings.ToLower(r.FormValue("finalize")) == "true"
} else {
data := struct {
Description string `json:"desc"`
Project string `json:"project"`
Patch string `json:"patch"`
Githash string `json:"githash"`
Variants string `json:"buildvariants"`
Tasks []string `json:"tasks"`
Finalize bool `json:"finalize"`
}{}
if err := util.ReadJSONInto(r.Body, &data); err != nil {
as.LoggedError(w, r, http.StatusBadRequest, err)
return
}
if len(data.Patch) > patch.SizeLimit {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Patch is too large."))
}
if len(data.Patch) == 0 {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
return
}
description = data.Description
projId = data.Project
finalize = data.Finalize
apiRequest = PatchAPIRequest{
ProjectFileName: data.Project,
ModuleName: r.FormValue("module"),
Githash: data.Githash,
PatchContent: data.Patch,
BuildVariants: strings.Split(data.Variants, ","),
Tasks: data.Tasks,
}
}
projectRef, err := model.FindOneProjectRef(projId)
if err != nil {
message := fmt.Errorf("Error locating project ref '%v': %v",
projId, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
project, err := model.FindProject("", projectRef)
if err != nil {
message := fmt.Errorf("Error locating project '%v' from '%v': %v",
projId, as.Settings.ConfigDir, err)
as.LoggedError(w, r, http.StatusInternalServerError, message)
return
}
if project == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("project %v not found", projId))
return
}
patchMetadata, err := apiRequest.Validate(as.Settings.Credentials["github"])
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid patch: %v", err))
return
}
if patchMetadata == nil {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("patch metadata is empty"))
return
}
if apiRequest.ModuleName != "" {
as.WriteJSON(w, http.StatusBadRequest,
"module not allowed when creating new patches (must be added in a subsequent request)")
return
}
patchProjectRef, err := model.FindOneProjectRef(patchMetadata.Project.Identifier)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid projectRef: %v", err))
return
}
if patchProjectRef == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("Empty patch project Ref"))
return
}
commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"],
patchProjectRef.Owner,
patchProjectRef.Repo,
apiRequest.Githash)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if commitInfo == nil {
as.WriteJSON(w, http.StatusBadRequest, "That commit doesn't seem to exist.")
return
}
createTime := time.Now()
// create a new object ID to use as reference for the patch data
patchFileId := bson.NewObjectId().Hex()
patchDoc := &patch.Patch{
Id: bson.NewObjectId(),
Description: description,
Author: user.Id,
Project: apiRequest.ProjectFileName,
Githash: apiRequest.Githash,
CreateTime: createTime,
Status: evergreen.PatchCreated,
BuildVariants: apiRequest.BuildVariants,
Tasks: apiRequest.Tasks,
Patches: []patch.ModulePatch{
patch.ModulePatch{
ModuleName: "",
Githash: apiRequest.Githash,
PatchSet: patch.PatchSet{
PatchFileId: patchFileId,
Summary: patchMetadata.Summaries,
},
},
},
}
// write the patch content into a GridFS file under a new ObjectId.
err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(apiRequest.PatchContent))
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err))
return
}
// Get and validate patched config and add it to the patch document
patchedProject, err := validator.GetPatchedProject(patchDoc, &as.Settings)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("invalid patched config: %v", err))
return
}
projectYamlBytes, err := yaml.Marshal(patchedProject)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error marshalling patched config: %v", err))
return
}
// set the patch number based on patch author
patchDoc.PatchNumber, err = user.IncPatchNumber()
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error computing patch num %v", err))
return
}
patchDoc.PatchedConfig = string(projectYamlBytes)
patchDoc.ClearPatchData()
//expand tasks and build variants and include dependencies
buildVariants := patchDoc.BuildVariants
if len(patchDoc.BuildVariants) == 1 && patchDoc.BuildVariants[0] == "all" {
buildVariants = make([]string, 0)
for _, buildVariant := range patchedProject.BuildVariants {
if buildVariant.Disabled {
continue
}
buildVariants = append(buildVariants, buildVariant.Name)
}
}
tasks := patchDoc.Tasks
if len(patchDoc.Tasks) == 1 && patchDoc.Tasks[0] == "all" {
tasks = make([]string, 0)
for _, t := range patchedProject.Tasks {
tasks = append(tasks, t.Name)
}
}
// update variant and tasks to include dependencies
patchDoc.BuildVariants, patchDoc.Tasks = model.IncludePatchDependencies(
project, buildVariants, tasks)
if err = patchDoc.Insert(); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error inserting patch: %v", err))
return
}
if finalize {
if _, err = model.FinalizePatch(patchDoc, &as.Settings); err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
}
as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc})
}
// CreatePatch checks an API request to see if it is safe and sane.
// Returns the relevant patch metadata, the patch document, and any errors that occur.
func (pr *PatchAPIRequest) CreatePatch(finalize bool, oauthToken string,
dbUser *user.DBUser, settings *evergreen.Settings) (*model.Project, *patch.Patch, error) {
var repoOwner, repo string
var module *model.Module
projectRef, err := model.FindOneProjectRef(pr.ProjectId)
if err != nil {
return nil, nil, fmt.Errorf("Could not find project ref %v : %v", pr.ProjectId, err)
}
repoOwner = projectRef.Owner
repo = projectRef.Repo
if len(pr.Githash) != 40 {
return nil, nil, fmt.Errorf("invalid githash")
}
gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash)
if err != nil {
return nil, nil, fmt.Errorf("could not find base revision %v for project %v: %v",
pr.Githash, projectRef.Identifier, err)
}
if gitCommit == nil {
return nil, nil, fmt.Errorf("commit hash %v doesn't seem to exist", pr.Githash)
}
summaries, err := getSummaries(pr.PatchContent)
if err != nil {
return nil, nil, err
}
isEmpty := (pr.PatchContent == "")
if finalize && (len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "") {
return nil, nil, fmt.Errorf("no buildvariants specified")
}
createTime := time.Now()
// create a new object ID to use as reference for the patch data
patchFileId := bson.NewObjectId().Hex()
patchDoc := &patch.Patch{
Id: bson.NewObjectId(),
Description: pr.Description,
Author: dbUser.Id,
Project: pr.ProjectId,
Githash: pr.Githash,
CreateTime: createTime,
Status: evergreen.PatchCreated,
BuildVariants: pr.BuildVariants,
Tasks: pr.Tasks,
IsEmpty: isEmpty,
Patches: []patch.ModulePatch{
{
ModuleName: "",
Githash: pr.Githash,
PatchSet: patch.PatchSet{
Patch: pr.PatchContent,
PatchFileId: patchFileId,
Summary: summaries,
},
},
},
}
// Get and validate patched config and add it to the patch document
project, err := validator.GetPatchedProject(patchDoc, settings)
if err != nil {
return nil, nil, fmt.Errorf("invalid patched config: %v", err)
}
if pr.ModuleName != "" {
// is there a module? validate it.
module, err = project.GetModuleByName(pr.ModuleName)
if err != nil {
return nil, nil, fmt.Errorf("could not find module %v: %v", pr.ModuleName, err)
}
if module == nil {
return nil, nil, fmt.Errorf("no module named %v", pr.ModuleName)
}
}
// verify that all variants exists
for _, buildVariant := range pr.BuildVariants {
if buildVariant == "all" || buildVariant == "" {
continue
}
bv := project.FindBuildVariant(buildVariant)
if bv == nil {
return nil, nil, fmt.Errorf("No such buildvariant: %v", buildVariant)
}
}
// write the patch content into a GridFS file under a new ObjectId after validating.
err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(pr.PatchContent))
if err != nil {
return nil, nil, fmt.Errorf("failed to write patch file to db: %v", err)
}
// add the project config
projectYamlBytes, err := yaml.Marshal(project)
if err != nil {
return nil, nil, fmt.Errorf("error marshalling patched config: %v", err)
}
// set the patch number based on patch author
patchDoc.PatchNumber, err = dbUser.IncPatchNumber()
if err != nil {
return nil, nil, fmt.Errorf("error computing patch num %v", err)
}
patchDoc.PatchedConfig = string(projectYamlBytes)
patchDoc.ClearPatchData()
return project, patchDoc, nil
}