Ejemplo n.º 1
0
func setupPatches(patchMode patchTestMode, b *build.Build, t *testing.T, patches ...patchRequest) {
	if patchMode == NoPatch {
		return
	}

	ptch := &patch.Patch{
		Status:  evergreen.PatchCreated,
		Version: b.Version,
		Patches: []patch.ModulePatch{},
	}

	for _, p := range patches {
		patchContent, err := ioutil.ReadFile(p.filePath)
		testutil.HandleTestingErr(err, t, "failed to read test patch file")

		if patchMode == InlinePatch {
			ptch.Patches = append(ptch.Patches, patch.ModulePatch{
				ModuleName: p.moduleName,
				Githash:    p.githash,
				PatchSet:   patch.PatchSet{Patch: string(patchContent)},
			})
		} else {
			pId := bson.NewObjectId().Hex()
			So(dbutil.WriteGridFile(patch.GridFSPrefix, pId, strings.NewReader(string(patchContent))), ShouldBeNil)
			ptch.Patches = append(ptch.Patches, patch.ModulePatch{
				ModuleName: p.moduleName,
				Githash:    p.githash,
				PatchSet:   patch.PatchSet{PatchFileId: pId},
			})
		}
	}
	testutil.HandleTestingErr(ptch.Insert(), t, "failed to insert patch")
}
Ejemplo n.º 2
0
func setupAPITestData(testConfig *evergreen.Settings, taskDisplayName string,
	variant string, patchMode patchTestMode, t *testing.T) (*model.Task, *build.Build, error) {
	//ignore errs here because the ns might just not exist.
	clearDataMsg := "Failed to clear test data collection"
	testCollections := []string{
		model.TasksCollection, build.Collection, host.Collection,
		distro.Collection, version.Collection, patch.Collection,
		model.PushlogCollection, model.ProjectVarsCollection, model.TaskQueuesCollection,
		manifest.Collection, model.ProjectRefCollection}
	testutil.HandleTestingErr(dbutil.ClearCollections(testCollections...), t, clearDataMsg)
	projectVars := &model.ProjectVars{
		Id: "evergreen-ci-render",
		Vars: map[string]string{
			"aws_key":    testConfig.Providers.AWS.Id,
			"aws_secret": testConfig.Providers.AWS.Secret,
			"fetch_key":  "fetch_expansion_value",
		},
	}
	_, err := projectVars.Upsert()
	testutil.HandleTestingErr(err, t, clearDataMsg)

	taskOne := &model.Task{
		Id:           "testTaskId",
		BuildId:      "testBuildId",
		DistroId:     "test-distro-one",
		BuildVariant: variant,
		Project:      "evergreen-ci-render",
		DisplayName:  taskDisplayName,
		HostId:       "testHost",
		Secret:       "testTaskSecret",
		Version:      "testVersionId",
		Status:       evergreen.TaskDispatched,
		Requester:    evergreen.RepotrackerVersionRequester,
	}

	taskTwo := &model.Task{
		Id:           "testTaskIdTwo",
		BuildId:      "testBuildId",
		DistroId:     "test-distro-one",
		BuildVariant: variant,
		Project:      "evergreen-ci-render",
		DisplayName:  taskDisplayName,
		HostId:       "",
		Secret:       "testTaskSecret",
		Activated:    true,
		Version:      "testVersionId",
		Status:       evergreen.TaskUndispatched,
		Requester:    evergreen.RepotrackerVersionRequester,
	}

	if patchMode != NoPatch {
		taskOne.Requester = evergreen.PatchVersionRequester
	}

	testutil.HandleTestingErr(taskOne.Insert(), t, "failed to insert taskOne")
	testutil.HandleTestingErr(taskTwo.Insert(), t, "failed to insert taskTwo")

	// set up task queue for task end tests
	taskQueue := &model.TaskQueue{
		Distro: "test-distro-one",
		Queue: []model.TaskQueueItem{
			model.TaskQueueItem{
				Id:          "testTaskIdTwo",
				DisplayName: taskDisplayName,
			},
		},
	}
	testutil.HandleTestingErr(taskQueue.Save(), t, "failed to insert taskqueue")
	workDir, err := ioutil.TempDir("", "agent_test_")
	testutil.HandleTestingErr(err, t, "failed to create working directory")

	host := &host.Host{
		Id:   "testHost",
		Host: "testHost",
		Distro: distro.Distro{
			Id:         "test-distro-one",
			WorkDir:    workDir,
			Expansions: []distro.Expansion{{"distro_exp", "DISTRO_EXP"}},
		},
		RunningTask:   "testTaskId",
		StartedBy:     evergreen.User,
		AgentRevision: agentRevision,
	}
	testutil.HandleTestingErr(host.Insert(), t, "failed to insert host")

	// read in the project configuration
	projectFile := "testdata/config_test_plugin/project/evergreen-ci-render.yml"
	projectConfig, err := ioutil.ReadFile(projectFile)
	testutil.HandleTestingErr(err, t, "failed to read project config")

	projectRef := &model.ProjectRef{
		Identifier: "evergreen-ci-render",
		Owner:      "evergreen-ci",
		Repo:       "render",
		RepoKind:   "github",
		Branch:     "master",
		Enabled:    true,
		BatchTime:  180,
	}

	testutil.HandleTestingErr(projectRef.Insert(), t, "failed to insert projectRef")

	err = testutil.CreateTestLocalConfig(testConfig, "evergreen-ci-render", "testdata/config_test_plugin/project/evergreen-ci-render.yml")
	testutil.HandleTestingErr(err, t, "failed to marshall project config")

	// unmarshall the project configuration into a struct
	project := &model.Project{}
	testutil.HandleTestingErr(yaml.Unmarshal(projectConfig, project), t, "failed to unmarshal project config")

	// now then marshall the project YAML for storage
	projectYamlBytes, err := yaml.Marshal(project)
	testutil.HandleTestingErr(err, t, "failed to marshall project config")

	// insert the version document
	v := &version.Version{
		Id:       "testVersionId",
		BuildIds: []string{taskOne.BuildId},
		Config:   string(projectYamlBytes),
	}

	testutil.HandleTestingErr(v.Insert(), t, "failed to insert version")
	if patchMode != NoPatch {
		mainPatchContent, err := ioutil.ReadFile("testdata/test.patch")
		testutil.HandleTestingErr(err, t, "failed to read test patch file")
		modulePatchContent, err := ioutil.ReadFile("testdata/testmodule.patch")
		testutil.HandleTestingErr(err, t, "failed to read test module patch file")

		ptch := &patch.Patch{
			Status:  evergreen.PatchCreated,
			Version: v.Id,
		}
		if patchMode == InlinePatch {
			ptch.Patches = []patch.ModulePatch{
				{
					ModuleName: "",
					Githash:    "1e5232709595db427893826ce19289461cba3f75",
					PatchSet:   patch.PatchSet{Patch: string(mainPatchContent)},
				},
				{
					ModuleName: "recursive",
					Githash:    "1e5232709595db427893826ce19289461cba3f75",
					PatchSet:   patch.PatchSet{Patch: string(modulePatchContent)},
				},
			}
		} else {
			p1Id, p2Id := bson.NewObjectId().Hex(), bson.NewObjectId().Hex()
			So(dbutil.WriteGridFile(patch.GridFSPrefix, p1Id, strings.NewReader(string(mainPatchContent))), ShouldBeNil)
			So(dbutil.WriteGridFile(patch.GridFSPrefix, p2Id, strings.NewReader(string(modulePatchContent))), ShouldBeNil)
			ptch.Patches = []patch.ModulePatch{
				{
					ModuleName: "",
					Githash:    "1e5232709595db427893826ce19289461cba3f75",
					PatchSet:   patch.PatchSet{PatchFileId: p1Id},
				},
				{
					ModuleName: "recursive",
					Githash:    "1e5232709595db427893826ce19289461cba3f75",
					PatchSet:   patch.PatchSet{PatchFileId: p2Id},
				},
			}
		}
		testutil.HandleTestingErr(ptch.Insert(), t, "failed to insert patch")
	}

	session, _, err := dbutil.GetGlobalSessionFactory().GetSession()
	testutil.HandleTestingErr(err, t, "couldn't get db session!")

	// Remove any logs for our test task from previous runs.
	_, err = session.DB(model.TaskLogDB).C(model.TaskLogCollection).
		RemoveAll(bson.M{"t_id": bson.M{"$in": []string{taskOne.Id, taskTwo.Id}}})
	testutil.HandleTestingErr(err, t, "failed to remove logs")

	build := &build.Build{
		Id: "testBuildId",
		Tasks: []build.TaskCache{
			build.NewTaskCache(taskOne.Id, taskOne.DisplayName, true),
			build.NewTaskCache(taskTwo.Id, taskTwo.DisplayName, true),
		},
		Version: "testVersionId",
	}

	testutil.HandleTestingErr(build.Insert(), t, "failed to insert build")
	return taskOne, build, nil
}
Ejemplo n.º 3
0
func (as *APIServer) updatePatchModule(w http.ResponseWriter, r *http.Request) {
	p, err := getPatchFromRequest(r)
	if err != nil {
		as.WriteJSON(w, http.StatusBadRequest, err.Error())
		return
	}

	var moduleName, patchContent, githash string

	if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
		moduleName, patchContent, githash = r.FormValue("module"), r.FormValue("patch"), r.FormValue("githash")
	} else {
		data := struct {
			Module  string `json:"module"`
			Patch   string `json:"patch"`
			Githash string `json:"githash"`
		}{}
		if err := util.ReadJSONInto(r.Body, &data); err != nil {
			as.LoggedError(w, r, http.StatusBadRequest, err)
			return
		}
		moduleName, patchContent, githash = data.Module, data.Patch, data.Githash
	}

	if len(patchContent) == 0 {
		as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
		return
	}

	projectRef, err := model.FindOneProjectRef(p.Project)
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting project ref with id %v: %v", p.Project, err))
		return
	}
	project, err := model.FindProject("", projectRef)
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Error getting patch: %v", err))
		return
	}
	if project == nil {
		as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("can't find project: %v", p.Project))
		return
	}

	module, err := project.GetModuleByName(moduleName)
	if err != nil || module == nil {
		as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("No such module", moduleName))
		return
	}

	gitOutput, err := thirdparty.GitApplyNumstat(patchContent)
	if err != nil {
		as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Invalid patch: %v", err))
		return
	}
	if gitOutput == nil {
		as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Empty diff"))
		return
	}

	summaries, err := thirdparty.ParseGitSummary(gitOutput)
	if err != nil {
		as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("Can't validate patch: %v", err))
		return
	}

	repoOwner, repo := module.GetRepoOwnerAndName()

	commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"], repoOwner, repo, githash)
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, err)
		return
	}

	if commitInfo == nil {
		as.WriteJSON(w, http.StatusBadRequest, fmt.Errorf("commit hash doesn't seem to exist"))
		return
	}

	// write the patch content into a GridFS file under a new ObjectId.
	patchFileId := bson.NewObjectId().Hex()
	err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(patchContent))
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err))
		return
	}

	modulePatch := patch.ModulePatch{
		ModuleName: moduleName,
		Githash:    githash,
		PatchSet: patch.PatchSet{
			PatchFileId: patchFileId,
			Summary:     summaries,
		},
	}

	if err = p.UpdateModulePatch(modulePatch); err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, err)
		return
	}

	as.WriteJSON(w, http.StatusOK, "Patch module updated")
	return
}
Ejemplo n.º 4
0
func (as *APIServer) submitPatch(w http.ResponseWriter, r *http.Request) {
	user := MustHaveUser(r)
	var apiRequest PatchAPIRequest
	var projId, description string
	var finalize bool
	if r.Header.Get("Content-Type") == "application/x-www-form-urlencoded" {
		patchContent := r.FormValue("patch")
		if patchContent == "" {
			as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
			return
		}
		apiRequest = PatchAPIRequest{
			ProjectFileName: r.FormValue("project"),
			ModuleName:      r.FormValue("module"),
			Githash:         r.FormValue("githash"),
			PatchContent:    r.FormValue("patch"),
			BuildVariants:   strings.Split(r.FormValue("buildvariants"), ","),
		}
		projId = r.FormValue("project")
		description = r.FormValue("desc")
		finalize = strings.ToLower(r.FormValue("finalize")) == "true"
	} else {
		data := struct {
			Description string   `json:"desc"`
			Project     string   `json:"project"`
			Patch       string   `json:"patch"`
			Githash     string   `json:"githash"`
			Variants    string   `json:"buildvariants"`
			Tasks       []string `json:"tasks"`
			Finalize    bool     `json:"finalize"`
		}{}
		if err := util.ReadJSONInto(r.Body, &data); err != nil {
			as.LoggedError(w, r, http.StatusBadRequest, err)
			return
		}
		if len(data.Patch) > patch.SizeLimit {
			as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Patch is too large."))
		}
		if len(data.Patch) == 0 {
			as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("Error: Patch must not be empty"))
			return
		}
		description = data.Description
		projId = data.Project
		finalize = data.Finalize

		apiRequest = PatchAPIRequest{
			ProjectFileName: data.Project,
			ModuleName:      r.FormValue("module"),
			Githash:         data.Githash,
			PatchContent:    data.Patch,
			BuildVariants:   strings.Split(data.Variants, ","),
			Tasks:           data.Tasks,
		}
	}

	projectRef, err := model.FindOneProjectRef(projId)
	if err != nil {
		message := fmt.Errorf("Error locating project ref '%v': %v",
			projId, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}
	project, err := model.FindProject("", projectRef)
	if err != nil {
		message := fmt.Errorf("Error locating project '%v' from '%v': %v",
			projId, as.Settings.ConfigDir, err)
		as.LoggedError(w, r, http.StatusInternalServerError, message)
		return
	}

	if project == nil {
		as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("project %v not found", projId))
		return
	}

	patchMetadata, err := apiRequest.Validate(as.Settings.Credentials["github"])
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid patch: %v", err))
		return
	}

	if patchMetadata == nil {
		as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("patch metadata is empty"))
		return
	}

	if apiRequest.ModuleName != "" {
		as.WriteJSON(w, http.StatusBadRequest,
			"module not allowed when creating new patches (must be added in a subsequent request)")
		return
	}
	patchProjectRef, err := model.FindOneProjectRef(patchMetadata.Project.Identifier)
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("Invalid projectRef: %v", err))
		return
	}
	if patchProjectRef == nil {
		as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("Empty patch project Ref"))
		return
	}

	commitInfo, err := thirdparty.GetCommitEvent(as.Settings.Credentials["github"],
		patchProjectRef.Owner,
		patchProjectRef.Repo,
		apiRequest.Githash)
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, err)
		return
	}
	if commitInfo == nil {
		as.WriteJSON(w, http.StatusBadRequest, "That commit doesn't seem to exist.")
		return
	}

	createTime := time.Now()

	// create a new object ID to use as reference for the patch data
	patchFileId := bson.NewObjectId().Hex()

	patchDoc := &patch.Patch{
		Id:            bson.NewObjectId(),
		Description:   description,
		Author:        user.Id,
		Project:       apiRequest.ProjectFileName,
		Githash:       apiRequest.Githash,
		CreateTime:    createTime,
		Status:        evergreen.PatchCreated,
		BuildVariants: apiRequest.BuildVariants,
		Tasks:         apiRequest.Tasks,
		Patches: []patch.ModulePatch{
			patch.ModulePatch{
				ModuleName: "",
				Githash:    apiRequest.Githash,
				PatchSet: patch.PatchSet{
					PatchFileId: patchFileId,
					Summary:     patchMetadata.Summaries,
				},
			},
		},
	}

	// write the patch content into a GridFS file under a new ObjectId.
	err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(apiRequest.PatchContent))
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("failed to write patch file to db: %v", err))
		return
	}

	// Get and validate patched config and add it to the patch document
	patchedProject, err := validator.GetPatchedProject(patchDoc, &as.Settings)
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("invalid patched config: %v", err))
		return
	}

	projectYamlBytes, err := yaml.Marshal(patchedProject)
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error marshalling patched config: %v", err))
		return
	}

	// set the patch number based on patch author
	patchDoc.PatchNumber, err = user.IncPatchNumber()
	if err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error computing patch num %v", err))
		return
	}
	patchDoc.PatchedConfig = string(projectYamlBytes)
	patchDoc.ClearPatchData()

	//expand tasks and build variants and include dependencies
	buildVariants := patchDoc.BuildVariants
	if len(patchDoc.BuildVariants) == 1 && patchDoc.BuildVariants[0] == "all" {
		buildVariants = make([]string, 0)
		for _, buildVariant := range patchedProject.BuildVariants {
			if buildVariant.Disabled {
				continue
			}
			buildVariants = append(buildVariants, buildVariant.Name)
		}
	}
	tasks := patchDoc.Tasks
	if len(patchDoc.Tasks) == 1 && patchDoc.Tasks[0] == "all" {
		tasks = make([]string, 0)
		for _, t := range patchedProject.Tasks {
			tasks = append(tasks, t.Name)
		}
	}
	// update variant and tasks to include dependencies
	patchDoc.BuildVariants, patchDoc.Tasks = model.IncludePatchDependencies(
		project, buildVariants, tasks)

	if err = patchDoc.Insert(); err != nil {
		as.LoggedError(w, r, http.StatusInternalServerError, fmt.Errorf("error inserting patch: %v", err))
		return
	}

	if finalize {
		if _, err = model.FinalizePatch(patchDoc, &as.Settings); err != nil {
			as.LoggedError(w, r, http.StatusInternalServerError, err)
			return
		}
	}

	as.WriteJSON(w, http.StatusCreated, PatchAPIResponse{Patch: patchDoc})
}
Ejemplo n.º 5
0
// CreatePatch checks an API request to see if it is safe and sane.
// Returns the relevant patch metadata, the patch document, and any errors that occur.
func (pr *PatchAPIRequest) CreatePatch(finalize bool, oauthToken string,
	dbUser *user.DBUser, settings *evergreen.Settings) (*model.Project, *patch.Patch, error) {
	var repoOwner, repo string
	var module *model.Module

	projectRef, err := model.FindOneProjectRef(pr.ProjectId)
	if err != nil {
		return nil, nil, fmt.Errorf("Could not find project ref %v : %v", pr.ProjectId, err)
	}

	repoOwner = projectRef.Owner
	repo = projectRef.Repo

	if len(pr.Githash) != 40 {
		return nil, nil, fmt.Errorf("invalid githash")
	}

	gitCommit, err := thirdparty.GetCommitEvent(oauthToken, repoOwner, repo, pr.Githash)
	if err != nil {
		return nil, nil, fmt.Errorf("could not find base revision %v for project %v: %v",
			pr.Githash, projectRef.Identifier, err)

	}
	if gitCommit == nil {
		return nil, nil, fmt.Errorf("commit hash %v doesn't seem to exist", pr.Githash)
	}

	summaries, err := getSummaries(pr.PatchContent)
	if err != nil {
		return nil, nil, err
	}

	isEmpty := (pr.PatchContent == "")

	if finalize && (len(pr.BuildVariants) == 0 || pr.BuildVariants[0] == "") {
		return nil, nil, fmt.Errorf("no buildvariants specified")
	}

	createTime := time.Now()

	// create a new object ID to use as reference for the patch data
	patchFileId := bson.NewObjectId().Hex()
	patchDoc := &patch.Patch{
		Id:            bson.NewObjectId(),
		Description:   pr.Description,
		Author:        dbUser.Id,
		Project:       pr.ProjectId,
		Githash:       pr.Githash,
		CreateTime:    createTime,
		Status:        evergreen.PatchCreated,
		BuildVariants: pr.BuildVariants,
		Tasks:         pr.Tasks,
		IsEmpty:       isEmpty,
		Patches: []patch.ModulePatch{
			{
				ModuleName: "",
				Githash:    pr.Githash,
				PatchSet: patch.PatchSet{
					Patch:       pr.PatchContent,
					PatchFileId: patchFileId,
					Summary:     summaries,
				},
			},
		},
	}

	// Get and validate patched config and add it to the patch document
	project, err := validator.GetPatchedProject(patchDoc, settings)
	if err != nil {
		return nil, nil, fmt.Errorf("invalid patched config: %v", err)
	}

	if pr.ModuleName != "" {
		// is there a module? validate it.
		module, err = project.GetModuleByName(pr.ModuleName)
		if err != nil {
			return nil, nil, fmt.Errorf("could not find module %v: %v", pr.ModuleName, err)
		}
		if module == nil {
			return nil, nil, fmt.Errorf("no module named %v", pr.ModuleName)
		}
	}

	// verify that all variants exists
	for _, buildVariant := range pr.BuildVariants {
		if buildVariant == "all" || buildVariant == "" {
			continue
		}
		bv := project.FindBuildVariant(buildVariant)
		if bv == nil {
			return nil, nil, fmt.Errorf("No such buildvariant: %v", buildVariant)
		}
	}

	// write the patch content into a GridFS file under a new ObjectId after validating.
	err = db.WriteGridFile(patch.GridFSPrefix, patchFileId, strings.NewReader(pr.PatchContent))
	if err != nil {
		return nil, nil, fmt.Errorf("failed to write patch file to db: %v", err)
	}

	// add the project config
	projectYamlBytes, err := yaml.Marshal(project)
	if err != nil {
		return nil, nil, fmt.Errorf("error marshalling patched config: %v", err)
	}

	// set the patch number based on patch author
	patchDoc.PatchNumber, err = dbUser.IncPatchNumber()
	if err != nil {
		return nil, nil, fmt.Errorf("error computing patch num %v", err)
	}
	patchDoc.PatchedConfig = string(projectYamlBytes)

	patchDoc.ClearPatchData()

	return project, patchDoc, nil
}