// validateProjectConfig returns a slice containing a list of any errors
// found in validating the given project configuration
func (as *APIServer) validateProjectConfig(w http.ResponseWriter, r *http.Request) {
defer r.Body.Close()
yamlBytes, err := ioutil.ReadAll(r.Body)
if err != nil {
as.WriteJSON(w, http.StatusBadRequest, fmt.Sprintf("Error reading request body: %v", err))
return
}
project := &model.Project{}
validationErr := validator.ValidationError{}
if err := model.LoadProjectInto(yamlBytes, "", project); err != nil {
validationErr.Message = err.Error()
as.WriteJSON(w, http.StatusBadRequest, []validator.ValidationError{validationErr})
return
}
syntaxErrs, err := validator.CheckProjectSyntax(project)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
semanticErrs := validator.CheckProjectSemantics(project)
if len(syntaxErrs)+len(semanticErrs) != 0 {
as.WriteJSON(w, http.StatusBadRequest, append(syntaxErrs, semanticErrs...))
return
}
as.WriteJSON(w, http.StatusOK, []validator.ValidationError{})
}
// GetRemoteConfig fetches the contents of a remote github repository's
// configuration data as at a given revision
func (gRepoPoller *GithubRepositoryPoller) GetRemoteConfig(
projectFileRevision string) (projectConfig *model.Project, err error) {
// find the project configuration file for the given repository revision
projectRef := gRepoPoller.ProjectRef
projectFileURL := thirdparty.GetGithubFileURL(
projectRef.Owner,
projectRef.Repo,
projectRef.RemotePath,
projectFileRevision,
)
githubFile, err := thirdparty.GetGithubFile(
gRepoPoller.OauthToken,
projectFileURL,
)
if err != nil {
return nil, err
}
projectFileBytes, err := base64.StdEncoding.DecodeString(githubFile.Content)
if err != nil {
return nil, thirdparty.FileDecodeError{err.Error()}
}
projectConfig = &model.Project{}
err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, projectConfig)
if err != nil {
return nil, thirdparty.YAMLFormatError{err.Error()}
}
return projectConfig, nil
}
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
if p.Version != "" {
return nil, fmt.Errorf("Patch %v already finalized", p.Version)
}
projectRef, err := model.FindOneProjectRef(p.Project)
if err != nil {
return nil, err
}
// try to get the remote project file data at the requested revision
var projectFileBytes []byte
projectFileURL := thirdparty.GetGithubFileURL(
projectRef.Owner,
projectRef.Repo,
projectRef.RemotePath,
p.Githash,
)
githubFile, err := thirdparty.GetGithubFile(settings.Credentials["github"], projectFileURL)
if err != nil {
// if the project file doesn't exist, but our patch includes a project file,
// we try to apply the diff and proceed.
if !(p.ConfigChanged(projectRef.RemotePath) && thirdparty.IsFileNotFound(err)) {
// return an error if the github error is network/auth-related or we aren't patching the config
return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
}
} else {
// we successfully got the project file in base64, so we decode it
projectFileBytes, err = base64.StdEncoding.DecodeString(githubFile.Content)
if err != nil {
return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
}
}
project := &model.Project{}
if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
return nil, err
}
// apply remote configuration patch if needed
if p.ConfigChanged(projectRef.RemotePath) {
project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
if err != nil {
return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
}
// overwrite project fields with the project ref to disallow tracking a
// different project or doing other crazy things via config patches
errs := CheckProjectSyntax(project)
if len(errs) != 0 {
var message string
for _, err := range errs {
message += fmt.Sprintf("\n\t=> %v", err)
}
return nil, fmt.Errorf(message)
}
}
return project, nil
}
// GetPatchedProject creates and validates a project created by fetching latest commit information from GitHub
// and applying the patch to the latest remote configuration. The error returned can be a validation error.
func GetPatchedProject(p *patch.Patch, settings *evergreen.Settings) (*model.Project, error) {
if p.Version != "" {
return nil, fmt.Errorf("Patch %v already finalized", p.Version)
}
projectRef, err := model.FindOneProjectRef(p.Project)
if err != nil {
return nil, err
}
// get the remote file at the requested revision
projectFileURL := thirdparty.GetGithubFileURL(
projectRef.Owner,
projectRef.Repo,
projectRef.RemotePath,
p.Githash,
)
githubFile, err := thirdparty.GetGithubFile(
settings.Credentials["github"],
projectFileURL,
)
if err != nil {
return nil, fmt.Errorf("Could not get github file at %v: %v", projectFileURL, err)
}
projectFileBytes, err := base64.StdEncoding.DecodeString(githubFile.Content)
if err != nil {
return nil, fmt.Errorf("Could not decode github file at %v: %v", projectFileURL, err)
}
project := &model.Project{}
if err = model.LoadProjectInto(projectFileBytes, projectRef.Identifier, project); err != nil {
return nil, err
}
// apply remote configuration patch if needed
if p.ConfigChanged(projectRef.RemotePath) {
project, err = model.MakePatchedConfig(p, projectRef.RemotePath, string(projectFileBytes))
if err != nil {
return nil, fmt.Errorf("Could not patch remote configuration file: %v", err)
}
// overwrite project fields with the project ref to disallow tracking a
// different project or doing other crazy things via config patches
errs := CheckProjectSyntax(project)
if len(errs) != 0 {
var message string
for _, err := range errs {
message += fmt.Sprintf("\n\t=> %v", err)
}
return nil, fmt.Errorf(message)
}
}
return project, nil
}
// GetProjectConfig loads the communicator's task's project from the API server.
func (h *HTTPCommunicator) GetProjectConfig() (*model.Project, error) {
projectConfig := &model.Project{}
retriableGet := util.RetriableFunc(
func() error {
resp, err := h.tryGet("version")
if resp != nil {
defer resp.Body.Close()
}
if resp != nil && resp.StatusCode == http.StatusConflict {
// Something very wrong, fail now with no retry.
return fmt.Errorf("conflict - wrong secret!")
}
if err != nil {
// Some generic error trying to connect - try again
return util.RetriableError{err}
}
if resp == nil {
return util.RetriableError{fmt.Errorf("empty response")}
} else {
v := &version.Version{}
err = util.ReadJSONInto(resp.Body, v)
if err != nil {
h.Logger.Errorf(slogger.ERROR,
"unable to read project version response: %v\n", err)
return util.RetriableError{fmt.Errorf("unable to read "+
"project version response: %v\n", err)}
}
err = model.LoadProjectInto([]byte(v.Config), v.Project, projectConfig)
if err != nil {
h.Logger.Errorf(slogger.ERROR,
"unable to unmarshal project config: %v\n", err)
return util.RetriableError{fmt.Errorf("unable to "+
"unmarshall project config: %v\n", err)}
}
return nil
}
},
)
retryFail, err := util.Retry(retriableGet, h.MaxAttempts, h.RetrySleep)
if retryFail {
return nil, fmt.Errorf("getting project configuration failed after %v "+
"tries: %v", h.MaxAttempts, err)
}
return projectConfig, nil
}
// Construct a map of project names to build variants for that project
func findProjectBuildVariants(configName string) (map[string][]string, error) {
projectNameToBuildVariants := make(map[string][]string)
allProjects, err := model.FindAllTrackedProjectRefs()
if err != nil {
return nil, err
}
for _, projectRef := range allProjects {
if !projectRef.Enabled {
continue
}
var buildVariants []string
var proj *model.Project
var err error
if projectRef.LocalConfig != "" {
proj, err = model.FindProject("", &projectRef)
if err != nil {
return nil, fmt.Errorf("unable to find project file: %v", err)
}
} else {
lastGood, err := version.FindOne(version.ByLastKnownGoodConfig(projectRef.Identifier))
if err != nil {
return nil, fmt.Errorf("unable to find last valid config: %v", err)
}
if lastGood == nil { // brand new project + no valid config yet, just return an empty map
return projectNameToBuildVariants, nil
}
proj = &model.Project{}
err = model.LoadProjectInto([]byte(lastGood.Config), projectRef.Identifier, proj)
if err != nil {
return nil, fmt.Errorf("error loading project '%v' from version: %v", projectRef.Identifier, err)
}
}
for _, buildVariant := range proj.BuildVariants {
buildVariants = append(buildVariants, buildVariant.Name)
}
projectNameToBuildVariants[projectRef.Identifier] = buildVariants
}
return projectNameToBuildVariants, nil
}
func (ec *EvaluateCommand) Execute(args []string) error {
if len(args) != 1 {
return fmt.Errorf("the evaluate command takes one project config path as an argument")
}
configBytes, err := ioutil.ReadFile(args[0])
if err != nil {
return fmt.Errorf("error reading project config: %v", err)
}
p := &model.Project{}
err = model.LoadProjectInto(configBytes, "", p)
if err != nil {
return fmt.Errorf("error loading project: %v", err)
}
var out interface{}
if ec.Tasks || ec.Variants {
tmp := struct {
Functions interface{} `yaml:"functions,omitempty"`
Tasks interface{} `yaml:"tasks,omitempty"`
Variants interface{} `yaml:"buildvariants,omitempty`
}{}
if ec.Tasks {
tmp.Functions = p.Functions
tmp.Tasks = p.Tasks
}
if ec.Variants {
tmp.Variants = p.BuildVariants
}
out = tmp
} else {
out = p
}
outYAML, err := yaml.Marshal(out)
if err != nil {
return fmt.Errorf("error marshalling evaluated project YAML: %v", err)
}
fmt.Println(string(outYAML))
return nil
}
// GetTaskConfig fetches task configuration data required to run the task from the API server.
func (agt *Agent) GetTaskConfig() (*model.TaskConfig, error) {
agt.logger.LogExecution(slogger.INFO, "Fetching distro configuration.")
confDistro, err := agt.GetDistro()
if err != nil {
return nil, err
}
agt.logger.LogExecution(slogger.INFO, "Fetching version.")
confVersion, err := agt.GetVersion()
if err != nil {
return nil, err
}
confProject := &model.Project{}
err = model.LoadProjectInto([]byte(confVersion.Config), confVersion.Identifier, confProject)
if err != nil {
return nil, fmt.Errorf("reading project config: %v", err)
}
agt.logger.LogExecution(slogger.INFO, "Fetching task configuration.")
confTask, err := agt.GetTask()
if err != nil {
return nil, err
}
agt.logger.LogExecution(slogger.INFO, "Fetching project ref.")
confRef, err := agt.GetProjectRef()
if err != nil {
return nil, err
}
if confRef == nil {
return nil, fmt.Errorf("agent retrieved an empty project ref")
}
agt.logger.LogExecution(slogger.INFO, "Constructing TaskConfig.")
return model.NewTaskConfig(confDistro, confVersion, confProject, confTask, confRef)
}
// Takes in a version id and a map of "key -> buildvariant" (where "key" is of
// type "versionBuildVariant") and updates the map with an entry for the
// buildvariants associated with "versionStr"
func (self *Scheduler) updateVersionBuildVarMap(versionStr string,
versionBuildVarMap map[versionBuildVariant]model.BuildVariant) (err error) {
version, err := version.FindOne(version.ById(versionStr))
if err != nil {
return
}
if version == nil {
return fmt.Errorf("nil version returned for version id '%v'", versionStr)
}
project := &model.Project{}
err = model.LoadProjectInto([]byte(version.Config), version.Identifier, project)
if err != nil {
return fmt.Errorf("unable to load project config for version %v: "+
"%v", versionStr, err)
}
// create buildvariant map (for accessing purposes)
for _, buildVariant := range project.BuildVariants {
key := versionBuildVariant{versionStr, buildVariant.Name}
versionBuildVarMap[key] = buildVariant
}
return
}
func setupAPITestData(testConfig *evergreen.Settings, taskDisplayName string,
variant string, projectFile string, patchMode patchTestMode, t *testing.T) (*task.Task, *build.Build, error) {
// Ignore errs here because the ns might just not exist.
clearDataMsg := "Failed to clear test data collection"
testCollections := []string{
task.Collection, build.Collection, host.Collection,
distro.Collection, version.Collection, patch.Collection,
model.PushlogCollection, model.ProjectVarsCollection, model.TaskQueuesCollection,
manifest.Collection, model.ProjectRefCollection}
testutil.HandleTestingErr(dbutil.ClearCollections(testCollections...), t, clearDataMsg)
// Read in the project configuration
projectConfig, err := ioutil.ReadFile(projectFile)
testutil.HandleTestingErr(err, t, "failed to read project config")
// Unmarshall the project configuration into a struct
project := &model.Project{}
testutil.HandleTestingErr(model.LoadProjectInto(projectConfig, "test", project),
t, "failed to unmarshal project config")
// Marshall the project YAML for storage
projectYamlBytes, err := yaml.Marshal(project)
testutil.HandleTestingErr(err, t, "failed to marshall project config")
// Create the ref for the project
projectRef := &model.ProjectRef{
Identifier: project.DisplayName,
Owner: project.Owner,
Repo: project.Repo,
RepoKind: project.RepoKind,
Branch: project.Branch,
Enabled: project.Enabled,
BatchTime: project.BatchTime,
LocalConfig: string(projectConfig),
}
testutil.HandleTestingErr(projectRef.Insert(), t, "failed to insert projectRef")
// Save the project variables
projectVars := &model.ProjectVars{
Id: project.DisplayName,
Vars: map[string]string{
"aws_key": testConfig.Providers.AWS.Id,
"aws_secret": testConfig.Providers.AWS.Secret,
"fetch_key": "fetch_expansion_value",
},
}
_, err = projectVars.Upsert()
testutil.HandleTestingErr(err, t, clearDataMsg)
// Create and insert two tasks
taskOne := &task.Task{
Id: "testTaskId",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: project.DisplayName,
DisplayName: taskDisplayName,
HostId: "testHost",
Secret: "testTaskSecret",
Version: "testVersionId",
Status: evergreen.TaskDispatched,
Requester: evergreen.RepotrackerVersionRequester,
}
if patchMode != NoPatch {
taskOne.Requester = evergreen.PatchVersionRequester
}
testutil.HandleTestingErr(taskOne.Insert(), t, "failed to insert taskOne")
taskTwo := &task.Task{
Id: "testTaskIdTwo",
BuildId: "testBuildId",
DistroId: "test-distro-one",
BuildVariant: variant,
Project: project.DisplayName,
DisplayName: taskDisplayName,
HostId: "",
Secret: "testTaskSecret",
Version: "testVersionId",
Status: evergreen.TaskUndispatched,
Requester: evergreen.RepotrackerVersionRequester,
Activated: true,
}
testutil.HandleTestingErr(taskTwo.Insert(), t, "failed to insert taskTwo")
// Set up a task queue for task end tests
taskQueue := &model.TaskQueue{
Distro: "test-distro-one",
Queue: []model.TaskQueueItem{
{
Id: "testTaskIdTwo",
DisplayName: taskDisplayName,
},
},
}
testutil.HandleTestingErr(taskQueue.Save(), t, "failed to insert taskqueue")
// Insert the version document
v := &version.Version{
Id: "testVersionId",
BuildIds: []string{taskOne.BuildId},
Config: string(projectYamlBytes),
}
testutil.HandleTestingErr(v.Insert(), t, "failed to insert version")
// Insert the build that contains the tasks
build := &build.Build{
Id: "testBuildId",
Tasks: []build.TaskCache{
build.NewTaskCache(taskOne.Id, taskOne.DisplayName, true),
build.NewTaskCache(taskTwo.Id, taskTwo.DisplayName, true),
},
Version: v.Id,
}
testutil.HandleTestingErr(build.Insert(), t, "failed to insert build")
workDir, err := ioutil.TempDir("", "agent_test_")
testutil.HandleTestingErr(err, t, "failed to create working directory")
// Insert the host info for running the tests
host := &host.Host{
Id: "testHost",
Host: "testHost",
Distro: distro.Distro{
Id: "test-distro-one",
WorkDir: workDir,
Expansions: []distro.Expansion{{"distro_exp", "DISTRO_EXP"}},
},
RunningTask: taskOne.Id,
StartedBy: evergreen.User,
AgentRevision: agentRevision,
}
testutil.HandleTestingErr(host.Insert(), t, "failed to insert host")
session, _, err := dbutil.GetGlobalSessionFactory().GetSession()
testutil.HandleTestingErr(err, t, "couldn't get db session!")
// Remove any logs for our test task from previous runs.
_, err = session.DB(model.TaskLogDB).C(model.TaskLogCollection).
RemoveAll(bson.M{"t_id": bson.M{"$in": []string{taskOne.Id, taskTwo.Id}}})
testutil.HandleTestingErr(err, t, "failed to remove logs")
return taskOne, build, nil
}