func TestArchiveExtract(t *testing.T) { Convey("After extracting a tarball", t, func() { testDir := testutil.GetDirectoryOfFile() //Remove the test output dir, in case it was left over from prior test err := os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_test")) testutil.HandleTestingErr(err, t, "Couldn't remove test dir") f, gz, tarReader, err := TarGzReader(filepath.Join(testDir, "testdata", "artifacts.tar.gz")) testutil.HandleTestingErr(err, t, "Couldn't open test tarball") defer f.Close() defer gz.Close() err = Extract(tarReader, filepath.Join(testDir, "testdata", "artifacts_test")) So(err, ShouldBeNil) Convey("extracted data should match the archive contents", func() { f, err := os.Open(filepath.Join(testDir, "testdata", "artifacts_test", "artifacts", "dir1", "dir2", "testfile.txt")) So(err, ShouldBeNil) defer f.Close() data, err := ioutil.ReadAll(f) So(err, ShouldBeNil) So(string(data), ShouldEqual, "test\n") }) }) }
func TestPatchPluginAPI(t *testing.T) { testConfig := evergreen.TestConfig() cwd := testutil.GetDirectoryOfFile() Convey("With a running api server and installed plugin", t, func() { registry := plugin.NewSimpleRegistry() gitPlugin := &GitPlugin{} err := registry.Register(gitPlugin) testutil.HandleTestingErr(err, t, "Couldn't register patch plugin") server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false) testutil.HandleTestingErr(err, t, "Couldn't set up testing server") taskConfig, _ := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t) testCommand := GitGetProjectCommand{Directory: "dir"} _, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t) testutil.HandleTestingErr(err, t, "Couldn't set up test documents") testTask, err := task.FindOne(task.ById("testTaskId")) testutil.HandleTestingErr(err, t, "Couldn't set up test patch task") sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}} logger := agentutil.NewTestLogger(sliceAppender) Convey("calls to existing tasks with patches should succeed", func() { httpCom := plugintest.TestAgentCommunicator(testTask.Id, testTask.Secret, server.URL) pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom} patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger) So(err, ShouldBeNil) So(patch, ShouldNotBeNil) testutil.HandleTestingErr(db.Clear(version.Collection), t, "unable to clear versions collection") }) Convey("calls to non-existing tasks should fail", func() { v := version.Version{Id: ""} testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version") httpCom := plugintest.TestAgentCommunicator("BAD_TASK_ID", "", server.URL) pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom} patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger) So(err.Error(), ShouldContainSubstring, "not found") So(err, ShouldNotBeNil) So(patch, ShouldBeNil) testutil.HandleTestingErr(db.Clear(version.Collection), t, "unable to clear versions collection") }) Convey("calls to existing tasks without patches should fail", func() { noPatchTask := task.Task{Id: "noPatchTask", BuildId: "a"} testutil.HandleTestingErr(noPatchTask.Insert(), t, "Couldn't insert patch task") noPatchVersion := version.Version{Id: "noPatchVersion", BuildIds: []string{"a"}} testutil.HandleTestingErr(noPatchVersion.Insert(), t, "Couldn't insert patch version") v := version.Version{Id: ""} testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version") httpCom := plugintest.TestAgentCommunicator(noPatchTask.Id, "", server.URL) pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom} patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger) So(err, ShouldNotBeNil) So(err.Error(), ShouldContainSubstring, "no patch found for task") So(patch, ShouldBeNil) testutil.HandleTestingErr(db.Clear(version.Collection), t, "unable to clear versions collection") }) }) }
func TestRegistry(t *testing.T) { Convey("With a SimpleRegistry", t, func() { Convey("Registering a plugin twice should return err", func() { registry := plugin.NewSimpleRegistry() err := registry.Register(&MockPlugin{}) testutil.HandleTestingErr(err, t, "Couldn't register plugin") err = registry.Register(&shell.ShellPlugin{}) testutil.HandleTestingErr(err, t, "Couldn't register plugin") err = registry.Register(&expansions.ExpansionsPlugin{}) testutil.HandleTestingErr(err, t, "Couldn't register plugin") }) Convey("with a project file containing references to a valid plugin", func() { registry := plugin.NewSimpleRegistry() registry.Register(&MockPlugin{}) registry.Register(&shell.ShellPlugin{}) registry.Register(&expansions.ExpansionsPlugin{}) data, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "plugin_project.yml")) testutil.HandleTestingErr(err, t, "failed to load test yaml file") project := &model.Project{} err = yaml.Unmarshal(data, project) Convey("all commands in project file should load parse successfully", func() { for _, newTask := range project.Tasks { for _, command := range newTask.Commands { pluginCmds, err := registry.GetCommands(command, project.Functions) testutil.HandleTestingErr(err, t, "Got error getting plugin commands: %v") So(pluginCmds, ShouldNotBeNil) So(err, ShouldBeNil) } } }) }) }) }
func TestParseOutputFiles(t *testing.T) { Convey("When parsing files containing go test output", t, func() { cwd := testutil.GetDirectoryOfFile() Convey("The output in all of the specified files should be parsed correctly", func() { // mock up a logger sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}} logger := agentutil.NewTestLogger(sliceAppender) // mock up a task config taskConfig := &model.TaskConfig{Task: &task.Task{Id: "taskOne", Execution: 1}} // the files we want to parse files := []string{ filepath.Join(cwd, "testdata", "monitor.suite"), filepath.Join(cwd, "testdata", "util.suite"), filepath.Join(cwd, "testdata", "test_output_dir", "monitor_fail.suite"), filepath.Join(cwd, "testdata", "test_output_dir", "evergreen.suite"), } logs, results, err := ParseTestOutputFiles(files, nil, logger, taskConfig) So(err, ShouldBeNil) So(logs, ShouldNotBeNil) So(results, ShouldNotBeNil) So(len(results), ShouldEqual, 4) }) }) }
func init() { dbutil.SetGlobalSessionProvider(dbutil.SessionFactoryFromConfig(testConfig)) testSetups = []testConfigPath{ {"With plugin mode test config", filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "config_test_plugin")}, } }
func TestMakePatchedConfig(t *testing.T) { Convey("With calling MakePatchedConfig with a config and remote configuration path", t, func() { cwd := testutil.GetDirectoryOfFile() Convey("the config should be patched correctly", func() { remoteConfigPath := filepath.Join("config", "evergreen.yml") fileBytes, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "patch.diff")) So(err, ShouldBeNil) // update patch with remove config path variable diffString := fmt.Sprintf(string(fileBytes), remoteConfigPath, remoteConfigPath, remoteConfigPath, remoteConfigPath) // the patch adds a new task p := &patch.Patch{ Patches: []patch.ModulePatch{{ Githash: "revision", PatchSet: patch.PatchSet{ Patch: diffString, Summary: []patch.Summary{{ Name: remoteConfigPath, Additions: 3, Deletions: 3, }}, }, }}, } projectBytes, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "project.config")) So(err, ShouldBeNil) project, err := MakePatchedConfig(p, remoteConfigPath, string(projectBytes)) So(err, ShouldBeNil) So(project, ShouldNotBeNil) So(len(project.Tasks), ShouldEqual, 2) }) Convey("an empty base config should be patched correctly", func() { remoteConfigPath := filepath.Join("model", "testdata", "project2.config") fileBytes, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "project.diff")) So(err, ShouldBeNil) p := &patch.Patch{ Patches: []patch.ModulePatch{{ Githash: "revision", PatchSet: patch.PatchSet{ Patch: string(fileBytes), Summary: []patch.Summary{{Name: remoteConfigPath}}, }, }}, } project, err := MakePatchedConfig(p, remoteConfigPath, "") So(err, ShouldBeNil) So(project, ShouldNotBeNil) So(len(project.Tasks), ShouldEqual, 1) So(project.Tasks[0].Name, ShouldEqual, "hello") Reset(func() { os.Remove(remoteConfigPath) }) }) }) }
func TestPythonMatrixIntegration(t *testing.T) { Convey("With a sample matrix project mocking up a python driver", t, func() { p := Project{} bytes, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "matrix_python.yml")) So(err, ShouldBeNil) Convey("the project should parse properly", func() { err := LoadProjectInto(bytes, "python", &p) So(err, ShouldBeNil) Convey("and contain the correct variants", func() { So(len(p.BuildVariants), ShouldEqual, (2*2*4 - 4)) Convey("so that excluded matrix cells are not created", func() { So(findMatrixVariant(p.BuildVariants, matrixValue{ "os": "windows", "python": "pypy", "c-extensions": "with-c", }), ShouldBeNil) So(findMatrixVariant(p.BuildVariants, matrixValue{ "os": "windows", "python": "jython", "c-extensions": "with-c", }), ShouldBeNil) So(findMatrixVariant(p.BuildVariants, matrixValue{ "os": "linux", "python": "pypy", "c-extensions": "with-c", }), ShouldBeNil) So(findMatrixVariant(p.BuildVariants, matrixValue{ "os": "linux", "python": "jython", "c-extensions": "with-c", }), ShouldBeNil) }) Convey("so that Windows builds without C extensions exclude LDAP tasks", func() { v := findMatrixVariant(p.BuildVariants, matrixValue{ "os": "windows", "python": "python3", "c-extensions": "without-c", }) So(v, ShouldNotBeNil) tasks := taskNames(v) So(len(tasks), ShouldEqual, 7) So(tasks, ShouldNotContain, "ldap_auth") So(v.DisplayName, ShouldEqual, "Windows 95 Python 3.0 (without C extensions)") So(v.RunOn, ShouldResemble, []string{"windows95-test"}) }) Convey("so that the linux/python3/c variant has a lint task", func() { v := findMatrixVariant(p.BuildVariants, matrixValue{ "os": "linux", "python": "python3", "c-extensions": "with-c", }) So(v, ShouldNotBeNil) tasks := taskNames(v) So(len(tasks), ShouldEqual, 9) So(tasks, ShouldContain, "ldap_auth") So(tasks, ShouldContain, "lint") So(v.DisplayName, ShouldEqual, "Linux Python 3.0 (with C extensions)") So(v.RunOn, ShouldResemble, []string{"centos6-perf"}) }) }) }) }) }
func TestMain(m *testing.M) { var err error testDirectory = testutil.GetDirectoryOfFile() if err != nil { panic(err) } exitCode := m.Run() os.Chdir(testDirectory) os.Exit(exitCode) }
func TestXMLToModelConversion(t *testing.T) { Convey("With a parsed XML file and a task", t, func() { file, err := os.Open(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "junit_3.xml")) testutil.HandleTestingErr(err, t, "Error reading file") defer file.Close() res, err := ParseXMLResults(file) So(err, ShouldBeNil) So(len(res), ShouldBeGreaterThan, 0) testTask := &task.Task{Id: "TEST", Execution: 5} Convey("when converting the results to model struct", func() { tests := []task.TestResult{} logs := []*model.TestLog{} for _, testCase := range res[0].TestCases { test, log := testCase.ToModelTestResultAndLog(testTask) if log != nil { logs = append(logs, log) } tests = append(tests, test) } Convey("the proper amount of each failure should be correct", func() { skipCount := 0 failCount := 0 passCount := 0 for _, t := range tests { switch t.Status { case evergreen.TestFailedStatus: failCount++ case evergreen.TestSkippedStatus: skipCount++ case evergreen.TestSucceededStatus: passCount++ } } So(failCount, ShouldEqual, res[0].Failures+res[0].Errors) So(skipCount, ShouldEqual, res[0].Skip) //make sure we didn't miss anything So(passCount+skipCount+failCount, ShouldEqual, len(tests)) Convey("and logs should be of the proper form", func() { So(logs[0].Name, ShouldNotEqual, "") So(len(logs[0].Lines), ShouldNotEqual, 0) So(logs[0].URL(), ShouldContainSubstring, "TEST/5/test.test_auth.TestAuthURIOptions.test_uri_options") }) }) }) }) }
func TestAllOutputFiles(t *testing.T) { Convey("When determining the name of all files to be parsed", t, func() { cwd := testutil.GetDirectoryOfFile() Convey("All specified files should be included", func() { pfCmd := &ParseFilesCommand{ Files: []string{ filepath.Join(cwd, "testdata", "monitor.suite"), filepath.Join(cwd, "testdata", "util.suite"), }, } files, err := pfCmd.AllOutputFiles() So(err, ShouldBeNil) So(len(files), ShouldEqual, 2) }) Convey("File patterns should be expanded correctly", func() { pfCmd := &ParseFilesCommand{ Files: []string{ filepath.Join(cwd, "testdata", "monitor.suite"), filepath.Join(cwd, "testdata", "test_output_dir", "*"), }, } files, err := pfCmd.AllOutputFiles() So(err, ShouldBeNil) So(len(files), ShouldEqual, 4) }) Convey("Duplicates should be removed", func() { pfCmd := &ParseFilesCommand{ Files: []string{ filepath.Join(cwd, "testdata", "monitor.suite"), filepath.Join(cwd, "testdata", "util.suite"), }, } files, err := pfCmd.AllOutputFiles() So(err, ShouldBeNil) So(len(files), ShouldEqual, 2) }) }) }
func TestAttachResults(t *testing.T) { resetTasks(t) testConfig := evergreen.TestConfig() cwd := testutil.GetDirectoryOfFile() Convey("With attachResults plugin installed into plugin registry", t, func() { registry := plugin.NewSimpleRegistry() attachPlugin := &AttachPlugin{} err := registry.Register(attachPlugin) testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v") server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true) testutil.HandleTestingErr(err, t, "Couldn't set up testing server") httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL) configFile := filepath.Join(cwd, "testdata", "plugin_attach_results.yml") resultsLoc := filepath.Join(cwd, "testdata", "plugin_attach_results.json") taskConfig, err := plugintest.CreateTestConfig(configFile, t) testutil.HandleTestingErr(err, t, "failed to create test config: %v") taskConfig.WorkDir = "." sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}} logger := agentutil.NewTestLogger(sliceAppender) Convey("all commands in test project should execute successfully", func() { for _, projTask := range taskConfig.Project.Tasks { So(len(projTask.Commands), ShouldNotEqual, 0) for _, command := range projTask.Commands { pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions) testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v") So(pluginCmds, ShouldNotBeNil) So(err, ShouldBeNil) pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom} err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool)) So(err, ShouldBeNil) testTask, err := task.FindOne(task.ById(httpCom.TaskId)) testutil.HandleTestingErr(err, t, "Couldn't find task") So(testTask, ShouldNotBeNil) // ensure test results are exactly as expected // attempt to open the file reportFile, err := os.Open(resultsLoc) testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err) results := &task.TestResults{} err = util.ReadJSONInto(reportFile, results) testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err) testResults := *results So(testTask.TestResults, ShouldResemble, testResults.Results) testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir") } } }) }) }
func setupCLITestHarness() cliTestHarness { // create a test API server testServer, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true) So(err, ShouldBeNil) So( db.ClearCollections( task.Collection, build.Collection, user.Collection, patch.Collection, model.ProjectRefCollection, artifact.Collection, ), ShouldBeNil) So(db.Clear(patch.Collection), ShouldBeNil) So(db.Clear(model.ProjectRefCollection), ShouldBeNil) So((&user.DBUser{Id: "testuser", APIKey: "testapikey", EmailAddress: "*****@*****.**"}).Insert(), ShouldBeNil) localConfBytes, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "sample.yml")) So(err, ShouldBeNil) projectRef := &model.ProjectRef{ Identifier: "sample", Owner: "evergreen-ci", Repo: "sample", RepoKind: "github", Branch: "master", RemotePath: "evergreen.yml", LocalConfig: string(localConfBytes), Enabled: true, BatchTime: 180, } So(projectRef.Insert(), ShouldBeNil) // create a settings file for the command line client settings := model.CLISettings{ APIServerHost: testServer.URL + "/api", UIServerHost: "http://dev-evg.mongodb.com", APIKey: "testapikey", User: "******", } settingsFile, err := ioutil.TempFile("", "settings") So(err, ShouldBeNil) settingsBytes, err := yaml.Marshal(settings) So(err, ShouldBeNil) _, err = settingsFile.Write(settingsBytes) So(err, ShouldBeNil) settingsFile.Close() return cliTestHarness{testServer, settingsFile.Name()} }
func TestPatchPlugin(t *testing.T) { cwd := testutil.GetDirectoryOfFile() testConfig := evergreen.TestConfig() db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig)) Convey("With patch plugin installed into plugin registry", t, func() { registry := plugin.NewSimpleRegistry() gitPlugin := &GitPlugin{} err := registry.Register(gitPlugin) testutil.HandleTestingErr(err, t, "Couldn't register plugin %v") testutil.HandleTestingErr(db.Clear(version.Collection), t, "unable to clear versions collection") version := &version.Version{ Id: "", } So(version.Insert(), ShouldBeNil) server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false) testutil.HandleTestingErr(err, t, "Couldn't set up testing server") httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL) //sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}} sliceAppender := slogger.StdOutAppender() logger := agentutil.NewTestLogger(sliceAppender) Convey("all commands in test project should execute successfully", func() { taskConfig, err := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t) testutil.HandleTestingErr(err, t, "could not create test config") taskConfig.Task.Requester = evergreen.PatchVersionRequester _, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t) testutil.HandleTestingErr(err, t, "Couldn't set up test documents") for _, task := range taskConfig.Project.Tasks { So(len(task.Commands), ShouldNotEqual, 0) for _, command := range task.Commands { pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions) testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v") So(pluginCmds, ShouldNotBeNil) So(err, ShouldBeNil) pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom} err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool)) So(err, ShouldBeNil) } } }) }) }
func TestMakeArchive(t *testing.T) { Convey("Making an archive should not return an error", t, func() { testDir := testutil.GetDirectoryOfFile() err := os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz")) testutil.HandleTestingErr(err, t, "Couldn't delete test tarball") f, gz, tarWriter, err := TarGzWriter(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz")) testutil.HandleTestingErr(err, t, "Couldn't open test tarball") defer f.Close() defer gz.Close() defer tarWriter.Close() includes := []string{"artifacts/dir1/**"} excludes := []string{"*.pdb"} _, err = BuildArchive(tarWriter, filepath.Join(testDir, "testdata", "artifacts_in"), includes, excludes, logger) So(err, ShouldBeNil) }) }
func TestArchiveRoundTrip(t *testing.T) { Convey("After building archive with include/exclude filters", t, func() { testDir := testutil.GetDirectoryOfFile() err := os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz")) testutil.HandleTestingErr(err, t, "Couldn't remove test tarball") err = os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_out")) testutil.HandleTestingErr(err, t, "Couldn't remove test tarball") f, gz, tarWriter, err := TarGzWriter(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz")) testutil.HandleTestingErr(err, t, "Couldn't open test tarball") includes := []string{"dir1/**"} excludes := []string{"*.pdb"} var found int found, err = BuildArchive(tarWriter, filepath.Join(testDir, "testdata", "artifacts_in"), includes, excludes, logger) So(err, ShouldBeNil) So(found, ShouldEqual, 2) tarWriter.Close() gz.Close() f.Close() f2, gz2, tarReader, err := TarGzReader(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz")) testutil.HandleTestingErr(err, t, "Couldn't open test tarball") err = Extract(tarReader, filepath.Join(testDir, "testdata", "artifacts_out")) defer f2.Close() defer gz2.Close() So(err, ShouldBeNil) exists, err := util.FileExists(filepath.Join(testDir, "testdata", "artifacts_out")) So(err, ShouldBeNil) So(exists, ShouldBeTrue) exists, err = util.FileExists(filepath.Join(testDir, "testdata", "artifacts_out", "dir1", "dir2", "test.pdb")) So(err, ShouldBeNil) So(exists, ShouldBeFalse) // Dereference symlinks exists, err = util.FileExists(filepath.Join(testDir, "testdata", "artifacts_out", "dir1", "dir2", "my_symlink.txt")) So(err, ShouldBeNil) So(exists, ShouldBeTrue) contents, err := ioutil.ReadFile(filepath.Join(testDir, "testdata", "artifacts_out", "dir1", "dir2", "my_symlink.txt")) So(err, ShouldBeNil) So(string(contents), ShouldEqual, "Hello, World\n") }) }
func TestPluginExecution(t *testing.T) { Convey("With a SimpleRegistry and test project file", t, func() { registry := plugin.NewSimpleRegistry() plugins := []plugin.CommandPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}, &shell.ShellPlugin{}} apiPlugins := []plugin.APIPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}} for _, p := range plugins { err := registry.Register(p) testutil.HandleTestingErr(err, t, "failed to register plugin") } testServer, err := service.CreateTestServer(evergreen.TestConfig(), nil, apiPlugins, false) testutil.HandleTestingErr(err, t, "Couldn't set up testing server") httpCom, err := comm.NewHTTPCommunicator(testServer.URL, "mocktaskid", "mocktasksecret", "", nil) So(err, ShouldBeNil) So(httpCom, ShouldNotBeNil) pluginConfigPath := filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "plugin_project.yml") taskConfig, err := createTestConfig(pluginConfigPath, t) testutil.HandleTestingErr(err, t, "failed to create test config: %v", err) sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}} logger := agentutil.NewTestLogger(sliceAppender) Convey("all commands in test project should execute successfully", func() { for _, newTask := range taskConfig.Project.Tasks { So(len(newTask.Commands), ShouldNotEqual, 0) for _, command := range newTask.Commands { pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions) testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v") So(pluginCmds, ShouldNotBeNil) So(err, ShouldBeNil) for _, c := range pluginCmds { pluginCom := &comm.TaskJSONCommunicator{c.Plugin(), httpCom} err = c.Execute(logger, pluginCom, taskConfig, make(chan bool)) So(err, ShouldBeNil) } } } }) }) }
func TestGitPlugin(t *testing.T) { testConfig := evergreen.TestConfig() Convey("With git plugin installed into plugin registry", t, func() { registry := plugin.NewSimpleRegistry() gitPlugin := &GitPlugin{} err := registry.Register(gitPlugin) testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v") server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false) testutil.HandleTestingErr(err, t, "Couldn't set up testing server") httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL) taskConfig, err := plugintest.CreateTestConfig( filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "plugin_clone.yml"), t) testutil.HandleTestingErr(err, t, "failed to create test config") sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}} logger := agentutil.NewTestLogger(sliceAppender) Convey("all commands in test project should execute successfully", func() { for _, task := range taskConfig.Project.Tasks { So(len(task.Commands), ShouldNotEqual, 0) for _, command := range task.Commands { pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions) testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v") So(pluginCmds, ShouldNotBeNil) So(err, ShouldBeNil) pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom} err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool)) So(err, ShouldBeNil) } } err = os.RemoveAll(taskConfig.WorkDir) testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir") }) }) }
func TestAttachRawResults(t *testing.T) { resetTasks(t) testConfig := evergreen.TestConfig() cwd := testutil.GetDirectoryOfFile() Convey("With attachResults plugin installed into plugin registry", t, func() { registry := plugin.NewSimpleRegistry() attachPlugin := &AttachPlugin{} err := registry.Register(attachPlugin) testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v") server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true) testutil.HandleTestingErr(err, t, "Couldn't set up testing server") httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL) configFile := filepath.Join(cwd, "testdata", "plugin_attach_results_raw.yml") resultsLoc := filepath.Join(cwd, "testdata", "plugin_attach_results_raw.json") taskConfig, err := plugintest.CreateTestConfig(configFile, t) testutil.HandleTestingErr(err, t, "failed to create test config: %v") taskConfig.WorkDir = "." sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}} logger := agentutil.NewTestLogger(sliceAppender) Convey("when attaching a raw log ", func() { for _, projTask := range taskConfig.Project.Tasks { So(len(projTask.Commands), ShouldNotEqual, 0) for _, command := range projTask.Commands { pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions) testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v") So(pluginCmds, ShouldNotBeNil) So(err, ShouldBeNil) // create a plugin communicator pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom} err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool)) So(err, ShouldBeNil) Convey("when retrieving task", func() { // fetch the task testTask, err := task.FindOne(task.ById(httpCom.TaskId)) testutil.HandleTestingErr(err, t, "Couldn't find task") So(testTask, ShouldNotBeNil) Convey("test results should match and raw log should be in appropriate collection", func() { reportFile, err := os.Open(resultsLoc) testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err) results := &task.TestResults{} err = util.ReadJSONInto(reportFile, results) testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err) testResults := *results So(len(testResults.Results), ShouldEqual, 3) So(len(testTask.TestResults), ShouldEqual, 3) firstResult := testTask.TestResults[0] So(firstResult.LogRaw, ShouldEqual, "") So(firstResult.LogId, ShouldNotEqual, "") testLog, err := model.FindOneTestLogById(firstResult.LogId) So(err, ShouldBeNil) So(testLog.Lines[0], ShouldEqual, testResults.Results[0].LogRaw) Convey("both URL and raw log should be stored appropriately if both exist", func() { urlResult := testTask.TestResults[2] So(urlResult.LogRaw, ShouldEqual, "") So(urlResult.URL, ShouldNotEqual, "") So(urlResult.LogId, ShouldNotEqual, "") testLog, err := model.FindOneTestLogById(urlResult.LogId) So(err, ShouldBeNil) So(testLog.Lines[0], ShouldEqual, testResults.Results[2].LogRaw) }) }) }) } } }) }) }
"github.com/evergreen-ci/evergreen/agent/comm" agentutil "github.com/evergreen-ci/evergreen/agent/testutil" "github.com/evergreen-ci/evergreen/model" "github.com/evergreen-ci/evergreen/model/task" "github.com/evergreen-ci/evergreen/plugin" . "github.com/evergreen-ci/evergreen/plugin/builtin/attach" "github.com/evergreen-ci/evergreen/plugin/plugintest" "github.com/evergreen-ci/evergreen/service" "github.com/evergreen-ci/evergreen/testutil" . "github.com/smartystreets/goconvey/convey" ) const TotalResultCount = 677 var ( workingDirectory = testutil.GetDirectoryOfFile() SingleFileConfig = filepath.Join(workingDirectory, "testdata", "plugin_attach_xunit.yml") WildcardConfig = filepath.Join(workingDirectory, "testdata", "plugin_attach_xunit_wildcard.yml") ) // runTest abstracts away common tests and setup between all attach xunit tests. // It also takes as an argument a function which runs any additional tests desired. func runTest(t *testing.T, configPath string, customTests func()) { resetTasks(t) testConfig := evergreen.TestConfig() Convey("With attachResults plugin installed into plugin registry", t, func() { registry := plugin.NewSimpleRegistry() attachPlugin := &AttachPlugin{} err := registry.Register(attachPlugin) testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")
func TestExpansions(t *testing.T) { Convey("With a set of expansions", t, func() { Convey("the expansions object should contain the values it is"+ " initialized with", func() { expansions := NewExpansions(map[string]string{ "key1": "val1", "key2": "val2", }) So((*expansions)["key1"], ShouldEqual, "val1") So((*expansions)["key2"], ShouldEqual, "val2") }) Convey("updating the expansions with a map should update all of the"+ " specified values", func() { expansions := NewExpansions(map[string]string{ "key1": "val1", "key2": "val2", }) expansions.Update(map[string]string{ "key2": "newval1", "key3": "newval2", }) So((*expansions)["key1"], ShouldEqual, "val1") So((*expansions)["key2"], ShouldEqual, "newval1") So((*expansions)["key3"], ShouldEqual, "newval2") }) Convey("fetching an expansion should return the appropriate value,"+ " or the empty string if the value is not present", func() { expansions := NewExpansions(map[string]string{ "key1": "val1", "key2": "val2", }) So(expansions.Get("key1"), ShouldEqual, "val1") So(expansions.Get("key2"), ShouldEqual, "val2") So(expansions.Get("key3"), ShouldEqual, "") }) Convey("checking for a key's existence should return whether it is"+ " specified as an expansion", func() { expansions := NewExpansions(map[string]string{ "key1": "val1", "key2": "val2", }) So(expansions.Exists("key1"), ShouldBeTrue) So(expansions.Exists("key2"), ShouldBeTrue) So(expansions.Exists("key3"), ShouldBeFalse) }) Convey("updating from a yaml file should get keys copied over", func() { expansions := NewExpansions(map[string]string{ "key1": "val1", "key2": "val2", }) err := expansions.UpdateFromYaml(filepath.Join( testutil.GetDirectoryOfFile(), "testdata", "expansions.yml")) So(err, ShouldBeNil) So(expansions.Get("marge"), ShouldEqual, "1") So(expansions.Get("lisa"), ShouldEqual, "2") So(expansions.Get("bart"), ShouldEqual, "3") So(expansions.Get("key1"), ShouldEqual, "blah") }) }) }
func TestParserFunctionality(t *testing.T) { var parser Parser cwd := testutil.GetDirectoryOfFile() Convey("With a simple log file and parser", t, func() { logdata, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "1_simple.log")) testutil.HandleTestingErr(err, t, "couldn't open log file") parser = &VanillaParser{Suite: "test"} Convey("running parse on the given log file should succeed", func() { err = parser.Parse(bytes.NewBuffer(logdata)) So(err, ShouldBeNil) Convey("and logs should be the correct length", func() { logs := parser.Logs() So(len(logs), ShouldEqual, 18) }) Convey("and there should be one test result", func() { results := parser.Results() So(len(results), ShouldEqual, 2) Convey("with the proper fields matching the original log file", func() { So(results[0].Name, ShouldEqual, "TestFailures") So(results[0].Status, ShouldEqual, FAIL) rTime, _ := time.ParseDuration("5.02s") So(results[0].RunTime, ShouldEqual, rTime) So(results[0].StartLine, ShouldEqual, 1) So(results[0].EndLine, ShouldEqual, 14) So(results[0].SuiteName, ShouldEqual, "test") So(results[1].Name, ShouldEqual, "TestFailures2") So(results[1].Status, ShouldEqual, FAIL) rTime, _ = time.ParseDuration("2.00s") So(results[1].RunTime, ShouldEqual, rTime) So(results[1].StartLine, ShouldEqual, 15) So(results[1].EndLine, ShouldEqual, 15) So(results[1].SuiteName, ShouldEqual, "test") }) }) }) }) Convey("With a gocheck log file and parser", t, func() { logdata, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "2_simple.log")) testutil.HandleTestingErr(err, t, "couldn't open log file") parser = &VanillaParser{Suite: "gocheck_test"} Convey("running parse on the given log file should succeed", func() { err = parser.Parse(bytes.NewBuffer(logdata)) So(err, ShouldBeNil) Convey("and logs should be the correct length", func() { logs := parser.Logs() So(len(logs), ShouldEqual, 15) }) Convey("and there should be three test results", func() { results := parser.Results() So(len(results), ShouldEqual, 3) Convey("with the proper fields matching the original log file", func() { So(results[0].Name, ShouldEqual, "MyTestName.SetUpTest") So(results[0].Status, ShouldEqual, PASS) rTime, _ := time.ParseDuration("0.576s") So(results[0].RunTime, ShouldEqual, rTime) So(results[0].StartLine, ShouldEqual, 2) So(results[0].EndLine, ShouldEqual, 4) So(results[0].SuiteName, ShouldEqual, "gocheck_test") }) }) }) }) }
func TestXMLParsing(t *testing.T) { cwd := testutil.GetDirectoryOfFile() Convey("With some test xml files", t, func() { Convey("with a basic test junit file", func() { file, err := os.Open(filepath.Join(cwd, "testdata", "junit_1.xml")) testutil.HandleTestingErr(err, t, "Error reading file") defer file.Close() Convey("the file should parse without error", func() { res, err := ParseXMLResults(file) So(err, ShouldBeNil) So(len(res), ShouldBeGreaterThan, 0) Convey("and have proper values decoded", func() { So(res[0].Errors, ShouldEqual, 1) So(res[0].Failures, ShouldEqual, 5) So(res[0].Name, ShouldEqual, "nose2-junit") So(res[0].TestCases[11].Name, ShouldEqual, "test_params_func:2") So(res[0].TestCases[11].Time, ShouldEqual, 0.000098) So(res[0].TestCases[11].Failure, ShouldNotBeNil) So(res[0].TestCases[11].Failure.Message, ShouldEqual, "test failure") }) }) }) Convey("with a more complex test junit file", func() { file, err := os.Open(filepath.Join(cwd, "testdata", "junit_2.xml")) testutil.HandleTestingErr(err, t, "Error reading file") defer file.Close() Convey("the file should parse without error", func() { res, err := ParseXMLResults(file) So(err, ShouldBeNil) So(len(res), ShouldBeGreaterThan, 0) Convey("and have proper values decoded", func() { So(res[0].Errors, ShouldEqual, 1) So(res[0].Failures, ShouldEqual, 1) So(res[0].Name, ShouldEqual, "tests.ATest") So(res[0].SysOut, ShouldContainSubstring, "here") So(res[0].TestCases[0].Name, ShouldEqual, "error") So(res[0].TestCases[0].ClassName, ShouldEqual, "tests.ATest") So(res[0].TestCases[0].Time, ShouldEqual, 0.0060) So(res[0].TestCases[0].Failure, ShouldBeNil) So(res[0].TestCases[0].Error, ShouldNotBeNil) So(res[0].TestCases[0].Error.Type, ShouldEqual, "java.lang.RuntimeException") }) }) }) Convey(`with a "real" pymongo xunit file`, func() { file, err := os.Open(filepath.Join(cwd, "testdata", "junit_3.xml")) testutil.HandleTestingErr(err, t, "Error reading file") defer file.Close() Convey("the file should parse without error", func() { res, err := ParseXMLResults(file) So(err, ShouldBeNil) So(len(res), ShouldBeGreaterThan, 0) Convey("and have proper values decoded", func() { So(res[0].Errors, ShouldEqual, 0) So(res[0].Failures, ShouldEqual, 1) So(res[0].Skip, ShouldEqual, 188) So(res[0].Name, ShouldEqual, "nosetests") So(res[0].TestCases[0].Name, ShouldEqual, "test_uri_options") So(res[0].TestCases[0].ClassName, ShouldEqual, "test.test_auth.TestAuthURIOptions") So(res[0].TestCases[0].Time, ShouldEqual, 0.002) So(res[0].TestCases[0].Failure, ShouldBeNil) So(res[0].TestCases[0].Error, ShouldBeNil) So(res[0].TestCases[0].Skipped, ShouldNotBeNil) So(res[0].TestCases[0].Skipped.Type, ShouldEqual, "unittest.case.SkipTest") So(res[0].TestCases[0].Skipped.Content, ShouldContainSubstring, "SkipTest: Authentication is not enabled on server") }) }) }) Convey(`with a "real" java driver xunit file`, func() { file, err := os.Open(filepath.Join(cwd, "testdata", "junit_4.xml")) testutil.HandleTestingErr(err, t, "Error reading file") defer file.Close() Convey("the file should parse without error", func() { res, err := ParseXMLResults(file) So(err, ShouldBeNil) So(len(res), ShouldBeGreaterThan, 0) Convey("and have proper values decoded", func() { So(res[0].Errors, ShouldEqual, 0) So(res[0].Failures, ShouldEqual, 0) So(res[0].Name, ShouldEqual, "com.mongodb.operation.InsertOperationSpecification") So(res[0].TestCases[0].Name, ShouldEqual, "should return correct result") So(res[0].SysOut, ShouldEqual, "out message") So(res[0].SysErr, ShouldEqual, "error message") }) }) }) }) }
func TestDepsMatrixIntegration(t *testing.T) { Convey("With a sample matrix project mocking up a python driver", t, func() { p := Project{} bytes, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "matrix_deps.yml")) So(err, ShouldBeNil) Convey("the project should parse properly", func() { err := LoadProjectInto(bytes, "deps", &p) So(err, ShouldBeNil) Convey("and contain the correct variants", func() { So(len(p.BuildVariants), ShouldEqual, (1 + 3*3)) Convey("including a non-matrix variant", func() { v := findRegularVariant(p.BuildVariants, "analysis") So(v, ShouldNotBeNil) ts := taskNames(v) So(ts, ShouldContain, "pre-task") So(ts, ShouldContain, "post-task") So(*v.Stepback, ShouldBeFalse) }) Convey("including linux/standalone", func() { v := findMatrixVariant(p.BuildVariants, matrixValue{ "os": "linux", "configuration": "standalone", }) So(v, ShouldNotBeNil) So(len(v.Tasks), ShouldEqual, 5) So(v.Tags, ShouldContain, "posix") Convey("which should contain a compile", func() { So(v.Tasks[4].Name, ShouldEqual, "compile") So(v.Tasks[4].Distros, ShouldResemble, []string{"linux_big"}) So(v.Tasks[4].DependsOn[0], ShouldResemble, TaskDependency{ Name: "pre-task", Variant: "analysis", }) }) }) Convey("including osx/repl", func() { v := findMatrixVariant(p.BuildVariants, matrixValue{ "os": "osx", "configuration": "repl", }) So(v, ShouldNotBeNil) So(len(v.Tasks), ShouldEqual, 4) So(v.Tags, ShouldContain, "posix") Convey("which should depend on another variant's compile", func() { So(v.Tasks[0].Name, ShouldEqual, "test1") So(v.Tasks[0].DependsOn[0].Name, ShouldEqual, "compile") So(v.Tasks[0].DependsOn[0].Variant, ShouldNotEqual, "") }) }) }) Convey("and contain the correct tasks", func() { So(len(p.Tasks), ShouldEqual, 7) Convey("such that post-task depends on everything", func() { pt := ProjectTask{} for _, t := range p.Tasks { if t.Name == "post-task" { pt = t } } So(pt.Name, ShouldEqual, "post-task") So(len(pt.DependsOn), ShouldEqual, 4*(3*3)) }) }) }) }) }