func TestArchiveExtract(t *testing.T) {
Convey("After extracting a tarball", t, func() {
testDir := testutil.GetDirectoryOfFile()
//Remove the test output dir, in case it was left over from prior test
err := os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_test"))
testutil.HandleTestingErr(err, t, "Couldn't remove test dir")
f, gz, tarReader, err := TarGzReader(filepath.Join(testDir, "testdata", "artifacts.tar.gz"))
testutil.HandleTestingErr(err, t, "Couldn't open test tarball")
defer f.Close()
defer gz.Close()
err = Extract(tarReader, filepath.Join(testDir, "testdata", "artifacts_test"))
So(err, ShouldBeNil)
Convey("extracted data should match the archive contents", func() {
f, err := os.Open(filepath.Join(testDir, "testdata", "artifacts_test", "artifacts", "dir1", "dir2", "testfile.txt"))
So(err, ShouldBeNil)
defer f.Close()
data, err := ioutil.ReadAll(f)
So(err, ShouldBeNil)
So(string(data), ShouldEqual, "test\n")
})
})
}
func TestPatchPluginAPI(t *testing.T) {
testConfig := evergreen.TestConfig()
cwd := testutil.GetDirectoryOfFile()
Convey("With a running api server and installed plugin", t, func() {
registry := plugin.NewSimpleRegistry()
gitPlugin := &GitPlugin{}
err := registry.Register(gitPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register patch plugin")
server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
taskConfig, _ := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t)
testCommand := GitGetProjectCommand{Directory: "dir"}
_, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t)
testutil.HandleTestingErr(err, t, "Couldn't set up test documents")
testTask, err := task.FindOne(task.ById("testTaskId"))
testutil.HandleTestingErr(err, t, "Couldn't set up test patch task")
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agentutil.NewTestLogger(sliceAppender)
Convey("calls to existing tasks with patches should succeed", func() {
httpCom := plugintest.TestAgentCommunicator(testTask.Id, testTask.Secret, server.URL)
pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
So(err, ShouldBeNil)
So(patch, ShouldNotBeNil)
testutil.HandleTestingErr(db.Clear(version.Collection), t,
"unable to clear versions collection")
})
Convey("calls to non-existing tasks should fail", func() {
v := version.Version{Id: ""}
testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
httpCom := plugintest.TestAgentCommunicator("BAD_TASK_ID", "", server.URL)
pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
So(err.Error(), ShouldContainSubstring, "not found")
So(err, ShouldNotBeNil)
So(patch, ShouldBeNil)
testutil.HandleTestingErr(db.Clear(version.Collection), t,
"unable to clear versions collection")
})
Convey("calls to existing tasks without patches should fail", func() {
noPatchTask := task.Task{Id: "noPatchTask", BuildId: "a"}
testutil.HandleTestingErr(noPatchTask.Insert(), t, "Couldn't insert patch task")
noPatchVersion := version.Version{Id: "noPatchVersion", BuildIds: []string{"a"}}
testutil.HandleTestingErr(noPatchVersion.Insert(), t, "Couldn't insert patch version")
v := version.Version{Id: ""}
testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
httpCom := plugintest.TestAgentCommunicator(noPatchTask.Id, "", server.URL)
pluginCom := &comm.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "no patch found for task")
So(patch, ShouldBeNil)
testutil.HandleTestingErr(db.Clear(version.Collection), t,
"unable to clear versions collection")
})
})
}
func TestRegistry(t *testing.T) {
Convey("With a SimpleRegistry", t, func() {
Convey("Registering a plugin twice should return err", func() {
registry := plugin.NewSimpleRegistry()
err := registry.Register(&MockPlugin{})
testutil.HandleTestingErr(err, t, "Couldn't register plugin")
err = registry.Register(&shell.ShellPlugin{})
testutil.HandleTestingErr(err, t, "Couldn't register plugin")
err = registry.Register(&expansions.ExpansionsPlugin{})
testutil.HandleTestingErr(err, t, "Couldn't register plugin")
})
Convey("with a project file containing references to a valid plugin", func() {
registry := plugin.NewSimpleRegistry()
registry.Register(&MockPlugin{})
registry.Register(&shell.ShellPlugin{})
registry.Register(&expansions.ExpansionsPlugin{})
data, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(),
"testdata", "plugin_project.yml"))
testutil.HandleTestingErr(err, t, "failed to load test yaml file")
project := &model.Project{}
err = yaml.Unmarshal(data, project)
Convey("all commands in project file should load parse successfully", func() {
for _, newTask := range project.Tasks {
for _, command := range newTask.Commands {
pluginCmds, err := registry.GetCommands(command, project.Functions)
testutil.HandleTestingErr(err, t, "Got error getting plugin commands: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
}
}
})
})
})
}
func TestParseOutputFiles(t *testing.T) {
Convey("When parsing files containing go test output", t, func() {
cwd := testutil.GetDirectoryOfFile()
Convey("The output in all of the specified files should be parsed correctly", func() {
// mock up a logger
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agentutil.NewTestLogger(sliceAppender)
// mock up a task config
taskConfig := &model.TaskConfig{Task: &task.Task{Id: "taskOne", Execution: 1}}
// the files we want to parse
files := []string{
filepath.Join(cwd, "testdata", "monitor.suite"),
filepath.Join(cwd, "testdata", "util.suite"),
filepath.Join(cwd, "testdata", "test_output_dir", "monitor_fail.suite"),
filepath.Join(cwd, "testdata", "test_output_dir", "evergreen.suite"),
}
logs, results, err := ParseTestOutputFiles(files, nil, logger, taskConfig)
So(err, ShouldBeNil)
So(logs, ShouldNotBeNil)
So(results, ShouldNotBeNil)
So(len(results), ShouldEqual, 4)
})
})
}
func init() {
dbutil.SetGlobalSessionProvider(dbutil.SessionFactoryFromConfig(testConfig))
testSetups = []testConfigPath{
{"With plugin mode test config",
filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "config_test_plugin")},
}
}
func TestMakePatchedConfig(t *testing.T) {
Convey("With calling MakePatchedConfig with a config and remote configuration path", t, func() {
cwd := testutil.GetDirectoryOfFile()
Convey("the config should be patched correctly", func() {
remoteConfigPath := filepath.Join("config", "evergreen.yml")
fileBytes, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "patch.diff"))
So(err, ShouldBeNil)
// update patch with remove config path variable
diffString := fmt.Sprintf(string(fileBytes),
remoteConfigPath, remoteConfigPath, remoteConfigPath, remoteConfigPath)
// the patch adds a new task
p := &patch.Patch{
Patches: []patch.ModulePatch{{
Githash: "revision",
PatchSet: patch.PatchSet{
Patch: diffString,
Summary: []patch.Summary{{
Name: remoteConfigPath,
Additions: 3,
Deletions: 3,
}},
},
}},
}
projectBytes, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "project.config"))
So(err, ShouldBeNil)
project, err := MakePatchedConfig(p, remoteConfigPath, string(projectBytes))
So(err, ShouldBeNil)
So(project, ShouldNotBeNil)
So(len(project.Tasks), ShouldEqual, 2)
})
Convey("an empty base config should be patched correctly", func() {
remoteConfigPath := filepath.Join("model", "testdata", "project2.config")
fileBytes, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "project.diff"))
So(err, ShouldBeNil)
p := &patch.Patch{
Patches: []patch.ModulePatch{{
Githash: "revision",
PatchSet: patch.PatchSet{
Patch: string(fileBytes),
Summary: []patch.Summary{{Name: remoteConfigPath}},
},
}},
}
project, err := MakePatchedConfig(p, remoteConfigPath, "")
So(err, ShouldBeNil)
So(project, ShouldNotBeNil)
So(len(project.Tasks), ShouldEqual, 1)
So(project.Tasks[0].Name, ShouldEqual, "hello")
Reset(func() {
os.Remove(remoteConfigPath)
})
})
})
}
func TestPythonMatrixIntegration(t *testing.T) {
Convey("With a sample matrix project mocking up a python driver", t, func() {
p := Project{}
bytes, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(),
"testdata", "matrix_python.yml"))
So(err, ShouldBeNil)
Convey("the project should parse properly", func() {
err := LoadProjectInto(bytes, "python", &p)
So(err, ShouldBeNil)
Convey("and contain the correct variants", func() {
So(len(p.BuildVariants), ShouldEqual, (2*2*4 - 4))
Convey("so that excluded matrix cells are not created", func() {
So(findMatrixVariant(p.BuildVariants, matrixValue{
"os": "windows", "python": "pypy", "c-extensions": "with-c",
}), ShouldBeNil)
So(findMatrixVariant(p.BuildVariants, matrixValue{
"os": "windows", "python": "jython", "c-extensions": "with-c",
}), ShouldBeNil)
So(findMatrixVariant(p.BuildVariants, matrixValue{
"os": "linux", "python": "pypy", "c-extensions": "with-c",
}), ShouldBeNil)
So(findMatrixVariant(p.BuildVariants, matrixValue{
"os": "linux", "python": "jython", "c-extensions": "with-c",
}), ShouldBeNil)
})
Convey("so that Windows builds without C extensions exclude LDAP tasks", func() {
v := findMatrixVariant(p.BuildVariants, matrixValue{
"os": "windows",
"python": "python3",
"c-extensions": "without-c",
})
So(v, ShouldNotBeNil)
tasks := taskNames(v)
So(len(tasks), ShouldEqual, 7)
So(tasks, ShouldNotContain, "ldap_auth")
So(v.DisplayName, ShouldEqual, "Windows 95 Python 3.0 (without C extensions)")
So(v.RunOn, ShouldResemble, []string{"windows95-test"})
})
Convey("so that the linux/python3/c variant has a lint task", func() {
v := findMatrixVariant(p.BuildVariants, matrixValue{
"os": "linux",
"python": "python3",
"c-extensions": "with-c",
})
So(v, ShouldNotBeNil)
tasks := taskNames(v)
So(len(tasks), ShouldEqual, 9)
So(tasks, ShouldContain, "ldap_auth")
So(tasks, ShouldContain, "lint")
So(v.DisplayName, ShouldEqual, "Linux Python 3.0 (with C extensions)")
So(v.RunOn, ShouldResemble, []string{"centos6-perf"})
})
})
})
})
}
func TestMain(m *testing.M) {
var err error
testDirectory = testutil.GetDirectoryOfFile()
if err != nil {
panic(err)
}
exitCode := m.Run()
os.Chdir(testDirectory)
os.Exit(exitCode)
}
func TestXMLToModelConversion(t *testing.T) {
Convey("With a parsed XML file and a task", t, func() {
file, err := os.Open(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "junit_3.xml"))
testutil.HandleTestingErr(err, t, "Error reading file")
defer file.Close()
res, err := ParseXMLResults(file)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
testTask := &task.Task{Id: "TEST", Execution: 5}
Convey("when converting the results to model struct", func() {
tests := []task.TestResult{}
logs := []*model.TestLog{}
for _, testCase := range res[0].TestCases {
test, log := testCase.ToModelTestResultAndLog(testTask)
if log != nil {
logs = append(logs, log)
}
tests = append(tests, test)
}
Convey("the proper amount of each failure should be correct", func() {
skipCount := 0
failCount := 0
passCount := 0
for _, t := range tests {
switch t.Status {
case evergreen.TestFailedStatus:
failCount++
case evergreen.TestSkippedStatus:
skipCount++
case evergreen.TestSucceededStatus:
passCount++
}
}
So(failCount, ShouldEqual, res[0].Failures+res[0].Errors)
So(skipCount, ShouldEqual, res[0].Skip)
//make sure we didn't miss anything
So(passCount+skipCount+failCount, ShouldEqual, len(tests))
Convey("and logs should be of the proper form", func() {
So(logs[0].Name, ShouldNotEqual, "")
So(len(logs[0].Lines), ShouldNotEqual, 0)
So(logs[0].URL(), ShouldContainSubstring,
"TEST/5/test.test_auth.TestAuthURIOptions.test_uri_options")
})
})
})
})
}
func TestAllOutputFiles(t *testing.T) {
Convey("When determining the name of all files to be parsed", t, func() {
cwd := testutil.GetDirectoryOfFile()
Convey("All specified files should be included", func() {
pfCmd := &ParseFilesCommand{
Files: []string{
filepath.Join(cwd, "testdata", "monitor.suite"),
filepath.Join(cwd, "testdata", "util.suite"),
},
}
files, err := pfCmd.AllOutputFiles()
So(err, ShouldBeNil)
So(len(files), ShouldEqual, 2)
})
Convey("File patterns should be expanded correctly", func() {
pfCmd := &ParseFilesCommand{
Files: []string{
filepath.Join(cwd, "testdata", "monitor.suite"),
filepath.Join(cwd, "testdata", "test_output_dir", "*"),
},
}
files, err := pfCmd.AllOutputFiles()
So(err, ShouldBeNil)
So(len(files), ShouldEqual, 4)
})
Convey("Duplicates should be removed", func() {
pfCmd := &ParseFilesCommand{
Files: []string{
filepath.Join(cwd, "testdata", "monitor.suite"),
filepath.Join(cwd, "testdata", "util.suite"),
},
}
files, err := pfCmd.AllOutputFiles()
So(err, ShouldBeNil)
So(len(files), ShouldEqual, 2)
})
})
}
func TestAttachResults(t *testing.T) {
resetTasks(t)
testConfig := evergreen.TestConfig()
cwd := testutil.GetDirectoryOfFile()
Convey("With attachResults plugin installed into plugin registry", t, func() {
registry := plugin.NewSimpleRegistry()
attachPlugin := &AttachPlugin{}
err := registry.Register(attachPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")
server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
configFile := filepath.Join(cwd, "testdata", "plugin_attach_results.yml")
resultsLoc := filepath.Join(cwd, "testdata", "plugin_attach_results.json")
taskConfig, err := plugintest.CreateTestConfig(configFile, t)
testutil.HandleTestingErr(err, t, "failed to create test config: %v")
taskConfig.WorkDir = "."
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agentutil.NewTestLogger(sliceAppender)
Convey("all commands in test project should execute successfully", func() {
for _, projTask := range taskConfig.Project.Tasks {
So(len(projTask.Commands), ShouldNotEqual, 0)
for _, command := range projTask.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
testTask, err := task.FindOne(task.ById(httpCom.TaskId))
testutil.HandleTestingErr(err, t, "Couldn't find task")
So(testTask, ShouldNotBeNil)
// ensure test results are exactly as expected
// attempt to open the file
reportFile, err := os.Open(resultsLoc)
testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err)
results := &task.TestResults{}
err = util.ReadJSONInto(reportFile, results)
testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err)
testResults := *results
So(testTask.TestResults, ShouldResemble, testResults.Results)
testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir")
}
}
})
})
}
func setupCLITestHarness() cliTestHarness {
// create a test API server
testServer, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
So(err, ShouldBeNil)
So(
db.ClearCollections(
task.Collection,
build.Collection,
user.Collection,
patch.Collection,
model.ProjectRefCollection,
artifact.Collection,
),
ShouldBeNil)
So(db.Clear(patch.Collection), ShouldBeNil)
So(db.Clear(model.ProjectRefCollection), ShouldBeNil)
So((&user.DBUser{Id: "testuser", APIKey: "testapikey", EmailAddress: "*****@*****.**"}).Insert(), ShouldBeNil)
localConfBytes, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "sample.yml"))
So(err, ShouldBeNil)
projectRef := &model.ProjectRef{
Identifier: "sample",
Owner: "evergreen-ci",
Repo: "sample",
RepoKind: "github",
Branch: "master",
RemotePath: "evergreen.yml",
LocalConfig: string(localConfBytes),
Enabled: true,
BatchTime: 180,
}
So(projectRef.Insert(), ShouldBeNil)
// create a settings file for the command line client
settings := model.CLISettings{
APIServerHost: testServer.URL + "/api",
UIServerHost: "http://dev-evg.mongodb.com",
APIKey: "testapikey",
User: "******",
}
settingsFile, err := ioutil.TempFile("", "settings")
So(err, ShouldBeNil)
settingsBytes, err := yaml.Marshal(settings)
So(err, ShouldBeNil)
_, err = settingsFile.Write(settingsBytes)
So(err, ShouldBeNil)
settingsFile.Close()
return cliTestHarness{testServer, settingsFile.Name()}
}
func TestPatchPlugin(t *testing.T) {
cwd := testutil.GetDirectoryOfFile()
testConfig := evergreen.TestConfig()
db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
Convey("With patch plugin installed into plugin registry", t, func() {
registry := plugin.NewSimpleRegistry()
gitPlugin := &GitPlugin{}
err := registry.Register(gitPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")
testutil.HandleTestingErr(db.Clear(version.Collection), t,
"unable to clear versions collection")
version := &version.Version{
Id: "",
}
So(version.Insert(), ShouldBeNil)
server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)
//sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
sliceAppender := slogger.StdOutAppender()
logger := agentutil.NewTestLogger(sliceAppender)
Convey("all commands in test project should execute successfully", func() {
taskConfig, err := plugintest.CreateTestConfig(filepath.Join(cwd, "testdata", "plugin_patch.yml"), t)
testutil.HandleTestingErr(err, t, "could not create test config")
taskConfig.Task.Requester = evergreen.PatchVersionRequester
_, _, err = plugintest.SetupAPITestData("testTask", filepath.Join(cwd, "testdata", "testmodule.patch"), t)
testutil.HandleTestingErr(err, t, "Couldn't set up test documents")
for _, task := range taskConfig.Project.Tasks {
So(len(task.Commands), ShouldNotEqual, 0)
for _, command := range task.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
}
}
})
})
}
func TestMakeArchive(t *testing.T) {
Convey("Making an archive should not return an error", t, func() {
testDir := testutil.GetDirectoryOfFile()
err := os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz"))
testutil.HandleTestingErr(err, t, "Couldn't delete test tarball")
f, gz, tarWriter, err := TarGzWriter(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz"))
testutil.HandleTestingErr(err, t, "Couldn't open test tarball")
defer f.Close()
defer gz.Close()
defer tarWriter.Close()
includes := []string{"artifacts/dir1/**"}
excludes := []string{"*.pdb"}
_, err = BuildArchive(tarWriter, filepath.Join(testDir, "testdata", "artifacts_in"), includes, excludes, logger)
So(err, ShouldBeNil)
})
}
func TestArchiveRoundTrip(t *testing.T) {
Convey("After building archive with include/exclude filters", t, func() {
testDir := testutil.GetDirectoryOfFile()
err := os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz"))
testutil.HandleTestingErr(err, t, "Couldn't remove test tarball")
err = os.RemoveAll(filepath.Join(testDir, "testdata", "artifacts_out"))
testutil.HandleTestingErr(err, t, "Couldn't remove test tarball")
f, gz, tarWriter, err := TarGzWriter(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz"))
testutil.HandleTestingErr(err, t, "Couldn't open test tarball")
includes := []string{"dir1/**"}
excludes := []string{"*.pdb"}
var found int
found, err = BuildArchive(tarWriter, filepath.Join(testDir, "testdata", "artifacts_in"), includes, excludes, logger)
So(err, ShouldBeNil)
So(found, ShouldEqual, 2)
tarWriter.Close()
gz.Close()
f.Close()
f2, gz2, tarReader, err := TarGzReader(filepath.Join(testDir, "testdata", "artifacts_out.tar.gz"))
testutil.HandleTestingErr(err, t, "Couldn't open test tarball")
err = Extract(tarReader, filepath.Join(testDir, "testdata", "artifacts_out"))
defer f2.Close()
defer gz2.Close()
So(err, ShouldBeNil)
exists, err := util.FileExists(filepath.Join(testDir, "testdata", "artifacts_out"))
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
exists, err = util.FileExists(filepath.Join(testDir, "testdata", "artifacts_out", "dir1", "dir2", "test.pdb"))
So(err, ShouldBeNil)
So(exists, ShouldBeFalse)
// Dereference symlinks
exists, err = util.FileExists(filepath.Join(testDir, "testdata", "artifacts_out", "dir1", "dir2", "my_symlink.txt"))
So(err, ShouldBeNil)
So(exists, ShouldBeTrue)
contents, err := ioutil.ReadFile(filepath.Join(testDir, "testdata", "artifacts_out", "dir1", "dir2", "my_symlink.txt"))
So(err, ShouldBeNil)
So(string(contents), ShouldEqual, "Hello, World\n")
})
}
func TestPluginExecution(t *testing.T) {
Convey("With a SimpleRegistry and test project file", t, func() {
registry := plugin.NewSimpleRegistry()
plugins := []plugin.CommandPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}, &shell.ShellPlugin{}}
apiPlugins := []plugin.APIPlugin{&MockPlugin{}, &expansions.ExpansionsPlugin{}}
for _, p := range plugins {
err := registry.Register(p)
testutil.HandleTestingErr(err, t, "failed to register plugin")
}
testServer, err := service.CreateTestServer(evergreen.TestConfig(), nil, apiPlugins, false)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom, err := comm.NewHTTPCommunicator(testServer.URL, "mocktaskid", "mocktasksecret", "", nil)
So(err, ShouldBeNil)
So(httpCom, ShouldNotBeNil)
pluginConfigPath := filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "plugin_project.yml")
taskConfig, err := createTestConfig(pluginConfigPath, t)
testutil.HandleTestingErr(err, t, "failed to create test config: %v", err)
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agentutil.NewTestLogger(sliceAppender)
Convey("all commands in test project should execute successfully", func() {
for _, newTask := range taskConfig.Project.Tasks {
So(len(newTask.Commands), ShouldNotEqual, 0)
for _, command := range newTask.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
for _, c := range pluginCmds {
pluginCom := &comm.TaskJSONCommunicator{c.Plugin(), httpCom}
err = c.Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
}
}
}
})
})
}
func TestGitPlugin(t *testing.T) {
testConfig := evergreen.TestConfig()
Convey("With git plugin installed into plugin registry", t, func() {
registry := plugin.NewSimpleRegistry()
gitPlugin := &GitPlugin{}
err := registry.Register(gitPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")
server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
taskConfig, err := plugintest.CreateTestConfig(
filepath.Join(testutil.GetDirectoryOfFile(), "testdata", "plugin_clone.yml"),
t)
testutil.HandleTestingErr(err, t, "failed to create test config")
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agentutil.NewTestLogger(sliceAppender)
Convey("all commands in test project should execute successfully", func() {
for _, task := range taskConfig.Project.Tasks {
So(len(task.Commands), ShouldNotEqual, 0)
for _, command := range task.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
}
}
err = os.RemoveAll(taskConfig.WorkDir)
testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir")
})
})
}
func TestAttachRawResults(t *testing.T) {
resetTasks(t)
testConfig := evergreen.TestConfig()
cwd := testutil.GetDirectoryOfFile()
Convey("With attachResults plugin installed into plugin registry", t, func() {
registry := plugin.NewSimpleRegistry()
attachPlugin := &AttachPlugin{}
err := registry.Register(attachPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")
server, err := service.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
configFile := filepath.Join(cwd, "testdata", "plugin_attach_results_raw.yml")
resultsLoc := filepath.Join(cwd, "testdata", "plugin_attach_results_raw.json")
taskConfig, err := plugintest.CreateTestConfig(configFile, t)
testutil.HandleTestingErr(err, t, "failed to create test config: %v")
taskConfig.WorkDir = "."
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agentutil.NewTestLogger(sliceAppender)
Convey("when attaching a raw log ", func() {
for _, projTask := range taskConfig.Project.Tasks {
So(len(projTask.Commands), ShouldNotEqual, 0)
for _, command := range projTask.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
// create a plugin communicator
pluginCom := &comm.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
Convey("when retrieving task", func() {
// fetch the task
testTask, err := task.FindOne(task.ById(httpCom.TaskId))
testutil.HandleTestingErr(err, t, "Couldn't find task")
So(testTask, ShouldNotBeNil)
Convey("test results should match and raw log should be in appropriate collection", func() {
reportFile, err := os.Open(resultsLoc)
testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err)
results := &task.TestResults{}
err = util.ReadJSONInto(reportFile, results)
testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err)
testResults := *results
So(len(testResults.Results), ShouldEqual, 3)
So(len(testTask.TestResults), ShouldEqual, 3)
firstResult := testTask.TestResults[0]
So(firstResult.LogRaw, ShouldEqual, "")
So(firstResult.LogId, ShouldNotEqual, "")
testLog, err := model.FindOneTestLogById(firstResult.LogId)
So(err, ShouldBeNil)
So(testLog.Lines[0], ShouldEqual, testResults.Results[0].LogRaw)
Convey("both URL and raw log should be stored appropriately if both exist", func() {
urlResult := testTask.TestResults[2]
So(urlResult.LogRaw, ShouldEqual, "")
So(urlResult.URL, ShouldNotEqual, "")
So(urlResult.LogId, ShouldNotEqual, "")
testLog, err := model.FindOneTestLogById(urlResult.LogId)
So(err, ShouldBeNil)
So(testLog.Lines[0], ShouldEqual, testResults.Results[2].LogRaw)
})
})
})
}
}
})
})
}
"github.com/evergreen-ci/evergreen/agent/comm"
agentutil "github.com/evergreen-ci/evergreen/agent/testutil"
"github.com/evergreen-ci/evergreen/model"
"github.com/evergreen-ci/evergreen/model/task"
"github.com/evergreen-ci/evergreen/plugin"
. "github.com/evergreen-ci/evergreen/plugin/builtin/attach"
"github.com/evergreen-ci/evergreen/plugin/plugintest"
"github.com/evergreen-ci/evergreen/service"
"github.com/evergreen-ci/evergreen/testutil"
. "github.com/smartystreets/goconvey/convey"
)
const TotalResultCount = 677
var (
workingDirectory = testutil.GetDirectoryOfFile()
SingleFileConfig = filepath.Join(workingDirectory, "testdata", "plugin_attach_xunit.yml")
WildcardConfig = filepath.Join(workingDirectory, "testdata", "plugin_attach_xunit_wildcard.yml")
)
// runTest abstracts away common tests and setup between all attach xunit tests.
// It also takes as an argument a function which runs any additional tests desired.
func runTest(t *testing.T, configPath string, customTests func()) {
resetTasks(t)
testConfig := evergreen.TestConfig()
Convey("With attachResults plugin installed into plugin registry", t, func() {
registry := plugin.NewSimpleRegistry()
attachPlugin := &AttachPlugin{}
err := registry.Register(attachPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")
func TestExpansions(t *testing.T) {
Convey("With a set of expansions", t, func() {
Convey("the expansions object should contain the values it is"+
" initialized with", func() {
expansions := NewExpansions(map[string]string{
"key1": "val1",
"key2": "val2",
})
So((*expansions)["key1"], ShouldEqual, "val1")
So((*expansions)["key2"], ShouldEqual, "val2")
})
Convey("updating the expansions with a map should update all of the"+
" specified values", func() {
expansions := NewExpansions(map[string]string{
"key1": "val1",
"key2": "val2",
})
expansions.Update(map[string]string{
"key2": "newval1",
"key3": "newval2",
})
So((*expansions)["key1"], ShouldEqual, "val1")
So((*expansions)["key2"], ShouldEqual, "newval1")
So((*expansions)["key3"], ShouldEqual, "newval2")
})
Convey("fetching an expansion should return the appropriate value,"+
" or the empty string if the value is not present", func() {
expansions := NewExpansions(map[string]string{
"key1": "val1",
"key2": "val2",
})
So(expansions.Get("key1"), ShouldEqual, "val1")
So(expansions.Get("key2"), ShouldEqual, "val2")
So(expansions.Get("key3"), ShouldEqual, "")
})
Convey("checking for a key's existence should return whether it is"+
" specified as an expansion", func() {
expansions := NewExpansions(map[string]string{
"key1": "val1",
"key2": "val2",
})
So(expansions.Exists("key1"), ShouldBeTrue)
So(expansions.Exists("key2"), ShouldBeTrue)
So(expansions.Exists("key3"), ShouldBeFalse)
})
Convey("updating from a yaml file should get keys copied over", func() {
expansions := NewExpansions(map[string]string{
"key1": "val1",
"key2": "val2",
})
err := expansions.UpdateFromYaml(filepath.Join(
testutil.GetDirectoryOfFile(), "testdata", "expansions.yml"))
So(err, ShouldBeNil)
So(expansions.Get("marge"), ShouldEqual, "1")
So(expansions.Get("lisa"), ShouldEqual, "2")
So(expansions.Get("bart"), ShouldEqual, "3")
So(expansions.Get("key1"), ShouldEqual, "blah")
})
})
}
func TestParserFunctionality(t *testing.T) {
var parser Parser
cwd := testutil.GetDirectoryOfFile()
Convey("With a simple log file and parser", t, func() {
logdata, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "1_simple.log"))
testutil.HandleTestingErr(err, t, "couldn't open log file")
parser = &VanillaParser{Suite: "test"}
Convey("running parse on the given log file should succeed", func() {
err = parser.Parse(bytes.NewBuffer(logdata))
So(err, ShouldBeNil)
Convey("and logs should be the correct length", func() {
logs := parser.Logs()
So(len(logs), ShouldEqual, 18)
})
Convey("and there should be one test result", func() {
results := parser.Results()
So(len(results), ShouldEqual, 2)
Convey("with the proper fields matching the original log file", func() {
So(results[0].Name, ShouldEqual, "TestFailures")
So(results[0].Status, ShouldEqual, FAIL)
rTime, _ := time.ParseDuration("5.02s")
So(results[0].RunTime, ShouldEqual, rTime)
So(results[0].StartLine, ShouldEqual, 1)
So(results[0].EndLine, ShouldEqual, 14)
So(results[0].SuiteName, ShouldEqual, "test")
So(results[1].Name, ShouldEqual, "TestFailures2")
So(results[1].Status, ShouldEqual, FAIL)
rTime, _ = time.ParseDuration("2.00s")
So(results[1].RunTime, ShouldEqual, rTime)
So(results[1].StartLine, ShouldEqual, 15)
So(results[1].EndLine, ShouldEqual, 15)
So(results[1].SuiteName, ShouldEqual, "test")
})
})
})
})
Convey("With a gocheck log file and parser", t, func() {
logdata, err := ioutil.ReadFile(filepath.Join(cwd, "testdata", "2_simple.log"))
testutil.HandleTestingErr(err, t, "couldn't open log file")
parser = &VanillaParser{Suite: "gocheck_test"}
Convey("running parse on the given log file should succeed", func() {
err = parser.Parse(bytes.NewBuffer(logdata))
So(err, ShouldBeNil)
Convey("and logs should be the correct length", func() {
logs := parser.Logs()
So(len(logs), ShouldEqual, 15)
})
Convey("and there should be three test results", func() {
results := parser.Results()
So(len(results), ShouldEqual, 3)
Convey("with the proper fields matching the original log file", func() {
So(results[0].Name, ShouldEqual, "MyTestName.SetUpTest")
So(results[0].Status, ShouldEqual, PASS)
rTime, _ := time.ParseDuration("0.576s")
So(results[0].RunTime, ShouldEqual, rTime)
So(results[0].StartLine, ShouldEqual, 2)
So(results[0].EndLine, ShouldEqual, 4)
So(results[0].SuiteName, ShouldEqual, "gocheck_test")
})
})
})
})
}
func TestXMLParsing(t *testing.T) {
cwd := testutil.GetDirectoryOfFile()
Convey("With some test xml files", t, func() {
Convey("with a basic test junit file", func() {
file, err := os.Open(filepath.Join(cwd, "testdata", "junit_1.xml"))
testutil.HandleTestingErr(err, t, "Error reading file")
defer file.Close()
Convey("the file should parse without error", func() {
res, err := ParseXMLResults(file)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
Convey("and have proper values decoded", func() {
So(res[0].Errors, ShouldEqual, 1)
So(res[0].Failures, ShouldEqual, 5)
So(res[0].Name, ShouldEqual, "nose2-junit")
So(res[0].TestCases[11].Name, ShouldEqual, "test_params_func:2")
So(res[0].TestCases[11].Time, ShouldEqual, 0.000098)
So(res[0].TestCases[11].Failure, ShouldNotBeNil)
So(res[0].TestCases[11].Failure.Message, ShouldEqual, "test failure")
})
})
})
Convey("with a more complex test junit file", func() {
file, err := os.Open(filepath.Join(cwd, "testdata", "junit_2.xml"))
testutil.HandleTestingErr(err, t, "Error reading file")
defer file.Close()
Convey("the file should parse without error", func() {
res, err := ParseXMLResults(file)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
Convey("and have proper values decoded", func() {
So(res[0].Errors, ShouldEqual, 1)
So(res[0].Failures, ShouldEqual, 1)
So(res[0].Name, ShouldEqual, "tests.ATest")
So(res[0].SysOut, ShouldContainSubstring, "here")
So(res[0].TestCases[0].Name, ShouldEqual, "error")
So(res[0].TestCases[0].ClassName, ShouldEqual, "tests.ATest")
So(res[0].TestCases[0].Time, ShouldEqual, 0.0060)
So(res[0].TestCases[0].Failure, ShouldBeNil)
So(res[0].TestCases[0].Error, ShouldNotBeNil)
So(res[0].TestCases[0].Error.Type, ShouldEqual, "java.lang.RuntimeException")
})
})
})
Convey(`with a "real" pymongo xunit file`, func() {
file, err := os.Open(filepath.Join(cwd, "testdata", "junit_3.xml"))
testutil.HandleTestingErr(err, t, "Error reading file")
defer file.Close()
Convey("the file should parse without error", func() {
res, err := ParseXMLResults(file)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
Convey("and have proper values decoded", func() {
So(res[0].Errors, ShouldEqual, 0)
So(res[0].Failures, ShouldEqual, 1)
So(res[0].Skip, ShouldEqual, 188)
So(res[0].Name, ShouldEqual, "nosetests")
So(res[0].TestCases[0].Name, ShouldEqual, "test_uri_options")
So(res[0].TestCases[0].ClassName, ShouldEqual, "test.test_auth.TestAuthURIOptions")
So(res[0].TestCases[0].Time, ShouldEqual, 0.002)
So(res[0].TestCases[0].Failure, ShouldBeNil)
So(res[0].TestCases[0].Error, ShouldBeNil)
So(res[0].TestCases[0].Skipped, ShouldNotBeNil)
So(res[0].TestCases[0].Skipped.Type, ShouldEqual, "unittest.case.SkipTest")
So(res[0].TestCases[0].Skipped.Content, ShouldContainSubstring,
"SkipTest: Authentication is not enabled on server")
})
})
})
Convey(`with a "real" java driver xunit file`, func() {
file, err := os.Open(filepath.Join(cwd, "testdata", "junit_4.xml"))
testutil.HandleTestingErr(err, t, "Error reading file")
defer file.Close()
Convey("the file should parse without error", func() {
res, err := ParseXMLResults(file)
So(err, ShouldBeNil)
So(len(res), ShouldBeGreaterThan, 0)
Convey("and have proper values decoded", func() {
So(res[0].Errors, ShouldEqual, 0)
So(res[0].Failures, ShouldEqual, 0)
So(res[0].Name, ShouldEqual, "com.mongodb.operation.InsertOperationSpecification")
So(res[0].TestCases[0].Name, ShouldEqual, "should return correct result")
So(res[0].SysOut, ShouldEqual, "out message")
So(res[0].SysErr, ShouldEqual, "error message")
})
})
})
})
}
func TestDepsMatrixIntegration(t *testing.T) {
Convey("With a sample matrix project mocking up a python driver", t, func() {
p := Project{}
bytes, err := ioutil.ReadFile(filepath.Join(testutil.GetDirectoryOfFile(),
"testdata", "matrix_deps.yml"))
So(err, ShouldBeNil)
Convey("the project should parse properly", func() {
err := LoadProjectInto(bytes, "deps", &p)
So(err, ShouldBeNil)
Convey("and contain the correct variants", func() {
So(len(p.BuildVariants), ShouldEqual, (1 + 3*3))
Convey("including a non-matrix variant", func() {
v := findRegularVariant(p.BuildVariants, "analysis")
So(v, ShouldNotBeNil)
ts := taskNames(v)
So(ts, ShouldContain, "pre-task")
So(ts, ShouldContain, "post-task")
So(*v.Stepback, ShouldBeFalse)
})
Convey("including linux/standalone", func() {
v := findMatrixVariant(p.BuildVariants, matrixValue{
"os": "linux",
"configuration": "standalone",
})
So(v, ShouldNotBeNil)
So(len(v.Tasks), ShouldEqual, 5)
So(v.Tags, ShouldContain, "posix")
Convey("which should contain a compile", func() {
So(v.Tasks[4].Name, ShouldEqual, "compile")
So(v.Tasks[4].Distros, ShouldResemble, []string{"linux_big"})
So(v.Tasks[4].DependsOn[0], ShouldResemble, TaskDependency{
Name: "pre-task",
Variant: "analysis",
})
})
})
Convey("including osx/repl", func() {
v := findMatrixVariant(p.BuildVariants, matrixValue{
"os": "osx",
"configuration": "repl",
})
So(v, ShouldNotBeNil)
So(len(v.Tasks), ShouldEqual, 4)
So(v.Tags, ShouldContain, "posix")
Convey("which should depend on another variant's compile", func() {
So(v.Tasks[0].Name, ShouldEqual, "test1")
So(v.Tasks[0].DependsOn[0].Name, ShouldEqual, "compile")
So(v.Tasks[0].DependsOn[0].Variant, ShouldNotEqual, "")
})
})
})
Convey("and contain the correct tasks", func() {
So(len(p.Tasks), ShouldEqual, 7)
Convey("such that post-task depends on everything", func() {
pt := ProjectTask{}
for _, t := range p.Tasks {
if t.Name == "post-task" {
pt = t
}
}
So(pt.Name, ShouldEqual, "post-task")
So(len(pt.DependsOn), ShouldEqual, 4*(3*3))
})
})
})
})
}
