// dBTestsWildcard are the database verification tests for globbed file execution
func dBTestsWildcard() {
task, err := model.FindTask("mocktaskid")
So(err, ShouldBeNil)
So(len(task.TestResults), ShouldEqual, TotalResultCount)
Convey("along with the proper logs", func() {
// junit_1.xml
tl := dBFindOneTestLog("pkg1.test.test_things.test_params_func:2")
So(tl.Lines[0], ShouldContainSubstring, "FAILURE")
So(tl.Lines[6], ShouldContainSubstring, "AssertionError")
tl = dBFindOneTestLog("pkg1.test.test_things.SomeTests.test_skippy")
So(tl.Lines[0], ShouldContainSubstring, "SKIPPED")
// junit_2.xml
tl = dBFindOneTestLog("tests.ATest.fail")
So(tl.Lines[0], ShouldContainSubstring, "FAILURE")
So(tl.Lines[1], ShouldContainSubstring, "AssertionFailedError")
// junit_3.xml
tl = dBFindOneTestLog(
"test.test_threads_replica_set_client.TestThreadsReplicaSet.test_safe_update",
)
So(tl.Lines[0], ShouldContainSubstring, "SKIPPED")
tl = dBFindOneTestLog("test.test_bson.TestBSON.test_basic_encode")
So(tl.Lines[0], ShouldContainSubstring, "AssertionError")
})
}
func TestTaskExecTimeout(t *testing.T) {
setupTlsConfigs(t)
for tlsString, tlsConfig := range tlsConfigs {
Convey("With agent running a slow test and live API server over "+tlsString, t, func() {
testTask, _, err := setupAPITestData(testConfig, "exec_timeout_task", "linux-64", NoPatch, t)
testutil.HandleTestingErr(err, t, "Failed to find test task")
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Expansions["api_httpscert"])
So(err, ShouldBeNil)
So(testAgent, ShouldNotBeNil)
Convey("after the slow test runs beyond the timeout threshold", func() {
// actually run the task.
// this function won't return until the whole thing is done.
testAgent.RunTask()
testAgent.APILogger.Flush()
time.Sleep(5 * time.Second)
printLogsForTask(testTask.Id)
Convey("the test should be marked as failed and timed out", func() {
So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the task-timeout script!"), ShouldBeTrue)
testTask, err = model.FindTask(testTask.Id)
So(testTask.Status, ShouldEqual, evergreen.TaskFailed)
So(testTask.Details.TimedOut, ShouldBeTrue)
So(testTask.Details.Description, ShouldEqual, "shell.exec")
})
})
})
}
}
func (as *APIServer) checkTask(checkSecret bool, next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
taskId := mux.Vars(r)["taskId"]
if taskId == "" {
as.LoggedError(w, r, http.StatusBadRequest, fmt.Errorf("missing task id"))
return
}
task, err := model.FindTask(taskId)
if err != nil {
as.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if task == nil {
as.LoggedError(w, r, http.StatusNotFound, fmt.Errorf("task not found"))
return
}
if checkSecret {
secret := r.Header.Get(evergreen.TaskSecretHeader)
// Check the secret - if it doesn't match, write error back to the client
if secret != task.Secret {
evergreen.Logger.Logf(slogger.ERROR, "Wrong secret sent for task %v: Expected %v but got %v", taskId, task.Secret, secret)
http.Error(w, "wrong secret!", http.StatusConflict)
return
}
}
context.Set(r, apiTaskKey, task)
// also set the task in the context visible to plugins
plugin.SetTask(r, task)
next(w, r)
}
}
func TestPatchPluginAPI(t *testing.T) {
testConfig := evergreen.TestConfig()
Convey("With a running api server and installed plugin", t, func() {
registry := plugin.NewSimpleRegistry()
gitPlugin := &GitPlugin{}
err := registry.Register(gitPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register patch plugin")
server, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, false)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
taskConfig, _ := plugintest.CreateTestConfig("testdata/plugin_patch.yml", t)
testCommand := GitApplyPatchCommand{"dir"}
_, _, err = plugintest.SetupAPITestData("testTask", true, t)
testutil.HandleTestingErr(err, t, "Couldn't set up test documents")
testTask, err := model.FindTask("testTaskId")
testutil.HandleTestingErr(err, t, "Couldn't set up test patch task")
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agent.NewTestLogger(sliceAppender)
Convey("calls to existing tasks with patches should succeed", func() {
httpCom := plugintest.TestAgentCommunicator(testTask.Id, testTask.Secret, server.URL)
pluginCom := &agent.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
So(err, ShouldBeNil)
So(patch, ShouldNotBeNil)
testutil.HandleTestingErr(db.Clear(version.Collection), t,
"unable to clear versions collection")
})
Convey("calls to non-existing tasks should fail", func() {
v := version.Version{Id: ""}
testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
httpCom := plugintest.TestAgentCommunicator("BAD_TASK_ID", "", server.URL)
pluginCom := &agent.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
So(err.Error(), ShouldContainSubstring, "not found")
So(err, ShouldNotBeNil)
So(patch, ShouldBeNil)
testutil.HandleTestingErr(db.Clear(version.Collection), t,
"unable to clear versions collection")
})
Convey("calls to existing tasks without patches should fail", func() {
noPatchTask := model.Task{Id: "noPatchTask", BuildId: "a"}
testutil.HandleTestingErr(noPatchTask.Insert(), t, "Couldn't insert patch task")
noPatchVersion := version.Version{Id: "noPatchVersion", BuildIds: []string{"a"}}
testutil.HandleTestingErr(noPatchVersion.Insert(), t, "Couldn't insert patch version")
v := version.Version{Id: ""}
testutil.HandleTestingErr(v.Insert(), t, "Couldn't insert dummy version")
httpCom := plugintest.TestAgentCommunicator(noPatchTask.Id, "", server.URL)
pluginCom := &agent.TaskJSONCommunicator{gitPlugin.Name(), httpCom}
patch, err := testCommand.GetPatch(taskConfig, pluginCom, logger)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "no patch found for task")
So(patch, ShouldBeNil)
testutil.HandleTestingErr(db.Clear(version.Collection), t,
"unable to clear versions collection")
})
})
}
func TestTaskEndEndpoint(t *testing.T) {
setupTlsConfigs(t)
for tlsString, tlsConfig := range tlsConfigs {
testTask, _, err := setupAPITestData(testConfig, "random", "linux-64", NoPatch, t)
testutil.HandleTestingErr(err, t, "Couldn't make test data: %v", err)
Convey("With a live api server, agent, and test task over "+tlsString, t, func() {
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := createAgent(testServer, testTask)
testutil.HandleTestingErr(err, t, "failed to create agent: %v")
testAgent.heartbeater.Interval = 10 * time.Second
testAgent.StartBackgroundActions(&NoopSignalHandler{})
Convey("calling end() should update task's/host's status properly "+
"and start running the next task", func() {
subsequentTaskId := testTask.Id + "Two"
details := &apimodels.TaskEndDetail{Status: evergreen.TaskSucceeded}
taskEndResp, err := testAgent.End(details)
time.Sleep(1 * time.Second)
So(err, ShouldBeNil)
taskUpdate, err := model.FindTask(testTask.Id)
So(err, ShouldBeNil)
So(taskUpdate.Status, ShouldEqual, evergreen.TaskSucceeded)
testHost, err := host.FindOne(host.ById(testTask.HostId))
So(err, ShouldBeNil)
So(testHost.RunningTask, ShouldEqual, subsequentTaskId)
taskUpdate, err = model.FindTask(subsequentTaskId)
So(err, ShouldBeNil)
So(taskUpdate.Status, ShouldEqual, evergreen.TaskDispatched)
So(taskEndResp, ShouldNotBeNil)
So(taskEndResp.RunNext, ShouldBeTrue)
So(taskEndResp.TaskId, ShouldEqual, subsequentTaskId)
})
})
}
}
func TestPatchTask(t *testing.T) {
setupTlsConfigs(t)
testConfig := evergreen.TestConfig()
db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
patchModes := []patchTestMode{InlinePatch, ExternalPatch}
testutil.ConfigureIntegrationTest(t, testConfig, "TestPatchTask")
for tlsString, tlsConfig := range tlsConfigs {
for _, testSetup := range testSetups {
Convey(testSetup.testSpec, t, func() {
Convey("With agent running a patched 'compile'"+tlsString, func() {
for _, mode := range patchModes {
Convey(fmt.Sprintf("Using patch mode %v", mode.String()), func() {
testTask, _, err := setupAPITestData(testConfig, "compile", "linux-64", mode, t)
testutil.HandleTestingErr(err, t, "Error setting up test data: %v", err)
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Expansions["api_httpscert"])
// actually run the task.
// this function won't return until the whole thing is done.
testAgent.RunTask()
time.Sleep(100 * time.Millisecond)
testAgent.APILogger.FlushAndWait()
printLogsForTask(testTask.Id)
Convey("all scripts in task should have been run successfully", func() {
So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "Cloning into") || // git 1.8
scanLogsForTask(testTask.Id, "Initialized empty Git repository"), // git 1.7
ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "i am patched!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "i am a patched module"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "i am compiling!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "i am sanity testing!"), ShouldBeTrue)
testTask, err = model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Error finding test task: %v", err)
So(testTask.Status, ShouldEqual, evergreen.TaskSucceeded)
})
})
}
})
})
}
}
}
func TestAttachResults(t *testing.T) {
resetTasks(t)
testConfig := evergreen.TestConfig()
Convey("With attachResults plugin installed into plugin registry", t, func() {
registry := plugin.NewSimpleRegistry()
attachPlugin := &AttachPlugin{}
err := registry.Register(attachPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")
server, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
configFile := "testdata/plugin_attach_results.yml"
resultsLoc := "testdata/plugin_attach_results.json"
taskConfig, err := plugintest.CreateTestConfig(configFile, t)
testutil.HandleTestingErr(err, t, "failed to create test config: %v")
taskConfig.WorkDir = "."
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agent.NewTestLogger(sliceAppender)
Convey("all commands in test project should execute successfully", func() {
for _, task := range taskConfig.Project.Tasks {
So(len(task.Commands), ShouldNotEqual, 0)
for _, command := range task.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
task, err := model.FindTask(httpCom.TaskId)
testutil.HandleTestingErr(err, t, "Couldn't find task")
So(task, ShouldNotBeNil)
// ensure test results are exactly as expected
// attempt to open the file
reportFile, err := os.Open(resultsLoc)
testutil.HandleTestingErr(err, t, "Couldn't open report file: '%v'", err)
results := &model.TestResults{}
err = util.ReadJSONInto(reportFile, results)
testutil.HandleTestingErr(err, t, "Couldn't read report file: '%v'", err)
testResults := *results
So(task.TestResults, ShouldResemble, testResults.Results)
testutil.HandleTestingErr(err, t, "Couldn't clean up test temp dir")
}
}
})
})
}
// DispatchTaskForHost assigns the task at the head of the task queue to the
// given host, dequeues the task and then marks it as dispatched for the host
func DispatchTaskForHost(taskQueue *model.TaskQueue, assignedHost *host.Host) (
nextTask *model.Task, err error) {
if assignedHost == nil {
return nil, fmt.Errorf("can not assign task to a nil host")
}
// only proceed if there are pending tasks left
for !taskQueue.IsEmpty() {
queueItem := taskQueue.NextTask()
// pin the task to the given host and fetch the full task document from
// the database
nextTask, err = model.FindTask(queueItem.Id)
if err != nil {
return nil, fmt.Errorf("error finding task with id %v: %v",
queueItem.Id, err)
}
if nextTask == nil {
return nil, fmt.Errorf("refusing to move forward because queued "+
"task with id %v does not exist", queueItem.Id)
}
// dequeue the task from the queue
if err = taskQueue.DequeueTask(nextTask.Id); err != nil {
return nil, fmt.Errorf("error pulling task with id %v from "+
"queue for distro %v: %v", nextTask.Id,
nextTask.DistroId, err)
}
// validate that the task can be run, if not fetch the next one in
// the queue
if shouldSkipTask(nextTask) {
evergreen.Logger.Logf(slogger.WARN, "Skipping task %v, which was "+
"picked up to be run but is not runnable - "+
"status (%v) activated (%v)", nextTask.Id, nextTask.Status,
nextTask.Activated)
continue
}
// record that the task was dispatched on the host
err = nextTask.MarkAsDispatched(assignedHost, time.Now())
if err != nil {
return nil, fmt.Errorf("error marking task %v as dispatched "+
"on host %v: %v", nextTask.Id, assignedHost.Id, err)
}
return nextTask, nil
}
return nil, nil
}
// dBTests are the database verification tests for standard one file execution
func dBTests() {
task, err := model.FindTask("mocktaskid")
So(err, ShouldBeNil)
So(len(task.TestResults), ShouldNotEqual, 0)
Convey("along with the proper logs", func() {
// junit_3.xml
tl := dBFindOneTestLog(
"test.test_threads_replica_set_client.TestThreadsReplicaSet.test_safe_update",
)
So(tl.Lines[0], ShouldContainSubstring, "SKIPPED")
tl = dBFindOneTestLog("test.test_bson.TestBSON.test_basic_encode")
So(tl.Lines[0], ShouldContainSubstring, "AssertionError")
})
}
func TestTaskFailures(t *testing.T) {
setupTlsConfigs(t)
testutil.ConfigureIntegrationTest(t, testConfig, "TestTaskFailures")
for tlsString, tlsConfig := range tlsConfigs {
for _, testSetup := range testSetups {
Convey(testSetup.testSpec, t, func() {
Convey("With agent running a failing test and live API server over "+tlsString, func() {
testTask, _, err := setupAPITestData(testConfig, "failing_task",
"linux-64", NoPatch, t)
testutil.HandleTestingErr(err, t, "Couldn't create test data: %v", err)
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := createAgent(testServer, testTask)
testutil.HandleTestingErr(err, t, "failed to create agent: %v")
// actually run the task.
// this function won't return until the whole thing is done.
testAgent.RunTask()
time.Sleep(100 * time.Millisecond)
testAgent.APILogger.FlushAndWait()
printLogsForTask(testTask.Id)
Convey("the pre and post-run scripts should have run", func() {
So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
Convey("the task should have run up until its first failure", func() {
So(scanLogsForTask(testTask.Id, "starting failing_task!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "done with failing_task!"), ShouldBeFalse)
})
Convey("the tasks's final status should be FAILED", func() {
testTask, err = model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Failed to find test task")
So(testTask.Status, ShouldEqual, evergreen.TaskFailed)
So(testTask.Details.Status, ShouldEqual, evergreen.TaskFailed)
So(testTask.Details.Description, ShouldEqual, "failing shell command")
So(testTask.Details.TimedOut, ShouldBeFalse)
So(testTask.Details.Type, ShouldEqual, model.SystemCommandType)
})
})
})
})
}
}
}
func TestTaskAbortion(t *testing.T) {
setupTlsConfigs(t)
testutil.ConfigureIntegrationTest(t, testConfig, "TestTaskAbortion")
for tlsString, tlsConfig := range tlsConfigs {
for _, testSetup := range testSetups {
Convey(testSetup.testSpec, t, func() {
Convey("With agent running a slow test and live API server over "+tlsString, func() {
testTask, _, err := setupAPITestData(testConfig, "very_slow_task", "linux-64", NoPatch, t)
testutil.HandleTestingErr(err, t, "Failed to find test task")
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := createAgent(testServer, testTask)
testutil.HandleTestingErr(err, t, "failed to create agent: %v")
Convey("when the abort signal is triggered on the task", func() {
go func() {
// Wait for a few seconds, then switch the task to aborted!
time.Sleep(3 * time.Second)
err := testTask.Abort("", true)
testutil.HandleTestingErr(err, t, "Failed to abort test task")
fmt.Println("aborted task.")
}()
// actually run the task.
// this function won't return until the whole thing is done.
_, err := testAgent.RunTask()
So(err, ShouldBeNil)
testAgent.APILogger.Flush()
time.Sleep(1 * time.Second)
printLogsForTask(testTask.Id)
Convey("the pre and post-run scripts should have run", func() {
So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "Received abort signal - stopping."), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "done with very_slow_task!"), ShouldBeFalse)
testTask, err = model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Failed to find test task")
So(testTask.Status, ShouldEqual, evergreen.TaskUndispatched)
})
})
})
})
}
}
}
// Returns a JSON response with the status of the specified task.
// The keys of the object are the test names.
func (restapi restAPI) getTaskStatus(w http.ResponseWriter, r *http.Request) {
taskId := mux.Vars(r)["task_id"]
task, err := model.FindTask(taskId)
if err != nil || task == nil {
msg := fmt.Sprintf("Error finding task '%v'", taskId)
statusCode := http.StatusNotFound
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
statusCode = http.StatusInternalServerError
}
restapi.WriteJSON(w, statusCode, responseError{Message: msg})
return
}
result := taskStatusContent{
Id: taskId,
Name: task.DisplayName,
Status: task.Status,
}
// Copy over the status details
result.StatusDetails.TimedOut = task.StatusDetails.TimedOut
result.StatusDetails.TimeoutStage = task.StatusDetails.TimeoutStage
// Copy over the test results
result.Tests = make(taskStatusByTest, len(task.TestResults))
for _, _testResult := range task.TestResults {
numSecs := _testResult.EndTime - _testResult.StartTime
testResult := taskTestResult{
Status: _testResult.Status,
TimeTaken: time.Duration(numSecs * float64(time.Second)),
Logs: taskTestLogURL{_testResult.URL},
}
result.Tests[_testResult.TestFile] = testResult
}
restapi.WriteJSON(w, http.StatusOK, result)
return
}
func (uis *UIServer) hostPage(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
vars := mux.Vars(r)
id := vars["host_id"]
h, err := host.FindOne(host.ById(id))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
if h == nil {
http.Error(w, "Host not found", http.StatusNotFound)
return
}
events, err := event.Find(event.MostRecentHostEvents(id, 50))
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
runningTask := &model.Task{}
if h.RunningTask != "" {
runningTask, err = model.FindTask(h.RunningTask)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
return
}
}
flashes := PopFlashes(uis.CookieStore, r, w)
uis.WriteHTML(w, http.StatusOK, struct {
Flashes []interface{}
Events []event.Event
Host *host.Host
RunningTask *model.Task
User *user.DBUser
ProjectData projectContext
}{flashes, events, h, runningTask, GetUser(r), projCtx},
"base", "host.html", "base_angular.html", "menu.html")
}
// populateTaskBuildVersion takes a task, build, and version ID and populates a projectContext
// with as many of the task, build, and version documents as possible.
// If any of the provided IDs is blank, they will be inferred from the more selective ones.
// Returns the project ID of the data found, which may be blank if the IDs are empty.
func (pc *projectContext) populateTaskBuildVersion(taskId, buildId, versionId string) (string, error) {
projectId := ""
var err error
// Fetch task if there's a task ID present; if we find one, populate build/version IDs from it
if len(taskId) > 0 {
pc.Task, err = model.FindTask(taskId)
if err != nil {
return "", err
}
if pc.Task != nil {
// override build and version ID with the ones this task belongs to
buildId = pc.Task.BuildId
versionId = pc.Task.Version
projectId = pc.Task.Project
}
}
// Fetch build if there's a build ID present; if we find one, populate version ID from it
if len(buildId) > 0 {
pc.Build, err = build.FindOne(build.ById(buildId))
if err != nil {
return "", err
}
if pc.Build != nil {
versionId = pc.Build.Version
projectId = pc.Build.Project
}
}
if len(versionId) > 0 {
pc.Version, err = version.FindOne(version.ById(versionId))
if err != nil {
return "", err
}
if pc.Version != nil {
projectId = pc.Version.Identifier
}
}
return projectId, nil
}
func getHostsData(includeSpawnedHosts bool) (*hostsData, error) {
data := &hostsData{}
// get all of the hosts
var dbHosts []host.Host
var err error
if includeSpawnedHosts {
dbHosts, err = host.Find(host.IsRunning)
} else {
dbHosts, err = host.Find(host.ByUserWithRunningStatus(evergreen.User))
}
if err != nil {
return nil, err
}
// convert the hosts to the ui models
uiHosts := make([]uiHost, len(dbHosts))
for idx, dbHost := range dbHosts {
// we only need the distro id for the hosts page
dbHost.Distro = distro.Distro{Id: dbHost.Distro.Id}
host := uiHost{
Host: dbHost,
RunningTask: nil,
}
uiHosts[idx] = host
// get the task running on this host
task, err := model.FindTask(dbHost.RunningTask)
if err != nil {
return nil, err
}
if task != nil {
uiHosts[idx].RunningTask = task
}
}
data.Hosts = uiHosts
return data, nil
}
// runTest abstracts away common tests and setup between all attach xunit tests.
// It also takes as an argument a function which runs any additional tests desired.
func runTest(t *testing.T, configPath string, customTests func()) {
resetTasks(t)
testConfig := evergreen.TestConfig()
Convey("With attachResults plugin installed into plugin registry", t, func() {
registry := plugin.NewSimpleRegistry()
attachPlugin := &AttachPlugin{}
err := registry.Register(attachPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin: %v")
server, err := apiserver.CreateTestServer(testConfig, nil, plugin.APIPlugins, true)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom := plugintest.TestAgentCommunicator("mocktaskid", "mocktasksecret", server.URL)
taskConfig, err := plugintest.CreateTestConfig(configPath, t)
testutil.HandleTestingErr(err, t, "failed to create test config: %v")
taskConfig.WorkDir = "."
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agent.NewTestLogger(sliceAppender)
Convey("all commands in test project should execute successfully", func() {
for _, task := range taskConfig.Project.Tasks {
So(len(task.Commands), ShouldNotEqual, 0)
for _, command := range task.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
task, err := model.FindTask(httpCom.TaskId)
testutil.HandleTestingErr(err, t, "Couldn't find task")
So(task, ShouldNotBeNil)
}
}
Convey("and the tests should be present in the db", customTests)
})
})
}
func TestTaskCallbackTimeout(t *testing.T) {
setupTlsConfigs(t)
for tlsString, tlsConfig := range tlsConfigs {
Convey("With an agent with callback_timeout_secs=1 and a live API server over "+tlsString, t, func() {
testTask, _, err := setupAPITestData(testConfig, "timeout_task", "linux-64", "testdata/config_test_plugin/project/evergreen-ci-render.yml", NoPatch, t)
testutil.HandleTestingErr(err, t, "Failed to find test task")
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Api.HttpsCert)
So(err, ShouldBeNil)
So(testAgent, ShouldNotBeNil)
prependConfigToVersion(t, testTask.Version, "callback_timeout_secs: 1\n")
Convey("after the slow test runs beyond the timeout threshold", func() {
// actually run the task.
// this function won't return until the whole thing is done.
testAgent.RunTask()
testAgent.APILogger.Flush()
time.Sleep(7 * time.Second)
//printLogsForTask(testTask.Id)
So(testAgent.taskConfig.Project.CallbackTimeout, ShouldEqual, 1)
Convey("the test should be marked as failed and timed out", func() {
So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the task-timeout script!"), ShouldBeTrue)
testTask, err = model.FindTask(testTask.Id)
So(testTask.Status, ShouldEqual, evergreen.TaskFailed)
So(testTask.Details.TimedOut, ShouldBeTrue)
So(testTask.Details.Description, ShouldEqual, "shell.exec")
Convey("and the timeout task should have failed", func() {
So(scanLogsForTask(testTask.Id, "running task-timeout commands in 1"), ShouldBeTrue)
})
})
})
})
}
}
// BuildFailuresSearchHandler handles the requests of searching jira in the build
// failures project
func (bbp *BuildBaronPlugin) buildFailuresSearch(w http.ResponseWriter, r *http.Request) {
taskId := mux.Vars(r)["task_id"]
task, err := model.FindTask(taskId)
if err != nil {
plugin.WriteJSON(w, http.StatusInternalServerError, err.Error())
return
}
jql := taskToJQL(task)
jiraHandler := thirdparty.NewJiraHandler(
bbp.opts.Host,
bbp.opts.Username,
bbp.opts.Password,
)
results, err := jiraHandler.JQLSearch(jql)
if err != nil {
message := fmt.Sprintf("%v: %v, %v", JIRAFailure, err, jql)
evergreen.Logger.Errorf(slogger.ERROR, message)
plugin.WriteJSON(w, http.StatusInternalServerError, message)
return
}
plugin.WriteJSON(w, http.StatusOK, results)
}
// loadAlertContext fetches details from the database for all documents that are associated with the
// AlertRequest. For example, it populates the task/build/version/project using the
// task/build/version/project ids in the alert request document.
func (qp *QueueProcessor) loadAlertContext(a *alert.AlertRequest) (*AlertContext, error) {
aCtx := &AlertContext{AlertRequest: a}
aCtx.Settings = qp.config
taskId, projectId, buildId, versionId := a.TaskId, a.ProjectId, a.BuildId, a.VersionId
patchId := a.PatchId
var err error
if len(a.HostId) > 0 {
aCtx.Host, err = host.FindOne(host.ById(a.HostId))
if err != nil {
return nil, err
}
}
// Fetch task if there's a task ID present; if we find one, populate build/version IDs from it
if len(taskId) > 0 {
aCtx.Task, err = model.FindTask(taskId)
if err != nil {
return nil, err
}
if aCtx.Task != nil && aCtx.Task.Execution != a.Execution {
oldTaskId := fmt.Sprintf("%s_%v", taskId, a.Execution)
aCtx.Task, err = model.FindOneOldTask(bson.M{"_id": oldTaskId}, db.NoProjection, db.NoSort)
if err != nil {
return nil, err
}
}
if aCtx.Task != nil {
// override build and version ID with the ones this task belongs to
buildId = aCtx.Task.BuildId
versionId = aCtx.Task.Version
projectId = aCtx.Task.Project
aCtx.FailedTests = []model.TestResult{}
for _, test := range aCtx.Task.TestResults {
if test.Status == "fail" {
aCtx.FailedTests = append(aCtx.FailedTests, test)
}
}
}
}
// Fetch build if there's a build ID present; if we find one, populate version ID from it
if len(buildId) > 0 {
aCtx.Build, err = build.FindOne(build.ById(buildId))
if err != nil {
return nil, err
}
if aCtx.Build != nil {
versionId = aCtx.Build.Version
projectId = aCtx.Build.Project
}
}
if len(versionId) > 0 {
aCtx.Version, err = version.FindOne(version.ById(versionId))
if err != nil {
return nil, err
}
if aCtx.Version != nil {
projectId = aCtx.Version.Identifier
}
}
if len(patchId) > 0 {
if !patch.IsValidId(patchId) {
return nil, fmt.Errorf("patch id '%v' is not an object id", patchId)
}
aCtx.Patch, err = patch.FindOne(patch.ById(patch.NewId(patchId)).Project(patch.ExcludePatchDiff))
} else if aCtx.Version != nil {
// patch isn't in URL but the version in context has one, get it
aCtx.Patch, err = patch.FindOne(patch.ByVersion(aCtx.Version.Id).Project(patch.ExcludePatchDiff))
}
// If there's a finalized patch loaded into context but not a version, load the version
// associated with the patch as the context's version.
if aCtx.Version == nil && aCtx.Patch != nil && aCtx.Patch.Version != "" {
aCtx.Version, err = version.FindOne(version.ById(aCtx.Patch.Version).WithoutFields(version.ConfigKey))
if err != nil {
return nil, err
}
}
if len(projectId) > 0 {
aCtx.ProjectRef, err = qp.findProject(projectId)
if err != nil {
return nil, err
}
}
return aCtx, nil
}
func TestPushTask(t *testing.T) {
testConfig := evergreen.TestConfig()
setupTlsConfigs(t)
db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
testutil.ConfigureIntegrationTest(t, testConfig, "TestPushTask")
for tlsString, tlsConfig := range tlsConfigs {
for _, testSetup := range testSetups {
Convey(testSetup.testSpec, t, func() {
Convey("With agent running a push task "+tlsString, func() {
testTask, _, err := setupAPITestData(testConfig, evergreen.PushStage,
"linux-64", false, t)
testutil.HandleTestingErr(err, t, "Error setting up test data: %v", err)
testutil.HandleTestingErr(db.ClearCollections(artifact.Collection), t, "can't clear files collection")
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.Published, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := New(testServer.URL, testTask.Id, testTask.Secret, "", testConfig.Expansions["api_httpscert"])
testutil.HandleTestingErr(err, t, "Error making test agent: %v", err)
// actually run the task.
// this function won't return until the whole thing is done.
testAgent.RunTask()
time.Sleep(100 * time.Millisecond)
testAgent.APILogger.FlushAndWait()
printLogsForTask(testTask.Id)
newDate := testAgent.taskConfig.Expansions.Get("new_date")
Convey("all scripts in task should have been run successfully", func() {
So(scanLogsForTask(testTask.Id, "executing the pre-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "push task pre-run!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "push task post-run!"), ShouldBeTrue)
Convey("s3.put attaches task file properly", func() {
entry, err := artifact.FindOne(artifact.ByTaskId(testTask.Id))
So(err, ShouldBeNil)
So(len(entry.Files), ShouldEqual, 2)
for _, element := range entry.Files {
So(element.Name, ShouldNotEqual, "")
}
So(entry.Files[0].Name, ShouldEqual, "push_file")
link := "https://s3.amazonaws.com/build-push-testing/pushtest-stage/unittest-testTaskId-DISTRO_EXP-BUILDVAR_EXP-FILE_EXP.txt"
So(entry.Files[0].Link, ShouldEqual, link)
})
Convey("s3.copy attached task file properly", func() {
entry, err := artifact.FindOne(artifact.ByTaskId(testTask.Id))
So(err, ShouldBeNil)
So(len(entry.Files), ShouldNotEqual, 0)
So(entry.Files[0].Name, ShouldEqual, "push_file")
So(entry.Files[1].Name, ShouldEqual, "copy_file")
So(entry.Files[0].Link, ShouldEqual, "https://s3.amazonaws.com/build-push-testing/pushtest-stage/unittest-testTaskId-DISTRO_EXP-BUILDVAR_EXP-FILE_EXP.txt")
So(entry.Files[1].Link, ShouldEqual,
"https://s3.amazonaws.com/build-push-testing/pushtest/unittest-DISTRO_EXP-BUILDVAR_EXP-FILE_EXP-latest.txt")
})
testTask, err = model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Error finding test task: %v", err)
So(testTask.Status, ShouldEqual, evergreen.TaskSucceeded)
// Check the file written to s3 is what we expected
auth := &aws.Auth{
AccessKey: testConfig.Providers.AWS.Id,
SecretKey: testConfig.Providers.AWS.Secret,
}
// check the staging location first
filebytes, err := getS3FileBytes(auth, "build-push-testing", "/pushtest-stage/unittest-testTaskId-DISTRO_EXP-BUILDVAR_EXP-FILE_EXP.txt")
testutil.HandleTestingErr(err, t, "Failed to get file from s3: %v", err)
So(string(filebytes), ShouldEqual, newDate+"\n")
// now check remote location (after copy)
filebytes, err = getS3FileBytes(auth, "build-push-testing", "/pushtest/unittest-DISTRO_EXP-BUILDVAR_EXP-FILE_EXP-latest.txt")
testutil.HandleTestingErr(err, t, "Failed to get remote file from s3: %v", err)
So(string(filebytes), ShouldEqual, newDate+"\n")
})
})
})
}
}
}
// avoids type-checking json params for the below function
func (uis *UIServer) taskModify(w http.ResponseWriter, r *http.Request) {
projCtx := MustHaveProjectContext(r)
if projCtx.Task == nil {
http.Error(w, "Not Found", http.StatusNotFound)
return
}
reqBody, err := ioutil.ReadAll(r.Body)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
defer r.Body.Close()
putParams := struct {
Action string `json:"action"`
Priority string `json:"priority"`
// for the set_active option
Active bool `json:"active"`
}{}
err = json.Unmarshal(reqBody, &putParams)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
authUser := GetUser(r)
authName := authUser.DisplayName()
// determine what action needs to be taken
switch putParams.Action {
case "restart":
if err := projCtx.Task.TryReset(authName, evergreen.UIPackage, projCtx.Project, nil); err != nil {
http.Error(w, fmt.Sprintf("Error restarting task %v: %v", projCtx.Task.Id, err), http.StatusInternalServerError)
return
}
// Reload the task from db, send it back
projCtx.Task, err = model.FindTask(projCtx.Task.Id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
}
uis.WriteJSON(w, http.StatusOK, projCtx.Task)
return
case "abort":
if err := projCtx.Task.Abort(authName, true); err != nil {
http.Error(w, fmt.Sprintf("Error aborting task %v: %v", projCtx.Task.Id, err), http.StatusInternalServerError)
return
}
// Reload the task from db, send it back
projCtx.Task, err = model.FindTask(projCtx.Task.Id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
}
uis.WriteJSON(w, http.StatusOK, projCtx.Task)
return
case "set_active":
active := putParams.Active
if err := model.SetTaskActivated(projCtx.Task.Id, authName, active); err != nil {
http.Error(w, fmt.Sprintf("Error activating task %v: %v", projCtx.Task.Id, err),
http.StatusInternalServerError)
return
}
// Reload the task from db, send it back
projCtx.Task, err = model.FindTask(projCtx.Task.Id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
}
uis.WriteJSON(w, http.StatusOK, projCtx.Task)
return
case "set_priority":
priority, err := strconv.ParseInt(putParams.Priority, 10, 64)
if err != nil {
http.Error(w, "Bad priority value, must be int", http.StatusBadRequest)
return
}
if err = projCtx.Task.SetPriority(priority); err != nil {
http.Error(w, fmt.Sprintf("Error setting task priority %v: %v", projCtx.Task.Id, err), http.StatusInternalServerError)
return
}
// Reload the task from db, send it back
projCtx.Task, err = model.FindTask(projCtx.Task.Id)
if err != nil {
uis.LoggedError(w, r, http.StatusInternalServerError, err)
}
uis.WriteJSON(w, http.StatusOK, projCtx.Task)
return
default:
uis.WriteJSON(w, http.StatusBadRequest, "Unrecognized action: "+putParams.Action)
}
}
func TestGotestPluginOnPassingTests(t *testing.T) {
SkipConvey("With gotest plugin installed into plugin registry", t, func() {
reset(t)
testConfig := evergreen.TestConfig()
testutil.ConfigureIntegrationTest(t, testConfig, "TestGotestPluginOnPassingTests")
registry := plugin.NewSimpleRegistry()
testPlugin := &GotestPlugin{}
err := registry.Register(testPlugin)
testutil.HandleTestingErr(err, t, "Couldn't register plugin %v")
server, err := apiserver.CreateTestServer(evergreen.TestConfig(), nil, plugin.APIPlugins, true)
testutil.HandleTestingErr(err, t, "Couldn't set up testing server")
httpCom := plugintest.TestAgentCommunicator("testTaskId", "testTaskSecret", server.URL)
sliceAppender := &evergreen.SliceAppender{[]*slogger.Log{}}
logger := agent.NewTestLogger(sliceAppender)
Convey("all commands in test project should execute successfully", func() {
curWD, err := os.Getwd()
testutil.HandleTestingErr(err, t, "Couldn't get working directory: %v")
taskConfig, err := plugintest.CreateTestConfig("testdata/good.yml", t)
// manually override working directory to the main repo, since this
// is much easier than copying over the required testing dependencies
// to a temporary directory
testutil.HandleTestingErr(err, t, "Couldn't set up test config %v")
taskConfig.WorkDir = curWD
task, _, err := plugintest.SetupAPITestData("testTask", false, t)
testutil.HandleTestingErr(err, t, "Couldn't set up test documents")
for _, task := range taskConfig.Project.Tasks {
So(len(task.Commands), ShouldNotEqual, 0)
for _, command := range task.Commands {
pluginCmds, err := registry.GetCommands(command, taskConfig.Project.Functions)
testutil.HandleTestingErr(err, t, "Couldn't get plugin command: %v")
So(pluginCmds, ShouldNotBeNil)
So(err, ShouldBeNil)
pluginCom := &agent.TaskJSONCommunicator{pluginCmds[0].Plugin(), httpCom}
err = pluginCmds[0].Execute(logger, pluginCom, taskConfig, make(chan bool))
So(err, ShouldBeNil)
}
}
Convey("and the tests in the task should be updated", func() {
updatedTask, err := model.FindTask(task.Id)
So(err, ShouldBeNil)
So(updatedTask, ShouldNotBeNil)
So(len(updatedTask.TestResults), ShouldEqual, 2)
So(updatedTask.TestResults[0].Status, ShouldEqual, "pass")
So(updatedTask.TestResults[1].Status, ShouldEqual, "pass")
So(updatedTask.TestResults[0].TestFile, ShouldEqual, "TestPass01")
So(updatedTask.TestResults[1].TestFile, ShouldEqual, "TestPass02")
So(updatedTask.TestResults[0].StartTime, ShouldBeLessThan,
updatedTask.TestResults[0].EndTime)
So(updatedTask.TestResults[1].StartTime, ShouldBeLessThan,
updatedTask.TestResults[1].EndTime)
Convey("with relevant logs present in the DB as well", func() {
log, err := model.FindOneTestLog("0_goodpkg", "testTaskId", 0)
So(log, ShouldNotBeNil)
So(err, ShouldBeNil)
So(log.Lines[0], ShouldContainSubstring, "TestPass01")
})
})
})
})
}
// Returns a JSON response with the marshalled output of the task
// specified in the request.
func (restapi restAPI) getTaskInfo(w http.ResponseWriter, r *http.Request) {
taskId := mux.Vars(r)["task_id"]
srcTask, err := model.FindTask(taskId)
if err != nil || srcTask == nil {
msg := fmt.Sprintf("Error finding task '%v'", taskId)
statusCode := http.StatusNotFound
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
statusCode = http.StatusInternalServerError
}
restapi.WriteJSON(w, statusCode, responseError{Message: msg})
return
}
destTask := &task{}
destTask.Id = srcTask.Id
destTask.CreateTime = srcTask.CreateTime
destTask.ScheduledTime = srcTask.ScheduledTime
destTask.DispatchTime = srcTask.DispatchTime
destTask.StartTime = srcTask.StartTime
destTask.FinishTime = srcTask.FinishTime
destTask.PushTime = srcTask.PushTime
destTask.Version = srcTask.Version
destTask.Project = srcTask.Project
destTask.Revision = srcTask.Revision
destTask.Priority = srcTask.Priority
destTask.LastHeartbeat = srcTask.LastHeartbeat
destTask.Activated = srcTask.Activated
destTask.BuildId = srcTask.BuildId
destTask.DistroId = srcTask.DistroId
destTask.BuildVariant = srcTask.BuildVariant
destTask.DependsOn = srcTask.DependsOn
destTask.DisplayName = srcTask.DisplayName
destTask.HostId = srcTask.HostId
destTask.Restarts = srcTask.Restarts
destTask.Execution = srcTask.Execution
destTask.Archived = srcTask.Archived
destTask.RevisionOrderNumber = srcTask.RevisionOrderNumber
destTask.Requester = srcTask.Requester
destTask.Status = srcTask.Status
destTask.Aborted = srcTask.Aborted
destTask.TimeTaken = srcTask.TimeTaken
destTask.ExpectedDuration = srcTask.ExpectedDuration
destTask.MinQueuePos = srcTask.MinQueuePos
// Copy over the status details
destTask.StatusDetails.TimedOut = srcTask.Details.TimedOut
destTask.StatusDetails.TimeoutStage = srcTask.Details.Description
// Copy over the test results
destTask.TestResults = make(taskTestResultsByName, len(srcTask.TestResults))
for _, _testResult := range srcTask.TestResults {
numSecs := _testResult.EndTime - _testResult.StartTime
testResult := taskTestResult{
Status: _testResult.Status,
TimeTaken: time.Duration(numSecs * float64(time.Second)),
Logs: taskTestLogURL{_testResult.URL},
}
destTask.TestResults[_testResult.TestFile] = testResult
}
// Copy over artifacts and binaries
entries, err := artifact.FindAll(artifact.ByTaskId(taskId))
if err != nil {
msg := fmt.Sprintf("Error finding task '%v'", taskId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
for _, entry := range entries {
for _, _file := range entry.Files {
file := taskFile{
Name: _file.Name,
URL: _file.Link,
}
destTask.Files = append(destTask.Files, file)
}
}
restapi.WriteJSON(w, http.StatusOK, destTask)
return
}
func TestTaskSuccess(t *testing.T) {
setupTlsConfigs(t)
testutil.ConfigureIntegrationTest(t, testConfig, "TestTaskSuccess")
for tlsString, tlsConfig := range tlsConfigs {
for _, testSetup := range testSetups {
for _, variant := range buildVariantsToTest {
Convey(testSetup.testSpec, t, func() {
Convey("With agent running 'compile' step and live API server over "+
tlsString+" with variant "+variant, func() {
testTask, _, err := setupAPITestData(testConfig, "compile", variant, NoPatch, t)
testutil.HandleTestingErr(err, t, "Couldn't create test task: %v", err)
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := createAgent(testServer, testTask)
testutil.HandleTestingErr(err, t, "failed to create agent: %v")
// actually run the task.
// this function won't return until the whole thing is done.
testAgent.RunTask()
Convey("expansions should be fetched", func() {
So(testAgent.taskConfig.Expansions.Get("aws_key"), ShouldEqual, testConfig.Providers.AWS.Id)
So(scanLogsForTask(testTask.Id, "fetch_expansion_value"), ShouldBeTrue)
})
time.Sleep(100 * time.Millisecond)
testAgent.APILogger.FlushAndWait()
printLogsForTask(testTask.Id)
Convey("all scripts in task should have been run successfully", func() {
So(scanLogsForTask(testTask.Id, "Executing script: echo \"predefined command!\""), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "predefined command!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "this should not end up in the logs"), ShouldBeFalse)
So(scanLogsForTask(testTask.Id, "Command timeout set to 21m40s"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "Command timeout set to 43m20s"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "Cloning into") || // git 1.8
scanLogsForTask(testTask.Id, "Initialized empty Git repository"), // git 1.7
ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "i am compiling!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "i am sanity testing!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "Skipping command 'git.apply_patch' on variant"), ShouldBeTrue)
// Check that functions with args are working correctly
So(scanLogsForTask(testTask.Id, "arg1 is FOO"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "arg2 is BAR"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "arg3 is Expanded: qux"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "arg4 is Default: default_value"), ShouldBeTrue)
// Check that multi-command functions are working correctly
So(scanLogsForTask(testTask.Id, "step 1 of multi-command func"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "step 2 of multi-command func"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "step 3 of multi-command func"), ShouldBeTrue)
// Check that logging output is only flushing on a newline
So(scanLogsForTask(testTask.Id, "this should be on the same line...as this."), ShouldBeTrue)
testTask, err = model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Couldn't find test task: %v", err)
So(testTask.Status, ShouldEqual, evergreen.TaskSucceeded)
// use function display name as description when none is specified in command
So(testTask.Details.Status, ShouldEqual, evergreen.TaskSucceeded)
So(testTask.Details.Description, ShouldEqual, `'shell.exec' in "silent shell test"`)
So(testTask.Details.TimedOut, ShouldBeFalse)
So(testTask.Details.Type, ShouldEqual, model.SystemCommandType)
})
Convey("manifest should be created", func() {
m, err := manifest.FindOne(manifest.ById(testTask.Version))
So(testTask.Version, ShouldEqual, "testVersionId")
So(err, ShouldBeNil)
So(m, ShouldNotBeNil)
So(m.ProjectName, ShouldEqual, testTask.Project)
So(m.Id, ShouldEqual, testTask.Version)
So(m.Modules, ShouldNotBeEmpty)
So(m.Modules["recursive"], ShouldNotBeNil)
So(m.Modules["recursive"].URL, ShouldNotEqual, "")
})
})
Convey("With agent running a regular test and live API server over "+
tlsString+" on variant "+variant, func() {
testTask, _, err := setupAPITestData(testConfig, "normal_task", variant, NoPatch, t)
testutil.HandleTestingErr(err, t, "Couldn't create test data: %v", err)
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := createAgent(testServer, testTask)
testutil.HandleTestingErr(err, t, "failed to create agent: %v")
// actually run the task.
// this function won't return until the whole thing is done.
testAgent.RunTask()
time.Sleep(100 * time.Millisecond)
testAgent.APILogger.FlushAndWait()
Convey("all scripts in task should have been run successfully", func() {
So(scanLogsForTask(testTask.Id, "executing the pre-run script"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "executing the post-run script!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "starting normal_task!"), ShouldBeTrue)
So(scanLogsForTask(testTask.Id, "done with normal_task!"), ShouldBeTrue)
testTask, err = model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Couldn't find test task: %v", err)
So(testTask.Status, ShouldEqual, evergreen.TaskSucceeded)
expectedResults := []model.TestResult{
model.TestResult{
Status: "success",
TestFile: "t1",
URL: "url",
ExitCode: 0,
StartTime: 0,
EndTime: 10,
},
}
So(testTask.TestResults, ShouldResemble, expectedResults)
})
})
})
}
}
}
}
func TestBasicEndpoints(t *testing.T) {
setupTlsConfigs(t)
for tlsString, tlsConfig := range tlsConfigs {
testTask, _, err := setupAPITestData(testConfig, "task", "linux-64", NoPatch, t)
testutil.HandleTestingErr(err, t, "Couldn't make test data: %v", err)
Convey("With a live api server, agent, and test task over "+tlsString, t, func() {
testServer, err := apiserver.CreateTestServer(testConfig, tlsConfig, plugin.APIPlugins, Verbose)
testutil.HandleTestingErr(err, t, "Couldn't create apiserver: %v", err)
testAgent, err := createAgent(testServer, testTask)
testutil.HandleTestingErr(err, t, "failed to create agent: %v")
Convey("sending logs should store the log messages properly", func() {
msg1 := "task logger initialized!"
msg2 := "system logger initialized!"
msg3 := "exec logger initialized!"
testAgent.logger.LogTask(slogger.INFO, msg1)
testAgent.logger.LogSystem(slogger.INFO, msg2)
testAgent.logger.LogExecution(slogger.INFO, msg3)
time.Sleep(100 * time.Millisecond)
testAgent.APILogger.FlushAndWait()
// This returns logs in order of NEWEST first.
logMessages, err := model.FindMostRecentLogMessages(testTask.Id, 0, 10, []string{}, []string{})
testutil.HandleTestingErr(err, t, "failed to get log msgs")
So(logMessages[2].Message, ShouldEndWith, msg1)
So(logMessages[1].Message, ShouldEndWith, msg2)
So(logMessages[0].Message, ShouldEndWith, msg3)
Convey("Task endpoints should work between agent and server", func() {
testAgent.StartBackgroundActions(&NoopSignalHandler{})
Convey("calling GetTask should get retrieve same task back", func() {
testTaskFromApi, err := testAgent.GetTask()
So(err, ShouldBeNil)
// ShouldResemble doesn't seem to work here, possibly because of
// omitempty? anyways, just assert equality of the important fields
So(testTaskFromApi.Id, ShouldEqual, testTask.Id)
So(testTaskFromApi.Status, ShouldEqual, testTask.Status)
So(testTaskFromApi.HostId, ShouldEqual, testTask.HostId)
})
Convey("calling start should flip the task's status to started", func() {
err := testAgent.Start("1")
testutil.HandleTestingErr(err, t, "Couldn't start task: %v", err)
testTask, err := model.FindTask(testTask.Id)
testutil.HandleTestingErr(err, t, "Couldn't refresh task from db: %v", err)
So(testTask.Status, ShouldEqual, evergreen.TaskStarted)
testHost, err := host.FindOne(host.ByRunningTaskId(testTask.Id))
So(err, ShouldBeNil)
So(testHost.Id, ShouldEqual, "testHost")
So(testHost.RunningTask, ShouldEqual, testTask.Id)
})
Convey("calling end() should update task status properly", func() {
commandType := model.SystemCommandType
description := "random"
details := &apimodels.TaskEndDetail{
Description: description,
Type: commandType,
TimedOut: true,
Status: evergreen.TaskSucceeded,
}
testAgent.End(details)
time.Sleep(100 * time.Millisecond)
taskUpdate, err := model.FindTask(testTask.Id)
So(err, ShouldBeNil)
So(taskUpdate.Status, ShouldEqual, evergreen.TaskSucceeded)
So(taskUpdate.Details.Description, ShouldEqual, description)
So(taskUpdate.Details.Type, ShouldEqual, commandType)
So(taskUpdate.Details.TimedOut, ShouldEqual, true)
})
Convey("no checkins should trigger timeout signal", func() {
testAgent.idleTimeoutWatcher.SetDuration(2 * time.Second)
testAgent.idleTimeoutWatcher.CheckIn()
// sleep long enough for the timeout watcher to time out
time.Sleep(3 * time.Second)
timeoutSignal, ok := <-testAgent.signalHandler.idleTimeoutChan
So(ok, ShouldBeTrue)
So(timeoutSignal, ShouldEqual, IdleTimeout)
})
})
})
})
}
}
// Returns a JSON response with the marshalled output of the task
// specified in the request.
func (restapi restAPI) getTaskInfo(w http.ResponseWriter, r *http.Request) {
taskId := mux.Vars(r)["task_id"]
srcTask, err := model.FindTask(taskId)
if err != nil || srcTask == nil {
msg := fmt.Sprintf("Error finding task '%v'", taskId)
statusCode := http.StatusNotFound
if err != nil {
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
statusCode = http.StatusInternalServerError
}
restapi.WriteJSON(w, statusCode, responseError{Message: msg})
return
}
destTask := &task{}
// Copy the contents from the database into our local task type
err = angier.TransferByFieldNames(srcTask, destTask)
if err != nil {
msg := fmt.Sprintf("Error finding task '%v'", taskId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
// Copy over the status details
destTask.StatusDetails.TimedOut = srcTask.StatusDetails.TimedOut
destTask.StatusDetails.TimeoutStage = srcTask.StatusDetails.TimeoutStage
// Copy over the test results
destTask.TestResults = make(taskTestResultsByName, len(srcTask.TestResults))
for _, _testResult := range srcTask.TestResults {
numSecs := _testResult.EndTime - _testResult.StartTime
testResult := taskTestResult{
Status: _testResult.Status,
TimeTaken: time.Duration(numSecs * float64(time.Second)),
Logs: taskTestLogURL{_testResult.URL},
}
destTask.TestResults[_testResult.TestFile] = testResult
}
// Copy over artifacts and binaries
entries, err := artifact.FindAll(artifact.ByTaskId(taskId))
if err != nil {
msg := fmt.Sprintf("Error finding task '%v'", taskId)
evergreen.Logger.Logf(slogger.ERROR, "%v: %v", msg, err)
restapi.WriteJSON(w, http.StatusInternalServerError, responseError{Message: msg})
return
}
for _, entry := range entries {
for _, _file := range entry.Files {
file := taskFile{
Name: _file.Name,
URL: _file.Link,
}
destTask.Files = append(destTask.Files, file)
}
}
restapi.WriteJSON(w, http.StatusOK, destTask)
return
}
func TestCleanupTask(t *testing.T) {
testConfig := evergreen.TestConfig()
db.SetGlobalSessionProvider(db.SessionFactoryFromConfig(testConfig))
Convey("When cleaning up a task", t, func() {
// reset the db
testutil.HandleTestingErr(db.ClearCollections(model.TasksCollection),
t, "error clearing tasks collection")
testutil.HandleTestingErr(db.ClearCollections(host.Collection),
t, "error clearing hosts collection")
Convey("an error should be thrown if the passed-in projects slice"+
" does not contain the task's project", func() {
wrapper := doomedTaskWrapper{
task: model.Task{
Project: "proj",
},
}
projects := map[string]model.Project{}
err := cleanUpTask(wrapper, projects)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "could not find project")
})
Convey("an error should be thrown if the task's host is marked with"+
" the wrong running task id", func() {
wrapper := doomedTaskWrapper{
task: model.Task{
Id: "t1",
HostId: "h1",
Project: "proj",
},
}
projects := map[string]model.Project{
"proj": model.Project{
Identifier: "proj",
},
}
host := &host.Host{
Id: "h1",
RunningTask: "nott1",
}
So(host.Insert(), ShouldBeNil)
err := cleanUpTask(wrapper, projects)
So(err, ShouldNotBeNil)
So(err.Error(), ShouldContainSubstring, "but the host thinks it"+
" is running task")
})
Convey("if the task's heartbeat timed out", func() {
// reset the db
testutil.HandleTestingErr(db.ClearCollections(model.TasksCollection),
t, "error clearing tasks collection")
testutil.HandleTestingErr(db.ClearCollections(host.Collection),
t, "error clearing hosts collection")
testutil.HandleTestingErr(db.ClearCollections(build.Collection),
t, "error clearing builds collection")
testutil.HandleTestingErr(db.ClearCollections(model.OldTasksCollection),
t, "error clearing old tasks collection")
testutil.HandleTestingErr(db.ClearCollections(version.Collection),
t, "error clearing versions collection")
Convey("the task should be reset", func() {
task := &model.Task{
Id: "t1",
Status: "started",
HostId: "h1",
BuildId: "b1",
Project: "proj",
Restarts: 1,
}
testutil.HandleTestingErr(task.Insert(), t, "error inserting task")
wrapper := doomedTaskWrapper{
reason: HeartbeatTimeout,
task: *task,
}
projects := map[string]model.Project{
"proj": model.Project{
Identifier: "proj",
Stepback: false,
},
}
host := &host.Host{
Id: "h1",
RunningTask: "t1",
}
So(host.Insert(), ShouldBeNil)
build := &build.Build{
Id: "b1",
Tasks: []build.TaskCache{{Id: "t1"}},
Version: "v1",
}
So(build.Insert(), ShouldBeNil)
v := &version.Version{
Id: "v1",
}
So(v.Insert(), ShouldBeNil)
// cleaning up the task should work
So(cleanUpTask(wrapper, projects), ShouldBeNil)
// refresh the task - it should be reset
task, err := model.FindTask("t1")
So(err, ShouldBeNil)
So(task.Status, ShouldEqual, evergreen.TaskUndispatched)
So(task.Restarts, ShouldEqual, 2)
})
Convey("the running task field on the task's host should be"+
" reset", func() {
task := &model.Task{
Id: "t1",
Status: "started",
HostId: "h1",
BuildId: "b1",
Project: "proj",
Restarts: 1,
}
testutil.HandleTestingErr(task.Insert(), t, "error inserting task")
wrapper := doomedTaskWrapper{
reason: HeartbeatTimeout,
task: *task,
}
projects := map[string]model.Project{
"proj": model.Project{
Identifier: "proj",
Stepback: false,
},
}
h := &host.Host{
Id: "h1",
RunningTask: "t1",
}
So(h.Insert(), ShouldBeNil)
build := &build.Build{
Id: "b1",
Tasks: []build.TaskCache{{Id: "t1"}},
Version: "v1",
}
So(build.Insert(), ShouldBeNil)
v := &version.Version{Id: "v1"}
So(v.Insert(), ShouldBeNil)
// cleaning up the task should work
So(cleanUpTask(wrapper, projects), ShouldBeNil)
// refresh the host, make sure its running task field has
// been reset
h, err := host.FindOne(host.ById("h1"))
So(err, ShouldBeNil)
So(h.RunningTask, ShouldEqual, "")
})
})
})
}
func (hwp *JSONPlugin) GetUIHandler() http.Handler {
r := mux.NewRouter()
r.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
plugin.WriteJSON(w, http.StatusOK, "1")
})
r.HandleFunc("/task/{task_id}/{name}/", func(w http.ResponseWriter, r *http.Request) {
var jsonForTask TaskJSON
err := db.FindOneQ(collection, db.Query(bson.M{"task_id": mux.Vars(r)["task_id"], "name": mux.Vars(r)["name"]}), &jsonForTask)
if err != nil {
if err != mgo.ErrNotFound {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Error(w, "{}", http.StatusNotFound)
return
}
plugin.WriteJSON(w, http.StatusOK, jsonForTask)
})
r.HandleFunc("/task/{task_id}/{name}/tags", func(w http.ResponseWriter, r *http.Request) {
t, err := model.FindTask(mux.Vars(r)["task_id"])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
tags := []struct {
Tag string `bson:"_id" json:"tag"`
}{}
err = db.Aggregate(collection, []bson.M{
{"$match": bson.M{"project_id": t.Project, "tag": bson.M{"$exists": true, "$ne": ""}}},
{"$project": bson.M{"tag": 1}}, bson.M{"$group": bson.M{"_id": "$tag"}},
}, &tags)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, tags)
})
r.HandleFunc("/task/{task_id}/{name}/tag", func(w http.ResponseWriter, r *http.Request) {
t, err := model.FindTask(mux.Vars(r)["task_id"])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if t == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
if r.Method == "DELETE" {
if _, err = db.UpdateAll(collection,
bson.M{"version_id": t.Version, "name": mux.Vars(r)["name"]},
bson.M{"$unset": bson.M{"tag": 1}}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, "")
}
inTag := struct {
Tag string `json:"tag"`
}{}
err = util.ReadJSONInto(r.Body, &inTag)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if len(inTag.Tag) == 0 {
http.Error(w, "tag must not be blank", http.StatusBadRequest)
return
}
_, err = db.UpdateAll(collection,
bson.M{"version_id": t.Version, "name": mux.Vars(r)["name"]},
bson.M{"$set": bson.M{"tag": inTag.Tag}})
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
plugin.WriteJSON(w, http.StatusOK, "")
})
r.HandleFunc("/tag/{project_id}/{tag}/{variant}/{task_name}/{name}", func(w http.ResponseWriter, r *http.Request) {
var jsonForTask TaskJSON
err := db.FindOneQ(collection,
db.Query(bson.M{"project_id": mux.Vars(r)["project_id"],
"tag": mux.Vars(r)["tag"],
"variant": mux.Vars(r)["variant"],
"task_name": mux.Vars(r)["task_name"],
"name": mux.Vars(r)["name"],
}), &jsonForTask)
if err != nil {
if err != mgo.ErrNotFound {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Error(w, "{}", http.StatusNotFound)
return
}
plugin.WriteJSON(w, http.StatusOK, jsonForTask)
})
r.HandleFunc("/commit/{project_id}/{revision}/{variant}/{task_name}/{name}", func(w http.ResponseWriter, r *http.Request) {
var jsonForTask TaskJSON
err := db.FindOneQ(collection,
db.Query(bson.M{"project_id": mux.Vars(r)["project_id"],
"revision": bson.RegEx{"^" + regexp.QuoteMeta(mux.Vars(r)["revision"]), "i"},
"variant": mux.Vars(r)["variant"],
"task_name": mux.Vars(r)["task_name"],
"name": mux.Vars(r)["name"],
"is_patch": false,
}), &jsonForTask)
if err != nil {
if err != mgo.ErrNotFound {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
http.Error(w, "{}", http.StatusNotFound)
return
}
plugin.WriteJSON(w, http.StatusOK, jsonForTask)
})
r.HandleFunc("/history/{task_id}/{name}", func(w http.ResponseWriter, r *http.Request) {
t, err := model.FindTask(mux.Vars(r)["task_id"])
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
if t == nil {
http.Error(w, "{}", http.StatusNotFound)
return
}
before := []TaskJSON{}
jsonQuery := db.Query(bson.M{
"project_id": t.Project,
"variant": t.BuildVariant,
"order": bson.M{"$lte": t.RevisionOrderNumber},
"task_name": t.DisplayName,
"is_patch": false,
"name": mux.Vars(r)["name"]})
jsonQuery = jsonQuery.Sort([]string{"-order"}).Limit(100)
err = db.FindAllQ(collection, jsonQuery, &before)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//reverse order of "before" because we had to sort it backwards to apply the limit correctly:
for i, j := 0, len(before)-1; i < j; i, j = i+1, j-1 {
before[i], before[j] = before[j], before[i]
}
after := []TaskJSON{}
jsonAfterQuery := db.Query(bson.M{
"project_id": t.Project,
"variant": t.BuildVariant,
"order": bson.M{"$gt": t.RevisionOrderNumber},
"task_name": t.DisplayName,
"is_patch": false,
"name": mux.Vars(r)["name"]}).Sort([]string{"order"}).Limit(100)
err = db.FindAllQ(collection, jsonAfterQuery, &after)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
//concatenate before + after
before = append(before, after...)
plugin.WriteJSON(w, http.StatusOK, before)
})
return r
}
